From 1f63eb947965876ec3d8f28027754b259d904e4f Mon Sep 17 00:00:00 2001 From: Naveen Swamy Date: Fri, 26 May 2017 17:11:50 -0700 Subject: [PATCH 001/834] Use sphinx==1.3.5 in Dockerfile.doc (#6470) changed PR name --- tests/ci_build/Dockerfile.doc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci_build/Dockerfile.doc b/tests/ci_build/Dockerfile.doc index a09adcba06ef..73f138139922 100644 --- a/tests/ci_build/Dockerfile.doc +++ b/tests/ci_build/Dockerfile.doc @@ -9,4 +9,4 @@ RUN wget http://downloads.lightbend.com/scala/2.11.8/scala-2.11.8.deb && \ dpkg -i scala-2.11.8.deb && rm scala-2.11.8.deb RUN apt-get install -y doxygen libatlas-base-dev graphviz pandoc -RUN pip install sphinx CommonMark==0.5.4 breathe mock recommonmark pypandoc +RUN pip install sphinx==1.3.5 CommonMark==0.5.4 breathe mock recommonmark pypandoc From 4f27d2324ce98001471db1e3e1a088559e768c00 Mon Sep 17 00:00:00 2001 From: Naveen Swamy Date: Fri, 26 May 2017 17:25:08 -0700 Subject: [PATCH 002/834] Add 0.10 release info to README.md and NEWS.md (#6471) @nswamy wants to merge it immediately, so i'm going to do it now. I also changed the PR title. --- NEWS.md | 9 +++++++++ README.md | 1 + 2 files changed, 10 insertions(+) diff --git a/NEWS.md b/NEWS.md index f29119be897e..2557aadfed27 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,14 @@ MXNet Change Log ================ +## 0.10.0 +- Overhauled documentation for commonly used Python APIs, Installation instructions, Tutorials, HowTos and MXNet Architecture. +- Updated mxnet.io for improved readability. +- Pad operator now support reflection padding. +- Fixed a memory corruption error in threadedengine. +- Added CTC loss layer to contrib package. See mx.contrib.sym.ctc_loss. +- Added new sampling operators for several distributions (normal,uniform,gamma,exponential,negative binomial). +- Added documentation for experimental RNN APIs. + ## 0.9.3 - Move symbolic API to NNVM @tqchen - Most front-end C API are backward compatible diff --git a/README.md b/README.md index 172d61d93529..a06c8f0682ff 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ deep learning systems, and interesting insights of DL systems for hackers. What's New ---------- +* [Version 0.10.0 Release](https://github.com/dmlc/mxnet/releases/tag/v0.10.0) - MXNet 0.10.0 Release. * [Version 0.9.3 Release](./docs/architecture/release_note_0_9.md) - First 0.9 official release. * [Version 0.9.1 Release (NNVM refactor)](./docs/architecture/release_note_0_9.md) - NNVM branch is merged into master now. An official release will be made soon. * [Version 0.8.0 Release](https://github.com/dmlc/mxnet/releases/tag/v0.8.0) From b35dc5645b64cc9834b76cd6f4d833fddb3017dd Mon Sep 17 00:00:00 2001 From: wenxuanxie Date: Sat, 27 May 2017 11:37:15 +0800 Subject: [PATCH 003/834] Update im2rec.py (#6473) Updated Line 107 of 'im2rec.py'. Read an image as binary. --- tools/im2rec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/im2rec.py b/tools/im2rec.py index 17b8754b1f63..380ad1e43162 100644 --- a/tools/im2rec.py +++ b/tools/im2rec.py @@ -104,7 +104,7 @@ def image_encode(args, i, item, q_out): if args.pass_through: try: - with open(fullpath) as fin: + with open(fullpath, 'rb') as fin: img = fin.read() s = mx.recordio.pack(header, img) q_out.put((i, s, item)) From 05e07287a7fde466e92f2874abcd5b684ba2a49e Mon Sep 17 00:00:00 2001 From: ziheng Date: Mon, 29 May 2017 23:37:08 -0700 Subject: [PATCH 004/834] Change Interface of NDArray & TBlob for DLPack Compatible (#6345) * Change Interface of NDArray & TBlob for DLPack Compatible Fix for cudnn operator Fix cpp tests * Update nnvm * Fix for MKL mem * Fix for windows macro * Bump up version number to 0.10.1 * Update NDArray Save&Load * trigger update * Add test for legacy data load * Use LegacyTShapeLoad * trigger update * Update tensor_blob.h --- .gitmodules | 3 + CMakeLists.txt | 1 + Makefile | 6 +- R-package/DESCRIPTION | 2 +- dlpack | 1 + include/mxnet/base.h | 2 +- include/mxnet/c_api.h | 2 +- include/mxnet/ndarray.h | 47 ++++--- include/mxnet/tensor_blob.h | 128 ++++++++++++------ nnvm | 2 +- python/mxnet/libinfo.py | 2 +- .../assembly/linux-x86_64-cpu/pom.xml | 4 +- .../assembly/linux-x86_64-gpu/pom.xml | 4 +- scala-package/assembly/osx-x86_64-cpu/pom.xml | 4 +- scala-package/assembly/pom.xml | 4 +- scala-package/core/pom.xml | 4 +- scala-package/examples/pom.xml | 6 +- .../init-native/linux-x86_64/pom.xml | 4 +- scala-package/init-native/osx-x86_64/pom.xml | 4 +- scala-package/init-native/pom.xml | 4 +- scala-package/init/pom.xml | 4 +- scala-package/macros/pom.xml | 4 +- scala-package/native/linux-x86_64-cpu/pom.xml | 4 +- scala-package/native/linux-x86_64-gpu/pom.xml | 4 +- scala-package/native/osx-x86_64-cpu/pom.xml | 4 +- scala-package/native/pom.xml | 4 +- scala-package/pom.xml | 2 +- scala-package/spark/pom.xml | 4 +- snapcraft.yaml | 2 +- src/c_api/c_api.cc | 6 +- src/c_api/c_api_common.h | 14 +- src/c_api/c_api_ndarray.cc | 1 - src/c_api/c_api_symbolic.cc | 20 +-- src/c_api/c_predict_api.cc | 14 +- src/executor/graph_executor.cc | 4 +- src/io/image_io.cc | 4 +- src/io/iter_batchloader.h | 2 +- src/io/iter_csv.cc | 2 +- src/io/iter_image_recordio_2.cc | 2 +- src/ndarray/ndarray.cc | 30 +++- src/operator/cudnn_convolution-inl.h | 59 +++++--- src/operator/cudnn_deconvolution-inl.h | 38 ++++-- src/operator/custom/custom-inl.h | 11 +- src/operator/custom/custom.cc | 9 +- src/operator/custom/native_op-inl.h | 19 ++- src/operator/custom/ndarray_op-inl.h | 9 +- src/operator/deconvolution-inl.h | 6 +- src/operator/tensor/control_flow_op.h | 2 +- src/operator/tensor/matrix_op-inl.h | 28 ++-- tests/cpp/include/test_util.h | 4 +- tests/python/unittest/legacy_ndarray.v0 | Bin 0 -> 3224 bytes tests/python/unittest/test_ndarray.py | 9 ++ 52 files changed, 384 insertions(+), 175 deletions(-) create mode 160000 dlpack create mode 100644 tests/python/unittest/legacy_ndarray.v0 diff --git a/.gitmodules b/.gitmodules index 08f2bc99f2aa..bfe84d7f0615 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,6 +10,9 @@ [submodule "nnvm"] path = nnvm url = https://github.com/dmlc/nnvm +[submodule "dlpack"] + path = dlpack + url = https://github.com/dmlc/dlpack [submodule "cub"] path = cub url = https://github.com/NVlabs/cub diff --git a/CMakeLists.txt b/CMakeLists.txt index c8260e94e9bc..d0835300edaa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -142,6 +142,7 @@ include_directories("mshadow") include_directories("cub") include_directories("nnvm/include") include_directories("dmlc-core/include") +include_directories("dlpack/include") if(NOT MSVC) set(BEGIN_WHOLE_ARCHIVE -Wl,--whole-archive) diff --git a/Makefile b/Makefile index c71cb1398963..501a170abeda 100644 --- a/Makefile +++ b/Makefile @@ -18,6 +18,10 @@ ifndef NNVM_PATH NNVM_PATH = $(ROOTDIR)/nnvm endif +ifndef DLPACK_PATH + DLPACK_PATH = $(ROOTDIR)/dlpack +endif + ifneq ($(USE_OPENMP), 1) export NO_OPENMP = 1 endif @@ -49,7 +53,7 @@ ifeq ($(DEBUG), 1) else CFLAGS += -O3 -DNDEBUG=1 endif -CFLAGS += -I$(ROOTDIR)/mshadow/ -I$(ROOTDIR)/dmlc-core/include -fPIC -I$(NNVM_PATH)/include -Iinclude $(MSHADOW_CFLAGS) +CFLAGS += -I$(ROOTDIR)/mshadow/ -I$(ROOTDIR)/dmlc-core/include -fPIC -I$(NNVM_PATH)/include -I$(DLPACK_PATH)/include -Iinclude $(MSHADOW_CFLAGS) LDFLAGS = -pthread $(MSHADOW_LDFLAGS) $(DMLC_LDFLAGS) ifeq ($(DEBUG), 1) NVCCFLAGS = -std=c++11 -Xcompiler -D_FORCE_INLINES -g -G -O0 -ccbin $(CXX) $(MSHADOW_NVCCFLAGS) diff --git a/R-package/DESCRIPTION b/R-package/DESCRIPTION index 6aed92808020..2c8c8aa04d8d 100644 --- a/R-package/DESCRIPTION +++ b/R-package/DESCRIPTION @@ -1,7 +1,7 @@ Package: mxnet Type: Package Title: MXNet -Version: 0.10.0 +Version: 0.10.1 Date: 2015-12-23 Author: Tianqi Chen, Qiang Kou, Tong He Maintainer: Qiang Kou diff --git a/dlpack b/dlpack new file mode 160000 index 000000000000..a6e09b58dc00 --- /dev/null +++ b/dlpack @@ -0,0 +1 @@ +Subproject commit a6e09b58dc00ee0065f5b7879800e646fbb01d1e diff --git a/include/mxnet/base.h b/include/mxnet/base.h index 8747109ce564..0c4c9d3daa77 100644 --- a/include/mxnet/base.h +++ b/include/mxnet/base.h @@ -87,7 +87,7 @@ /*! \brief minor version */ #define MXNET_MINOR 10 /*! \brief patch version */ -#define MXNET_PATCH 0 +#define MXNET_PATCH 1 /*! \brief mxnet version */ #define MXNET_VERSION (MXNET_MAJOR*10000 + MXNET_MINOR*100 + MXNET_PATCH) /*! \brief helper for making version number */ diff --git a/include/mxnet/c_api.h b/include/mxnet/c_api.h index 1b112abe2ba9..4508a51e64d4 100644 --- a/include/mxnet/c_api.h +++ b/include/mxnet/c_api.h @@ -390,7 +390,7 @@ MXNET_DLL int MXNDArrayGetShape(NDArrayHandle handle, const mx_uint **out_pdata); /*! * \brief get the content of the data in NDArray - * \param handle the handle to the narray + * \param handle the handle to the ndarray * \param out_pdata pointer holder to get pointer of data * \return 0 when success, -1 when failure happens */ diff --git a/include/mxnet/ndarray.h b/include/mxnet/ndarray.h index ea38909d07f1..b8cd550118d3 100644 --- a/include/mxnet/ndarray.h +++ b/include/mxnet/ndarray.h @@ -57,10 +57,10 @@ class AutogradRuntime; */ class NDArray { public: - /*! \brief default cosntructor */ + /*! \brief default constructor */ NDArray() { #if MKL_EXPERIMENTAL == 1 - Mkl_mem_ = MKLMemHolder::create(); + Mkl_mem_ = MKLMemHolder::create(); #endif } /*! @@ -75,7 +75,7 @@ class NDArray { : ptr_(std::make_shared(shape.Size(), ctx, delay_alloc, dtype)), shape_(shape), offset_(0), dtype_(dtype), entry_({nullptr, 0, 0}) { #if MKL_EXPERIMENTAL == 1 - Mkl_mem_ = std::make_shared(); + Mkl_mem_ = std::make_shared(); #endif } /*! @@ -89,29 +89,32 @@ class NDArray { : ptr_(std::make_shared(data, dev_id)), shape_(data.shape_), offset_(0), dtype_(data.type_flag_), entry_({nullptr, 0, 0}) { #if MKL_EXPERIMENTAL == 1 - Mkl_mem_ = std::make_shared(); + Mkl_mem_ = std::make_shared(); #endif } /*! * \return the shape of current NDArray */ - inline const TShape &shape() const { + inline const TShape& shape() const { return shape_; } /*! * \return the data TBlob */ - inline TBlob data() const { + inline const TBlob& data() const { CheckAndAlloc(); - TBlob res; +#if MKL_EXPERIMENTAL == 1 MSHADOW_TYPE_SWITCH(dtype_, DType, { - res = TBlob(static_cast(ptr_->shandle.dptr) - + offset_, shape_, ptr_->shandle.ctx.dev_mask()); + tblob_ = TBlob(static_cast(ptr_->shandle.dptr) + offset_, + shape_, ptr_->shandle.ctx.dev_mask(), ptr_->shandle.ctx.dev_id, Mkl_mem_); + }); +#else + MSHADOW_TYPE_SWITCH(dtype_, DType, { + tblob_ = TBlob(static_cast(ptr_->shandle.dptr) + offset_, + shape_, ptr_->shandle.ctx.dev_mask(), ptr_->shandle.ctx.dev_id); }); -#if MKL_EXPERIMENTAL == 1 - res.Mkl_mem_ = Mkl_mem_; #endif - return res; + return tblob_; } /*! * \return a chunk of raw data in TBlob @@ -122,8 +125,8 @@ class NDArray { TShape raw_shape(1); raw_shape[0] = length; MSHADOW_TYPE_SWITCH(dtype_, DType, { - res = TBlob(static_cast(ptr_->shandle.dptr) - + offset_ + offset, raw_shape, ptr_->shandle.ctx.dev_mask()); + res = TBlob(static_cast(ptr_->shandle.dptr) + offset_ + offset, + raw_shape, ptr_->shandle.ctx.dev_mask(), ptr_->shandle.ctx.dev_id); }); #if MKL_EXPERIMENTAL == 1 res.Mkl_mem_ = Mkl_mem_; @@ -326,7 +329,7 @@ class NDArray { ptr_->CheckAndAlloc(); } /*! - * \brief Save list of narray into the Stream.x + * \brief Save list of ndarray into the Stream.x * \param fo The stream of output. * \param data the NDArrays to be saved. * \param names the name of the NDArray, optional, can be zero length. @@ -335,7 +338,7 @@ class NDArray { const std::vector& data, const std::vector& names); /*! - * \brief Load list of narray into from the stream. + * \brief Load list of ndarray into from the stream. * \param fi The stream of the input file. * \param data the NDArrays to be loaded * \param keys the name of the NDArray, if saved in the file. @@ -368,10 +371,10 @@ class NDArray { : static_data(true), delay_alloc(false) { var = Engine::Get()->NewVariable(); - if (data.dev_mask_ == cpu::kDevMask) { + if (data.dev_mask() == cpu::kDevMask) { shandle.ctx = Context::CPU(); } else { - CHECK_EQ(data.dev_mask_, gpu::kDevMask); + CHECK_EQ(data.dev_mask(), gpu::kDevMask); shandle.ctx = Context::GPU(dev_id); } shandle.dptr = data.dptr_; @@ -418,6 +421,14 @@ class NDArray { int dtype_ = -1; /*! \brief node entry for autograd */ autograd::AGNodeEntry entry_; + /*! + * \brief internal TBlob + * \note When user access tblob_ by some const methods like + * NDArray::data(), the dptr in tblob_ still need to be updated + * in case that allocation happens. So we make it mutable for + * this situation. + */ + mutable TBlob tblob_; }; /*! diff --git a/include/mxnet/tensor_blob.h b/include/mxnet/tensor_blob.h index e4e335666d80..d142c20aa30a 100755 --- a/include/mxnet/tensor_blob.h +++ b/include/mxnet/tensor_blob.h @@ -11,6 +11,7 @@ #include #include +#include #include #include #include @@ -39,12 +40,6 @@ class TBlob { void *dptr_; /*! \brief shape of the tensor */ TShape shape_; - /*! - * \brief storing the stride information in x dimension - */ - index_t stride_; - /*! \brief device mask of the corresponding device */ - int dev_mask_; /*! \brief type flag of the tensor blob */ int type_flag_; @@ -54,49 +49,61 @@ class TBlob { #endif /*! \brief default constructor, default copy assign will work */ TBlob(void) - : dptr_(NULL), dev_mask_(cpu::kDevMask), + : dptr_(NULL), type_flag_(mshadow::DataType::kFlag) { #if MKL_EXPERIMENTAL == 1 - Mkl_mem_ = NULL; + Mkl_mem_ = NULL; #endif + SetDLTensor(cpu::kDevMask, 0); } /*! * \brief constructor that construct TBlob from contiguous memory * \param dptr the pointer to the memory * \param shape the shape of the data * \param dev_mask the device mask, can be cpu::kDevMask or gpu::kDevMask + * \param dev_id the device id */ template - TBlob(DType *dptr, - const TShape &shape, - int dev_mask) + TBlob(DType *dptr, const TShape &shape, int dev_mask, int dev_id = -1) : dptr_(dptr), shape_(shape), - stride_(shape[shape.ndim() - 1]), - dev_mask_(dev_mask), type_flag_(mshadow::DataType::kFlag) { #if MKL_EXPERIMENTAL == 1 - Mkl_mem_ = NULL; + Mkl_mem_ = NULL; #endif + SetDLTensor(dev_mask, dev_id); } - +#if MKL_EXPERIMENTAL == 1 /*! * \brief constructor that construct TBlob from contiguous memory * \param dptr the pointer to the memory * \param shape the shape of the data * \param dev_mask the device mask, can be cpu::kDevMask or gpu::kDevMask - * \param type_flag the type flag. Can be one of enum mshadow::dtype + * \param dev_id the device id + * \param Mkl_mem the mkl memory */ - TBlob(void *dptr, - const TShape &shape, - int dev_mask, - int type_flag) + template + TBlob(DType *dptr, const TShape &shape, int dev_mask, int dev_id, + std::shared_ptr Mkl_mem) : dptr_(dptr), shape_(shape), - stride_(shape[shape.ndim() - 1]), - dev_mask_(dev_mask), - type_flag_(type_flag) { + type_flag_(mshadow::DataType::kFlag), + Mkl_mem_(Mkl_mem) { + SetDLTensor(dev_mask, dev_id); + } +#endif + /*! + * \brief constructor that construct TBlob from contiguous memory + * \param dptr the pointer to the memory + * \param shape the shape of the data + * \param dev_mask the device mask, can be cpu::kDevMask or gpu::kDevMask + * \param type_flag the type flag. Can be one of enum mshadow::dtype + * \param dev_id the device id + */ + TBlob(void *dptr, const TShape &shape, int dev_mask, int type_flag, int dev_id = -1) + : dptr_(dptr), shape_(shape), type_flag_(type_flag) { #if MKL_EXPERIMENTAL == 1 - Mkl_mem_ = NULL; + Mkl_mem_ = NULL; #endif + SetDLTensor(dev_mask, dev_id); } /*! * \brief constructor from tensor @@ -108,9 +115,6 @@ class TBlob { template TBlob(const mshadow::Tensor &src) { // NOLINT(*) *this = src; -#if MKL_EXPERIMENTAL == 1 - Mkl_mem_ = NULL; -#endif } /*! * \brief assignment from tensor @@ -121,20 +125,21 @@ class TBlob { * \return reference of self */ template - inline TBlob - &operator=(const mshadow::Tensor &src) { + inline TBlob &operator=(const mshadow::Tensor &src) { dptr_ = src.dptr_; shape_ = src.shape_; - stride_ = src.stride_; - dev_mask_ = Device::kDevMask; type_flag_ = mshadow::DataType::kFlag; + SetDLTensor(Device::kDevMask, -1); +#if MKL_EXPERIMENTAL == 1 + Mkl_mem_ = NULL; +#endif return *this; } /*! * \return whether the tensor's memory is continuous */ inline bool CheckContiguous(void) const { - return shape_[shape_.ndim() - 1] == stride_; + return true; } /*! * \brief reshape to shape @@ -144,7 +149,7 @@ class TBlob { inline TBlob reshape(const TShape& shape) const { CHECK_EQ(this->shape_.Size(), shape.Size()) << "Shape size mismatch " << this->shape_.Size() << " v.s. " << shape.Size(); - TBlob ret(this->dptr_, shape, this->dev_mask_, this->type_flag_); + TBlob ret(this->dptr_, shape, this->dev_mask(), this->type_flag_, this->dev_id()); return ret; } /*! @@ -157,7 +162,7 @@ class TBlob { template inline mshadow::Tensor FlatTo2D( mshadow::Stream *stream = NULL) const { - CHECK(Device::kDevMask == dev_mask_) + CHECK(Device::kDevMask == this->dev_mask()) << "TBlob.get: device type do not match specified type"; CHECK(mshadow::DataType::kFlag == type_flag_) << "TBlob.get_with_shape: data type do not match specified type." @@ -168,7 +173,9 @@ class TBlob { } #endif return mshadow::Tensor(static_cast(dptr_), - shape_.FlatTo2D(), stride_, stream); + shape_.FlatTo2D(), + shape_[shape_.ndim() - 1], + stream); } /*! * \brief flatten the tensor to 1 dimension, collapse all the dimensions together. @@ -212,6 +219,22 @@ class TBlob { #endif return static_cast(dptr_); } + /*! \brief device mask of the corresponding device */ + inline int dev_mask() const { + return dltensor_.ctx.device_type; + } + /*! \brief device index of the corresponding device */ + inline int dev_id() const { + return dltensor_.ctx.device_id; + } + /*! + * \brief return the corresponding DLTensor + * \return the address of internal DLTensor + */ + inline const DLTensor& dltensor() { + return dltensor_; + } + /*! * \brief fetch the tensor, with respect to specific dimension * if dim do not match the stored dimension, an error will be issued @@ -223,9 +246,10 @@ class TBlob { */ template inline mshadow::Tensor get(mshadow::Stream *stream = NULL) const { - CHECK(Device::kDevMask == dev_mask_) + CHECK(Device::kDevMask == this->dev_mask()) << "TBlob.get: device type do not match specified type"; - return mshadow::Tensor(dptr(), shape_.get(), stride_, stream); + return mshadow::Tensor(dptr(), + shape_.get(), shape_[shape_.ndim() - 1], stream); } /*! * \brief fetch a tensor in given shape @@ -241,7 +265,7 @@ class TBlob { inline mshadow::Tensor get_with_shape( const mshadow::Shape &shape, mshadow::Stream *stream = NULL) const { - CHECK(Device ::kDevMask == dev_mask_) + CHECK(Device::kDevMask == this->dev_mask()) << "TBlob.get: device type do not match specified type"; CHECK_EQ(this->CheckContiguous(), true) << "TBlob.get_reshape: must be contiguous"; CHECK_EQ(this->shape_.Size(), shape.Size()) @@ -281,6 +305,34 @@ class TBlob { return this->get_with_shape( this->shape_.FlatTo3D(axis_begin, axis_end), stream); } + + private: + static DLDataType DTypeTransform(int type_flag) { + static std::unordered_map + MSHADOW_DTYPE_TO_DLPACK_DTYPE = { + {0, {2, 32, 1}}, // Float32 + {1, {2, 64, 1}}, // Float64 + {2, {2, 16, 1}}, // Float16 + {3, {1, 8, 1}}, // UInt8 + {4, {0, 32, 1}}, // Int32 + {5, {0, 8, 1}} // Int8 + }; + return MSHADOW_DTYPE_TO_DLPACK_DTYPE[type_flag]; + } + + inline void SetDLTensor(int dev_mask, int dev_id) { + dltensor_.data = dptr_; + dltensor_.ctx = DLContext{static_cast(dev_mask), dev_id}; + dltensor_.ndim = shape_.ndim(); + dltensor_.dtype = DTypeTransform(type_flag_); + dltensor_.shape = shape_.data(); + dltensor_.strides = NULL; + dltensor_.byte_offset = 0; + } + + private: + /*! \brief corresponding DLTensor of this TBlob */ + DLTensor dltensor_; }; } // namespace mxnet diff --git a/nnvm b/nnvm index b279286304ac..93072dc8733a 160000 --- a/nnvm +++ b/nnvm @@ -1 +1 @@ -Subproject commit b279286304ac954098d94a2695bca599e832effb +Subproject commit 93072dc8733aa2a89459ecf16413d96ad0b998db diff --git a/python/mxnet/libinfo.py b/python/mxnet/libinfo.py index 57b0a2c18130..a24756632c10 100644 --- a/python/mxnet/libinfo.py +++ b/python/mxnet/libinfo.py @@ -44,4 +44,4 @@ def find_lib_path(): # current version -__version__ = "0.10.0" +__version__ = "0.10.1" diff --git a/scala-package/assembly/linux-x86_64-cpu/pom.xml b/scala-package/assembly/linux-x86_64-cpu/pom.xml index d6639973d5c4..138c5c84304f 100644 --- a/scala-package/assembly/linux-x86_64-cpu/pom.xml +++ b/scala-package/assembly/linux-x86_64-cpu/pom.xml @@ -6,13 +6,13 @@ ml.dmlc.mxnet mxnet-full-parent_2.11 - 0.10.0-SNAPSHOT + 0.10.1-SNAPSHOT ../pom.xml ml.dmlc.mxnet mxnet-full_2.11-linux-x86_64-cpu - 0.10.0-SNAPSHOT + 0.10.1-SNAPSHOT MXNet Scala Package - Full Linux-x86_64 CPU-only jar diff --git a/scala-package/assembly/linux-x86_64-gpu/pom.xml b/scala-package/assembly/linux-x86_64-gpu/pom.xml index 38b2bd623865..7e818cb28123 100644 --- a/scala-package/assembly/linux-x86_64-gpu/pom.xml +++ b/scala-package/assembly/linux-x86_64-gpu/pom.xml @@ -6,13 +6,13 @@ ml.dmlc.mxnet mxnet-full-parent_2.11 - 0.10.0-SNAPSHOT + 0.10.1-SNAPSHOT ../pom.xml ml.dmlc.mxnet mxnet-full_2.11-linux-x86_64-gpu - 0.10.0-SNAPSHOT + 0.10.1-SNAPSHOT MXNet Scala Package - Full Linux-x86_64 GPU jar diff --git a/scala-package/assembly/osx-x86_64-cpu/pom.xml b/scala-package/assembly/osx-x86_64-cpu/pom.xml index f72be6dc17ff..ead035668892 100644 --- a/scala-package/assembly/osx-x86_64-cpu/pom.xml +++ b/scala-package/assembly/osx-x86_64-cpu/pom.xml @@ -6,13 +6,13 @@ ml.dmlc.mxnet mxnet-full-parent_2.11 - 0.10.0-SNAPSHOT + 0.10.1-SNAPSHOT ../pom.xml ml.dmlc.mxnet mxnet-full_2.11-osx-x86_64-cpu - 0.10.0-SNAPSHOT + 0.10.1-SNAPSHOT MXNet Scala Package - Full OSX-x86_64 CPU-only jar diff --git a/scala-package/assembly/pom.xml b/scala-package/assembly/pom.xml index b5b52ff35646..a1009ae6b08c 100644 --- a/scala-package/assembly/pom.xml +++ b/scala-package/assembly/pom.xml @@ -6,13 +6,13 @@ ml.dmlc.mxnet mxnet-parent_2.11 - 0.10.0-SNAPSHOT + 0.10.1-SNAPSHOT ../pom.xml ml.dmlc.mxnet mxnet-full-parent_2.11 - 0.10.0-SNAPSHOT + 0.10.1-SNAPSHOT MXNet Scala Package - Full Parent pom diff --git a/scala-package/core/pom.xml b/scala-package/core/pom.xml index 592e15b5c49e..7f639b9a8d39 100644 --- a/scala-package/core/pom.xml +++ b/scala-package/core/pom.xml @@ -6,13 +6,13 @@ ml.dmlc.mxnet mxnet-parent_2.11 - 0.10.0-SNAPSHOT + 0.10.1-SNAPSHOT ../pom.xml ml.dmlc.mxnet mxnet-core_2.11 - 0.10.0-SNAPSHOT + 0.10.1-SNAPSHOT MXNet Scala Package - Core diff --git a/scala-package/examples/pom.xml b/scala-package/examples/pom.xml index fa99ed9c44da..bda4fcdab5c4 100644 --- a/scala-package/examples/pom.xml +++ b/scala-package/examples/pom.xml @@ -6,12 +6,12 @@ ml.dmlc.mxnet mxnet-parent_2.11 - 0.10.0-SNAPSHOT + 0.10.1-SNAPSHOT ../pom.xml mxnet-examples_2.11 - 0.10.0-SNAPSHOT + 0.10.1-SNAPSHOT MXNet Scala Package - Examples @@ -84,7 +84,7 @@ package copy-dependencies - + ${project.build.outputDirectory}/lib runtime diff --git a/scala-package/init-native/linux-x86_64/pom.xml b/scala-package/init-native/linux-x86_64/pom.xml index 61f49e598a3d..7e6c02aefd83 100644 --- a/scala-package/init-native/linux-x86_64/pom.xml +++ b/scala-package/init-native/linux-x86_64/pom.xml @@ -6,12 +6,12 @@ ml.dmlc.mxnet mxnet-scala-init-native-parent - 0.10.0-SNAPSHOT + 0.10.1-SNAPSHOT ../pom.xml libmxnet-init-scala-linux-x86_64 - 0.10.0-SNAPSHOT + 0.10.1-SNAPSHOT MXNet Scala Package - Initializer Native Linux-x86_64 http://maven.apache.org diff --git a/scala-package/init-native/osx-x86_64/pom.xml b/scala-package/init-native/osx-x86_64/pom.xml index 449f66e3ba7f..4f5125c06f15 100644 --- a/scala-package/init-native/osx-x86_64/pom.xml +++ b/scala-package/init-native/osx-x86_64/pom.xml @@ -6,12 +6,12 @@ ml.dmlc.mxnet mxnet-scala-init-native-parent - 0.10.0-SNAPSHOT + 0.10.1-SNAPSHOT ../pom.xml libmxnet-init-scala-osx-x86_64 - 0.10.0-SNAPSHOT + 0.10.1-SNAPSHOT MXNet Scala Package - Initializer Native OSX-x86_64 http://maven.apache.org diff --git a/scala-package/init-native/pom.xml b/scala-package/init-native/pom.xml index 8e02d45d015f..3ce227a9b587 100644 --- a/scala-package/init-native/pom.xml +++ b/scala-package/init-native/pom.xml @@ -6,12 +6,12 @@ ml.dmlc.mxnet mxnet-parent_2.11 - 0.10.0-SNAPSHOT + 0.10.1-SNAPSHOT ../pom.xml mxnet-scala-init-native-parent - 0.10.0-SNAPSHOT + 0.10.1-SNAPSHOT MXNet Scala Package - Initializer Native Parent pom diff --git a/scala-package/init/pom.xml b/scala-package/init/pom.xml index 44bf7a677abc..9f079565874e 100644 --- a/scala-package/init/pom.xml +++ b/scala-package/init/pom.xml @@ -6,12 +6,12 @@ ml.dmlc.mxnet mxnet-parent_2.11 - 0.10.0-SNAPSHOT + 0.10.1-SNAPSHOT ../pom.xml mxnet-init_2.11 - 0.10.0-SNAPSHOT + 0.10.1-SNAPSHOT MXNet Scala Package - Initializer diff --git a/scala-package/macros/pom.xml b/scala-package/macros/pom.xml index aec0c2897fe5..fd7fe3e4ab7b 100644 --- a/scala-package/macros/pom.xml +++ b/scala-package/macros/pom.xml @@ -6,12 +6,12 @@ ml.dmlc.mxnet mxnet-parent_2.11 - 0.10.0-SNAPSHOT + 0.10.1-SNAPSHOT ../pom.xml mxnet-macros_2.11 - 0.10.0-SNAPSHOT + 0.10.1-SNAPSHOT MXNet Scala Package - Macros diff --git a/scala-package/native/linux-x86_64-cpu/pom.xml b/scala-package/native/linux-x86_64-cpu/pom.xml index 4aae3d8f1bf9..b2cfa4263cda 100644 --- a/scala-package/native/linux-x86_64-cpu/pom.xml +++ b/scala-package/native/linux-x86_64-cpu/pom.xml @@ -6,13 +6,13 @@ ml.dmlc.mxnet mxnet-scala-native-parent - 0.10.0-SNAPSHOT + 0.10.1-SNAPSHOT ../pom.xml ml.dmlc.mxnet libmxnet-scala-linux-x86_64-cpu - 0.10.0-SNAPSHOT + 0.10.1-SNAPSHOT MXNet Scala Package - Native Linux-x86_64 CPU-only http://maven.apache.org diff --git a/scala-package/native/linux-x86_64-gpu/pom.xml b/scala-package/native/linux-x86_64-gpu/pom.xml index f0a158031ded..27f9221c3bad 100644 --- a/scala-package/native/linux-x86_64-gpu/pom.xml +++ b/scala-package/native/linux-x86_64-gpu/pom.xml @@ -6,13 +6,13 @@ ml.dmlc.mxnet mxnet-scala-native-parent - 0.10.0-SNAPSHOT + 0.10.1-SNAPSHOT ../pom.xml ml.dmlc.mxnet libmxnet-scala-linux-x86_64-gpu - 0.10.0-SNAPSHOT + 0.10.1-SNAPSHOT MXNet Scala Package - Native Linux-x86_64 GPU http://maven.apache.org diff --git a/scala-package/native/osx-x86_64-cpu/pom.xml b/scala-package/native/osx-x86_64-cpu/pom.xml index fa82d31ee386..f924106a605c 100644 --- a/scala-package/native/osx-x86_64-cpu/pom.xml +++ b/scala-package/native/osx-x86_64-cpu/pom.xml @@ -6,12 +6,12 @@ ml.dmlc.mxnet mxnet-scala-native-parent - 0.10.0-SNAPSHOT + 0.10.1-SNAPSHOT ../pom.xml libmxnet-scala-osx-x86_64-cpu - 0.10.0-SNAPSHOT + 0.10.1-SNAPSHOT MXNet Scala Package - Native OSX-x86_64 CPU-only http://maven.apache.org diff --git a/scala-package/native/pom.xml b/scala-package/native/pom.xml index dbf286c633e6..0af9e087f906 100644 --- a/scala-package/native/pom.xml +++ b/scala-package/native/pom.xml @@ -6,12 +6,12 @@ ml.dmlc.mxnet mxnet-parent_2.11 - 0.10.0-SNAPSHOT + 0.10.1-SNAPSHOT ../pom.xml mxnet-scala-native-parent - 0.10.0-SNAPSHOT + 0.10.1-SNAPSHOT MXNet Scala Package - Native Parent pom diff --git a/scala-package/pom.xml b/scala-package/pom.xml index 1eae0b9eb6ed..86d8cfc16a43 100644 --- a/scala-package/pom.xml +++ b/scala-package/pom.xml @@ -5,7 +5,7 @@ 4.0.0 ml.dmlc.mxnet mxnet-parent_2.11 - 0.10.0-SNAPSHOT + 0.10.1-SNAPSHOT MXNet Scala Package - Parent https://github.com/dmlc/mxnet/tree/master/scala-package MXNet Scala Package diff --git a/scala-package/spark/pom.xml b/scala-package/spark/pom.xml index 9d7b31909dfb..f35cbe45d9de 100644 --- a/scala-package/spark/pom.xml +++ b/scala-package/spark/pom.xml @@ -6,12 +6,12 @@ ml.dmlc.mxnet mxnet-parent_2.11 - 0.10.0-SNAPSHOT + 0.10.1-SNAPSHOT ../pom.xml mxnet-spark_2.11 - 0.10.0-SNAPSHOT + 0.10.1-SNAPSHOT MXNet Scala Package - Spark ML diff --git a/snapcraft.yaml b/snapcraft.yaml index a0073f2d4f1a..b9329a0ccd41 100644 --- a/snapcraft.yaml +++ b/snapcraft.yaml @@ -1,5 +1,5 @@ name: mxnet -version: '0.10.0' +version: '0.10.1' summary: MXNet is a deep learning framework designed for efficiency and flexibility. description: | MXNet is a deep learning framework designed for both efficiency and diff --git a/src/c_api/c_api.cc b/src/c_api/c_api.cc index ae7af5bad129..41986a0d577b 100644 --- a/src/c_api/c_api.cc +++ b/src/c_api/c_api.cc @@ -336,12 +336,16 @@ MXNET_DLL int MXNDArrayReshape(NDArrayHandle handle, int MXNDArrayGetShape(NDArrayHandle handle, mx_uint *out_dim, const mx_uint **out_pdata) { + MXAPIThreadLocalEntry *ret = MXAPIThreadLocalStore::Get(); API_BEGIN(); NDArray *arr = static_cast(handle); if (!arr->is_none()) { const TShape &s = arr->shape(); *out_dim = s.ndim(); - *out_pdata = s.data(); + std::vector& buffer = ret->arg_shape_buffer; + buffer.resize(s.ndim()); + nnvm::ShapeTypeCast(s.begin(), s.end(), buffer.data()); + *out_pdata = buffer.data(); } else { *out_dim = 0; } diff --git a/src/c_api/c_api_common.h b/src/c_api/c_api_common.h index e2e739ae62a4..d8857f80635d 100644 --- a/src/c_api/c_api_common.h +++ b/src/c_api/c_api_common.h @@ -62,16 +62,24 @@ struct MXAPIThreadLocalEntry { std::vector arg_shape_ndim, out_shape_ndim, aux_shape_ndim; /*! \brief result holder for returning shape pointer */ std::vector arg_shape_data, out_shape_data, aux_shape_data; + /*! \brief uint32_t buffer for returning shape pointer */ + std::vector arg_shape_buffer, out_shape_buffer, aux_shape_buffer; // helper function to setup return value of shape array - inline static void SetupShapeArrayReturn( + inline static void SetupShapeArrayReturnWithBuffer( const std::vector &shapes, std::vector *ndim, - std::vector *data) { + std::vector *data, + std::vector *buffer) { ndim->resize(shapes.size()); data->resize(shapes.size()); + size_t size = 0; + for (const auto& s : shapes) size += s.ndim(); + buffer->resize(size); + uint32_t *ptr = buffer->data(); for (size_t i = 0; i < shapes.size(); ++i) { ndim->at(i) = shapes[i].ndim(); - data->at(i) = shapes[i].data(); + data->at(i) = ptr; + ptr = nnvm::ShapeTypeCast(shapes[i].begin(), shapes[i].end(), ptr); } } }; diff --git a/src/c_api/c_api_ndarray.cc b/src/c_api/c_api_ndarray.cc index c633e8609cd4..66a237a4bd36 100644 --- a/src/c_api/c_api_ndarray.cc +++ b/src/c_api/c_api_ndarray.cc @@ -396,7 +396,6 @@ int MXImperativeInvoke(AtomicSymbolCreator creator, } } - if (outarray == nullptr) { ret->ret_handles.clear(); for (int i = 0; i < num_visible_outputs; ++i) { diff --git a/src/c_api/c_api_symbolic.cc b/src/c_api/c_api_symbolic.cc index f7281c999e6a..fdf095b09361 100644 --- a/src/c_api/c_api_symbolic.cc +++ b/src/c_api/c_api_symbolic.cc @@ -429,14 +429,14 @@ int MXSymbolInferShape(SymbolHandle sym, std::vector read_only_args = mxnet::ReadOnlyArgIndices(g.indexed_graph()); CHECK_LE(num_args, read_only_args.size()); for (mx_uint i = 0; i < num_args; ++i) { - arg_shapes[read_only_args[i]] = TShape(arg_shape_data + arg_ind_ptr[i], - arg_shape_data + arg_ind_ptr[i+1]); + arg_shapes[read_only_args[i]] = nnvm::ShapeTypeCast( + arg_shape_data + arg_ind_ptr[i], arg_shape_data + arg_ind_ptr[i+1]); } } else { std::unordered_map kwargs; for (mx_uint i = 0; i < num_args; ++i) { - kwargs[keys[i]] = TShape(arg_shape_data + arg_ind_ptr[i], - arg_shape_data + arg_ind_ptr[i+1]); + kwargs[keys[i]] = nnvm::ShapeTypeCast( + arg_shape_data + arg_ind_ptr[i], arg_shape_data + arg_ind_ptr[i+1]); } mxnet::MatchArguments(g.indexed_graph(), kwargs, &arg_shapes, "InferShape"); } @@ -452,12 +452,12 @@ int MXSymbolInferShape(SymbolHandle sym, &(ret->arg_shapes), &(ret->out_shapes), &(ret->aux_shapes)); // copy data back - MXAPIThreadLocalEntry::SetupShapeArrayReturn( - ret->arg_shapes, &(ret->arg_shape_ndim), &(ret->arg_shape_data)); - MXAPIThreadLocalEntry::SetupShapeArrayReturn( - ret->out_shapes, &(ret->out_shape_ndim), &(ret->out_shape_data)); - MXAPIThreadLocalEntry::SetupShapeArrayReturn( - ret->aux_shapes, &(ret->aux_shape_ndim), &(ret->aux_shape_data)); + MXAPIThreadLocalEntry::SetupShapeArrayReturnWithBuffer(ret->arg_shapes, + &(ret->arg_shape_ndim), &(ret->arg_shape_data), &(ret->arg_shape_buffer)); + MXAPIThreadLocalEntry::SetupShapeArrayReturnWithBuffer(ret->out_shapes, + &(ret->out_shape_ndim), &(ret->out_shape_data), &(ret->out_shape_buffer)); + MXAPIThreadLocalEntry::SetupShapeArrayReturnWithBuffer(ret->aux_shapes, + &(ret->aux_shape_ndim), &(ret->aux_shape_data), &(ret->aux_shape_buffer)); *in_shape_size = static_cast(ret->arg_shapes.size()); *in_shape_ndim = dmlc::BeginPtr(ret->arg_shape_ndim); *in_shape_data = dmlc::BeginPtr(ret->arg_shape_data); diff --git a/src/c_api/c_predict_api.cc b/src/c_api/c_predict_api.cc index 26bc44b701e5..1dd784ba2249 100644 --- a/src/c_api/c_predict_api.cc +++ b/src/c_api/c_predict_api.cc @@ -25,6 +25,8 @@ struct MXAPIPredictor { std::vector arg_arrays; // output shapes std::vector out_shapes; + // uint32_t buffer for output shapes + std::vector out_shapes_buffer; // key to arguments std::unordered_map key2arg; // executor @@ -34,6 +36,7 @@ struct MXAPIPredictor { struct MXAPINDList { std::vector keys; std::vector shapes; + std::vector shapes_buffer; std::vector indptr; std::vector data; }; @@ -228,7 +231,11 @@ int MXPredGetOutputShape(PredictorHandle handle, API_BEGIN(); CHECK_LT(out_index, p->out_arrays.size()) << "Index exceed number of outputs"; - *shape_data = p->out_shapes[out_index].data(); + + const TShape& s = p->out_shapes[out_index]; + p->out_shapes_buffer.resize(s.ndim()); + nnvm::ShapeTypeCast(s.begin(), s.end(), p->out_shapes_buffer.data()); + *shape_data = p->out_shapes_buffer.data(); *shape_ndim = p->out_shapes[out_index].ndim(); API_END(); } @@ -322,7 +329,10 @@ int MXNDListGet(NDListHandle handle, << "Index out of range"; *out_key = p->keys[index].c_str(); *out_data = dmlc::BeginPtr(p->data) + p->indptr[index]; - *out_shape = p->shapes[index].data(); + const TShape& s = p->shapes[index]; + p->shapes_buffer.resize(s.ndim()); + nnvm::ShapeTypeCast(s.begin(), s.end(), p->shapes_buffer.data()); + *out_shape = p->shapes_buffer.data(); *out_ndim = p->shapes[index].ndim(); API_END(); } diff --git a/src/executor/graph_executor.cc b/src/executor/graph_executor.cc index 6ba0ff96b382..cdbb129304b1 100644 --- a/src/executor/graph_executor.cc +++ b/src/executor/graph_executor.cc @@ -538,9 +538,9 @@ void GraphExecutor::InitDataEntryMemory(std::vector* shared_pool) { } if (!allocated) { size_t nword = (bytes + 3) / 4; - CHECK_LE(nword, std::numeric_limits::max()); + CHECK_LE(nword, std::numeric_limits::max()); // allocate float arrays - TShape shape{index_t(nword)}; + TShape shape{static_cast(nword)}; NDArray nd(shape, ctx); data_pool_[i] = nd; // put the new allocated arrays to shared pool diff --git a/src/io/image_io.cc b/src/io/image_io.cc index 9c65edd1aa87..1ef1df1b74bd 100644 --- a/src/io/image_io.cc +++ b/src/io/image_io.cc @@ -26,7 +26,7 @@ namespace io { // http://www.64lines.com/jpeg-width-height // Gets the JPEG size from the array of data passed to the function, // file reference: http://www.obrador.com/essentialjpeg/headerinfo.htm -bool get_jpeg_size(const uint8_t* data, uint32_t data_size, uint32_t *width, uint32_t *height) { +bool get_jpeg_size(const uint8_t* data, uint32_t data_size, int64_t *width, int64_t *height) { // Check for valid JPEG image uint32_t i = 0; // Keeps track of the position within the file if (data[i] == 0xFF && data[i+1] == 0xD8 && data[i+2] == 0xFF && data[i+3] == 0xE0) { @@ -63,7 +63,7 @@ bool get_jpeg_size(const uint8_t* data, uint32_t data_size, uint32_t *width, uin } } -bool get_png_size(const uint8_t* data, uint32_t data_size, uint32_t *width, uint32_t *height) { +bool get_png_size(const uint8_t* data, uint32_t data_size, int64_t *width, int64_t *height) { if (data[0] == 0x89 && data[1] == 0x50 && data[2] ==0x4E && data[3] == 0x47) { uint8_t const* p = data + 16; *width = ((p[0]*256 + p[1])*256 + p[2])*256 + p[3]; diff --git a/src/io/iter_batchloader.h b/src/io/iter_batchloader.h index 2b53393679c6..a51e24503785 100644 --- a/src/io/iter_batchloader.h +++ b/src/io/iter_batchloader.h @@ -145,7 +145,7 @@ class BatchLoader : public IIterator { shape_[i] = dst_shape; data_[i].resize(mshadow::Shape1(dst_shape.Size()), src_type_flag); unit_size_[i] = src_shape.Size(); - out_.data.push_back(TBlob(data_[i].dptr_, dst_shape, cpu::kDevMask, src_type_flag)); + out_.data.push_back(TBlob(data_[i].dptr_, dst_shape, cpu::kDevMask, src_type_flag, 0)); } } }; // class BatchLoader diff --git a/src/io/iter_csv.cc b/src/io/iter_csv.cc index 2817b4d8ff51..c43f99911f69 100644 --- a/src/io/iter_csv.cc +++ b/src/io/iter_csv.cc @@ -107,7 +107,7 @@ class CSVIter: public IIterator { << "The data size in CSV do not match size of shape: " << "specified shape=" << shape << ", the csv row-length=" << row.length; const real_t* ptr = row.value; - return TBlob((real_t*)ptr, shape, cpu::kDevMask); // NOLINT(*) + return TBlob((real_t*)ptr, shape, cpu::kDevMask, 0); // NOLINT(*) } CSVIterParam param_; diff --git a/src/io/iter_image_recordio_2.cc b/src/io/iter_image_recordio_2.cc index 94019fe293df..ace42855b6a7 100644 --- a/src/io/iter_image_recordio_2.cc +++ b/src/io/iter_image_recordio_2.cc @@ -266,7 +266,7 @@ inline bool ImageRecordIOParser2::ParseNext(DataBatch *out) { auto dtype = prefetch_param_.dtype ? prefetch_param_.dtype.value() : first_batch.data[i].type_flag_; - out->data.at(i) = NDArray(dst_shape, Context::CPU(), false , src_type_flag); + out->data.at(i) = NDArray(dst_shape, Context::CPU(), false, src_type_flag); unit_size_[i] = src_shape.Size(); } } diff --git a/src/ndarray/ndarray.cc b/src/ndarray/ndarray.cc index c19a82b164c4..717ba170aaf7 100644 --- a/src/ndarray/ndarray.cc +++ b/src/ndarray/ndarray.cc @@ -4,6 +4,7 @@ * \brief ndarry module of mxnet */ #include +#include #include #include #include @@ -613,8 +614,11 @@ NDArray &NDArray::operator/=(const real_t &src) { return ScalarOpApply(this, src); } +/* magic number for ndarray version 1, with int64_t TShape */ +static const uint32_t NDARRAY_V1_MAGIC = 0xF993fac8; + void NDArray::Save(dmlc::Stream *strm) const { - // save shape + strm->Write(NDARRAY_V1_MAGIC); shape_.Save(strm); if (is_none()) return; // save context @@ -638,10 +642,28 @@ void NDArray::Save(dmlc::Stream *strm) const { strm->Write(save_data.dptr_, type_size * shape_.Size()); } +bool LegacyTShapeLoad(dmlc::Stream *strm, TShape *shape) { + uint32_t magic; + if (strm->Read(&magic, sizeof(uint32_t)) != sizeof(uint32_t)) return false; + switch (magic) { + case NDARRAY_V1_MAGIC: + return shape->Load(strm); + default: + // meet legacy TShape, magic is ndim here + uint32_t ndim = magic; + *shape = TShape(ndim); + std::vector buffer(ndim); + size_t nread = ndim * sizeof(uint32_t); + if (strm->Read(buffer.data(), nread) != nread) return false; + nnvm::ShapeTypeCast(buffer.begin(), buffer.end(), shape->begin()); + return true; + } +} + bool NDArray::Load(dmlc::Stream *strm) { // load shape TShape shape; - if (!shape.Load(strm)) return false; + if (!LegacyTShapeLoad(strm, &shape)) return false; if (shape.ndim() == 0) { *this = NDArray(); return true; } @@ -710,7 +732,7 @@ void NDArray::SyncCopyFromCPU(const void *data, size_t size) const { TShape dshape = this->shape(); CHECK_EQ(dshape.Size(), size) << "Memory size do not match"; - TBlob src((void*)data, dshape, cpu::kDevMask, this->dtype_); // NOLINT(*) + TBlob src((void*)data, dshape, cpu::kDevMask, this->dtype_, 0); // NOLINT(*) if (this->ctx().dev_mask() == cpu::kDevMask) { this->WaitToWrite(); @@ -739,7 +761,7 @@ void NDArray::SyncCopyToCPU(void *data, size_t size) const { TShape dshape = this->shape(); CHECK_EQ(dshape.Size(), size) << "Memory size do not match"; - TBlob dst(data, dshape, cpu::kDevMask, this->dtype_); // NOLINT(*) + TBlob dst(data, dshape, cpu::kDevMask, this->dtype_, 0); // NOLINT(*) if (this->ctx().dev_mask() == cpu::kDevMask) { this->WaitToRead(); diff --git a/src/operator/cudnn_convolution-inl.h b/src/operator/cudnn_convolution-inl.h index 96eadcdcca4c..508b1f8be84d 100644 --- a/src/operator/cudnn_convolution-inl.h +++ b/src/operator/cudnn_convolution-inl.h @@ -33,6 +33,7 @@ class CuDNNConvolutionOp : public Operator { const Context& ctx) { using namespace mshadow; this->param_ = param; + InitBufferForParam(); auto cudnn_forward_compute_type = convertToCuDNNDataType(forward_compute_type); auto cudnn_backward_compute_type = convertToCuDNNDataType(backward_compute_type); // convert MB to words @@ -426,27 +427,28 @@ class CuDNNConvolutionOp : public Operator { // 3d conv #if CUDNN_MAJOR >= 5 CHECK_EQ(param_.layout.value(), kNCDHW) << "CuDNN only support 3D conv with NCDHW layout"; + std::vector wshape_buffer(wshape.ndim()); CUDNN_CALL(cudnnSetFilterNdDescriptor(filter_desc_, dtype_, CUDNN_TENSOR_NCHW, static_cast(wshape.ndim()), - reinterpret_cast(&wshape[0]))); + CastTShapeToIntPtr(wshape, &wshape_buffer))); #else LOG(FATAL) << "Only support CUDNN V5 for 3D convolution"; #endif CUDNN_CALL(cudnnSetConvolutionNdDescriptor(forward_conv_desc_, 3, - reinterpret_cast(¶m_.pad[0]), - reinterpret_cast(¶m_.stride[0]), - reinterpret_cast(¶m_.dilate[0]), + param_pad_.data(), + param_stride_.data(), + param_dilate_.data(), CUDNN_CROSS_CORRELATION, cudnn_forward_compute_type)); CUDNN_CALL(cudnnSetConvolutionNdDescriptor(backward_conv_desc_, 3, - reinterpret_cast(¶m_.pad[0]), - reinterpret_cast(¶m_.stride[0]), - reinterpret_cast(¶m_.dilate[0]), + param_pad_.data(), + param_stride_.data(), + param_dilate_.data(), CUDNN_CROSS_CORRELATION, cudnn_backward_compute_type)); @@ -472,17 +474,26 @@ class CuDNNConvolutionOp : public Operator { data_offset_ = dstride[1] * dshape[1]; out_offset_ = ostride[1] * oshape[1]; - CUDNN_CALL(cudnnSetTensorNdDescriptor(in_desc_, - dtype_, - static_cast(dshape.ndim()), - reinterpret_cast(&dshape[0]), - reinterpret_cast(&dstride[0]))); + std::vector dshape_buffer(dshape.ndim()); + nnvm::ShapeTypeCast(dshape.begin(), dshape.end(), dshape_buffer.data()); + std::vector dstride_buffer(dstride.ndim()); + nnvm::ShapeTypeCast(dstride.begin(), dstride.end(), dstride_buffer.data()); + CUDNN_CALL(cudnnSetTensorNdDescriptor(in_desc_, + dtype_, + static_cast(dshape.ndim()), + dshape_buffer.data(), + dstride_buffer.data())); + + std::vector oshape_buffer(oshape.ndim()); + nnvm::ShapeTypeCast(oshape.begin(), oshape.end(), oshape_buffer.data()); + std::vector ostride_buffer(ostride.ndim()); + nnvm::ShapeTypeCast(ostride.begin(), ostride.end(), ostride_buffer.data()); CUDNN_CALL(cudnnSetTensorNdDescriptor(out_desc_, - dtype_, - static_cast(oshape.ndim()), - reinterpret_cast(&oshape[0]), - reinterpret_cast(&ostride[0]))); + dtype_, + static_cast(oshape.ndim()), + oshape_buffer.data(), + ostride_buffer.data())); if (!param_.no_bias) { TShape bias = in_shape[conv::kBias]; @@ -661,6 +672,22 @@ class CuDNNConvolutionOp : public Operator { init_temp_size_ = true; } + int *CastTShapeToIntPtr(const TShape& s, std::vector *buffer) { + buffer->resize(s.ndim()); + nnvm::ShapeTypeCast(s.begin(), s.end(), buffer->data()); + return buffer->data(); + } + + void InitBufferForParam() { + CastTShapeToIntPtr(param_.stride, ¶m_stride_); + CastTShapeToIntPtr(param_.dilate, ¶m_dilate_); + CastTShapeToIntPtr(param_.pad, ¶m_pad_); + } + + std::vector param_stride_; + std::vector param_dilate_; + std::vector param_pad_; + bool init_cudnn_; bool init_temp_size_; size_t forward_workspace_; diff --git a/src/operator/cudnn_deconvolution-inl.h b/src/operator/cudnn_deconvolution-inl.h index 99426531beb0..5bba1e5278fa 100644 --- a/src/operator/cudnn_deconvolution-inl.h +++ b/src/operator/cudnn_deconvolution-inl.h @@ -30,6 +30,7 @@ class CuDNNDeconvolutionOp : public Operator { const Context& ctx) { using namespace mshadow; this->param_ = param; + InitBufferForParam(); auto cudnn_forward_compute_type = convertToCuDNNDataType(forward_compute_type); auto cudnn_backward_compute_type = convertToCuDNNDataType(backward_compute_type); // convert MB to words @@ -449,27 +450,28 @@ class CuDNNDeconvolutionOp : public Operator { #if CUDNN_MAJOR >= 5 CHECK_EQ(param_.layout.value(), kNCDHW) << "CuDNN only support 3D conv with NCDHW layout"; + std::vector wshape_buffer(wshape.ndim()); CUDNN_CALL(cudnnSetFilterNdDescriptor(filter_desc_, dtype_, CUDNN_TENSOR_NCHW, static_cast(wshape.ndim()), - reinterpret_cast(&wshape[0]))); + CastTShapeToIntPtr(wshape, &wshape_buffer))); #else LOG(FATAL) << "Only support CUDNN V5 for 3D convolution"; #endif CUDNN_CALL(cudnnSetConvolutionNdDescriptor(forward_conv_desc_, 3, reinterpret_cast(&o_pad[0]), - reinterpret_cast(¶m_.stride[0]), - reinterpret_cast(¶m_.dilate[0]), + param_stride_.data(), + param_dilate_.data(), CUDNN_CROSS_CORRELATION, cudnn_forward_compute_type)); CUDNN_CALL(cudnnSetConvolutionNdDescriptor(backward_conv_desc_, 3, reinterpret_cast(&o_pad[0]), - reinterpret_cast(¶m_.stride[0]), - reinterpret_cast(¶m_.dilate[0]), + param_stride_.data(), + param_dilate_.data(), CUDNN_CROSS_CORRELATION, cudnn_backward_compute_type)); @@ -495,17 +497,21 @@ class CuDNNDeconvolutionOp : public Operator { data_offset_ = dstride[1] * dshape[1]; out_offset_ = ostride[1] * oshape[1]; + std::vector dshape_buffer(dshape.ndim()); + std::vector dstride_buffer(dstride.ndim()); CUDNN_CALL(cudnnSetTensorNdDescriptor(in_desc_, dtype_, static_cast(dshape.ndim()), - reinterpret_cast(&dshape[0]), - reinterpret_cast(&dstride[0]))); + CastTShapeToIntPtr(dshape, &dshape_buffer), + CastTShapeToIntPtr(dstride, &dstride_buffer))) + std::vector oshape_buffer(oshape.ndim()); + std::vector ostride_buffer(ostride.ndim()); CUDNN_CALL(cudnnSetTensorNdDescriptor(out_desc_, dtype_, static_cast(oshape.ndim()), - reinterpret_cast(&oshape[0]), - reinterpret_cast(&ostride[0]))); + CastTShapeToIntPtr(oshape, &oshape_buffer), + CastTShapeToIntPtr(ostride, &ostride_buffer))); if (!param_.no_bias) { TShape bias = in_shape[deconv::kBias]; @@ -687,6 +693,20 @@ class CuDNNDeconvolutionOp : public Operator { init_temp_size_ = true; } + int *CastTShapeToIntPtr(const TShape& s, std::vector *buffer) { + buffer->resize(s.ndim()); + nnvm::ShapeTypeCast(s.begin(), s.end(), buffer->data()); + return buffer->data(); + } + + void InitBufferForParam() { + CastTShapeToIntPtr(param_.stride, ¶m_stride_); + CastTShapeToIntPtr(param_.dilate, ¶m_dilate_); + } + + std::vector param_stride_; + std::vector param_dilate_; + bool init_cudnn_; bool init_temp_size_; size_t forward_workspace_; diff --git a/src/operator/custom/custom-inl.h b/src/operator/custom/custom-inl.h index b9224cd30f48..f640c3abd7a6 100644 --- a/src/operator/custom/custom-inl.h +++ b/src/operator/custom/custom-inl.h @@ -184,11 +184,17 @@ class CustomOpProp : public OperatorProperty { bool InferShape(std::vector *in_shape, std::vector *out_shape, std::vector *aux_shape) const override { - std::vector shapes; + std::vector shapes; std::vector ndims; + size_t size = 0; + for (const auto& s : *in_shape) size += s.ndim(); + std::vector shapes_buffer(size); + shapes_buffer.resize(size); + uint32_t *ptr = shapes_buffer.data(); for (auto iter = in_shape->begin(); iter != in_shape->end(); ++iter) { - shapes.push_back(iter->data()); + shapes.push_back(ptr); ndims.push_back(iter->ndim()); + ptr = nnvm::ShapeTypeCast(iter->begin(), iter->end(), ptr); } shapes.resize(num_inputs_+num_outputs_+num_auxs_); ndims.resize(num_inputs_+num_outputs_+num_auxs_); @@ -284,6 +290,7 @@ class CustomOpProp : public OperatorProperty { std::shared_ptr info_; std::vector > kwargs_; unsigned num_inputs_, num_outputs_, num_auxs_; + mutable std::vector shapes_buffer_; }; // class CustomOpProp } // namespace op } // namespace mxnet diff --git a/src/operator/custom/custom.cc b/src/operator/custom/custom.cc index 06330a4a062e..29f624ead2ad 100644 --- a/src/operator/custom/custom.cc +++ b/src/operator/custom/custom.cc @@ -160,11 +160,16 @@ void CustomOp::Backward(const OpContext &ctx, Operator* CustomOpProp::CreateOperatorEx(Context ctx, std::vector *in_shape, std::vector *in_type) const { - std::vector shapes; + std::vector shapes; std::vector ndims; + size_t size = 0; + for (const auto& s : *in_shape) size += s.ndim(); + shapes_buffer_.resize(size); + uint32_t *ptr = shapes_buffer_.data(); for (auto iter = in_shape->begin(); iter != in_shape->end(); ++iter) { - shapes.push_back(iter->data()); + shapes.push_back(ptr); ndims.push_back(iter->ndim()); + ptr = nnvm::ShapeTypeCast(iter->begin(), iter->end(), ptr); } std::string str_ctx; if (ctx.dev_mask() == cpu::kDevMask) { diff --git a/src/operator/custom/native_op-inl.h b/src/operator/custom/native_op-inl.h index b5706205c82b..780b0ae41f67 100644 --- a/src/operator/custom/native_op-inl.h +++ b/src/operator/custom/native_op-inl.h @@ -108,7 +108,8 @@ class NativeOp : public Operator { NativeOpParam param_; std::vector ptrs; std::vector ndims; - std::vector shapes; + std::vector shapes; + std::vector shapes_buffer_; std::vector tags; std::map > > buffer_map; @@ -137,13 +138,18 @@ class NativeOp : public Operator { const std::string &prefix, mshadow::Stream *stream, int tag) { + size_t size = 0; + for (const auto& tblob : vec) size += tblob.shape_.ndim(); + shapes_buffer_.resize(size); + uint32_t *ptr = shapes_buffer_.data(); for (size_t i = 0; i < vec.size(); ++i) { std::stringstream name; name << prefix << i; SyncBuffer(vec[i], name.str(), stream); ptrs.push_back(buffer_map[name.str()].second.dptr_); ndims.push_back(vec[i].ndim()); - shapes.push_back(const_cast(vec[i].shape_.data())); + shapes.push_back(ptr); + ptr = nnvm::ShapeTypeCast(vec[i].shape_.begin(), vec[i].shape_.end(), ptr); tags.push_back(tag); } } @@ -198,11 +204,16 @@ class NativeOpProp : public OperatorProperty { bool InferShape(std::vector *in_shape, std::vector *out_shape, std::vector *aux_shape) const override { - std::vector shapes; + std::vector shapes; std::vector ndims; + size_t size = 0; + for (const auto& s : *in_shape) size += s.ndim(); + std::vector shapes_buffer(size); + uint32_t *ptr = shapes_buffer.data(); for (auto iter = in_shape->begin(); iter != in_shape->end(); ++iter) { - shapes.push_back(iter->data()); + shapes.push_back(ptr); ndims.push_back(iter->ndim()); + ptr = nnvm::ShapeTypeCast(iter->begin(), iter->end(), ptr); } shapes.resize(param_.num_inputs_+param_.num_outputs_); ndims.resize(param_.num_inputs_+param_.num_outputs_); diff --git a/src/operator/custom/ndarray_op-inl.h b/src/operator/custom/ndarray_op-inl.h index a07a7f781d2d..05b1a3a902e8 100644 --- a/src/operator/custom/ndarray_op-inl.h +++ b/src/operator/custom/ndarray_op-inl.h @@ -110,11 +110,16 @@ class NDArrayOpProp : public OperatorProperty { bool InferShape(std::vector *in_shape, std::vector *out_shape, std::vector *aux_shape) const override { - std::vector shapes; + std::vector shapes; std::vector ndims; + size_t size = 0; + for (const auto& s : *in_shape) size += s.ndim(); + std::vector shapes_buffer(size); + uint32_t *ptr = shapes_buffer.data(); for (auto iter = in_shape->begin(); iter != in_shape->end(); ++iter) { - shapes.push_back(iter->data()); + shapes.push_back(ptr); ndims.push_back(iter->ndim()); + ptr = nnvm::ShapeTypeCast(iter->begin(), iter->end(), ptr); } shapes.resize(param_.num_inputs_+param_.num_outputs_); ndims.resize(param_.num_inputs_+param_.num_outputs_); diff --git a/src/operator/deconvolution-inl.h b/src/operator/deconvolution-inl.h index 771f0e217073..4edeb6979222 100644 --- a/src/operator/deconvolution-inl.h +++ b/src/operator/deconvolution-inl.h @@ -151,7 +151,8 @@ class DeconvolutionOp : public Operator { Tensor out = out_data[deconv::kOut].get(s); index_t o_pad[2], o_adj[2]; - TShape dshape = {data.size(2), data.size(3)}; + TShape dshape = {static_cast(data.size(2)), + static_cast(data.size(3))}; param_.InferPad(dshape, o_pad, o_adj); Shape<3> wmat_shape = @@ -268,7 +269,8 @@ class DeconvolutionOp : public Operator { << "Must init CuBLAS handle in stream"; #endif index_t o_pad[2], o_adj[2]; - TShape dshape = {data.size(2), data.size(3)}; + TShape dshape = {static_cast(data.size(2)), + static_cast(data.size(3))}; param_.InferPad(dshape, o_pad, o_adj); const index_t nbatch = data.size(0); diff --git a/src/operator/tensor/control_flow_op.h b/src/operator/tensor/control_flow_op.h index 0ab24899042d..c7fcda0f0c01 100644 --- a/src/operator/tensor/control_flow_op.h +++ b/src/operator/tensor/control_flow_op.h @@ -108,7 +108,7 @@ inline bool WhereOpShape(const nnvm::NodeAttrs& attrs, SHAPE_ASSIGN_CHECK(*in_attrs, 0, tshape); return true; } else if ((*in_attrs)[0].ndim() == 1) { - return (*in_attrs)[0].Size() == tshape[0]; + return (*in_attrs)[0].Size() == static_cast(tshape[0]); } return false; } diff --git a/src/operator/tensor/matrix_op-inl.h b/src/operator/tensor/matrix_op-inl.h index d7a591944e47..cdc8819da18e 100644 --- a/src/operator/tensor/matrix_op-inl.h +++ b/src/operator/tensor/matrix_op-inl.h @@ -283,8 +283,8 @@ inline bool TransposeShape(const nnvm::NodeAttrs& attrs, } } else { CHECK_EQ(shp.ndim(), param.axes.ndim()); - for (index_t i = 0; i < shp.ndim(); ++i) { - CHECK(param.axes[i] < shp.ndim()); + for (size_t i = 0; i < shp.ndim(); ++i) { + CHECK(param.axes[i] < static_cast(shp.ndim())); ret[i] = shp[param.axes[i]]; } } @@ -1387,11 +1387,13 @@ void RepeatOpForward(const nnvm::NodeAttrs& attrs, std::pair rshapes = ReshapeInputOutputForRepeatOp(ishape, axisOpt, repeats); // reshaped input tblob - TBlob iblob(inputs[0].dptr_, rshapes.first, inputs[0].dev_mask_, inputs[0].type_flag_); + TBlob iblob(inputs[0].dptr_, rshapes.first, inputs[0].dev_mask(), + inputs[0].type_flag_, inputs[0].dev_id()); std::vector newInputs = {iblob}; // reshaped output tblob - TBlob oblob(outputs[0].dptr_, rshapes.second, outputs[0].dev_mask_, outputs[0].type_flag_); + TBlob oblob(outputs[0].dptr_, rshapes.second, outputs[0].dev_mask(), + outputs[0].type_flag_, outputs[0].dev_id()); std::vector newOutputs = {oblob}; BroadcastCompute(attrs, ctx, newInputs, req, newOutputs); @@ -1429,11 +1431,13 @@ void RepeatOpBackward(const nnvm::NodeAttrs& attrs, ReshapeInputOutputForRepeatOp(oshape, axisOpt, repeats); // reshaped output grad tblob - TBlob oblob(outputs[0].dptr_, rshapes.first, outputs[0].dev_mask_, outputs[0].type_flag_); + TBlob oblob(outputs[0].dptr_, rshapes.first, outputs[0].dev_mask(), + outputs[0].type_flag_, outputs[0].dev_id()); std::vector newOutputs = {oblob}; // reshaped input grad tblob - TBlob iblob(inputs[0].dptr_, rshapes.second, inputs[0].dev_mask_, inputs[0].type_flag_); + TBlob iblob(inputs[0].dptr_, rshapes.second, inputs[0].dev_mask(), + inputs[0].type_flag_, inputs[0].dev_id()); std::vector newInputs = {iblob}; ReduceAxesComputeImpl( @@ -1563,10 +1567,12 @@ void TileOpForward(const nnvm::NodeAttrs& attrs, std::pair rshapes = ReshapeInputOutputForTileOp(ishape, reps); // reshaped input tblob - TBlob iblob(inputs[0].dptr_, rshapes.first, inputs[0].dev_mask_, inputs[0].type_flag_); + TBlob iblob(inputs[0].dptr_, rshapes.first, inputs[0].dev_mask(), + inputs[0].type_flag_, inputs[0].dev_id()); std::vector newInputs = {iblob}; // reshaped output tblob - TBlob oblob(outputs[0].dptr_, rshapes.second, outputs[0].dev_mask_, outputs[0].type_flag_); + TBlob oblob(outputs[0].dptr_, rshapes.second, outputs[0].dev_mask(), + outputs[0].type_flag_, outputs[0].dev_id()); std::vector newOutputs = {oblob}; BroadcastCompute(attrs, ctx, newInputs, req, newOutputs); @@ -1603,10 +1609,12 @@ void TileOpBackward(const nnvm::NodeAttrs& attrs, std::pair rshapes = ReshapeInputOutputForTileOp(oshape, reps); // reshaped output grad tblob - TBlob oblob(outputs[0].dptr_, rshapes.first, outputs[0].dev_mask_, outputs[0].type_flag_); + TBlob oblob(outputs[0].dptr_, rshapes.first, outputs[0].dev_mask(), + outputs[0].type_flag_, outputs[0].dev_id()); std::vector newOutputs = {oblob}; // reshaped input grad tblob - TBlob iblob(inputs[0].dptr_, rshapes.second, inputs[0].dev_mask_, inputs[0].type_flag_); + TBlob iblob(inputs[0].dptr_, rshapes.second, inputs[0].dev_mask(), + inputs[0].type_flag_, inputs[0].dev_id()); std::vector newInputs = {iblob}; ReduceAxesComputeImpl( diff --git a/tests/cpp/include/test_util.h b/tests/cpp/include/test_util.h index 6b87312e174a..b0e4c866f9de 100644 --- a/tests/cpp/include/test_util.h +++ b/tests/cpp/include/test_util.h @@ -160,14 +160,14 @@ inline StreamType& print_blob(StreamType *_os, const TBlob &blob, if (dim == 1) { // probably a tensor (mshadow::Tensor is deprecated) - TBlob changed(blob.dptr(), TShape(3), blob.dev_mask_); + TBlob changed(blob.dptr(), TShape(3), blob.dev_mask(), blob.dev_id()); changed.shape_[0] = 1; changed.shape_[1] = 1; changed.shape_[2] = blob.shape_[0]; return print_blob(&os, changed, false, false); } else if (dim == 2) { // probably a tensor (mshadow::Tensor is deprecated) - TBlob changed(blob.dptr(), TShape(4), blob.dev_mask_); + TBlob changed(blob.dptr(), TShape(4), blob.dev_mask(), blob.dev_id()); changed.shape_[0] = 1; changed.shape_[1] = 1; changed.shape_[2] = blob.shape_[0]; diff --git a/tests/python/unittest/legacy_ndarray.v0 b/tests/python/unittest/legacy_ndarray.v0 new file mode 100644 index 0000000000000000000000000000000000000000..f4306d8372021bec350f9faea44ef0b07c96d939 GIT binary patch literal 3224 zcmeI!u}d3a7{K8Rh&VWe;?Th%gF^=ghYk*cF?4X~kinrthYSu54h{|u4o8s^q?8~< zf|L?Ukx)trA|fIs6cG^-5h)^4LXi@rNDwKZNa+*MA^8{jecW@$eK+2Z_wGR=YFpiR zcdhN1j>oOtQKz(nm=aZLL}O8uB*hSEvgDW{CQpGPCCXITq{<#goKmAsgCY}3DDj>TtWxG9pZH9LHP+c+lP$LSLY1%Vu*)9%9B{}H$DHtuQ_eW&J2ifA!B6U3 z@{21PTyw*3n*8CGznO#=5YxK0uh2wjqP+;EggFuBL`M{%i7+QRk_dCcw>A7vv{w=4 L#Q)mFeFy&lZ0mW> literal 0 HcmV?d00001 diff --git a/tests/python/unittest/test_ndarray.py b/tests/python/unittest/test_ndarray.py index 7f0a1d2b6301..fcc7d70f20fe 100644 --- a/tests/python/unittest/test_ndarray.py +++ b/tests/python/unittest/test_ndarray.py @@ -230,6 +230,15 @@ def test_ndarray_saveload(): assert np.sum(x.asnumpy() != y.asnumpy()) == 0 os.remove(fname) +def test_ndarray_legacy_load(): + data = [] + for i in range(6): + data.append(mx.nd.arange(128)) + path = os.path.dirname(os.path.realpath(__file__)) + legacy_data = mx.nd.load(os.path.join(path, 'legacy_ndarray.v0')) + assert len(data) == len(legacy_data) + for i in range(len(data)): + assert same(data[i].asnumpy(), legacy_data[i].asnumpy()) def test_ndarray_slice(): shape = (10,) From 4fb4a208c6f98a0c4c7b12d8474dd95d3c3c166e Mon Sep 17 00:00:00 2001 From: Jian Guo Date: Tue, 30 May 2017 12:57:39 -0500 Subject: [PATCH 005/834] update faster rcnn example with logging and cpu make (#6486) * update rcnn for logging and cpu make * remove deprecated files * update pycocoutils * use logging for output * support cpu make and setup.py * fix proposal op --- example/rcnn/README.md | 16 +- example/rcnn/demo.py | 11 +- example/rcnn/rcnn/core/tester.py | 12 +- example/rcnn/rcnn/cython/setup.py | 49 ++-- example/rcnn/rcnn/dataset/coco.py | 22 +- example/rcnn/rcnn/dataset/imdb.py | 14 +- example/rcnn/rcnn/dataset/pascal_voc.py | 22 +- example/rcnn/rcnn/dataset/pascal_voc_eval.py | 6 +- example/rcnn/rcnn/io/rpn.py | 52 ++-- example/rcnn/rcnn/logger.py | 6 + .../rcnn/rcnn/processing/bbox_regression.py | 9 +- .../rcnn/rcnn/processing/image_processing.py | 83 ------ example/rcnn/rcnn/processing/nms.py | 10 +- example/rcnn/rcnn/processing/roidb.py | 91 ------ example/rcnn/rcnn/pycocotools/UPSTREAM_REV | 2 +- example/rcnn/rcnn/pycocotools/_mask.pyx | 37 ++- example/rcnn/rcnn/pycocotools/coco.py | 207 +++++++++----- example/rcnn/rcnn/pycocotools/cocoeval.py | 261 ++++++++++++------ example/rcnn/rcnn/pycocotools/mask.py | 31 ++- example/rcnn/rcnn/pycocotools/maskApi.c | 74 +++-- example/rcnn/rcnn/pycocotools/maskApi.h | 35 ++- example/rcnn/rcnn/symbol/proposal.py | 19 +- example/rcnn/rcnn/symbol/proposal_target.py | 23 +- example/rcnn/rcnn/tools/reeval.py | 4 +- example/rcnn/rcnn/tools/test_rcnn.py | 4 +- example/rcnn/rcnn/tools/test_rpn.py | 4 +- example/rcnn/rcnn/tools/train_rcnn.py | 17 +- example/rcnn/rcnn/tools/train_rpn.py | 19 +- example/rcnn/rcnn/utils/caffe_convert.py | 75 ----- example/rcnn/rcnn/utils/load_data.py | 4 +- example/rcnn/test.py | 4 +- example/rcnn/train_alternate.py | 21 +- example/rcnn/train_end2end.py | 19 +- src/operator/contrib/proposal-inl.h | 2 +- 34 files changed, 616 insertions(+), 649 deletions(-) create mode 100644 example/rcnn/rcnn/logger.py delete mode 100644 example/rcnn/rcnn/processing/image_processing.py delete mode 100644 example/rcnn/rcnn/processing/roidb.py delete mode 100644 example/rcnn/rcnn/utils/caffe_convert.py diff --git a/example/rcnn/README.md b/example/rcnn/README.md index 43cd054cb876..282a1aebe9a9 100644 --- a/example/rcnn/README.md +++ b/example/rcnn/README.md @@ -1,5 +1,7 @@ # Faster R-CNN in MXNet with distributed implementation and data parallelization +![example detections](https://cloud.githubusercontent.com/assets/13162287/22101032/92085dc0-de6c-11e6-9228-67e72606ddbc.png) + ## Why? There exist good implementations of Faster R-CNN yet they lack support for recent ConvNet architectures. The aim of reproducing it from scratch is to fully utilize @@ -43,9 +45,8 @@ MXNet engines and parallelization for object detection. | Faster R-CNN end-to-end | VGG16 | COCO train | COCO val | 21.2 | 22.8 | | Faster R-CNN end-to-end | ResNet-101 | COCO train | COCO val | 27.2 | 26.1 | -All reference results are from original publications. -All VOC experiments are conducted in MXNet-v0.9.1-nnvm. MXNet-v0.8 have similar results. -All COCO experiments are conducted in MXNet-v0.8. +The above experiments were conducted at [mx-rcnn](https://github.com/precedenceguo/mx-rcnn/tree/6a1ab0eec5035a10a1efb5fc8c9d6c54e101b4d0) +using [a MXNet fork, based on MXNet 0.9.1 nnvm pre-release](https://github.com/precedenceguo/mxnet/tree/simple). ## I'm Feeling Lucky * Prepare: `bash script/additional_deps.sh` @@ -56,9 +57,8 @@ All COCO experiments are conducted in MXNet-v0.8. ## Getting started See if `bash script/additional_deps.sh` will do the following for you. * Suppose `HOME` represents where this file is located. All commands, unless stated otherwise, should be started from `HOME`. - Executing scripts in `script` must also be from `HOME`. * Install python package `cython easydict matplotlib scikit-image`. -* Install MXNet Python Interface. Open `python` type `import mxnet` to confirm. +* Install MXNet version v0.9.5 or higher and MXNet Python Interface. Open `python` type `import mxnet` to confirm. * Run `make` in `HOME`. Command line arguments have the same meaning as in mxnet/example/image-classification. @@ -82,7 +82,7 @@ Refer to `script/vgg_voc07.sh` and other experiments for examples. ### Prepare Training Data See `bash script/get_voc.sh` and `bash script/get_coco.sh` will do the following for you. -* Make a folder `data` in `HOME`. `data` folder will be used to place the training data folder `VOCdevkit` and `coco`. +* Make a folder `data` in `HOME`. `data` folder will be used to place the training data folder `VOCdevkit` and `coco`. * Download and extract [Pascal VOC data](http://host.robots.ox.ac.uk/pascal/VOC/), place the `VOCdevkit` folder in `HOME/data`. * Download and extract [coco dataset](http://mscoco.org/dataset/), place all images to `coco/images` and annotation jsons to `data/annotations`. @@ -94,6 +94,7 @@ See `bash script/get_voc.sh` and `bash script/get_coco.sh` will do the following ### Prepare Pretrained Models See if `bash script/get_pretrained_model.sh` will do this for you. If not, * Make a folder `model` in `HOME`. `model` folder will be used to place model checkpoints along the training process. + It is recommended to set `model` as a symbolic link to somewhere else in hard disk. * Download VGG16 pretrained model `vgg16-0000.params` from [MXNet model gallery](https://github.com/dmlc/mxnet-model-gallery/blob/master/imagenet-1k-vgg.md) to `model` folder. * Download ResNet pretrained model `resnet-101-0000.params` from [ResNet](https://github.com/tornadomeet/ResNet) to `model` folder. @@ -174,7 +175,7 @@ History of this implementation is: * Faster R-CNN with end-to-end training and module testing (v4) * Faster R-CNN with accelerated training and resnet (v5) -mxnet/example/rcnn was v1, v2 and v3.5. +mxnet/example/rcnn was v1, v2, v3.5 and now v5. ## References 1. Tianqi Chen, Mu Li, Yutian Li, Min Lin, Naiyan Wang, Minjie Wang, Tianjun Xiao, Bing Xu, Chiyuan Zhang, and Zheng Zhang. MXNet: A Flexible and Efficient Machine Learning Library for Heterogeneous Distributed Systems. In Neural Information Processing Systems, Workshop on Machine Learning Systems, 2015 @@ -186,3 +187,4 @@ mxnet/example/rcnn was v1, v2 and v3.5. 7. Karen Simonyan, and Andrew Zisserman. "Very deep convolutional networks for large-scale image recognition." arXiv preprint arXiv:1409.1556 (2014). 8. Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun. "Deep Residual Learning for Image Recognition". In Computer Vision and Pattern Recognition, IEEE Conference on, 2016. 9. Tsung-Yi Lin, Michael Maire, Serge Belongie, James Hays, Pietro Perona, Deva Ramanan, Piotr Dollár, and C. Lawrence Zitnick. "Microsoft COCO: Common Objects in Context" In European Conference on Computer Vision, pp. 740-755. Springer International Publishing, 2014. + diff --git a/example/rcnn/demo.py b/example/rcnn/demo.py index 9c01b48fd1bd..34ea327cffac 100644 --- a/example/rcnn/demo.py +++ b/example/rcnn/demo.py @@ -1,9 +1,9 @@ -from __future__ import print_function import argparse import os import cv2 import mxnet as mx import numpy as np +from rcnn.logger import logger from rcnn.config import config from rcnn.symbol import get_vgg_test, get_vgg_rpn_test from rcnn.io.image import resize, transform @@ -104,17 +104,18 @@ def demo_net(predictor, image_name, vis=False): boxes_this_image = [[]] + [all_boxes[j] for j in range(1, len(CLASSES))] # print results - print('class ---- [[x1, x2, y1, y2, confidence]]') + logger.info('---class---') + logger.info('[[x1, x2, y1, y2, confidence]]') for ind, boxes in enumerate(boxes_this_image): if len(boxes) > 0: - print('---------', CLASSES[ind], '---------') - print(boxes) + logger.info('---%s---' % CLASSES[ind]) + logger.info('%s' % boxes) if vis: vis_all_detection(data_dict['data'].asnumpy(), boxes_this_image, CLASSES, im_scale) else: result_file = image_name.replace('.', '_result.') - print('results saved to %s' % result_file) + logger.info('results saved to %s' % result_file) im = draw_all_detection(data_dict['data'].asnumpy(), boxes_this_image, CLASSES, im_scale) cv2.imwrite(result_file, im) diff --git a/example/rcnn/rcnn/core/tester.py b/example/rcnn/rcnn/core/tester.py index a99614b370b5..0ccc47df71eb 100644 --- a/example/rcnn/rcnn/core/tester.py +++ b/example/rcnn/rcnn/core/tester.py @@ -1,4 +1,3 @@ -from __future__ import print_function import cPickle import os import time @@ -6,6 +5,7 @@ import numpy as np from module import MutableModule +from rcnn.logger import logger from rcnn.config import config from rcnn.io import image from rcnn.processing.bbox_transform import bbox_pred, clip_boxes @@ -79,9 +79,9 @@ def generate_proposals(predictor, test_data, imdb, vis=False, thresh=0.): if vis: vis_all_detection(data_dict['data'].asnumpy(), [dets], ['obj'], scale) - print('generating %d/%d' % (i + 1, imdb.num_images), - 'proposal %d' % (dets.shape[0]), - 'data %.4fs net %.4fs' % (t1, t2)) + logger.info('generating %d/%d ' % (i + 1, imdb.num_images) + + 'proposal %d ' % (dets.shape[0]) + + 'data %.4fs net %.4fs' % (t1, t2)) i += 1 assert len(imdb_boxes) == imdb.num_images, 'calculations not complete' @@ -100,7 +100,7 @@ def generate_proposals(predictor, test_data, imdb, vis=False, thresh=0.): with open(full_rpn_file, 'wb') as f: cPickle.dump(original_boxes, f, cPickle.HIGHEST_PROTOCOL) - print('wrote rpn proposals to {}'.format(rpn_file)) + logger.info('wrote rpn proposals to %s' % rpn_file) return imdb_boxes @@ -189,7 +189,7 @@ def pred_eval(predictor, test_data, imdb, vis=False, thresh=1e-3): t3 = time.time() - t t = time.time() - print('testing {}/{} data {:.4f}s net {:.4f}s post {:.4f}s'.format(i, imdb.num_images, t1, t2, t3)) + logger.info('testing %d/%d data %.4fs net %.4fs post %.4fs' % (i, imdb.num_images, t1, t2, t3)) i += 1 det_file = os.path.join(imdb.cache_path, imdb.name + '_detections.pkl') diff --git a/example/rcnn/rcnn/cython/setup.py b/example/rcnn/rcnn/cython/setup.py index 330373dddb72..786460798fd2 100644 --- a/example/rcnn/rcnn/cython/setup.py +++ b/example/rcnn/rcnn/cython/setup.py @@ -55,7 +55,13 @@ def locate_cuda(): raise EnvironmentError('The CUDA %s path could not be located in %s' % (k, v)) return cudaconfig -CUDA = locate_cuda() + + +# Test if cuda could be foun +try: + CUDA = locate_cuda() +except EnvironmentError: + CUDA = None # Obtain the numpy include directory. This logic works across numpy versions. @@ -123,25 +129,32 @@ def build_extensions(self): extra_compile_args={'gcc': ["-Wno-cpp", "-Wno-unused-function"]}, include_dirs = [numpy_include] ), - Extension('gpu_nms', - ['nms_kernel.cu', 'gpu_nms.pyx'], - library_dirs=[CUDA['lib64']], - libraries=['cudart'], - language='c++', - runtime_library_dirs=[CUDA['lib64']], - # this syntax is specific to this build system - # we're only going to use certain compiler args with nvcc and not with - # gcc the implementation of this trick is in customize_compiler() below - extra_compile_args={'gcc': ["-Wno-unused-function"], - 'nvcc': ['-arch=sm_35', - '--ptxas-options=-v', - '-c', - '--compiler-options', - "'-fPIC'"]}, - include_dirs = [numpy_include, CUDA['include']] - ), ] +if CUDA is not None: + ext_modules.append( + Extension('gpu_nms', + ['nms_kernel.cu', 'gpu_nms.pyx'], + library_dirs=[CUDA['lib64']], + libraries=['cudart'], + language='c++', + runtime_library_dirs=[CUDA['lib64']], + # this syntax is specific to this build system + # we're only going to use certain compiler args with nvcc and not with + # gcc the implementation of this trick is in customize_compiler() below + extra_compile_args={'gcc': ["-Wno-unused-function"], + 'nvcc': ['-arch=sm_35', + '--ptxas-options=-v', + '-c', + '--compiler-options', + "'-fPIC'"]}, + include_dirs = [numpy_include, CUDA['include']] + ) + ) +else: + print('Skipping GPU_NMS') + + setup( name='frcnn_cython', ext_modules=ext_modules, diff --git a/example/rcnn/rcnn/dataset/coco.py b/example/rcnn/rcnn/dataset/coco.py index 8026071a90c3..00c4c41cf3ce 100644 --- a/example/rcnn/rcnn/dataset/coco.py +++ b/example/rcnn/rcnn/dataset/coco.py @@ -1,10 +1,10 @@ -from __future__ import print_function import cPickle import cv2 import os import json import numpy as np +from ..logger import logger from imdb import IMDB # coco api @@ -38,7 +38,7 @@ def __init__(self, image_set, root_path, data_path): # load image file names self.image_set_index = self._load_image_set_index() self.num_images = len(self.image_set_index) - print('num_images', self.num_images) + logger.info('%s num_images %d' % (self.name, self.num_images)) # deal with data name view_map = {'minival2014': 'val2014', @@ -68,13 +68,13 @@ def gt_roidb(self): if os.path.exists(cache_file): with open(cache_file, 'rb') as fid: roidb = cPickle.load(fid) - print('{} gt roidb loaded from {}'.format(self.name, cache_file)) + logger.info('%s gt roidb loaded from %s' % (self.name, cache_file)) return roidb gt_roidb = [self._load_coco_annotation(index) for index in self.image_set_index] with open(cache_file, 'wb') as fid: cPickle.dump(gt_roidb, fid, cPickle.HIGHEST_PROTOCOL) - print('wrote gt roidb to {}'.format(cache_file)) + logger.info('%s wrote gt roidb to %s' % (self.name, cache_file)) return gt_roidb @@ -155,10 +155,10 @@ def _write_coco_results(self, detections, res_file): for cls_ind, cls in enumerate(self.classes): if cls == '__background__': continue - print('Collecting %s results (%d/%d)' % (cls, cls_ind, self.num_classes - 1)) + logger.info('collecting %s results (%d/%d)' % (cls, cls_ind, self.num_classes - 1)) coco_cat_id = self._class_to_coco_ind[cls] results.extend(self._coco_results_one_category(detections[cls_ind], coco_cat_id)) - print('Writing results json to %s' % res_file) + logger.info('writing results json to %s' % res_file) with open(res_file, 'w') as f: json.dump(results, f, sort_keys=True, indent=4) @@ -192,7 +192,7 @@ def _do_python_eval(self, res_file, res_folder): eval_file = os.path.join(res_folder, 'detections_%s_results.pkl' % self.image_set) with open(eval_file, 'wb') as f: cPickle.dump(coco_eval, f, cPickle.HIGHEST_PROTOCOL) - print('coco eval results saved to %s' % eval_file) + logger.info('eval results saved to %s' % eval_file) def _print_detection_metrics(self, coco_eval): IoU_lo_thresh = 0.5 @@ -214,15 +214,15 @@ def _get_thr_ind(coco_eval, thr): precision = \ coco_eval.eval['precision'][ind_lo:(ind_hi + 1), :, :, 0, 2] ap_default = np.mean(precision[precision > -1]) - print('~~~~ Mean and per-category AP @ IoU=%.2f,%.2f] ~~~~' % (IoU_lo_thresh, IoU_hi_thresh)) - print('%-15s %5.1f' % ('all', 100 * ap_default)) + logger.info('~~~~ Mean and per-category AP @ IoU=%.2f,%.2f] ~~~~' % (IoU_lo_thresh, IoU_hi_thresh)) + logger.info('%-15s %5.1f' % ('all', 100 * ap_default)) for cls_ind, cls in enumerate(self.classes): if cls == '__background__': continue # minus 1 because of __background__ precision = coco_eval.eval['precision'][ind_lo:(ind_hi + 1), :, cls_ind - 1, 0, 2] ap = np.mean(precision[precision > -1]) - print('%-15s %5.1f' % (cls, 100 * ap)) + logger.info('%-15s %5.1f' % (cls, 100 * ap)) - print('~~~~ Summary metrics ~~~~') + logger.info('~~~~ Summary metrics ~~~~') coco_eval.summarize() diff --git a/example/rcnn/rcnn/dataset/imdb.py b/example/rcnn/rcnn/dataset/imdb.py index 1ad18dbc29bc..acdcd50f8208 100644 --- a/example/rcnn/rcnn/dataset/imdb.py +++ b/example/rcnn/rcnn/dataset/imdb.py @@ -9,7 +9,7 @@ 'boxes', 'gt_classes', 'gt_overlaps', 'max_classes', 'max_overlaps', 'bbox_targets'] """ -from __future__ import print_function +from ..logger import logger import os import cPickle import numpy as np @@ -70,8 +70,8 @@ def load_rpn_data(self, full=False): rpn_file = os.path.join(self.root_path, 'rpn_data', self.name + '_full_rpn.pkl') else: rpn_file = os.path.join(self.root_path, 'rpn_data', self.name + '_rpn.pkl') - print('loading {}'.format(rpn_file)) - assert os.path.exists(rpn_file), 'rpn data not found at {}'.format(rpn_file) + assert os.path.exists(rpn_file), '%s rpn data not found at %s' % (self.name, rpn_file) + logger.info('%s loading rpn data from %s' % (self.name, rpn_file)) with open(rpn_file, 'rb') as f: box_list = cPickle.load(f) return box_list @@ -93,7 +93,7 @@ def rpn_roidb(self, gt_roidb, append_gt=False): :return: roidb of rpn """ if append_gt: - print('appending ground truth annotations') + logger.info('%s appending ground truth annotations' % self.name) rpn_roidb = self.load_rpn_roidb(gt_roidb) roidb = IMDB.merge_roidbs(gt_roidb, rpn_roidb) else: @@ -156,7 +156,7 @@ def append_flipped_images(self, roidb): :param roidb: [image_index]['boxes', 'gt_classes', 'gt_overlaps', 'flipped'] :return: roidb: [image_index]['boxes', 'gt_classes', 'gt_overlaps', 'flipped'] """ - print('append flipped images to roidb') + logger.info('%s append flipped images to roidb' % self.name) assert self.num_images == len(roidb) for i in range(self.num_images): roi_rec = roidb[i] @@ -211,8 +211,8 @@ def evaluate_recall(self, roidb, candidate_boxes=None, thresholds=None): area_counts.append(area_count) total_counts = float(sum(area_counts)) for area_name, area_count in zip(area_names[1:], area_counts): - print('percentage of', area_name, area_count / total_counts) - print('average number of proposal', total_counts / self.num_images) + logger.info('percentage of %s is %f' % (area_name, area_count / total_counts)) + logger.info('average number of proposal is %f' % (total_counts / self.num_images)) for area_name, area_range in zip(area_names, area_ranges): gt_overlaps = np.zeros(0) num_pos = 0 diff --git a/example/rcnn/rcnn/dataset/pascal_voc.py b/example/rcnn/rcnn/dataset/pascal_voc.py index 268399316162..2135971faadf 100644 --- a/example/rcnn/rcnn/dataset/pascal_voc.py +++ b/example/rcnn/rcnn/dataset/pascal_voc.py @@ -6,12 +6,12 @@ criterion. """ -from __future__ import print_function import cPickle import cv2 import os import numpy as np +from ..logger import logger from imdb import IMDB from pascal_voc_eval import voc_eval from ds_utils import unique_boxes, filter_small_boxes @@ -42,7 +42,7 @@ def __init__(self, image_set, root_path, devkit_path): self.num_classes = len(self.classes) self.image_set_index = self.load_image_set_index() self.num_images = len(self.image_set_index) - print('num_images', self.num_images) + logger.info('%s num_images %d' % (self.name, self.num_images)) self.config = {'comp_id': 'comp4', 'use_diff': False, @@ -78,13 +78,13 @@ def gt_roidb(self): if os.path.exists(cache_file): with open(cache_file, 'rb') as fid: roidb = cPickle.load(fid) - print('{} gt roidb loaded from {}'.format(self.name, cache_file)) + logger.info('%s gt roidb loaded from %s' % (self.name, cache_file)) return roidb gt_roidb = [self.load_pascal_annotation(index) for index in self.image_set_index] with open(cache_file, 'wb') as fid: cPickle.dump(gt_roidb, fid, cPickle.HIGHEST_PROTOCOL) - print('wrote gt roidb to {}'.format(cache_file)) + logger.info('%s wrote gt roidb to %s' % (self.name, cache_file)) return gt_roidb @@ -168,18 +168,18 @@ def selective_search_roidb(self, gt_roidb, append_gt=False): if os.path.exists(cache_file): with open(cache_file, 'rb') as fid: roidb = cPickle.load(fid) - print('{} ss roidb loaded from {}'.format(self.name, cache_file)) + logger.info('%s ss roidb loaded from %s' % (self.name, cache_file)) return roidb if append_gt: - print('appending ground truth annotations') + logger.info('%s appending ground truth annotations' % self.name) ss_roidb = self.load_selective_search_roidb(gt_roidb) roidb = IMDB.merge_roidbs(gt_roidb, ss_roidb) else: roidb = self.load_selective_search_roidb(gt_roidb) with open(cache_file, 'wb') as fid: cPickle.dump(roidb, fid, cPickle.HIGHEST_PROTOCOL) - print('wrote ss roidb to {}'.format(cache_file)) + logger.info('%s wrote ss roidb to %s' % (self.name, cache_file)) return roidb @@ -224,7 +224,7 @@ def write_pascal_results(self, all_boxes): for cls_ind, cls in enumerate(self.classes): if cls == '__background__': continue - print('Writing {} VOC results file'.format(cls)) + logger.info('Writing %s VOC results file' % cls) filename = self.get_result_file_template().format(cls) with open(filename, 'wt') as f: for im_ind, index in enumerate(self.image_set_index): @@ -248,7 +248,7 @@ def do_python_eval(self): aps = [] # The PASCAL VOC metric changed in 2010 use_07_metric = True if int(self.year) < 2010 else False - print('VOC07 metric? ' + ('Y' if use_07_metric else 'No')) + logger.info('VOC07 metric? ' + ('Y' if use_07_metric else 'No')) for cls_ind, cls in enumerate(self.classes): if cls == '__background__': continue @@ -256,5 +256,5 @@ def do_python_eval(self): rec, prec, ap = voc_eval(filename, annopath, imageset_file, cls, annocache, ovthresh=0.5, use_07_metric=use_07_metric) aps += [ap] - print('AP for {} = {:.4f}'.format(cls, ap)) - print('Mean AP = {:.4f}'.format(np.mean(aps))) + logger.info('AP for {} = {:.4f}'.format(cls, ap)) + logger.info('Mean AP = {:.4f}'.format(np.mean(aps))) diff --git a/example/rcnn/rcnn/dataset/pascal_voc_eval.py b/example/rcnn/rcnn/dataset/pascal_voc_eval.py index 295b866bb697..54fa12ddccd8 100644 --- a/example/rcnn/rcnn/dataset/pascal_voc_eval.py +++ b/example/rcnn/rcnn/dataset/pascal_voc_eval.py @@ -2,7 +2,7 @@ given a pascal voc imdb, compute mAP """ -from __future__ import print_function +from ..logger import logger import numpy as np import os import cPickle @@ -86,8 +86,8 @@ def voc_eval(detpath, annopath, imageset_file, classname, annocache, ovthresh=0. for ind, image_filename in enumerate(image_filenames): recs[image_filename] = parse_voc_rec(annopath.format(image_filename)) if ind % 100 == 0: - print('reading annotations for {:d}/{:d}'.format(ind + 1, len(image_filenames))) - print('saving annotations cache to {:s}'.format(annocache)) + logger.info('reading annotations for %d/%d' % (ind + 1, len(image_filenames))) + logger.info('saving annotations cache to %s' % annocache) with open(annocache, 'wb') as f: cPickle.dump(recs, f, protocol=cPickle.HIGHEST_PROTOCOL) else: diff --git a/example/rcnn/rcnn/io/rpn.py b/example/rcnn/rcnn/io/rpn.py index c813e4ab06f6..52fe1a50c276 100644 --- a/example/rcnn/rcnn/io/rpn.py +++ b/example/rcnn/rcnn/io/rpn.py @@ -10,10 +10,11 @@ 'bbox_weight': [batch_size, num_anchors, feat_height, feat_width]} """ -from __future__ import print_function +import logging import numpy as np import numpy.random as npr +from ..logger import logger from ..config import config from .image import get_image, tensor_vstack from ..processing.generate_anchor import generate_anchors @@ -94,23 +95,19 @@ def _unmap(data, count, inds, fill=0): ret[inds, :] = data return ret - DEBUG = False im_info = im_info[0] scales = np.array(scales, dtype=np.float32) base_anchors = generate_anchors(base_size=feat_stride, ratios=list(ratios), scales=scales) num_anchors = base_anchors.shape[0] feat_height, feat_width = feat_shape[-2:] - if DEBUG: - print('anchors:') - print(base_anchors) - print('anchor shapes:') - print(np.hstack((base_anchors[:, 2::4] - base_anchors[:, 0::4], - base_anchors[:, 3::4] - base_anchors[:, 1::4]))) - print('im_info', im_info) - print('height', feat_height, 'width', feat_width) - print('gt_boxes shape', gt_boxes.shape) - print('gt_boxes', gt_boxes) + logger.debug('anchors: %s' % base_anchors) + logger.debug('anchor shapes: %s' % np.hstack((base_anchors[:, 2::4] - base_anchors[:, 0::4], + base_anchors[:, 3::4] - base_anchors[:, 1::4]))) + logger.debug('im_info %s' % im_info) + logger.debug('height %d width %d' % (feat_height, feat_width)) + logger.debug('gt_boxes shape %s' % np.array(gt_boxes.shape)) + logger.debug('gt_boxes %s' % gt_boxes) # 1. generate proposals from bbox deltas and shifted anchors shift_x = np.arange(0, feat_width) * feat_stride @@ -132,14 +129,12 @@ def _unmap(data, count, inds, fill=0): (all_anchors[:, 1] >= -allowed_border) & (all_anchors[:, 2] < im_info[1] + allowed_border) & (all_anchors[:, 3] < im_info[0] + allowed_border))[0] - if DEBUG: - print('total_anchors', total_anchors) - print('inds_inside', len(inds_inside)) + logger.debug('total_anchors %d' % total_anchors) + logger.debug('inds_inside %d' % len(inds_inside)) # keep only inside anchors anchors = all_anchors[inds_inside, :] - if DEBUG: - print('anchors shape', anchors.shape) + logger.debug('anchors shape %s' % np.array(anchors.shape)) # label: 1 is positive, 0 is negative, -1 is dont care labels = np.empty((len(inds_inside),), dtype=np.float32) @@ -176,7 +171,7 @@ def _unmap(data, count, inds, fill=0): fg_inds = np.where(labels == 1)[0] if len(fg_inds) > num_fg: disable_inds = npr.choice(fg_inds, size=(len(fg_inds) - num_fg), replace=False) - if DEBUG: + if logger.level == logging.INFO: disable_inds = fg_inds[:(len(fg_inds) - num_fg)] labels[disable_inds] = -1 @@ -185,7 +180,7 @@ def _unmap(data, count, inds, fill=0): bg_inds = np.where(labels == 0)[0] if len(bg_inds) > num_bg: disable_inds = npr.choice(bg_inds, size=(len(bg_inds) - num_bg), replace=False) - if DEBUG: + if logger.level == logging.INFO: disable_inds = bg_inds[:(len(bg_inds) - num_bg)] labels[disable_inds] = -1 @@ -196,29 +191,30 @@ def _unmap(data, count, inds, fill=0): bbox_weights = np.zeros((len(inds_inside), 4), dtype=np.float32) bbox_weights[labels == 1, :] = np.array(config.TRAIN.RPN_BBOX_WEIGHTS) - if DEBUG: + if logger.level == logging.DEBUG: _sums = bbox_targets[labels == 1, :].sum(axis=0) _squared_sums = (bbox_targets[labels == 1, :] ** 2).sum(axis=0) _counts = np.sum(labels == 1) means = _sums / (_counts + 1e-14) stds = np.sqrt(_squared_sums / _counts - means ** 2) - print('means', means) - print('stdevs', stds) + logger.debug('means %s' % means) + logger.debug('stdevs %s' % stds) # map up to original set of anchors labels = _unmap(labels, total_anchors, inds_inside, fill=-1) bbox_targets = _unmap(bbox_targets, total_anchors, inds_inside, fill=0) bbox_weights = _unmap(bbox_weights, total_anchors, inds_inside, fill=0) - if DEBUG: - print('rpn: max max_overlaps', np.max(max_overlaps)) - print('rpn: num_positives', np.sum(labels == 1)) - print('rpn: num_negatives', np.sum(labels == 0)) + if logger.level == logging.DEBUG: + if gt_boxes.size > 0: + logger.debug('rpn: max max_overlaps %f' % np.max(max_overlaps)) + logger.debug('rpn: num_positives %f' % np.sum(labels == 1)) + logger.debug('rpn: num_negatives %f' % np.sum(labels == 0)) _fg_sum = np.sum(labels == 1) _bg_sum = np.sum(labels == 0) _count = 1 - print('rpn: num_positive avg', _fg_sum / _count) - print('rpn: num_negative avg', _bg_sum / _count) + logger.debug('rpn: num_positive avg %f' % (_fg_sum / _count)) + logger.debug('rpn: num_negative avg %f' % (_bg_sum / _count)) labels = labels.reshape((1, feat_height, feat_width, A)).transpose(0, 3, 1, 2) labels = labels.reshape((1, A * feat_height * feat_width)) diff --git a/example/rcnn/rcnn/logger.py b/example/rcnn/rcnn/logger.py new file mode 100644 index 000000000000..2806e1add180 --- /dev/null +++ b/example/rcnn/rcnn/logger.py @@ -0,0 +1,6 @@ +import logging + +# set up logger +logging.basicConfig() +logger = logging.getLogger() +logger.setLevel(logging.INFO) diff --git a/example/rcnn/rcnn/processing/bbox_regression.py b/example/rcnn/rcnn/processing/bbox_regression.py index 46969aa0ec5e..d5b48a71b754 100644 --- a/example/rcnn/rcnn/processing/bbox_regression.py +++ b/example/rcnn/rcnn/processing/bbox_regression.py @@ -2,9 +2,9 @@ This file has functions about generating bounding box regression targets """ -from __future__ import print_function import numpy as np +from ..logger import logger from bbox_transform import bbox_overlaps, bbox_transform from rcnn.config import config @@ -22,12 +22,13 @@ def compute_bbox_regression_targets(rois, overlaps, labels): # Sanity check if len(rois) != len(overlaps): - print('bbox regression: this should not happen') + logger.warning('bbox regression: len(rois) != len(overlaps)') # Indices of ground-truth ROIs gt_inds = np.where(overlaps == 1)[0] if len(gt_inds) == 0: - print('something wrong : zero ground truth rois') + logger.warning('bbox regression: len(gt_inds) == 0') + # Indices of examples for which we try to make predictions ex_inds = np.where(overlaps >= config.TRAIN.BBOX_REGRESSION_THRESH)[0] @@ -52,7 +53,7 @@ def add_bbox_regression_targets(roidb): :param roidb: roidb to be processed. must have gone through imdb.prepare_roidb :return: means, std variances of targets """ - print('add bounding box regression targets') + logger.info('bbox regression: add bounding box regression targets') assert len(roidb) > 0 assert 'max_classes' in roidb[0] diff --git a/example/rcnn/rcnn/processing/image_processing.py b/example/rcnn/rcnn/processing/image_processing.py deleted file mode 100644 index dafca3c15850..000000000000 --- a/example/rcnn/rcnn/processing/image_processing.py +++ /dev/null @@ -1,83 +0,0 @@ -import numpy as np -import cv2 - - -def resize(im, target_size, max_size): - """ - only resize input image to target size and return scale - :param im: BGR image input by opencv - :param target_size: one dimensional size (the short side) - :param max_size: one dimensional max size (the long side) - :return: - """ - im_shape = im.shape - im_size_min = np.min(im_shape[0:2]) - im_size_max = np.max(im_shape[0:2]) - im_scale = float(target_size) / float(im_size_min) - # prevent bigger axis from being more than max_size: - if np.round(im_scale * im_size_max) > max_size: - im_scale = float(max_size) / float(im_size_max) - im = cv2.resize(im, None, None, fx=im_scale, fy=im_scale, interpolation=cv2.INTER_LINEAR) - return im, im_scale - - -def transform(im, pixel_means, need_mean=False): - """ - transform into mxnet tensor - subtract pixel size and transform to correct format - :param im: [height, width, channel] in BGR - :param pixel_means: [[[R, G, B pixel means]]] - :return: [batch, channel, height, width] - """ - im = im.copy() - im[:, :, (0, 1, 2)] = im[:, :, (2, 1, 0)] - im = im.astype(float) - if need_mean: - im -= pixel_means - im_tensor = im[np.newaxis, :] - # put channel first - channel_swap = (0, 3, 1, 2) - im_tensor = im_tensor.transpose(channel_swap) - return im_tensor - - -def transform_inverse(im_tensor, pixel_means): - """ - transform from mxnet im_tensor to ordinary RGB image - im_tensor is limited to one image - :param im_tensor: [batch, channel, height, width] - :param pixel_means: [[[R, G, B pixel means]]] - :return: im [height, width, channel(RGB)] - """ - assert im_tensor.shape[0] == 1 - im_tensor = im_tensor.copy() - # put channel back - channel_swap = (0, 2, 3, 1) - im_tensor = im_tensor.transpose(channel_swap) - im = im_tensor[0] - assert im.shape[2] == 3 - im += pixel_means - im = im.astype(np.uint8) - return im - - -def tensor_vstack(tensor_list, pad=0): - """ - vertically stack tensors - :param tensor_list: list of tensor to be stacked vertically - :param pad: label to pad with - :return: tensor with max shape - """ - ndim = len(tensor_list[0].shape) - if ndim == 1: - return np.hstack(tensor_list) - dimensions = [0] - for dim in range(1, ndim): - dimensions.append(max([tensor.shape[dim] for tensor in tensor_list])) - for ind, tensor in enumerate(tensor_list): - pad_shape = [(0, 0)] - for dim in range(1, ndim): - pad_shape.append((0, dimensions[dim] - tensor.shape[dim])) - tensor_list[ind] = np.lib.pad(tensor, pad_shape, 'constant', constant_values=pad) - all_tensor = np.vstack(tensor_list) - return all_tensor diff --git a/example/rcnn/rcnn/processing/nms.py b/example/rcnn/rcnn/processing/nms.py index cab093c51152..230139c413ec 100644 --- a/example/rcnn/rcnn/processing/nms.py +++ b/example/rcnn/rcnn/processing/nms.py @@ -1,6 +1,9 @@ import numpy as np from ..cython.cpu_nms import cpu_nms -from ..cython.gpu_nms import gpu_nms +try: + from ..cython.gpu_nms import gpu_nms +except ImportError: + gpu_nms = None def py_nms_wrapper(thresh): @@ -18,7 +21,10 @@ def _nms(dets): def gpu_nms_wrapper(thresh, device_id): def _nms(dets): return gpu_nms(dets, thresh, device_id) - return _nms + if gpu_nms is not None: + return _nms + else: + return cpu_nms_wrapper(thresh) def nms(dets, thresh): diff --git a/example/rcnn/rcnn/processing/roidb.py b/example/rcnn/rcnn/processing/roidb.py deleted file mode 100644 index 8dddc27f60c9..000000000000 --- a/example/rcnn/rcnn/processing/roidb.py +++ /dev/null @@ -1,91 +0,0 @@ -""" -roidb -basic format [image_index]['boxes', 'gt_classes', 'gt_overlaps', 'flipped'] -extended ['image', 'max_classes', 'max_overlaps', 'bbox_targets'] -""" - -from __future__ import print_function -import cv2 -import numpy as np - -from bbox_regression import compute_bbox_regression_targets -from rcnn.config import config - - -def prepare_roidb(imdb, roidb): - """ - add image path, max_classes, max_overlaps to roidb - :param imdb: image database, provide path - :param roidb: roidb - :return: None - """ - print('prepare roidb') - for i in range(len(roidb)): # image_index - roidb[i]['image'] = imdb.image_path_from_index(imdb.image_set_index[i]) - if config.TRAIN.ASPECT_GROUPING: - size = cv2.imread(roidb[i]['image']).shape - roidb[i]['height'] = size[0] - roidb[i]['width'] = size[1] - gt_overlaps = roidb[i]['gt_overlaps'].toarray() - max_overlaps = gt_overlaps.max(axis=1) - max_classes = gt_overlaps.argmax(axis=1) - roidb[i]['max_overlaps'] = max_overlaps - roidb[i]['max_classes'] = max_classes - - # background roi => background class - zero_indexes = np.where(max_overlaps == 0)[0] - assert all(max_classes[zero_indexes] == 0) - # foreground roi => foreground class - nonzero_indexes = np.where(max_overlaps > 0)[0] - assert all(max_classes[nonzero_indexes] != 0) - - -def add_bbox_regression_targets(roidb): - """ - given roidb, add ['bbox_targets'] and normalize bounding box regression targets - :param roidb: roidb to be processed. must have gone through imdb.prepare_roidb - :return: means, std variances of targets - """ - print('add bounding box regression targets') - assert len(roidb) > 0 - assert 'max_classes' in roidb[0] - - num_images = len(roidb) - num_classes = roidb[0]['gt_overlaps'].shape[1] - for im_i in range(num_images): - rois = roidb[im_i]['boxes'] - max_overlaps = roidb[im_i]['max_overlaps'] - max_classes = roidb[im_i]['max_classes'] - roidb[im_i]['bbox_targets'] = compute_bbox_regression_targets(rois, max_overlaps, max_classes) - - if config.TRAIN.BBOX_NORMALIZATION_PRECOMPUTED: - # use fixed / precomputed means and stds instead of empirical values - means = np.tile(np.array(config.TRAIN.BBOX_MEANS), (num_classes, 1)) - stds = np.tile(np.array(config.TRAIN.BBOX_STDS), (num_classes, 1)) - else: - # compute mean, std values - class_counts = np.zeros((num_classes, 1)) + config.EPS - sums = np.zeros((num_classes, 4)) - squared_sums = np.zeros((num_classes, 4)) - for im_i in range(num_images): - targets = roidb[im_i]['bbox_targets'] - for cls in range(1, num_classes): - cls_indexes = np.where(targets[:, 0] == cls)[0] - if cls_indexes.size > 0: - class_counts[cls] += cls_indexes.size - sums[cls, :] += targets[cls_indexes, 1:].sum(axis=0) - squared_sums[cls, :] += (targets[cls_indexes, 1:] ** 2).sum(axis=0) - - means = sums / class_counts - # var(x) = E(x^2) - E(x)^2 - stds = np.sqrt(squared_sums / class_counts - means ** 2) - - # normalized targets - for im_i in range(num_images): - targets = roidb[im_i]['bbox_targets'] - for cls in range(1, num_classes): - cls_indexes = np.where(targets[:, 0] == cls)[0] - roidb[im_i]['bbox_targets'][cls_indexes, 1:] -= means[cls, :] - roidb[im_i]['bbox_targets'][cls_indexes, 1:] /= stds[cls, :] - - return means.ravel(), stds.ravel() diff --git a/example/rcnn/rcnn/pycocotools/UPSTREAM_REV b/example/rcnn/rcnn/pycocotools/UPSTREAM_REV index 706219b77d90..9613b145b237 100644 --- a/example/rcnn/rcnn/pycocotools/UPSTREAM_REV +++ b/example/rcnn/rcnn/pycocotools/UPSTREAM_REV @@ -1 +1 @@ -https://github.com/pdollar/coco/commit/3ac47c77ebd5a1ed4254a98b7fbf2ef4765a3574 +https://github.com/pdollar/coco/commit/336d2a27c91e3c0663d2dcf0b13574674d30f88e diff --git a/example/rcnn/rcnn/pycocotools/_mask.pyx b/example/rcnn/rcnn/pycocotools/_mask.pyx index 4e9278af2a03..1c3e127a1c05 100644 --- a/example/rcnn/rcnn/pycocotools/_mask.pyx +++ b/example/rcnn/rcnn/pycocotools/_mask.pyx @@ -10,6 +10,9 @@ __author__ = 'tsungyi' +import sys +PYTHON_VERSION = sys.version_info[0] + # import both Python-level and C-level symbols of Numpy # the API uses Numpy to interface C and Python import numpy as np @@ -38,7 +41,7 @@ cdef extern from "maskApi.h": void rlesInit( RLE **R, siz n ) void rleEncode( RLE *R, const byte *M, siz h, siz w, siz n ) void rleDecode( const RLE *R, byte *mask, siz n ) - void rleMerge( const RLE *R, RLE *M, siz n, bint intersect ) + void rleMerge( const RLE *R, RLE *M, siz n, int intersect ) void rleArea( const RLE *R, siz n, uint *a ) void rleIou( RLE *dt, RLE *gt, siz m, siz n, byte *iscrowd, double *o ) void bbIou( BB dt, BB gt, siz m, siz n, byte *iscrowd, double *o ) @@ -119,7 +122,12 @@ def _frString(rleObjs): cdef bytes py_string cdef char* c_string for i, obj in enumerate(rleObjs): - py_string = str(obj['counts']) + if PYTHON_VERSION == 2: + py_string = str(obj['counts']).encode('utf8') + elif PYTHON_VERSION == 3: + py_string = str.encode(obj['counts']) if type(obj['counts']) == str else obj['counts'] + else: + raise Exception('Python version must be 2 or 3') c_string = py_string rleFrString( &Rs._R[i], c_string, obj['size'][0], obj['size'][1] ) return Rs @@ -138,10 +146,10 @@ def decode(rleObjs): cdef RLEs Rs = _frString(rleObjs) h, w, n = Rs._R[0].h, Rs._R[0].w, Rs._n masks = Masks(h, w, n) - rleDecode( Rs._R, masks._mask, n ); + rleDecode(Rs._R, masks._mask, n); return np.array(masks) -def merge(rleObjs, bint intersect=0): +def merge(rleObjs, intersect=0): cdef RLEs Rs = _frString(rleObjs) cdef RLEs R = RLEs(1) rleMerge(Rs._R, R._R, Rs._n, intersect) @@ -255,7 +263,7 @@ def frPoly( poly, siz h, siz w ): Rs = RLEs(n) for i, p in enumerate(poly): np_poly = np.array(p, dtype=np.double, order='F') - rleFrPoly( &Rs._R[i], np_poly.data, len(np_poly)/2, h, w ) + rleFrPoly( &Rs._R[i], np_poly.data, int(len(p)/2), h, w ) objs = _toString(Rs) return objs @@ -277,15 +285,24 @@ def frUncompressedRLE(ucRles, siz h, siz w): objs.append(_toString(Rs)[0]) return objs -def frPyObjects(pyobj, siz h, w): +def frPyObjects(pyobj, h, w): + # encode rle from a list of python objects if type(pyobj) == np.ndarray: - objs = frBbox(pyobj, h, w ) + objs = frBbox(pyobj, h, w) elif type(pyobj) == list and len(pyobj[0]) == 4: - objs = frBbox(pyobj, h, w ) + objs = frBbox(pyobj, h, w) elif type(pyobj) == list and len(pyobj[0]) > 4: - objs = frPoly(pyobj, h, w ) - elif type(pyobj) == list and type(pyobj[0]) == dict: + objs = frPoly(pyobj, h, w) + elif type(pyobj) == list and type(pyobj[0]) == dict \ + and 'counts' in pyobj[0] and 'size' in pyobj[0]: objs = frUncompressedRLE(pyobj, h, w) + # encode rle from single python object + elif type(pyobj) == list and len(pyobj) == 4: + objs = frBbox([pyobj], h, w)[0] + elif type(pyobj) == list and len(pyobj) > 4: + objs = frPoly([pyobj], h, w)[0] + elif type(pyobj) == dict and 'counts' in pyobj and 'size' in pyobj: + objs = frUncompressedRLE([pyobj], h, w)[0] else: raise Exception('input type is not supported.') return objs diff --git a/example/rcnn/rcnn/pycocotools/coco.py b/example/rcnn/rcnn/pycocotools/coco.py index 44158d21d5a4..ca35cc0b053b 100644 --- a/example/rcnn/rcnn/pycocotools/coco.py +++ b/example/rcnn/rcnn/pycocotools/coco.py @@ -1,5 +1,5 @@ __author__ = 'tylin' -__version__ = '1.0.1' +__version__ = '2.0' # Interface for accessing the Microsoft COCO dataset. # Microsoft COCO is a large image dataset designed for object detection, @@ -27,7 +27,7 @@ # loadAnns - Load anns with the specified ids. # loadCats - Load cats with the specified ids. # loadImgs - Load imgs with the specified ids. -# segToMask - Convert polygon segmentation to binary mask. +# annToMask - Convert segmentation in an annotation to binary mask. # showAnns - Display the specified annotations. # loadRes - Load algorithm results and create API for accessing them. # download - Download COCO images from mscoco.org server. @@ -37,27 +37,30 @@ # See also COCO>decodeMask, # COCO>encodeMask, COCO>getAnnIds, COCO>getCatIds, # COCO>getImgIds, COCO>loadAnns, COCO>loadCats, -# COCO>loadImgs, COCO>segToMask, COCO>showAnns +# COCO>loadImgs, COCO>annToMask, COCO>showAnns # Microsoft COCO Toolbox. version 2.0 # Data, paper, and tutorials available at: http://mscoco.org/ # Code written by Piotr Dollar and Tsung-Yi Lin, 2014. # Licensed under the Simplified BSD License [see bsd.txt] -from __future__ import print_function import json -import datetime import time import matplotlib.pyplot as plt from matplotlib.collections import PatchCollection from matplotlib.patches import Polygon import numpy as np -from skimage.draw import polygon -import urllib import copy import itertools -import mask +from . import mask as maskUtils import os +from collections import defaultdict +import sys +PYTHON_VERSION = sys.version_info[0] +if PYTHON_VERSION == 2: + from urllib import urlretrieve +elif PYTHON_VERSION == 3: + from urllib.request import urlretrieve class COCO: def __init__(self, annotation_file=None): @@ -68,47 +71,38 @@ def __init__(self, annotation_file=None): :return: """ # load dataset - self.dataset = {} - self.anns = [] - self.imgToAnns = {} - self.catToImgs = {} - self.imgs = {} - self.cats = {} - if annotation_file is not None: + self.dataset,self.anns,self.cats,self.imgs = dict(),dict(),dict(),dict() + self.imgToAnns, self.catToImgs = defaultdict(list), defaultdict(list) + if not annotation_file == None: print('loading annotations into memory...') tic = time.time() dataset = json.load(open(annotation_file, 'r')) - print('Done (t=%0.2fs)'%(time.time()- tic)) + assert type(dataset)==dict, 'annotation file format {} not supported'.format(type(dataset)) + print('Done (t={:0.2f}s)'.format(time.time()- tic)) self.dataset = dataset self.createIndex() def createIndex(self): # create index print('creating index...') - anns = {} - imgToAnns = {} - catToImgs = {} - cats = {} - imgs = {} + anns, cats, imgs = {}, {}, {} + imgToAnns,catToImgs = defaultdict(list),defaultdict(list) if 'annotations' in self.dataset: - imgToAnns = {ann['image_id']: [] for ann in self.dataset['annotations']} - anns = {ann['id']: [] for ann in self.dataset['annotations']} for ann in self.dataset['annotations']: - imgToAnns[ann['image_id']] += [ann] + imgToAnns[ann['image_id']].append(ann) anns[ann['id']] = ann if 'images' in self.dataset: - imgs = {im['id']: {} for im in self.dataset['images']} for img in self.dataset['images']: imgs[img['id']] = img if 'categories' in self.dataset: - cats = {cat['id']: [] for cat in self.dataset['categories']} for cat in self.dataset['categories']: cats[cat['id']] = cat - catToImgs = {cat['id']: [] for cat in self.dataset['categories']} + + if 'annotations' in self.dataset and 'categories' in self.dataset: for ann in self.dataset['annotations']: - catToImgs[ann['category_id']] += [ann['image_id']] + catToImgs[ann['category_id']].append(ann['image_id']) print('index created!') @@ -125,7 +119,7 @@ def info(self): :return: """ for key, value in self.dataset['info'].items(): - print('%s: %s'%(key, value)) + print('{}: {}'.format(key, value)) def getAnnIds(self, imgIds=[], catIds=[], areaRng=[], iscrowd=None): """ @@ -143,14 +137,13 @@ def getAnnIds(self, imgIds=[], catIds=[], areaRng=[], iscrowd=None): anns = self.dataset['annotations'] else: if not len(imgIds) == 0: - # this can be changed by defaultdict lists = [self.imgToAnns[imgId] for imgId in imgIds if imgId in self.imgToAnns] anns = list(itertools.chain.from_iterable(lists)) else: anns = self.dataset['annotations'] anns = anns if len(catIds) == 0 else [ann for ann in anns if ann['category_id'] in catIds] anns = anns if len(areaRng) == 0 else [ann for ann in anns if ann['area'] > areaRng[0] and ann['area'] < areaRng[1]] - if iscrowd is not None: + if not iscrowd == None: ids = [ann['id'] for ann in anns if ann['iscrowd'] == iscrowd] else: ids = [ann['id'] for ann in anns] @@ -240,39 +233,57 @@ def showAnns(self, anns): """ if len(anns) == 0: return 0 - if 'segmentation' in anns[0]: + if 'segmentation' in anns[0] or 'keypoints' in anns[0]: datasetType = 'instances' elif 'caption' in anns[0]: datasetType = 'captions' + else: + raise Exception('datasetType not supported') if datasetType == 'instances': ax = plt.gca() + ax.set_autoscale_on(False) polygons = [] color = [] for ann in anns: - c = np.random.random((1, 3)).tolist()[0] - if type(ann['segmentation']) == list: - # polygon - for seg in ann['segmentation']: - poly = np.array(seg).reshape((len(seg)/2, 2)) - polygons.append(Polygon(poly, True,alpha=0.4)) - color.append(c) - else: - # mask - t = self.imgs[ann['image_id']] - if type(ann['segmentation']['counts']) == list: - rle = mask.frPyObjects([ann['segmentation']], t['height'], t['width']) + c = (np.random.random((1, 3))*0.6+0.4).tolist()[0] + if 'segmentation' in ann: + if type(ann['segmentation']) == list: + # polygon + for seg in ann['segmentation']: + poly = np.array(seg).reshape((int(len(seg)/2), 2)) + polygons.append(Polygon(poly)) + color.append(c) else: - rle = [ann['segmentation']] - m = mask.decode(rle) - img = np.ones( (m.shape[0], m.shape[1], 3) ) - if ann['iscrowd'] == 1: - color_mask = np.array([2.0,166.0,101.0])/255 - if ann['iscrowd'] == 0: - color_mask = np.random.random((1, 3)).tolist()[0] - for i in range(3): - img[:,:,i] = color_mask[i] - ax.imshow(np.dstack( (img, m*0.5) )) - p = PatchCollection(polygons, facecolors=color, edgecolors=(0,0,0,1), linewidths=3, alpha=0.4) + # mask + t = self.imgs[ann['image_id']] + if type(ann['segmentation']['counts']) == list: + rle = maskUtils.frPyObjects([ann['segmentation']], t['height'], t['width']) + else: + rle = [ann['segmentation']] + m = maskUtils.decode(rle) + img = np.ones( (m.shape[0], m.shape[1], 3) ) + if ann['iscrowd'] == 1: + color_mask = np.array([2.0,166.0,101.0])/255 + if ann['iscrowd'] == 0: + color_mask = np.random.random((1, 3)).tolist()[0] + for i in range(3): + img[:,:,i] = color_mask[i] + ax.imshow(np.dstack( (img, m*0.5) )) + if 'keypoints' in ann and type(ann['keypoints']) == list: + # turn skeleton into zero-based index + sks = np.array(self.loadCats(ann['category_id'])[0]['skeleton'])-1 + kp = np.array(ann['keypoints']) + x = kp[0::3] + y = kp[1::3] + v = kp[2::3] + for sk in sks: + if np.all(v[sk]>0): + plt.plot(x[sk],y[sk], linewidth=3, color=c) + plt.plot(x[v>0], y[v>0],'o',markersize=8, markerfacecolor=c, markeredgecolor='k',markeredgewidth=2) + plt.plot(x[v>1], y[v>1],'o',markersize=8, markerfacecolor=c, markeredgecolor=c, markeredgewidth=2) + p = PatchCollection(polygons, facecolor=color, linewidths=0, alpha=0.4) + ax.add_collection(p) + p = PatchCollection(polygons, facecolor='none', edgecolors=color, linewidths=2) ax.add_collection(p) elif datasetType == 'captions': for ann in anns: @@ -286,12 +297,15 @@ def loadRes(self, resFile): """ res = COCO() res.dataset['images'] = [img for img in self.dataset['images']] - # res.dataset['info'] = copy.deepcopy(self.dataset['info']) - # res.dataset['licenses'] = copy.deepcopy(self.dataset['licenses']) - print('Loading and preparing results... ') + print('Loading and preparing results...') tic = time.time() - anns = json.load(open(resFile)) + if type(resFile) == str or type(resFile) == unicode: + anns = json.load(open(resFile)) + elif type(resFile) == np.ndarray: + anns = self.loadNumpyAnnotations(resFile) + else: + anns = resFile assert type(anns) == list, 'results in not an array of objects' annsImgIds = [ann['image_id'] for ann in anns] assert set(annsImgIds) == (set(annsImgIds) & set(self.getImgIds())), \ @@ -315,18 +329,28 @@ def loadRes(self, resFile): res.dataset['categories'] = copy.deepcopy(self.dataset['categories']) for id, ann in enumerate(anns): # now only support compressed RLE format as segmentation results - ann['area'] = mask.area([ann['segmentation']])[0] + ann['area'] = maskUtils.area(ann['segmentation']) if not 'bbox' in ann: - ann['bbox'] = mask.toBbox([ann['segmentation']])[0] + ann['bbox'] = maskUtils.toBbox(ann['segmentation']) ann['id'] = id+1 ann['iscrowd'] = 0 - print('DONE (t=%0.2fs)'%(time.time()- tic)) + elif 'keypoints' in anns[0]: + res.dataset['categories'] = copy.deepcopy(self.dataset['categories']) + for id, ann in enumerate(anns): + s = ann['keypoints'] + x = s[0::3] + y = s[1::3] + x0,x1,y0,y1 = np.min(x), np.max(x), np.min(y), np.max(y) + ann['area'] = (x1-x0)*(y1-y0) + ann['id'] = id + 1 + ann['bbox'] = [x0,y0,x1-x0,y1-y0] + print('DONE (t={:0.2f}s)'.format(time.time()- tic)) res.dataset['annotations'] = anns res.createIndex() return res - def download(self, tarDir=None, imgIds=[]): + def download(self, tarDir = None, imgIds = [] ): ''' Download COCO images from mscoco.org server. :param tarDir (str): COCO results directory name @@ -347,5 +371,58 @@ def download(self, tarDir=None, imgIds=[]): tic = time.time() fname = os.path.join(tarDir, img['file_name']) if not os.path.exists(fname): - urllib.urlretrieve(img['coco_url'], fname) - print('downloaded %d/%d images (t=%.1fs)'%(i, N, time.time()- tic)) + urlretrieve(img['coco_url'], fname) + print('downloaded {}/{} images (t={:0.1f}s)'.format(i, N, time.time()- tic)) + + def loadNumpyAnnotations(self, data): + """ + Convert result data from a numpy array [Nx7] where each row contains {imageID,x1,y1,w,h,score,class} + :param data (numpy.ndarray) + :return: annotations (python nested list) + """ + print('Converting ndarray to lists...') + assert(type(data) == np.ndarray) + print(data.shape) + assert(data.shape[1] == 7) + N = data.shape[0] + ann = [] + for i in range(N): + if i % 1000000 == 0: + print('{}/{}'.format(i,N)) + ann += [{ + 'image_id' : int(data[i, 0]), + 'bbox' : [ data[i, 1], data[i, 2], data[i, 3], data[i, 4] ], + 'score' : data[i, 5], + 'category_id': int(data[i, 6]), + }] + return ann + + def annToRLE(self, ann): + """ + Convert annotation which can be polygons, uncompressed RLE to RLE. + :return: binary mask (numpy 2D array) + """ + t = self.imgs[ann['image_id']] + h, w = t['height'], t['width'] + segm = ann['segmentation'] + if type(segm) == list: + # polygon -- a single object might consist of multiple parts + # we merge all parts into one mask rle code + rles = maskUtils.frPyObjects(segm, h, w) + rle = maskUtils.merge(rles) + elif type(segm['counts']) == list: + # uncompressed RLE + rle = maskUtils.frPyObjects(segm, h, w) + else: + # rle + rle = ann['segmentation'] + return rle + + def annToMask(self, ann): + """ + Convert annotation which can be polygons, uncompressed RLE, or RLE to binary mask. + :return: binary mask (numpy 2D array) + """ + rle = self.annToRLE(ann) + m = maskUtils.decode(rle) + return m \ No newline at end of file diff --git a/example/rcnn/rcnn/pycocotools/cocoeval.py b/example/rcnn/rcnn/pycocotools/cocoeval.py index 015c9f4ff8cc..a5dd1852912d 100644 --- a/example/rcnn/rcnn/pycocotools/cocoeval.py +++ b/example/rcnn/rcnn/pycocotools/cocoeval.py @@ -1,11 +1,10 @@ __author__ = 'tsungyi' -from __future__ import print_function import numpy as np import datetime import time from collections import defaultdict -import mask +import mask as maskUtils import copy class COCOeval: @@ -27,8 +26,9 @@ class COCOeval: # recThrs - [0:.01:1] R=101 recall thresholds for evaluation # areaRng - [...] A=4 object area ranges for evaluation # maxDets - [1 10 100] M=3 thresholds on max detections per image - # useSegm - [1] if true evaluate against ground-truth segments - # useCats - [1] if true use category labels for evaluation # Note: if useSegm=0 the evaluation is run on bounding boxes. + # iouType - ['segm'] set iouType to 'segm', 'bbox' or 'keypoints' + # iouType replaced the now DEPRECATED useSegm parameter. + # useCats - [1] if true use category labels for evaluation # Note: if useCats=0 category labels are ignored as in proposal scoring. # Note: multiple areaRngs [Ax2] and maxDets [Mx1] can be specified. # @@ -57,13 +57,15 @@ class COCOeval: # Data, paper, and tutorials available at: http://mscoco.org/ # Code written by Piotr Dollar and Tsung-Yi Lin, 2015. # Licensed under the Simplified BSD License [see coco/license.txt] - def __init__(self, cocoGt=None, cocoDt=None): + def __init__(self, cocoGt=None, cocoDt=None, iouType='segm'): ''' Initialize CocoEval using coco APIs for gt and dt :param cocoGt: coco object with ground truth annotations :param cocoDt: coco object with detection results :return: None ''' + if not iouType: + print('iouType not specified. use default iouType segm') self.cocoGt = cocoGt # ground truth COCO API self.cocoDt = cocoDt # detections COCO API self.params = {} # evaluation parameters @@ -71,7 +73,7 @@ def __init__(self, cocoGt=None, cocoDt=None): self.eval = {} # accumulated evaluation results self._gts = defaultdict(list) # gt for evaluation self._dts = defaultdict(list) # dt for evaluation - self.params = Params() # parameters + self.params = Params(iouType=iouType) # parameters self._paramsEval = {} # parameters for evaluation self.stats = [] # result summarization self.ious = {} # ious between all gts and dts @@ -85,28 +87,11 @@ def _prepare(self): Prepare ._gts and ._dts for evaluation based on params :return: None ''' - # - def _toMask(objs, coco): - # modify segmentation by reference - for obj in objs: - t = coco.imgs[obj['image_id']] - if type(obj['segmentation']) == list: - if type(obj['segmentation'][0]) == dict: - print('debug') - obj['segmentation'] = mask.frPyObjects(obj['segmentation'],t['height'],t['width']) - if len(obj['segmentation']) == 1: - obj['segmentation'] = obj['segmentation'][0] - else: - # an object can have multiple polygon regions - # merge them into one RLE mask - obj['segmentation'] = mask.merge(obj['segmentation']) - elif type(obj['segmentation']) == dict and type(obj['segmentation']['counts']) == list: - obj['segmentation'] = mask.frPyObjects([obj['segmentation']],t['height'],t['width'])[0] - elif type(obj['segmentation']) == dict and \ - type(obj['segmentation']['counts'] == unicode or type(obj['segmentation']['counts']) == str): - pass - else: - raise Exception('segmentation format not supported.') + def _toMask(anns, coco): + # modify ann['segmentation'] by reference + for ann in anns: + rle = coco.annToRLE(ann) + ann['segmentation'] = rle p = self.params if p.useCats: gts=self.cocoGt.loadAnns(self.cocoGt.getAnnIds(imgIds=p.imgIds, catIds=p.catIds)) @@ -115,9 +100,16 @@ def _toMask(objs, coco): gts=self.cocoGt.loadAnns(self.cocoGt.getAnnIds(imgIds=p.imgIds)) dts=self.cocoDt.loadAnns(self.cocoDt.getAnnIds(imgIds=p.imgIds)) - if p.useSegm: + # convert ground truth to mask if iouType == 'segm' + if p.iouType == 'segm': _toMask(gts, self.cocoGt) _toMask(dts, self.cocoDt) + # set ignore flag + for gt in gts: + gt['ignore'] = gt['ignore'] if 'ignore' in gt else 0 + gt['ignore'] = 'iscrowd' in gt and gt['iscrowd'] + if p.iouType == 'keypoints': + gt['ignore'] = (gt['num_keypoints'] == 0) or gt['ignore'] self._gts = defaultdict(list) # gt for evaluation self._dts = defaultdict(list) # dt for evaluation for gt in gts: @@ -133,8 +125,13 @@ def evaluate(self): :return: None ''' tic = time.time() - print('Running per image evaluation... ') + print('Running per image evaluation...') p = self.params + # add backward compatibility if useSegm is specified in params + if not p.useSegm is None: + p.iouType = 'segm' if p.useSegm == 1 else 'bbox' + print('useSegm (deprecated) is not None. Running {} evaluation'.format(p.iouType)) + print('Evaluate annotation type *{}*'.format(p.iouType)) p.imgIds = list(np.unique(p.imgIds)) if p.useCats: p.catIds = list(np.unique(p.catIds)) @@ -145,7 +142,10 @@ def evaluate(self): # loop through images, area range, max detection number catIds = p.catIds if p.useCats else [-1] - computeIoU = self.computeIoU + if p.iouType == 'segm' or p.iouType == 'bbox': + computeIoU = self.computeIoU + elif p.iouType == 'keypoints': + computeIoU = self.computeOks self.ious = {(imgId, catId): computeIoU(imgId, catId) \ for imgId in p.imgIds for catId in catIds} @@ -159,7 +159,7 @@ def evaluate(self): ] self._paramsEval = copy.deepcopy(self.params) toc = time.time() - print('DONE (t=%0.2fs).'%(toc-tic)) + print('DONE (t={:0.2f}s).'.format(toc-tic)) def computeIoU(self, imgId, catId): p = self.params @@ -171,20 +171,66 @@ def computeIoU(self, imgId, catId): dt = [_ for cId in p.catIds for _ in self._dts[imgId,cId]] if len(gt) == 0 and len(dt) ==0: return [] - dt = sorted(dt, key=lambda x: -x['score']) + inds = np.argsort([-d['score'] for d in dt], kind='mergesort') + dt = [dt[i] for i in inds] if len(dt) > p.maxDets[-1]: dt=dt[0:p.maxDets[-1]] - if p.useSegm: + if p.iouType == 'segm': g = [g['segmentation'] for g in gt] d = [d['segmentation'] for d in dt] - else: + elif p.iouType == 'bbox': g = [g['bbox'] for g in gt] d = [d['bbox'] for d in dt] + else: + raise Exception('unknown iouType for iou computation') # compute iou between each dt and gt region iscrowd = [int(o['iscrowd']) for o in gt] - ious = mask.iou(d,g,iscrowd) + ious = maskUtils.iou(d,g,iscrowd) + return ious + + def computeOks(self, imgId, catId): + p = self.params + # dimention here should be Nxm + gts = self._gts[imgId, catId] + dts = self._dts[imgId, catId] + inds = np.argsort([-d['score'] for d in dts], kind='mergesort') + dts = [dts[i] for i in inds] + if len(dts) > p.maxDets[-1]: + dts = dts[0:p.maxDets[-1]] + # if len(gts) == 0 and len(dts) == 0: + if len(gts) == 0 or len(dts) == 0: + return [] + ious = np.zeros((len(dts), len(gts))) + sigmas = np.array([.26, .25, .25, .35, .35, .79, .79, .72, .72, .62,.62, 1.07, 1.07, .87, .87, .89, .89])/10.0 + vars = (sigmas * 2)**2 + k = len(sigmas) + # compute oks between each detection and ground truth object + for j, gt in enumerate(gts): + # create bounds for ignore regions(double the gt bbox) + g = np.array(gt['keypoints']) + xg = g[0::3]; yg = g[1::3]; vg = g[2::3] + k1 = np.count_nonzero(vg > 0) + bb = gt['bbox'] + x0 = bb[0] - bb[2]; x1 = bb[0] + bb[2] * 2 + y0 = bb[1] - bb[3]; y1 = bb[1] + bb[3] * 2 + for i, dt in enumerate(dts): + d = np.array(dt['keypoints']) + xd = d[0::3]; yd = d[1::3] + if k1>0: + # measure the per-keypoint distance if keypoints visible + dx = xd - xg + dy = yd - yg + else: + # measure minimum distance to keypoints in (x0,y0) & (x1,y1) + z = np.zeros((k)) + dx = np.max((z, x0-xd),axis=0)+np.max((z, xd-x1),axis=0) + dy = np.max((z, y0-yd),axis=0)+np.max((z, yd-y1),axis=0) + e = (dx**2 + dy**2) / vars / (gt['area']+np.spacing(1)) / 2 + if k1 > 0: + e=e[vg > 0] + ious[i, j] = np.sum(np.exp(-e)) / e.shape[0] return ious def evaluateImg(self, imgId, catId, aRng, maxDet): @@ -192,7 +238,6 @@ def evaluateImg(self, imgId, catId, aRng, maxDet): perform evaluation for single category and image :return: dict (single image results) ''' - # p = self.params if p.useCats: gt = self._gts[imgId,catId] @@ -204,23 +249,19 @@ def evaluateImg(self, imgId, catId, aRng, maxDet): return None for g in gt: - if 'ignore' not in g: - g['ignore'] = 0 - if g['iscrowd'] == 1 or g['ignore'] or (g['area']aRng[1]): + if g['ignore'] or (g['area']aRng[1]): g['_ignore'] = 1 else: g['_ignore'] = 0 # sort dt highest score first, sort gt ignore last - # gt = sorted(gt, key=lambda x: x['_ignore']) - gtind = [ind for (ind, g) in sorted(enumerate(gt), key=lambda (ind, g): g['_ignore']) ] - - gt = [gt[ind] for ind in gtind] - dt = sorted(dt, key=lambda x: -x['score'])[0:maxDet] + gtind = np.argsort([g['_ignore'] for g in gt], kind='mergesort') + gt = [gt[i] for i in gtind] + dtind = np.argsort([-d['score'] for d in dt], kind='mergesort') + dt = [dt[i] for i in dtind[0:maxDet]] iscrowd = [int(o['iscrowd']) for o in gt] # load computed ious - N_iou = len(self.ious[imgId, catId]) - ious = self.ious[imgId, catId][0:maxDet, np.array(gtind)] if N_iou >0 else self.ious[imgId, catId] + ious = self.ious[imgId, catId][:, gtind] if len(self.ious[imgId, catId]) > 0 else self.ious[imgId, catId] T = len(p.iouThrs) G = len(gt) @@ -245,7 +286,7 @@ def evaluateImg(self, imgId, catId, aRng, maxDet): # continue to next gt unless better match made if ious[dind,gind] < iou: continue - # match successful and best so far, store appropriately + # if match successful and best so far, store appropriately iou=ious[dind,gind] m=gind # if match made store id of match for both dt and gt @@ -278,7 +319,7 @@ def accumulate(self, p = None): :param p: input params for evaluation :return: None ''' - print('Accumulating evaluation results... ') + print('Accumulating evaluation results...') tic = time.time() if not self.evalImgs: print('Please run evaluate() first') @@ -306,7 +347,6 @@ def accumulate(self, p = None): m_list = [m for n, m in enumerate(p.maxDets) if m in setM] a_list = [n for n, a in enumerate(map(lambda x: tuple(x), p.areaRng)) if a in setA] i_list = [n for n, i in enumerate(p.imgIds) if i in setI] - # K0 = len(_pe.catIds) I0 = len(_pe.imgIds) A0 = len(_pe.areaRng) # retrieve E at each category, area range, and max number of detections @@ -315,8 +355,8 @@ def accumulate(self, p = None): for a, a0 in enumerate(a_list): Na = a0*I0 for m, maxDet in enumerate(m_list): - E = [self.evalImgs[Nk+Na+i] for i in i_list] - E = filter(None, E) + E = [self.evalImgs[Nk + Na + i] for i in i_list] + E = [e for e in E if not e is None] if len(E) == 0: continue dtScores = np.concatenate([e['dtScores'][0:maxDet] for e in E]) @@ -327,8 +367,8 @@ def accumulate(self, p = None): dtm = np.concatenate([e['dtMatches'][:,0:maxDet] for e in E], axis=1)[:,inds] dtIg = np.concatenate([e['dtIgnore'][:,0:maxDet] for e in E], axis=1)[:,inds] - gtIg = np.concatenate([e['gtIgnore'] for e in E]) - npig = len([ig for ig in gtIg if ig == 0]) + gtIg = np.concatenate([e['gtIgnore'] for e in E]) + npig = np.count_nonzero(gtIg==0 ) if npig == 0: continue tps = np.logical_and( dtm, np.logical_not(dtIg) ) @@ -357,7 +397,7 @@ def accumulate(self, p = None): if pr[i] > pr[i-1]: pr[i-1] = pr[i] - inds = np.searchsorted(rc, p.recThrs) + inds = np.searchsorted(rc, p.recThrs, side='left') try: for ri, pi in enumerate(inds): q[ri] = pr[pi] @@ -367,12 +407,12 @@ def accumulate(self, p = None): self.eval = { 'params': p, 'counts': [T, R, K, A, M], - 'date': datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + 'date': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), 'precision': precision, 'recall': recall, } toc = time.time() - print('DONE (t=%0.2fs).'%( toc-tic )) + print('DONE (t={:0.2f}s).'.format( toc-tic)) def summarize(self): ''' @@ -381,15 +421,14 @@ def summarize(self): ''' def _summarize( ap=1, iouThr=None, areaRng='all', maxDets=100 ): p = self.params - iStr = ' {:<18} {} @[ IoU={:<9} | area={:>6} | maxDets={:>3} ] = {}' - titleStr = 'Average Precision' if ap == 1 else 'Average Recall' - typeStr = '(AP)' if ap==1 else '(AR)' - iouStr = '%0.2f:%0.2f'%(p.iouThrs[0], p.iouThrs[-1]) if iouThr is None else '%0.2f'%(iouThr) - areaStr = areaRng - maxDetsStr = '%d'%(maxDets) - - aind = [i for i, aRng in enumerate(['all', 'small', 'medium', 'large']) if aRng == areaRng] - mind = [i for i, mDet in enumerate([1, 10, 100]) if mDet == maxDets] + iStr = ' {:<18} {} @[ IoU={:<9} | area={:>6s} | maxDets={:>3d} ] = {:0.3f}' + titleStr = 'Average Precision' if ap == 1 else 'Average Recall' + typeStr = '(AP)' if ap==1 else '(AR)' + iouStr = '{:0.2f}:{:0.2f}'.format(p.iouThrs[0], p.iouThrs[-1]) \ + if iouThr is None else '{:0.2f}'.format(iouThr) + + aind = [i for i, aRng in enumerate(p.areaRngLbl) if aRng == areaRng] + mind = [i for i, mDet in enumerate(p.maxDets) if mDet == maxDets] if ap == 1: # dimension of precision: [TxRxKxAxM] s = self.eval['precision'] @@ -397,34 +436,56 @@ def _summarize( ap=1, iouThr=None, areaRng='all', maxDets=100 ): if iouThr is not None: t = np.where(iouThr == p.iouThrs)[0] s = s[t] - # areaRng s = s[:,:,:,aind,mind] else: # dimension of recall: [TxKxAxM] s = self.eval['recall'] + if iouThr is not None: + t = np.where(iouThr == p.iouThrs)[0] + s = s[t] s = s[:,:,aind,mind] if len(s[s>-1])==0: mean_s = -1 else: mean_s = np.mean(s[s>-1]) - print(iStr.format(titleStr, typeStr, iouStr, areaStr, maxDetsStr, '%.3f'%(float(mean_s)))) + print(iStr.format(titleStr, typeStr, iouStr, areaRng, maxDets, mean_s)) return mean_s - + def _summarizeDets(): + stats = np.zeros((12,)) + stats[0] = _summarize(1) + stats[1] = _summarize(1, iouThr=.5, maxDets=self.params.maxDets[2]) + stats[2] = _summarize(1, iouThr=.75, maxDets=self.params.maxDets[2]) + stats[3] = _summarize(1, areaRng='small', maxDets=self.params.maxDets[2]) + stats[4] = _summarize(1, areaRng='medium', maxDets=self.params.maxDets[2]) + stats[5] = _summarize(1, areaRng='large', maxDets=self.params.maxDets[2]) + stats[6] = _summarize(0, maxDets=self.params.maxDets[0]) + stats[7] = _summarize(0, maxDets=self.params.maxDets[1]) + stats[8] = _summarize(0, maxDets=self.params.maxDets[2]) + stats[9] = _summarize(0, areaRng='small', maxDets=self.params.maxDets[2]) + stats[10] = _summarize(0, areaRng='medium', maxDets=self.params.maxDets[2]) + stats[11] = _summarize(0, areaRng='large', maxDets=self.params.maxDets[2]) + return stats + def _summarizeKps(): + stats = np.zeros((10,)) + stats[0] = _summarize(1, maxDets=20) + stats[1] = _summarize(1, maxDets=20, iouThr=.5) + stats[2] = _summarize(1, maxDets=20, iouThr=.75) + stats[3] = _summarize(1, maxDets=20, areaRng='medium') + stats[4] = _summarize(1, maxDets=20, areaRng='large') + stats[5] = _summarize(0, maxDets=20) + stats[6] = _summarize(0, maxDets=20, iouThr=.5) + stats[7] = _summarize(0, maxDets=20, iouThr=.75) + stats[8] = _summarize(0, maxDets=20, areaRng='medium') + stats[9] = _summarize(0, maxDets=20, areaRng='large') + return stats if not self.eval: raise Exception('Please run accumulate() first') - self.stats = np.zeros((12,)) - self.stats[0] = _summarize(1) - self.stats[1] = _summarize(1,iouThr=.5) - self.stats[2] = _summarize(1,iouThr=.75) - self.stats[3] = _summarize(1,areaRng='small') - self.stats[4] = _summarize(1,areaRng='medium') - self.stats[5] = _summarize(1,areaRng='large') - self.stats[6] = _summarize(0,maxDets=1) - self.stats[7] = _summarize(0,maxDets=10) - self.stats[8] = _summarize(0,maxDets=100) - self.stats[9] = _summarize(0,areaRng='small') - self.stats[10] = _summarize(0,areaRng='medium') - self.stats[11] = _summarize(0,areaRng='large') + iouType = self.params.iouType + if iouType == 'segm' or iouType == 'bbox': + summarize = _summarizeDets + elif iouType == 'keypoints': + summarize = _summarizeKps + self.stats = summarize() def __str__(self): self.summarize() @@ -433,13 +494,35 @@ class Params: ''' Params for coco evaluation api ''' - def __init__(self): + def setDetParams(self): + self.imgIds = [] + self.catIds = [] + # np.arange causes trouble. the data point on arange is slightly larger than the true value + self.iouThrs = np.linspace(.5, 0.95, np.round((0.95 - .5) / .05) + 1, endpoint=True) + self.recThrs = np.linspace(.0, 1.00, np.round((1.00 - .0) / .01) + 1, endpoint=True) + self.maxDets = [1, 10, 100] + self.areaRng = [[0 ** 2, 1e5 ** 2], [0 ** 2, 32 ** 2], [32 ** 2, 96 ** 2], [96 ** 2, 1e5 ** 2]] + self.areaRngLbl = ['all', 'small', 'medium', 'large'] + self.useCats = 1 + + def setKpParams(self): self.imgIds = [] self.catIds = [] # np.arange causes trouble. the data point on arange is slightly larger than the true value - self.iouThrs = np.linspace(.5, 0.95, np.round((0.95-.5)/.05)+1, endpoint=True) - self.recThrs = np.linspace(.0, 1.00, np.round((1.00-.0)/.01)+1, endpoint=True) - self.maxDets = [1,10,100] - self.areaRng = [ [0**2,1e5**2], [0**2, 32**2], [32**2, 96**2], [96**2, 1e5**2] ] - self.useSegm = 0 - self.useCats = 1 \ No newline at end of file + self.iouThrs = np.linspace(.5, 0.95, np.round((0.95 - .5) / .05) + 1, endpoint=True) + self.recThrs = np.linspace(.0, 1.00, np.round((1.00 - .0) / .01) + 1, endpoint=True) + self.maxDets = [20] + self.areaRng = [[0 ** 2, 1e5 ** 2], [32 ** 2, 96 ** 2], [96 ** 2, 1e5 ** 2]] + self.areaRngLbl = ['all', 'medium', 'large'] + self.useCats = 1 + + def __init__(self, iouType='segm'): + if iouType == 'segm' or iouType == 'bbox': + self.setDetParams() + elif iouType == 'keypoints': + self.setKpParams() + else: + raise Exception('iouType not supported') + self.iouType = iouType + # useSegm is deprecated + self.useSegm = None \ No newline at end of file diff --git a/example/rcnn/rcnn/pycocotools/mask.py b/example/rcnn/rcnn/pycocotools/mask.py index c00e09b6e46e..f49b8736b280 100644 --- a/example/rcnn/rcnn/pycocotools/mask.py +++ b/example/rcnn/rcnn/pycocotools/mask.py @@ -1,6 +1,6 @@ __author__ = 'tsungyi' -import _mask as _mask +import _mask # Interface for manipulating masks stored in RLE format. # @@ -73,10 +73,31 @@ # Code written by Piotr Dollar and Tsung-Yi Lin, 2015. # Licensed under the Simplified BSD License [see coco/license.txt] -encode = _mask.encode -decode = _mask.decode iou = _mask.iou merge = _mask.merge -area = _mask.area -toBbox = _mask.toBbox frPyObjects = _mask.frPyObjects + +def encode(bimask): + if len(bimask.shape) == 3: + return _mask.encode(bimask) + elif len(bimask.shape) == 2: + h, w = bimask.shape + return _mask.encode(bimask.reshape((h, w, 1), order='F'))[0] + +def decode(rleObjs): + if type(rleObjs) == list: + return _mask.decode(rleObjs) + else: + return _mask.decode([rleObjs])[:,:,0] + +def area(rleObjs): + if type(rleObjs) == list: + return _mask.area(rleObjs) + else: + return _mask.area([rleObjs])[0] + +def toBbox(rleObjs): + if type(rleObjs) == list: + return _mask.toBbox(rleObjs) + else: + return _mask.toBbox([rleObjs])[0] \ No newline at end of file diff --git a/example/rcnn/rcnn/pycocotools/maskApi.c b/example/rcnn/rcnn/pycocotools/maskApi.c index 2b2d89116574..85e397918278 100644 --- a/example/rcnn/rcnn/pycocotools/maskApi.c +++ b/example/rcnn/rcnn/pycocotools/maskApi.c @@ -13,7 +13,7 @@ uint umax( uint a, uint b ) { return (a>b) ? a : b; } void rleInit( RLE *R, siz h, siz w, siz m, uint *cnts ) { R->h=h; R->w=w; R->m=m; R->cnts=(m==0)?0:malloc(sizeof(uint)*m); - if(cnts) for(siz j=0; jcnts[j]=cnts[j]; + siz j; if(cnts) for(j=0; jcnts[j]=cnts[j]; } void rleFree( RLE *R ) { @@ -21,12 +21,12 @@ void rleFree( RLE *R ) { } void rlesInit( RLE **R, siz n ) { - *R = (RLE*) malloc(sizeof(RLE)*n); - for(siz i=0; i0) { crowd=iscrowd!=NULL && iscrowd[g]; if(dt[d].h!=gt[g].h || dt[d].w!=gt[g].w) { o[g*m+d]=-1; continue; } - siz ka, kb, a, b; uint c, ca, cb, ct, i, u; bool va, vb; + siz ka, kb, a, b; uint c, ca, cb, ct, i, u; int va, vb; ca=dt[d].cnts[0]; ka=dt[d].m; va=vb=0; cb=gt[g].cnts[0]; kb=gt[g].m; a=b=1; i=u=0; ct=1; while( ct>0 ) { @@ -95,8 +95,19 @@ void rleIou( RLE *dt, RLE *gt, siz m, siz n, byte *iscrowd, double *o ) { } } +void rleNms( RLE *dt, siz n, uint *keep, double thr ) { + siz i, j; double u; + for( i=0; ithr) keep[j]=0; + } + } +} + void bbIou( BB dt, BB gt, siz m, siz n, byte *iscrowd, double *o ) { - double h, w, i, u, ga, da; siz g, d; bool crowd; + double h, w, i, u, ga, da; siz g, d; int crowd; for( g=0; gthr) keep[j]=0; + } + } +} + void rleToBbox( const RLE *R, BB bb, siz n ) { - for( siz i=0; i=dy && xs>xe) || (dxye); if(flip) { t=xs; xs=xe; xe=t; t=ys; ys=ye; ye=t; } s = dx>=dy ? (double)(ye-ys)/dx : (double)(xe-xs)/dy; - if(dx>=dy) for( int d=0; d<=dx; d++ ) { + if(dx>=dy) for( d=0; d<=dx; d++ ) { t=flip?dx-d:d; u[m]=t+xs; v[m]=(int)(ys+s*t+.5); m++; - } else for( int d=0; d<=dy; d++ ) { + } else for( d=0; d<=dy; d++ ) { t=flip?dy-d:d; v[m]=t+ys; u[m]=(int)(xs+s*t+.5); m++; } } - // get points along y-boundary and downsample + /* get points along y-boundary and downsample */ free(x); free(y); k=m; m=0; double xd, yd; x=malloc(sizeof(int)*k); y=malloc(sizeof(int)*k); for( j=1; jh) yd=h; yd=ceil(yd); x[m]=(int) xd; y[m]=(int) yd; m++; } - // compute rle encoding given y-boundary points + /* compute rle encoding given y-boundary points */ k=m; a=malloc(sizeof(uint)*(k+1)); for( j=0; jm, p=0; long x; bool more; + /* Similar to LEB128 but using 6 bits/char and ascii chars 48-111. */ + siz i, m=R->m, p=0; long x; int more; char *s=malloc(sizeof(char)*m*6); for( i=0; icnts[i]; if(i>2) x-=(long) R->cnts[i-2]; more=1; @@ -193,7 +215,7 @@ char* rleToString( const RLE *R ) { } void rleFrString( RLE *R, char *s, siz h, siz w ) { - siz m=0, p=0, k; long x; bool more; uint *cnts; + siz m=0, p=0, k; long x; int more; uint *cnts; while( s[m] ) m++; cnts=malloc(sizeof(uint)*m); m=0; while( s[p] ) { x=0; k=0; more=1; diff --git a/example/rcnn/rcnn/pycocotools/maskApi.h b/example/rcnn/rcnn/pycocotools/maskApi.h index ff16116c4781..ebc7892da382 100644 --- a/example/rcnn/rcnn/pycocotools/maskApi.h +++ b/example/rcnn/rcnn/pycocotools/maskApi.h @@ -5,7 +5,6 @@ * Licensed under the Simplified BSD License [see coco/license.txt] **************************************************************************/ #pragma once -#include typedef unsigned int uint; typedef unsigned long siz; @@ -13,43 +12,49 @@ typedef unsigned char byte; typedef double* BB; typedef struct { siz h, w, m; uint *cnts; } RLE; -// Initialize/destroy RLE. +/* Initialize/destroy RLE. */ void rleInit( RLE *R, siz h, siz w, siz m, uint *cnts ); void rleFree( RLE *R ); -// Initialize/destroy RLE array. +/* Initialize/destroy RLE array. */ void rlesInit( RLE **R, siz n ); void rlesFree( RLE **R, siz n ); -// Encode binary masks using RLE. +/* Encode binary masks using RLE. */ void rleEncode( RLE *R, const byte *mask, siz h, siz w, siz n ); -// Decode binary masks encoded via RLE. +/* Decode binary masks encoded via RLE. */ void rleDecode( const RLE *R, byte *mask, siz n ); -// Compute union or intersection of encoded masks. -void rleMerge( const RLE *R, RLE *M, siz n, bool intersect ); +/* Compute union or intersection of encoded masks. */ +void rleMerge( const RLE *R, RLE *M, siz n, int intersect ); -// Compute area of encoded masks. +/* Compute area of encoded masks. */ void rleArea( const RLE *R, siz n, uint *a ); -// Compute intersection over union between masks. +/* Compute intersection over union between masks. */ void rleIou( RLE *dt, RLE *gt, siz m, siz n, byte *iscrowd, double *o ); -// Compute intersection over union between bounding boxes. +/* Compute non-maximum suppression between bounding masks */ +void rleNms( RLE *dt, siz n, uint *keep, double thr ); + +/* Compute intersection over union between bounding boxes. */ void bbIou( BB dt, BB gt, siz m, siz n, byte *iscrowd, double *o ); -// Get bounding boxes surrounding encoded masks. +/* Compute non-maximum suppression between bounding boxes */ +void bbNms( BB dt, siz n, uint *keep, double thr ); + +/* Get bounding boxes surrounding encoded masks. */ void rleToBbox( const RLE *R, BB bb, siz n ); -// Convert bounding boxes to encoded masks. +/* Convert bounding boxes to encoded masks. */ void rleFrBbox( RLE *R, const BB bb, siz h, siz w, siz n ); -// Convert polygon to encoded mask. +/* Convert polygon to encoded mask. */ void rleFrPoly( RLE *R, const double *xy, siz k, siz h, siz w ); -// Get compressed string representation of encoded mask. +/* Get compressed string representation of encoded mask. */ char* rleToString( const RLE *R ); -// Convert from compressed string representation of encoded mask. +/* Convert from compressed string representation of encoded mask. */ void rleFrString( RLE *R, char *s, siz h, siz w ); diff --git a/example/rcnn/rcnn/symbol/proposal.py b/example/rcnn/rcnn/symbol/proposal.py index 397030db6d7c..dd0bb15f5168 100644 --- a/example/rcnn/rcnn/symbol/proposal.py +++ b/example/rcnn/rcnn/symbol/proposal.py @@ -3,18 +3,16 @@ classification probability and bounding box prediction results, and image size and scale information. """ -from __future__ import print_function import mxnet as mx import numpy as np import numpy.random as npr from distutils.util import strtobool +from rcnn.logger import logger from rcnn.processing.bbox_transform import bbox_pred, clip_boxes from rcnn.processing.generate_anchor import generate_anchors from rcnn.processing.nms import py_nms_wrapper, cpu_nms_wrapper, gpu_nms_wrapper -DEBUG = False - class ProposalOperator(mx.operator.CustomOp): def __init__(self, feat_stride, scales, ratios, output_score, @@ -31,10 +29,8 @@ def __init__(self, feat_stride, scales, ratios, output_score, self._threshold = threshold self._rpn_min_size = rpn_min_size - if DEBUG: - print('feat_stride: {}'.format(self._feat_stride)) - print('anchors:') - print(self._anchors) + logger.debug('feat_stride: %s' % self._feat_stride) + logger.debug('anchors:\n%s' % self._anchors) def forward(self, is_train, req, in_data, out_data, aux): nms = gpu_nms_wrapper(self._threshold, in_data[0].context.device_id) @@ -64,17 +60,14 @@ def forward(self, is_train, req, in_data, out_data, aux): bbox_deltas = in_data[1].asnumpy() im_info = in_data[2].asnumpy()[0, :] - if DEBUG: - print('im_size: ({}, {})'.format(im_info[0], im_info[1])) - print('scale: {}'.format(im_info[2])) + logger.debug('im_info: %s' % im_info) # 1. Generate proposals from bbox_deltas and shifted anchors # use real image size instead of padded feature map sizes height, width = int(im_info[0] / self._feat_stride), int(im_info[1] / self._feat_stride) - if DEBUG: - print('score map size: {}'.format(scores.shape)) - print("resudial: {}".format((scores.shape[2] - height, scores.shape[3] - width))) + logger.debug('score map size: (%d, %d)' % (scores.shape[2], scores.shape[3])) + logger.debug('resudial: (%d, %d)' % (scores.shape[2] - height, scores.shape[3] - width)) # Enumerate all shifts shift_x = np.arange(0, width) * self._feat_stride diff --git a/example/rcnn/rcnn/symbol/proposal_target.py b/example/rcnn/rcnn/symbol/proposal_target.py index 3f28cb2cbebb..6f1a6ffbc440 100644 --- a/example/rcnn/rcnn/symbol/proposal_target.py +++ b/example/rcnn/rcnn/symbol/proposal_target.py @@ -2,15 +2,14 @@ Proposal Target Operator selects foreground and background roi and assigns label, bbox_transform to them. """ -from __future__ import print_function +import logging import mxnet as mx import numpy as np from distutils.util import strtobool +from ..logger import logger from rcnn.io.rcnn import sample_rois -DEBUG = False - class ProposalTargetOperator(mx.operator.CustomOp): def __init__(self, num_classes, batch_images, batch_rois, fg_fraction): @@ -20,7 +19,7 @@ def __init__(self, num_classes, batch_images, batch_rois, fg_fraction): self._batch_rois = batch_rois self._fg_fraction = fg_fraction - if DEBUG: + if logger.level == logging.DEBUG: self._count = 0 self._fg_num = 0 self._bg_num = 0 @@ -43,17 +42,17 @@ def forward(self, is_train, req, in_data, out_data, aux): rois, labels, bbox_targets, bbox_weights = \ sample_rois(all_rois, fg_rois_per_image, rois_per_image, self._num_classes, gt_boxes=gt_boxes) - if DEBUG: - print("labels=", labels) - print('num fg: {}'.format((labels > 0).sum())) - print('num bg: {}'.format((labels == 0).sum())) + if logger.level == logging.DEBUG: + logger.debug("labels: %s" % labels) + logger.debug('num fg: {}'.format((labels > 0).sum())) + logger.debug('num bg: {}'.format((labels == 0).sum())) self._count += 1 self._fg_num += (labels > 0).sum() self._bg_num += (labels == 0).sum() - print("self._count=", self._count) - print('num fg avg: {}'.format(self._fg_num / self._count)) - print('num bg avg: {}'.format(self._bg_num / self._count)) - print('ratio: {:.3f}'.format(float(self._fg_num) / float(self._bg_num))) + logger.debug("self._count: %d" % self._count) + logger.debug('num fg avg: %d' % (self._fg_num / self._count)) + logger.debug('num bg avg: %d' % (self._bg_num / self._count)) + logger.debug('ratio: %.3f' % (float(self._fg_num) / float(self._bg_num))) for ind, val in enumerate([rois, labels, bbox_targets, bbox_weights]): self.assign(out_data[ind], req[ind], val) diff --git a/example/rcnn/rcnn/tools/reeval.py b/example/rcnn/rcnn/tools/reeval.py index a2e6264942de..22e5e206f4d0 100644 --- a/example/rcnn/rcnn/tools/reeval.py +++ b/example/rcnn/rcnn/tools/reeval.py @@ -1,9 +1,9 @@ -from __future__ import print_function import argparse import cPickle import os import mxnet as mx +from ..logger import logger from ..config import config, default, generate_config from ..dataset import * @@ -39,7 +39,7 @@ def parse_args(): def main(): args = parse_args() - print('Called with argument:', args) + logger.info('Called with argument: %s' % args) reeval(args) diff --git a/example/rcnn/rcnn/tools/test_rcnn.py b/example/rcnn/rcnn/tools/test_rcnn.py index 65dca7a6d0f4..83a9fac03e67 100644 --- a/example/rcnn/rcnn/tools/test_rcnn.py +++ b/example/rcnn/rcnn/tools/test_rcnn.py @@ -1,8 +1,8 @@ -from __future__ import print_function import argparse import pprint import mxnet as mx +from ..logger import logger from ..config import config, default, generate_config from ..symbol import * from ..dataset import * @@ -99,8 +99,8 @@ def parse_args(): def main(): args = parse_args() + logger.info('Called with argument: %s' % args) ctx = mx.gpu(args.gpu) - print(args) test_rcnn(args.network, args.dataset, args.image_set, args.root_path, args.dataset_path, ctx, args.prefix, args.epoch, args.vis, args.shuffle, args.has_rpn, args.proposal, args.thresh) diff --git a/example/rcnn/rcnn/tools/test_rpn.py b/example/rcnn/rcnn/tools/test_rpn.py index 9d0ff198e1b4..09f6af74368f 100644 --- a/example/rcnn/rcnn/tools/test_rpn.py +++ b/example/rcnn/rcnn/tools/test_rpn.py @@ -1,8 +1,8 @@ -from __future__ import print_function import argparse import pprint import mxnet as mx +from ..logger import logger from ..config import config, default, generate_config from ..symbol import * from ..dataset import * @@ -89,7 +89,7 @@ def parse_args(): def main(): args = parse_args() - print('Called with argument:', args) + logger.info('Called with argument: %s' % args) ctx = mx.gpu(args.gpu) test_rpn(args.network, args.dataset, args.image_set, args.root_path, args.dataset_path, ctx, args.prefix, args.epoch, diff --git a/example/rcnn/rcnn/tools/train_rcnn.py b/example/rcnn/rcnn/tools/train_rcnn.py index 0669af047819..3f1cde380e8c 100644 --- a/example/rcnn/rcnn/tools/train_rcnn.py +++ b/example/rcnn/rcnn/tools/train_rcnn.py @@ -1,8 +1,8 @@ import argparse -import logging import pprint import mxnet as mx +from ..logger import logger from ..config import config, default, generate_config from ..symbol import * from ..core import callback, metric @@ -17,11 +17,6 @@ def train_rcnn(network, dataset, image_set, root_path, dataset_path, frequent, kvstore, work_load_list, no_flip, no_shuffle, resume, ctx, pretrained, epoch, prefix, begin_epoch, end_epoch, train_shared, lr, lr_step, proposal): - # set up logger - logging.basicConfig() - logger = logging.getLogger() - logger.setLevel(logging.INFO) - # set up config config.TRAIN.BATCH_IMAGES = 2 config.TRAIN.BATCH_ROIS = 128 @@ -36,7 +31,7 @@ def train_rcnn(network, dataset, image_set, root_path, dataset_path, input_batch_size = config.TRAIN.BATCH_IMAGES * batch_size # print config - pprint.pprint(config) + logger.info(pprint.pformat(config)) # load dataset and prepare imdb for training image_sets = [iset for iset in image_set.split('+')] @@ -53,6 +48,7 @@ def train_rcnn(network, dataset, image_set, root_path, dataset_path, # infer max shape max_data_shape = [('data', (input_batch_size, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES])))] + logger.info('providing maximum shape %s' % max_data_shape) # infer shape data_shape_dict = dict(train_data.provide_data + train_data.provide_label) @@ -60,8 +56,7 @@ def train_rcnn(network, dataset, image_set, root_path, dataset_path, arg_shape_dict = dict(zip(sym.list_arguments(), arg_shape)) out_shape_dict = dict(zip(sym.list_outputs(), out_shape)) aux_shape_dict = dict(zip(sym.list_auxiliary_states(), aux_shape)) - print('output shape') - pprint.pprint(out_shape_dict) + logger.info('output shape %s' % pprint.pformat(out_shape_dict)) # load and initialize params if resume: @@ -115,7 +110,7 @@ def train_rcnn(network, dataset, image_set, root_path, dataset_path, lr_epoch_diff = [epoch - begin_epoch for epoch in lr_epoch if epoch > begin_epoch] lr = base_lr * (lr_factor ** (len(lr_epoch) - len(lr_epoch_diff))) lr_iters = [int(epoch * len(roidb) / batch_size) for epoch in lr_epoch_diff] - print('lr', lr, 'lr_epoch_diff', lr_epoch_diff, 'lr_iters', lr_iters) + logger.info('lr %f lr_epoch_diff %s lr_iters %s' % (lr, lr_epoch_diff, lr_iters)) lr_scheduler = mx.lr_scheduler.MultiFactorScheduler(lr_iters, lr_factor) # optimizer optimizer_params = {'momentum': 0.9, @@ -166,7 +161,7 @@ def parse_args(): def main(): args = parse_args() - print('Called with argument:', args) + logger.info('Called with argument: %s' % args) ctx = [mx.gpu(int(i)) for i in args.gpus.split(',')] train_rcnn(args.network, args.dataset, args.image_set, args.root_path, args.dataset_path, args.frequent, args.kvstore, args.work_load_list, args.no_flip, args.no_shuffle, args.resume, diff --git a/example/rcnn/rcnn/tools/train_rpn.py b/example/rcnn/rcnn/tools/train_rpn.py index 2c7267ea36ef..87b92c8229ef 100644 --- a/example/rcnn/rcnn/tools/train_rpn.py +++ b/example/rcnn/rcnn/tools/train_rpn.py @@ -1,9 +1,8 @@ -from __future__ import print_function import argparse -import logging import pprint import mxnet as mx +from ..logger import logger from ..config import config, default, generate_config from ..symbol import * from ..core import callback, metric @@ -17,11 +16,6 @@ def train_rpn(network, dataset, image_set, root_path, dataset_path, frequent, kvstore, work_load_list, no_flip, no_shuffle, resume, ctx, pretrained, epoch, prefix, begin_epoch, end_epoch, train_shared, lr, lr_step): - # set up logger - logging.basicConfig() - logger = logging.getLogger() - logger.setLevel(logging.INFO) - # setup config config.TRAIN.BATCH_IMAGES = 1 @@ -34,7 +28,7 @@ def train_rpn(network, dataset, image_set, root_path, dataset_path, input_batch_size = config.TRAIN.BATCH_IMAGES * batch_size # print config - pprint.pprint(config) + logger.info(pprint.pformat(config)) # load dataset and prepare imdb for training image_sets = [iset for iset in image_set.split('+')] @@ -53,7 +47,7 @@ def train_rpn(network, dataset, image_set, root_path, dataset_path, # infer max shape max_data_shape = [('data', (input_batch_size, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES])))] max_data_shape, max_label_shape = train_data.infer_shape(max_data_shape) - print('providing maximum shape', max_data_shape, max_label_shape) + logger.info('providing maximum shape %s %s' % (max_data_shape, max_label_shape)) # infer shape data_shape_dict = dict(train_data.provide_data + train_data.provide_label) @@ -61,8 +55,7 @@ def train_rpn(network, dataset, image_set, root_path, dataset_path, arg_shape_dict = dict(zip(sym.list_arguments(), arg_shape)) out_shape_dict = dict(zip(sym.list_outputs(), out_shape)) aux_shape_dict = dict(zip(sym.list_auxiliary_states(), aux_shape)) - print('output shape') - pprint.pprint(out_shape_dict) + logger.info('output shape %s' % pprint.pformat(out_shape_dict)) # load and initialize params if resume: @@ -118,7 +111,7 @@ def train_rpn(network, dataset, image_set, root_path, dataset_path, lr_epoch_diff = [epoch - begin_epoch for epoch in lr_epoch if epoch > begin_epoch] lr = base_lr * (lr_factor ** (len(lr_epoch) - len(lr_epoch_diff))) lr_iters = [int(epoch * len(roidb) / batch_size) for epoch in lr_epoch_diff] - print('lr', lr, 'lr_epoch_diff', lr_epoch_diff, 'lr_iters', lr_iters) + logger.info('lr %f lr_epoch_diff %s lr_iters %s' % (lr, lr_epoch_diff, lr_iters)) lr_scheduler = mx.lr_scheduler.MultiFactorScheduler(lr_iters, lr_factor) # optimizer optimizer_params = {'momentum': 0.9, @@ -168,7 +161,7 @@ def parse_args(): def main(): args = parse_args() - print('Called with argument:', args) + logger.info('Called with argument: %s' % args) ctx = [mx.gpu(int(i)) for i in args.gpus.split(',')] train_rpn(args.network, args.dataset, args.image_set, args.root_path, args.dataset_path, args.frequent, args.kvstore, args.work_load_list, args.no_flip, args.no_shuffle, args.resume, diff --git a/example/rcnn/rcnn/utils/caffe_convert.py b/example/rcnn/rcnn/utils/caffe_convert.py deleted file mode 100644 index b5f0fbe27d14..000000000000 --- a/example/rcnn/rcnn/utils/caffe_convert.py +++ /dev/null @@ -1,75 +0,0 @@ -# This script will not work unless all paths are set right - -from __future__ import print_function -import os -import sys -import mxnet as mx -import numpy as np -fast_rcnn_path = None -sys.path.insert(0, os.path.join(fast_rcnn_path, 'caffe-fast-rcnn', 'python')) -sys.path.insert(0, os.path.join(fast_rcnn_path, 'lib')) -import caffe -from rcnn.symbol import get_symbol_vgg_test - -def load_model(caffeproto, caffemodel, arg_shape_dic): - def get_caffe_iter(layer_names, layers): - for layer_idx, layer in enumerate(layers): - layer_name = layer_names[layer_idx].replace('/', '_') - layer_type = layer.type - layer_blobs = layer.blobs - yield (layer_name, layer_type, layer_blobs) - - net_caffe = caffe.Net(caffeproto, caffemodel, caffe.TEST) - layer_names = net_caffe._layer_names - layers = net_caffe.layers - iter = '' - iter = get_caffe_iter(layer_names, layers) - first_conv = True - - arg_params = {} - for layer_name, layer_type, layer_blobs in iter: - if layer_type == 'Convolution' or layer_type == 'InnerProduct' or layer_type == 4 or layer_type == 14: - assert(len(layer_blobs) == 2) - wmat = np.array(layer_blobs[0].data).reshape(layer_blobs[0].num, layer_blobs[0].channels, layer_blobs[0].height, layer_blobs[0].width) - bias = np.array(layer_blobs[1].data) - if first_conv: - print('Swapping BGR of caffe into RGB in mxnet') - wmat[:, [0, 2], :, :] = wmat[:, [2, 0], :, :] - - assert(wmat.flags['C_CONTIGUOUS'] is True) - assert(bias.flags['C_CONTIGUOUS'] is True) - print('converting layer {0}, wmat shape = {1}, bias shape = {2}'.format(layer_name, wmat.shape, bias.shape)) - wmat = wmat.reshape((wmat.shape[0], -1)) - bias = bias.reshape((bias.shape[0], 1)) - weight_name = layer_name + "_weight" - bias_name = layer_name + "_bias" - - if weight_name not in arg_shape_dic: - print(weight_name + ' not found in arg_shape_dic.') - continue - wmat = wmat.reshape(arg_shape_dic[weight_name]) - arg_params[weight_name] = mx.nd.zeros(wmat.shape) - arg_params[weight_name][:] = wmat - - bias = bias.reshape(arg_shape_dic[bias_name]) - arg_params[bias_name] = mx.nd.zeros(bias.shape) - arg_params[bias_name][:] = bias - - if first_conv and (layer_type == 'Convolution' or layer_type == 4): - first_conv = False - - return arg_params - -proto_path = os.path.join(fast_rcnn_path, 'models', 'VGG16', 'test.prototxt') -model_path = os.path.join(fast_rcnn_path, 'data', 'fast_rcnn_models', 'vgg16_fast_rcnn_iter_40000.caffemodel') - -symbol = get_symbol_vgg_test() -arg_shapes, out_shapes, aux_shapes = symbol.infer_shape(**{'data': (1, 3, 224, 224), 'rois': (1, 5)}) -arg_shape_dic = { name: shape for name, shape in zip(symbol.list_arguments(), arg_shapes) } - -arg_params = load_model(proto_path, model_path, arg_shape_dic) - -model = mx.model.FeedForward(ctx=mx.cpu(), symbol=symbol, arg_params=arg_params, - aux_params={}, num_epoch=1, - learning_rate=0.01, momentum=0.9, wd=0.0001) -model.save('model/ref') diff --git a/example/rcnn/rcnn/utils/load_data.py b/example/rcnn/rcnn/utils/load_data.py index d56882a5c9d8..4700229e65af 100644 --- a/example/rcnn/rcnn/utils/load_data.py +++ b/example/rcnn/rcnn/utils/load_data.py @@ -1,5 +1,5 @@ -from __future__ import print_function import numpy as np +from ..logger import logger from ..config import config from ..dataset import * @@ -47,6 +47,6 @@ def is_valid(entry): num = len(roidb) filtered_roidb = [entry for entry in roidb if is_valid(entry)] num_after = len(filtered_roidb) - print('filtered %d roidb entries: %d -> %d' % (num - num_after, num, num_after)) + logger.info('load data: filtered %d roidb entries: %d -> %d' % (num - num_after, num, num_after)) return filtered_roidb diff --git a/example/rcnn/test.py b/example/rcnn/test.py index 708efc8c7ddb..12fe6973fbcf 100644 --- a/example/rcnn/test.py +++ b/example/rcnn/test.py @@ -1,6 +1,6 @@ -from __future__ import print_function import argparse import mxnet as mx +from rcnn.logger import logger from rcnn.config import config, default, generate_config from rcnn.tools.test_rcnn import test_rcnn @@ -31,8 +31,8 @@ def parse_args(): def main(): args = parse_args() + logger.info('Called with argument: %s' % args) ctx = mx.gpu(args.gpu) - print(args) test_rcnn(args.network, args.dataset, args.image_set, args.root_path, args.dataset_path, ctx, args.prefix, args.epoch, args.vis, args.shuffle, args.has_rpn, args.proposal, args.thresh) diff --git a/example/rcnn/train_alternate.py b/example/rcnn/train_alternate.py index 991fb237d085..74f16b9980aa 100644 --- a/example/rcnn/train_alternate.py +++ b/example/rcnn/train_alternate.py @@ -1,9 +1,7 @@ -from __future__ import print_function import argparse -import logging - import mxnet as mx +from rcnn.logger import logger from rcnn.config import config, default, generate_config from rcnn.tools.train_rpn import train_rpn from rcnn.tools.test_rpn import test_rpn @@ -14,41 +12,36 @@ def alternate_train(args, ctx, pretrained, epoch, rpn_epoch, rpn_lr, rpn_lr_step, rcnn_epoch, rcnn_lr, rcnn_lr_step): - # set up logger - logging.basicConfig() - logger = logging.getLogger() - logger.setLevel(logging.INFO) - # basic config begin_epoch = 0 config.TRAIN.BG_THRESH_LO = 0.0 - logging.info('########## TRAIN RPN WITH IMAGENET INIT') + logger.info('########## TRAIN RPN WITH IMAGENET INIT') train_rpn(args.network, args.dataset, args.image_set, args.root_path, args.dataset_path, args.frequent, args.kvstore, args.work_load_list, args.no_flip, args.no_shuffle, args.resume, ctx, pretrained, epoch, 'model/rpn1', begin_epoch, rpn_epoch, train_shared=False, lr=rpn_lr, lr_step=rpn_lr_step) - logging.info('########## GENERATE RPN DETECTION') + logger.info('########## GENERATE RPN DETECTION') image_sets = [iset for iset in args.image_set.split('+')] for image_set in image_sets: test_rpn(args.network, args.dataset, image_set, args.root_path, args.dataset_path, ctx[0], 'model/rpn1', rpn_epoch, vis=False, shuffle=False, thresh=0) - logging.info('########## TRAIN RCNN WITH IMAGENET INIT AND RPN DETECTION') + logger.info('########## TRAIN RCNN WITH IMAGENET INIT AND RPN DETECTION') train_rcnn(args.network, args.dataset, args.image_set, args.root_path, args.dataset_path, args.frequent, args.kvstore, args.work_load_list, args.no_flip, args.no_shuffle, args.resume, ctx, pretrained, epoch, 'model/rcnn1', begin_epoch, rcnn_epoch, train_shared=False, lr=rcnn_lr, lr_step=rcnn_lr_step, proposal='rpn') - logging.info('########## TRAIN RPN WITH RCNN INIT') + logger.info('########## TRAIN RPN WITH RCNN INIT') train_rpn(args.network, args.dataset, args.image_set, args.root_path, args.dataset_path, args.frequent, args.kvstore, args.work_load_list, args.no_flip, args.no_shuffle, args.resume, ctx, 'model/rcnn1', rcnn_epoch, 'model/rpn2', begin_epoch, rpn_epoch, train_shared=True, lr=rpn_lr, lr_step=rpn_lr_step) - logging.info('########## GENERATE RPN DETECTION') + logger.info('########## GENERATE RPN DETECTION') image_sets = [iset for iset in args.image_set.split('+')] for image_set in image_sets: test_rpn(args.network, args.dataset, image_set, args.root_path, args.dataset_path, @@ -101,7 +94,7 @@ def parse_args(): def main(): args = parse_args() - print('Called with argument:', args) + logger.info('Called with argument: %s' % args) ctx = [mx.gpu(int(i)) for i in args.gpus.split(',')] alternate_train(args, ctx, args.pretrained, args.pretrained_epoch, args.rpn_epoch, args.rpn_lr, args.rpn_lr_step, diff --git a/example/rcnn/train_end2end.py b/example/rcnn/train_end2end.py index ac00120131c9..b8b1c5c3a410 100644 --- a/example/rcnn/train_end2end.py +++ b/example/rcnn/train_end2end.py @@ -1,10 +1,9 @@ -from __future__ import print_function import argparse -import logging import pprint import mxnet as mx import numpy as np +from rcnn.logger import logger from rcnn.config import config, default, generate_config from rcnn.symbol import * from rcnn.core import callback, metric @@ -16,11 +15,6 @@ def train_net(args, ctx, pretrained, epoch, prefix, begin_epoch, end_epoch, lr=0.001, lr_step='5'): - # set up logger - logging.basicConfig() - logger = logging.getLogger() - logger.setLevel(logging.INFO) - # setup config config.TRAIN.BATCH_IMAGES = 1 config.TRAIN.BATCH_ROIS = 128 @@ -36,7 +30,7 @@ def train_net(args, ctx, pretrained, epoch, prefix, begin_epoch, end_epoch, input_batch_size = config.TRAIN.BATCH_IMAGES * batch_size # print config - pprint.pprint(config) + logger.info(pprint.pformat(config)) # load dataset and prepare imdb for training image_sets = [iset for iset in args.image_set.split('+')] @@ -56,7 +50,7 @@ def train_net(args, ctx, pretrained, epoch, prefix, begin_epoch, end_epoch, max_data_shape = [('data', (input_batch_size, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES])))] max_data_shape, max_label_shape = train_data.infer_shape(max_data_shape) max_data_shape.append(('gt_boxes', (input_batch_size, 100, 5))) - print('providing maximum shape', max_data_shape, max_label_shape) + logger.info('providing maximum shape %s %s' % (max_data_shape, max_label_shape)) # infer shape data_shape_dict = dict(train_data.provide_data + train_data.provide_label) @@ -64,8 +58,7 @@ def train_net(args, ctx, pretrained, epoch, prefix, begin_epoch, end_epoch, arg_shape_dict = dict(zip(sym.list_arguments(), arg_shape)) out_shape_dict = dict(zip(sym.list_outputs(), out_shape)) aux_shape_dict = dict(zip(sym.list_auxiliary_states(), aux_shape)) - print('output shape') - pprint.pprint(out_shape_dict) + logger.info('output shape %s' % pprint.pformat(out_shape_dict)) # load and initialize params if args.resume: @@ -127,7 +120,7 @@ def train_net(args, ctx, pretrained, epoch, prefix, begin_epoch, end_epoch, lr_epoch_diff = [epoch - begin_epoch for epoch in lr_epoch if epoch > begin_epoch] lr = base_lr * (lr_factor ** (len(lr_epoch) - len(lr_epoch_diff))) lr_iters = [int(epoch * len(roidb) / batch_size) for epoch in lr_epoch_diff] - print('lr', lr, 'lr_epoch_diff', lr_epoch_diff, 'lr_iters', lr_iters) + logger.info('lr %f lr_epoch_diff %s lr_iters %s' % (lr, lr_epoch_diff, lr_iters)) lr_scheduler = mx.lr_scheduler.MultiFactorScheduler(lr_iters, lr_factor) # optimizer optimizer_params = {'momentum': 0.9, @@ -176,7 +169,7 @@ def parse_args(): def main(): args = parse_args() - print('Called with argument:', args) + logger.info('Called with argument: %s' % args) ctx = [mx.gpu(int(i)) for i in args.gpus.split(',')] train_net(args, ctx, args.pretrained, args.pretrained_epoch, args.prefix, args.begin_epoch, args.end_epoch, lr=args.lr, lr_step=args.lr_step) diff --git a/src/operator/contrib/proposal-inl.h b/src/operator/contrib/proposal-inl.h index ed0ec826588f..686a8a354ff9 100644 --- a/src/operator/contrib/proposal-inl.h +++ b/src/operator/contrib/proposal-inl.h @@ -267,7 +267,7 @@ inline void _Transform(float scale, float ratio, const std::vector& base_anchor, std::vector *out_anchors) { - float w = base_anchor[2] - base_anchor[1] + 1.0f; + float w = base_anchor[2] - base_anchor[0] + 1.0f; float h = base_anchor[3] - base_anchor[1] + 1.0f; float x_ctr = base_anchor[0] + 0.5 * (w - 1.0f); float y_ctr = base_anchor[1] + 0.5 * (h - 1.0f); From 1a10c2d92859cac7d0c3bc7080c1f07868489c84 Mon Sep 17 00:00:00 2001 From: Yao Wang Date: Tue, 30 May 2017 11:48:02 -0700 Subject: [PATCH 006/834] Remove horizontal scroll bar (#6491) --- docs/_static/mxnet.css | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/_static/mxnet.css b/docs/_static/mxnet.css index 8ed40a1144a1..7358f0ae5e75 100644 --- a/docs/_static/mxnet.css +++ b/docs/_static/mxnet.css @@ -644,6 +644,7 @@ div.content { right: 200px; margin-right: 5%; padding: 40px 0 0 0; + overflow-x: hidden; z-index: -1; } From 215ae4a0dd1a96ce75e1c451809dbfade286cfcd Mon Sep 17 00:00:00 2001 From: "Qiang Kou (KK)" Date: Tue, 30 May 2017 13:31:29 -0700 Subject: [PATCH 007/834] [R][DOC] fix R tutorials (#6472) --- docs/tutorials/index.md | 17 +++++ docs/tutorials/r/CallbackFunctionTutorial.md | 20 ++--- ...tsDogsFinetune.rmd => CatsDogsFinetune.md} | 74 +++++++++---------- docs/tutorials/r/CustomIteratorTutorial.md | 33 +++++---- docs/tutorials/r/CustomLossFunction.md | 62 ---------------- docs/tutorials/r/charRnnModel.md | 5 +- .../r/classifyRealImageWithPretrainedModel.md | 22 +++--- docs/tutorials/r/fiveMinutesNeuralNetwork.md | 72 ++---------------- docs/tutorials/r/mnistCompetition.md | 62 +++++----------- docs/tutorials/r/ndarray.md | 10 +-- docs/tutorials/r/symbol.md | 10 +-- 11 files changed, 123 insertions(+), 264 deletions(-) rename docs/tutorials/r/{CatsDogsFinetune.rmd => CatsDogsFinetune.md} (88%) delete mode 100644 docs/tutorials/r/CustomLossFunction.md diff --git a/docs/tutorials/index.md b/docs/tutorials/index.md index aed11a4bebf1..ce00b74ed1bd 100644 --- a/docs/tutorials/index.md +++ b/docs/tutorials/index.md @@ -27,6 +27,23 @@ These tutorials introduce a few fundamental concepts in deep learning and how to python/predict_image vision/large_scale_classification ``` + +## R + +```eval_rst +.. toctree:: + :maxdepth: 1 + + r/ndarray + r/symbol + r/fiveMinutesNeuralNetwork + r/classifyRealImageWithPretrainedModel + r/CatsDogsFinetune + r/mnistCompetition + r/charRnnModel + r/CallbackFunctionTutorial + r/CustomIteratorTutorial +```
More tutorials and examples are available in the GitHub [repository](https://github.com/dmlc/mxnet/tree/master/example). diff --git a/docs/tutorials/r/CallbackFunctionTutorial.md b/docs/tutorials/r/CallbackFunctionTutorial.md index 103352dd2907..3290831b46c2 100644 --- a/docs/tutorials/r/CallbackFunctionTutorial.md +++ b/docs/tutorials/r/CallbackFunctionTutorial.md @@ -1,11 +1,11 @@ -Callback Function +Callback function Tutorial ====================================== This tutorial provides guidelines for using and writing callback functions, which can very useful in model training. -Model Training Example ----------- +## Model Training Example + Let's begin with a small example. We can build and train a model with the following code: @@ -56,8 +56,8 @@ Let's begin with a small example. We can build and train a model with the follow We also provide two optional parameters, `batch.end.callback` and `epoch.end.callback`, which can provide great flexibility in model training. -How to Use Callback Functions ---------- +## How to Use Callback Functions + This package provides two callback functions: @@ -148,8 +148,8 @@ You also can save the training and evaluation errors for later use by passing a ## [1] 12.715069 14.810532 15.840361 10.898733 9.349706 9.363087 ``` -How to Write Your Own Callback Functions ----------- +## How to Write Your Own Callback Functions + You can find the source code for the two callback functions on [GitHub](https://github.com/dmlc/mxnet/blob/master/R-package/R/callback.R) and use it as a template: @@ -247,8 +247,4 @@ Yes! You can stop the training early with `return(FALSE)`. See the following exa When the validation metric dips below the threshold we set, the training process stops. -## Next Steps -* [Neural Networks with MXNet in Five Minutes](http://mxnet.io/tutorials/r/fiveMinutesNeuralNetwork.html) -* [Classify Real-World Images with a Pretrained Model](http://mxnet.io/tutorials/r/classifyRealImageWithPretrainedModel.html) -* [Handwritten Digits Classification Competition](http://mxnet.io/tutorials/r/mnistCompetition.html) -* [Character Language Model Using RNN](http://mxnet.io/tutorials/r/charRnnModel.html) + diff --git a/docs/tutorials/r/CatsDogsFinetune.rmd b/docs/tutorials/r/CatsDogsFinetune.md similarity index 88% rename from docs/tutorials/r/CatsDogsFinetune.rmd rename to docs/tutorials/r/CatsDogsFinetune.md index a99e7042804e..18cd3bb8164a 100644 --- a/docs/tutorials/r/CatsDogsFinetune.rmd +++ b/docs/tutorials/r/CatsDogsFinetune.md @@ -1,31 +1,27 @@ ---- -title: "Dogs vs. Cats classification with mxnet and R" -author: "Andrey Ogurtsov (https://github.com/statist-bhfz/)" -date: "February 25, 2017" ---- +Dogs vs. Cats classification with MXNet and R +====================================== -## 1. Packages and prerequisites +## Packages and prerequisites -Ubuntu 16, **mxnet** 0.9.4 (compiled with GPU support), **imager** for image processind, **abind** for manipulations with arrays. It is almost end-to-end R solution for Kaggle competition https://www.kaggle.com/c/dogs-vs-cats-redux-kernels-edition/, we will use Python only for creating .rec-files. +Ubuntu 16.04, **mxnet** (compiled with GPU support), **imager** for image processind, +**abind** for manipulations with arrays. It is almost end-to-end R solution for Kaggle +competition , +we will use Python only for creating .rec-files. -Thanks to [jeremiedb](https://github.com/jeremiedb), my code for fine-tuning is largely based on his [answers](https://github.com/dmlc/mxnet/issues/4817). +Thanks to [jeremiedb](https://github.com/jeremiedb), my code for fine-tuning is +largely based on his [answers](https://github.com/dmlc/mxnet/issues/4817). -```{r} -knitr::opts_chunk$set(eval = FALSE) -``` - -```{r} +```r library(imager) library(mxnet) library(abind) ``` +## Image processing -## 2. Image processing - -### 2.1. Renaming train files +### Renaming train files -```{r} +```r files <- list.files("train") old_names <- sapply(files, strsplit, split = ".", fixed = TRUE) max_length <- max(sapply(old_names, function(x) nchar(x[[2]]))) @@ -50,9 +46,9 @@ dir.create("./train/dog") Map(function(x, y) file.rename(from = x, to = y), files, new_names) ``` -### 2.2. Train images: 224x224, padded with empty space +### Train images: 224x224, padded with empty space -```{r} +```r files <- list.files("train", recursive = TRUE) new_names <- paste0("train_pad_224x224/", files) files <- paste0("./train/", files) @@ -76,9 +72,9 @@ Map(function(x, y) { }, x = files, y = new_names) ``` -### 2.3. Renaming test files +### Renaming test files -```{r} +```r files <- list.files("test") max_length <- max(sapply(files, nchar)) zeros <- max_length - sapply(files, nchar) @@ -91,9 +87,9 @@ Map(function(x, y) file.rename(from = x, to = y), files, newnames) ``` -### 2.4. Test images: 224x224, padded with empty space +### Test images: 224x224, padded with empty space -```{r} +```r files <- list.files("test") new_names <- paste0("test_pad_224x224/", files) files <- paste0("./test/", files) @@ -106,18 +102,18 @@ Map(function(x, y) { }, x = files, y = new_names) ``` -### 2.5. Creating .rec files +### Creating .rec files -```{bash, eval = FALSE} +``` python ~/mxnet/tools/im2rec.py --list=1 --recursive=1 --train-ratio=0.8 cats_dogs train_pad_224x224 python ~/mxnet/tools/im2rec.py --num-thread=4 --pass-through=1 cats_dogs_train.lst train_pad_224x224 python ~/mxnet/tools/im2rec.py --num-thread=4 --pass-through=1 cats_dogs_val.lst train_pad_224x224 ``` -## 3. Iterators +## Iterators -```{r} +```r get_iterator <- function(data_shape, train_data, val_data, @@ -142,7 +138,7 @@ get_iterator <- function(data_shape, ``` -```{r} +```r data <- get_iterator(data_shape = c(224, 224, 3), train_data = "/media/andrey/Data/KAGGLE/cats_dogs/cats_dogs_train.rec", val_data = "/media/andrey/Data/KAGGLE/cats_dogs/cats_dogs_val.rec", @@ -152,13 +148,13 @@ val <- data$val ``` -## 4. Load pretrained model +## Load pretrained model Model from http://data.dmlc.ml/models/imagenet/ Last fully connected layes for 1000 classes replaced with new layer for 2 classes. -```{r} +```r inception_bn <- mx.model.load("models/inception_bn/Inception-BN", iteration = 126) @@ -193,9 +189,9 @@ arg_params_new[["fc1_bias"]] <- fc1_bias_new ``` -## 5. Train (fine-tune) model +## Train (fine-tune) model -```{r} +```r model <- mx.model.FeedForward.create( symbol = new_soft, X = train, @@ -217,13 +213,13 @@ model <- mx.model.FeedForward.create( ) ``` -```{r} +```r model <- mx.model.load("inception_bn", 1) ``` Continue training with decreased speed (`learning.rate = 0.03`): -```{r} +```r model <- mx.model.FeedForward.create( symbol = model$symbol, X = train, @@ -245,16 +241,16 @@ model <- mx.model.FeedForward.create( ) ``` -```{r} +```r model <- mx.model.load("inception_bn", 1) ``` My R session crashed after each iteration, so I made some iterations manually. -## 6. Make predictions +## Make predictions -```{r} +```r preprocImage<- function(src, # URL or file location height = 224, width = 224, @@ -279,7 +275,7 @@ preprocImage<- function(src, # URL or file location } ``` -```{r} +```r files <- list.files("test_pad_224x224/") files <- paste0("./test_pad_224x224/", files) @@ -303,3 +299,5 @@ probs <- t(do.call(cbind, probs)) preds <- data.frame(id = 1:12500, label = probs[, 2]) write.csv(preds, "subm.csv", row.names = FALSE, quote = FALSE) ``` + + diff --git a/docs/tutorials/r/CustomIteratorTutorial.md b/docs/tutorials/r/CustomIteratorTutorial.md index 1ad634bcd669..b35b8926f359 100644 --- a/docs/tutorials/r/CustomIteratorTutorial.md +++ b/docs/tutorials/r/CustomIteratorTutorial.md @@ -1,23 +1,26 @@ Custom Iterator Tutorial ====================================== -This tutorial provides a guideline on how to use and write custom iterators, which can very useful when having a dataset that does not fit into memory. +This tutorial provides a guideline on how to use and write custom iterators, +which can very useful when having a dataset that does not fit into memory. -Getting the data ----------- -The data we are going to use is the [MNIST dataset](http://yann.lecun.com/exdb/mnist/) in CSV format, the data can be found in this [web](http://pjreddie.com/projects/mnist-in-csv/). +## Getting the data + + +The data we are going to use is the [MNIST dataset](http://yann.lecun.com/exdb/mnist/) in +CSV format, the data can be found in this [web](http://pjreddie.com/projects/mnist-in-csv/). To download the data: -```bash +``` wget http://pjreddie.com/media/files/mnist_train.csv wget http://pjreddie.com/media/files/mnist_test.csv ``` You'll get two files, `mnist_train.csv` that contains 60.000 examples of hand written numbers and `mxnist_test.csv` that contains 10.000 examples. The first element of each line in the CSV is the label, which is a number between 0 and 9. The rest of the line are 784 numbers between 0 and 255, corresponding to the levels of grey of a matrix of 28x28. Therefore, each line contains an image of 28x28 pixels of a hand written number and its true label. -Custom CSV Iterator ----------- +## Custom CSV Iterator + Next we are going to create a custom CSV Iterator based on the [C++ CSVIterator class](https://github.com/dmlc/mxnet/blob/master/src/io/iter_csv.cc). For that we are going to use the R function `mx.io.CSVIter` as a base class. This class has as parameters `data.csv, data.shape, batch.size` and two main functions, `iter.next()` that calls the iterator in the next batch of data and `value()` that returns the train data and the label. @@ -132,8 +135,8 @@ batch.size <- 100 train.iter <- CustomCSVIter$new(iter = NULL, data.csv = "mnist_train.csv", data.shape = 28, batch.size = batch.size) ``` -CNN Model ----------- +## CNN Model + For this tutorial we are going to use the known LeNet architecture: @@ -156,8 +159,8 @@ lenet.model <- function(){ network <- lenet.model() ``` -Training with the Custom Iterator ----------- +## Training with the Custom Iterator + Finally, we can directly add the custom iterator as the training data source. ```r @@ -176,7 +179,7 @@ model <- mx.model.FeedForward.create(symbol=network, The last 2 iterations with a K80 GPU looks like this: -```bash +``` [8] Train-accuracy=0.998866666666667 Batch [100] Speed: 15413.0104454713 samples/sec Train-accuracy=0.999 Batch [200] Speed: 16629.3412459049 samples/sec Train-accuracy=0.99935 @@ -194,9 +197,11 @@ Batch [600] Speed: 13818.7899518255 samples/sec Train-accuracy=0.99975 [10] Train-accuracy=0.99975 ``` -Conclusion ----------- +## Conclusion + We have shown how to create a custom CSV Iterator by extending the class `mx.io.CSVIter`. In our class, we iteratively read from a CSV file a batch of data that will be transformed and then processed in the stochastic gradient descent optimization. That way, we are able to manage CSV files that are bigger than the memory of the machine we are using. Based of this custom iterator, we can also create data loaders that internally transform or expand the data, allowing to manage files of any size. + + diff --git a/docs/tutorials/r/CustomLossFunction.md b/docs/tutorials/r/CustomLossFunction.md deleted file mode 100644 index a7104803cacb..000000000000 --- a/docs/tutorials/r/CustomLossFunction.md +++ /dev/null @@ -1,62 +0,0 @@ -Customized loss function -====================================== - -This tutorial provides guidelines for using customized loss function in network construction. - - -Model Training Example ----------- - -Let's begin with a small regression example. We can build and train a regression model with the following code: - - - ```r - library(mxnet) - data(BostonHousing, package="mlbench") - train.ind = seq(1, 506, 3) - train.x = data.matrix(BostonHousing[train.ind, -14]) - train.y = BostonHousing[train.ind, 14] - test.x = data.matrix(BostonHousing[-train.ind, -14]) - test.y = BostonHousing[-train.ind, 14] - data <- mx.symbol.Variable("data") - fc1 <- mx.symbol.FullyConnected(data, num_hidden=1) - lro <- mx.symbol.LinearRegressionOutput(fc1) - mx.set.seed(0) - model <- mx.model.FeedForward.create( - lro, X=train.x, y=train.y, - eval.data=list(data=test.x, label=test.y), - ctx=mx.cpu(), num.round=10, array.batch.size=20, - learning.rate=2e-6, momentum=0.9, eval.metric=mx.metric.rmse) - ``` - -Besides the `LinearRegressionOutput`, we also provide `LogisticRegressionOutput` and `MAERegressionOutput`. -However, this might not be enough for real-world models. You can provide your own loss function -by using `mx.symbol.MakeLoss` when constructing the network. - - -How to Use Your Own Loss Function ---------- - -We still use our previous example. - - ```r - library(mxnet) - data <- mx.symbol.Variable("data") - fc1 <- mx.symbol.FullyConnected(data, num_hidden=1) - lro <- mx.symbol.MakeLoss(mx.symbol.square(mx.symbol.Reshape(fc1, shape = 0) - label)) - ``` - -In the last line of network definition, we do not use the predefined loss function. We define the loss -by ourselves, which is `(pred-label)^2`. - -We have provided many operations on the symbols, so you can also define `|pred-label|` using the line below. - - ```r - lro <- mx.symbol.MakeLoss(mx.symbol.abs(mx.symbol.Reshape(fc1, shape = 0) - label)) - ``` - -## Next Steps -* [Neural Networks with MXNet in Five Minutes](http://mxnet.io/tutorials/r/fiveMinutesNeuralNetwork.html) -* [Classify Real-World Images with a PreTrained Model](http://mxnet.io/tutorials/r/classifyRealImageWithPretrainedModel.html) -* [Handwritten Digits Classification Competition](http://mxnet.io/tutorials/r/mnistCompetition.html) -* [Character Language Model Using RNN](http://mxnet.io/tutorials/r/charRnnModel.html) diff --git a/docs/tutorials/r/charRnnModel.md b/docs/tutorials/r/charRnnModel.md index 82e10a11f8d5..e4d6eb354b96 100644 --- a/docs/tutorials/r/charRnnModel.md +++ b/docs/tutorials/r/charRnnModel.md @@ -1,4 +1,4 @@ -Char RNN Example +Character language model using RNN ============================================= This tutorial shows how to use an LSTM model to build a char-level language model, and generate text from it. For demonstration purposes, we use a Shakespearean text. You can find the data on [GitHub](https://github.com/dmlc/web-data/tree/master/mxnet/tinyshakespeare). @@ -310,5 +310,4 @@ In `mxnet`, other RNN models, like custom RNN and GRU, are also provided: - For a custom RNN model, you can replace `mx.lstm` with `mx.rnn` to train an RNN model. You can replace `mx.lstm.inference` and `mx.lstm.forward` with `mx.rnn.inference` and `mx.rnn.forward` to build inference from an RNN model and get the forward result from the inference model. - For a GRU model, you can replace `mx.lstm` with `mx.gru` to train a GRU model. You can replace `mx.lstm.inference` and `mx.lstm.forward` with `mx.gru.inference` and `mx.gru.forward` to build inference from a GRU model and get the forward result from the inference model. -## Next Steps -* [MXNet tutorials index](http://mxnet.io/tutorials/index.html) + diff --git a/docs/tutorials/r/classifyRealImageWithPretrainedModel.md b/docs/tutorials/r/classifyRealImageWithPretrainedModel.md index 4276fdeef282..1272d611fb4a 100644 --- a/docs/tutorials/r/classifyRealImageWithPretrainedModel.md +++ b/docs/tutorials/r/classifyRealImageWithPretrainedModel.md @@ -9,8 +9,8 @@ image. For information about the network architecture, see [1]. The pre-trained Inception-BatchNorm network is able to be downloaded from [this link](http://data.mxnet.io/mxnet/data/Inception.zip) This model gives the recent state-of-art prediction accuracy on image net dataset. -Load the MXNet Package ---------------- +## Load the MXNet Package + To get started, load the mxnet package: ```r @@ -60,8 +60,8 @@ Now load the imager package to load and preprocess the images in R: ## save.image ``` -Load the PreTrained Model -------------------------- +## Load the PreTrained Model + Make sure you unzip the pre-trained model in the current folder. Use the model loading function to load the model into R: @@ -76,8 +76,8 @@ Load in the mean image, which is used for preprocessing using: mean.img = as.array(mx.nd.load("Inception/mean_224.nd")[["mean_img"]]) ``` -Load and Preprocess the Image ------------------------------ +## Load and Preprocess the Image + Now, we are ready to classify a real image. In this example, we simply take the parrots image from the imager package. You can use another image, if you prefer. @@ -89,7 +89,7 @@ Load and plot the image: plot(im) ``` -![plot of chunk unnamed-chunk-5](../../web-data/mxnet/knitr/classifyRealImageWithPretrainedModel-unnamed-chunk-5-1.png) +![](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/knitr/classifyRealImageWithPretrainedModel-unnamed-chunk-5-1.png) Before feeding the image to the deep network, we need to perform some preprocessing to make the image meet the deep network input requirements. Preprocessing @@ -125,8 +125,8 @@ Use the defined preprocessing function to get the normalized image: normed <- preproc.image(im, mean.img) ``` -Classify the Image ------------------- +## Classify the Image + Now we are ready to classify the image! Use the ```predict``` function to get the probability over classes: @@ -179,6 +179,4 @@ Reference --------- [1] Ioffe, Sergey, and Christian Szegedy. "Batch normalization: Accelerating deep network training by reducing internal covariate shift." arXiv preprint arXiv:1502.03167 (2015). -## Next Steps -* [Handwritten Digits Classification Competition](http://mxnet.io/tutorials/r/mnistCompetition.html) -* [Character Language Model using RNN](http://mxnet.io/tutorials/r/charRnnModel.html) + diff --git a/docs/tutorials/r/fiveMinutesNeuralNetwork.md b/docs/tutorials/r/fiveMinutesNeuralNetwork.md index 9104e8f05c2f..d328d95e0f37 100644 --- a/docs/tutorials/r/fiveMinutesNeuralNetwork.md +++ b/docs/tutorials/r/fiveMinutesNeuralNetwork.md @@ -1,4 +1,4 @@ -Develop a Neural Network with MXNet in Five Minutes +Neural networks with MXNet in five minutes ============================================= This tutorial is designed for new users of the `mxnet` package for R. It shows how to construct a neural network to do regression in 5 minutes. It shows how to perform classification and regression tasks, respectively. The data we use is in the `mlbench` package. @@ -169,35 +169,9 @@ Next, make prediction with this structure and other parameters with `mx.model.Fe ## [8] Train-rmse=9.10463850277417 ## [9] Train-rmse=9.03977049028532 ## [10] Train-rmse=8.96870685004475 - ## [11] Train-rmse=8.93113287361574 - ## [12] Train-rmse=8.89937257821847 - ## [13] Train-rmse=8.87182096922953 - ## [14] Train-rmse=8.84476075083586 - ## [15] Train-rmse=8.81464673014974 - ## [16] Train-rmse=8.78672567900196 - ## [17] Train-rmse=8.76265872846474 - ## [18] Train-rmse=8.73946101419974 - ## [19] Train-rmse=8.71651926303267 - ## [20] Train-rmse=8.69457600919277 - ## [21] Train-rmse=8.67354928674563 - ## [22] Train-rmse=8.65328755392436 - ## [23] Train-rmse=8.63378039680078 - ## [24] Train-rmse=8.61488162586984 - ## [25] Train-rmse=8.5965105183022 - ## [26] Train-rmse=8.57868133563275 - ## [27] Train-rmse=8.56135851937663 - ## [28] Train-rmse=8.5444819772098 - ## [29] Train-rmse=8.52802114610432 - ## [30] Train-rmse=8.5119504512622 - ## [31] Train-rmse=8.49624261719241 - ## [32] Train-rmse=8.48087453238701 - ## [33] Train-rmse=8.46582689119887 - ## [34] Train-rmse=8.45107881002491 - ## [35] Train-rmse=8.43661331401712 - ## [36] Train-rmse=8.42241575909639 - ## [37] Train-rmse=8.40847217331365 - ## [38] Train-rmse=8.39476931796395 - ## [39] Train-rmse=8.38129658373974 + + ............ + ## [40] Train-rmse=8.36804269059018 ## [41] Train-rmse=8.35499817678397 ## [42] Train-rmse=8.34215505742154 @@ -263,35 +237,9 @@ This is an example of the mean absolute error metric. Simply plug it into the tr ## [8] Train-mae=7.02742733055105 ## [9] Train-mae=7.00618194618469 ## [10] Train-mae=6.92541576984028 - ## [11] Train-mae=6.87530243690643 - ## [12] Train-mae=6.84757369098564 - ## [13] Train-mae=6.82966501611388 - ## [14] Train-mae=6.81151759574811 - ## [15] Train-mae=6.78394182841811 - ## [16] Train-mae=6.75914719419347 - ## [17] Train-mae=6.74180388773481 - ## [18] Train-mae=6.725853071279 - ## [19] Train-mae=6.70932178215848 - ## [20] Train-mae=6.6928868798746 - ## [21] Train-mae=6.6769521329138 - ## [22] Train-mae=6.66184809505939 - ## [23] Train-mae=6.64754504809777 - ## [24] Train-mae=6.63358514060577 - ## [25] Train-mae=6.62027640889088 - ## [26] Train-mae=6.60738245232238 - ## [27] Train-mae=6.59505546771818 - ## [28] Train-mae=6.58346195800437 - ## [29] Train-mae=6.57285477783945 - ## [30] Train-mae=6.56259003960424 - ## [31] Train-mae=6.5527790788975 - ## [32] Train-mae=6.54353428422991 - ## [33] Train-mae=6.5344172368447 - ## [34] Train-mae=6.52557652526432 - ## [35] Train-mae=6.51697905850079 - ## [36] Train-mae=6.50847898812758 - ## [37] Train-mae=6.50014844106303 - ## [38] Train-mae=6.49207674844397 - ## [39] Train-mae=6.48412070125341 + + ............ + ## [40] Train-mae=6.47650500999557 ## [41] Train-mae=6.46893867486053 ## [42] Train-mae=6.46142131653097 @@ -307,8 +255,4 @@ This is an example of the mean absolute error metric. Simply plug it into the tr Congratulations! You've learned the basics for using MXNet in R. To learn how to use MXNet's advanced features, see the other tutorials. - -## Next Steps -* [Classify Real-World Images with Pre-trained Model](http://mxnet.io/tutorials/r/classifyRealImageWithPretrainedModel.html) -* [Handwritten Digits Classification Competition](http://mxnet.io/tutorials/r/mnistCompetition.html) -* [Character Language Model using RNN](http://mxnet.io/tutorials/r/charRnnModel.html) + diff --git a/docs/tutorials/r/mnistCompetition.md b/docs/tutorials/r/mnistCompetition.md index 04e68cf41ce2..cfac8316ac82 100644 --- a/docs/tutorials/r/mnistCompetition.md +++ b/docs/tutorials/r/mnistCompetition.md @@ -1,4 +1,4 @@ -Handwritten Digits Classification Competition +Handwritten digits classification competition ============================================= [MNIST](http://yann.lecun.com/exdb/mnist/) is a handwritten digits image data set created by Yann LeCun. Every digit is represented by a 28 x 28 pixel image. It's become a standard data set for testing classifiers on simple image input. A neural network is a strong model for image classification tasks. There's a [long-term hosted competition](https://www.kaggle.com/c/digit-recognizer) on Kaggle using this data set. @@ -93,7 +93,7 @@ We assign CPU to `mxnet`. Now, you can run the following command to train the ne ctx=devices, num.round=10, array.batch.size=100, learning.rate=0.07, momentum=0.9, eval.metric=mx.metric.accuracy, initializer=mx.init.uniform(0.07), - epoch.end.callback=mx.callback.log.train.metric(100)) + epoch.end.callback=mx.callback.log.train.metric(100)) ``` ``` @@ -108,35 +108,9 @@ We assign CPU to `mxnet`. Now, you can run the following command to train the ne ## Batch [300] Train-accuracy=0.955866666666666 ## Batch [400] Train-accuracy=0.957525000000001 ## [2] Train-accuracy=0.958309523809525 - ## Batch [100] Train-accuracy=0.968 - ## Batch [200] Train-accuracy=0.9677 - ## Batch [300] Train-accuracy=0.9696 - ## Batch [400] Train-accuracy=0.970650000000002 - ## [3] Train-accuracy=0.970809523809526 - ## Batch [100] Train-accuracy=0.973 - ## Batch [200] Train-accuracy=0.974249999999999 - ## Batch [300] Train-accuracy=0.976 - ## Batch [400] Train-accuracy=0.977100000000003 - ## [4] Train-accuracy=0.977452380952384 - ## Batch [100] Train-accuracy=0.9834 - ## Batch [200] Train-accuracy=0.981949999999999 - ## Batch [300] Train-accuracy=0.981900000000001 - ## Batch [400] Train-accuracy=0.982600000000003 - ## [5] Train-accuracy=0.983000000000003 - ## Batch [100] Train-accuracy=0.983399999999999 - ## Batch [200] Train-accuracy=0.98405 - ## Batch [300] Train-accuracy=0.985000000000001 - ## Batch [400] Train-accuracy=0.985725000000003 - ## [6] Train-accuracy=0.985952380952384 - ## Batch [100] Train-accuracy=0.988999999999999 - ## Batch [200] Train-accuracy=0.9876 - ## Batch [300] Train-accuracy=0.988100000000001 - ## Batch [400] Train-accuracy=0.988750000000003 - ## [7] Train-accuracy=0.988880952380955 - ## Batch [100] Train-accuracy=0.991999999999999 - ## Batch [200] Train-accuracy=0.9912 - ## Batch [300] Train-accuracy=0.990066666666668 - ## Batch [400] Train-accuracy=0.990275000000003 + + ............ + ## [8] Train-accuracy=0.990452380952384 ## Batch [100] Train-accuracy=0.9937 ## Batch [200] Train-accuracy=0.99235 @@ -200,12 +174,12 @@ data <- mx.symbol.Variable('data') conv1 <- mx.symbol.Convolution(data=data, kernel=c(5,5), num_filter=20) tanh1 <- mx.symbol.Activation(data=conv1, act_type="tanh") pool1 <- mx.symbol.Pooling(data=tanh1, pool_type="max", - kernel=c(2,2), stride=c(2,2)) + kernel=c(2,2), stride=c(2,2)) # second conv conv2 <- mx.symbol.Convolution(data=pool1, kernel=c(5,5), num_filter=50) tanh2 <- mx.symbol.Activation(data=conv2, act_type="tanh") pool2 <- mx.symbol.Pooling(data=tanh2, pool_type="max", - kernel=c(2,2), stride=c(2,2)) + kernel=c(2,2), stride=c(2,2)) # first fullc flatten <- mx.symbol.Flatten(data=pool2) fc1 <- mx.symbol.FullyConnected(data=flatten, num_hidden=500) @@ -247,10 +221,10 @@ Start by training on the CPU first. Because this takes a bit time, we run it for mx.set.seed(0) tic <- proc.time() model <- mx.model.FeedForward.create(lenet, X=train.array, y=train.y, - ctx=device.cpu, num.round=1, array.batch.size=100, - learning.rate=0.05, momentum=0.9, wd=0.00001, - eval.metric=mx.metric.accuracy, - epoch.end.callback=mx.callback.log.train.metric(100)) + ctx=device.cpu, num.round=1, array.batch.size=100, + learning.rate=0.05, momentum=0.9, wd=0.00001, + eval.metric=mx.metric.accuracy, + epoch.end.callback=mx.callback.log.train.metric(100)) ``` ``` @@ -278,10 +252,10 @@ Train on a GPU: mx.set.seed(0) tic <- proc.time() model <- mx.model.FeedForward.create(lenet, X=train.array, y=train.y, - ctx=device.gpu, num.round=5, array.batch.size=100, - learning.rate=0.05, momentum=0.9, wd=0.00001, - eval.metric=mx.metric.accuracy, - epoch.end.callback=mx.callback.log.train.metric(100)) + ctx=device.gpu, num.round=5, array.batch.size=100, + learning.rate=0.05, momentum=0.9, wd=0.00001, + eval.metric=mx.metric.accuracy, + epoch.end.callback=mx.callback.log.train.metric(100)) ``` ``` @@ -333,7 +307,7 @@ Now, we can submit the result to Kaggle to see the improvement of our ranking! write.csv(submission, file='submission.csv', row.names=FALSE, quote=FALSE) ``` -![](../../web-data/mxnet/knitr/mnistCompetition-kaggle-submission.png) +![](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/knitr/mnistCompetition-kaggle-submission.png) + -## Next Steps -* [Character Language Model using RNN](http://mxnet.io/tutorials/r/charRnnModel.html) + diff --git a/docs/tutorials/r/ndarray.md b/docs/tutorials/r/ndarray.md index e00f9470688b..4d5379ffa3b3 100644 --- a/docs/tutorials/r/ndarray.md +++ b/docs/tutorials/r/ndarray.md @@ -1,4 +1,4 @@ -# NDArray: Vectorized Tensor Computations on CPUs and GPUs +# NDArray - Imperative tensor operations on CPU/GPU `NDArray` is the basic vectorized operation unit in MXNet for matrix and tensor computations. Users can perform usual calculations as on an R"s array, but with two additional features: @@ -197,10 +197,4 @@ parallel. The actual computations are finished, allowing us to copy the results someplace else, such as `as.array(a)` or `mx.nd.save(a, "temp.dat")`. To write highly parallelized codes, we only need to postpone when we need the results. -## Next Steps -* [Symbol](http://mxnet.io/tutorials/r/symbol.html) -* [Write and use callback functions](http://mxnet.io/tutorials/r/CallbackFunctionTutorial.html) -* [Neural Networks with MXNet in Five Minutes](http://mxnet.io/tutorials/r/fiveMinutesNeuralNetwork.html) -* [Classify Real-World Images with Pre-trained Model](http://mxnet.io/tutorials/r/classifyRealImageWithPretrainedModel.html) -* [Handwritten Digits Classification Competition](http://mxnet.io/tutorials/r/mnistCompetition.html) -* [Character Language Model using RNN](http://mxnet.io/tutorials/r/charRnnModel.html) \ No newline at end of file + diff --git a/docs/tutorials/r/symbol.md b/docs/tutorials/r/symbol.md index 8333265e1273..1a8f46b9f70f 100644 --- a/docs/tutorials/r/symbol.md +++ b/docs/tutorials/r/symbol.md @@ -1,4 +1,5 @@ -# Symbol and Automatic Differentiation +Symbol - Neural network graphs and auto-differentiation +====================================== The computational unit `NDArray` requires a way to construct neural networks. MXNet provides a symbolic interface, named Symbol, to do this. Symbol combines both flexibility and efficiency. @@ -122,9 +123,4 @@ composition. Because MXNet does more in-place memory allocation, it can be more memory efficient than CXXNet and gets to the same runtime with greater flexibility. -## Next Steps -* [Write and use callback functions](http://mxnet.io/tutorials/r/CallbackFunctionTutorial.html) -* [Neural Networks with MXNet in Five Minutes](http://mxnet.io/tutorials/r/fiveMinutesNeuralNetwork.html) -* [Classify Real-World Images with Pre-trained Model](http://mxnet.io/tutorials/r/classifyRealImageWithPretrainedModel.html) -* [Handwritten Digits Classification Competition](http://mxnet.io/tutorials/r/mnistCompetition.html) -* [Character Language Model using RNN](http://mxnet.io/tutorials/r/charRnnModel.html) + From 32ced389c1e52e2874c75ef75b09973835b15138 Mon Sep 17 00:00:00 2001 From: Eric Junyuan Xie Date: Tue, 30 May 2017 19:02:08 -0700 Subject: [PATCH 008/834] backward headgrads and detach (#6332) * backward headgrads and detach lint fix add updated grad add retain grad exclude reduce cpp cached invoke cached symbol move symbol init module symbol cython udpate updated_grad->fresh_grad fix * fix --- include/mxnet/c_api.h | 68 +++++++ include/mxnet/ndarray.h | 13 ++ nnvm | 2 +- python/mxnet/_ctypes/common.py | 30 ++++ python/mxnet/_ctypes/ndarray.py | 33 +++- python/mxnet/_ctypes/symbol.py | 163 ++++------------- python/mxnet/base.py | 1 + python/mxnet/contrib/autograd.py | 38 +++- python/mxnet/cython/base.pyi | 73 ++++++++ python/mxnet/cython/ndarray.pyx | 45 ++++- python/mxnet/cython/symbol.pyx | 193 +++++++------------- python/mxnet/ndarray.py | 38 +++- python/mxnet/symbol.py | 207 +++++++++++++++++++++- src/c_api/c_api.cc | 21 +++ src/c_api/c_api_ndarray.cc | 95 ++++++++-- src/c_api/c_api_symbolic.cc | 16 ++ src/ndarray/autograd.cc | 79 ++++++--- src/ndarray/autograd.h | 8 +- src/ndarray/ndarray.cc | 27 ++- src/operator/custom/custom.cc | 4 +- src/operator/custom/native_op.cc | 1 + src/operator/custom/ndarray_op.cc | 1 + src/operator/tensor/broadcast_reduce_op.h | 121 +++++++------ src/operator/tensor/elemwise_unary_op.cc | 8 +- tests/python/unittest/test_autograd.py | 84 ++++++++- tests/python/unittest/test_init.py | 2 +- tests/python/unittest/test_ndarray.py | 11 ++ tests/python/unittest/test_operator.py | 81 ++------- tests/python/unittest/test_symbol.py | 29 +-- 29 files changed, 1018 insertions(+), 474 deletions(-) create mode 100644 python/mxnet/_ctypes/common.py diff --git a/include/mxnet/c_api.h b/include/mxnet/c_api.h index 4508a51e64d4..d2efdf585e88 100644 --- a/include/mxnet/c_api.h +++ b/include/mxnet/c_api.h @@ -48,6 +48,8 @@ typedef void *NDArrayHandle; typedef const void *FunctionHandle; /*! \brief handle to a function that takes param and creates symbol */ typedef void *AtomicSymbolCreator; +/*! \brief handle to cached operator */ +typedef void *CachedOpHandle; /*! \brief handle to a symbol that can be bind as operator */ typedef void *SymbolHandle; /*! \brief handle to a AtomicSymbol */ @@ -414,6 +416,26 @@ MXNET_DLL int MXNDArrayGetDType(NDArrayHandle handle, MXNET_DLL int MXNDArrayGetContext(NDArrayHandle handle, int *out_dev_type, int *out_dev_id); +/*! + * \brief detach and ndarray from computation graph by clearing entry_ + * \param handle NDArray handle + * \return 0 when success, -1 when failure happens + */ +MXNET_DLL int MXNDArrayDetach(NDArrayHandle handle, NDArrayHandle *out); +/*! + * \brief set the flag for gradient array state. + * \param handle NDArray handle + * \param state the new state. + * \return 0 when success, -1 when failure happens + */ +MXNET_DLL int MXNDArraySetGradState(NDArrayHandle handle, int state); +/*! + * \brief set the flag for gradient array state. + * \param handle NDArray handle + * \param state the new state. + * \return 0 when success, -1 when failure happens + */ +MXNET_DLL int MXNDArrayGetGradState(NDArrayHandle handle, int *out); //-------------------------------- // Part 2: functions on NDArray //-------------------------------- @@ -548,6 +570,39 @@ MXNET_DLL int MXAutogradMarkVariables(mx_uint num_var, */ MXNET_DLL int MXAutogradComputeGradient(mx_uint num_output, NDArrayHandle* output_handles); +/*! + * \brief compute the gradient of outputs w.r.t variabels + * \param num_output number of output NDArray + * \param output_handles output NDArrays + * \param ograd_handles head gradient for NDArrays + * \param retain_graph whether to keep the graph after backward + * \return 0 when success, -1 when failure happens + */ +MXNET_DLL int MXAutogradBackward(mx_uint num_output, + NDArrayHandle* output_handles, + NDArrayHandle* ograd_handles, + int retain_graph); +/*! + * \brief create cached operator + */ +MXNET_DLL int MXCachedCreateOp(AtomicSymbolCreator creator, + int num_inputs, + int num_params, + const char **param_keys, + const char **param_vals, + CachedOpHandle *out); +/*! + * \brief free cached operator + */ +MXNET_DLL int MXCachedFree(CachedOpHandle handle); +/*! + * \brief invoke cached operator + */ +MXNET_DLL int MXCachedInvoke(CachedOpHandle handle, + int num_inputs, + NDArrayHandle *inputs, + int *num_outputs, + NDArrayHandle **outputs); //-------------------------------------------- // Part 3: symbolic configuration generation //-------------------------------------------- @@ -615,6 +670,19 @@ MXNET_DLL int MXSymbolCreateAtomicSymbol(AtomicSymbolCreator creator, const char **keys, const char **vals, SymbolHandle *out); +/*! + * \brief Create an AtomicSymbol from cached op. + * \param handle cached node attribute. + * \param name name of new symbol. + * \param num_args the number of symbol arguments + * \param args symbol arguments + * \return 0 when success, -1 when failure happens + */ +MXNET_DLL int MXCachedCreateSymbol(CachedOpHandle handle, + const char* name, + mx_uint num_args, + SymbolHandle* args, + SymbolHandle* out); /*! * \brief Create a Variable Symbol. * \param name name of the variable diff --git a/include/mxnet/ndarray.h b/include/mxnet/ndarray.h index b8cd550118d3..f30b09a054f0 100644 --- a/include/mxnet/ndarray.h +++ b/include/mxnet/ndarray.h @@ -47,6 +47,7 @@ class AGNodeEntry { } nnvm::NodeEntry nn_entry() const; + bool is_none() const; }; class AutogradRuntime; @@ -149,6 +150,10 @@ class NDArray { inline bool is_none() const { return ptr_.get() == nullptr; } + /*! \return updated grad state in entry_ */ + bool fresh_out_grad() const; + /*! \return updated grad state in entry_ */ + void set_fresh_out_grad(bool state) const; /*! * \brief Block until all the pending write operations with respect * to current NDArray are finished, and read can be performed. @@ -321,6 +326,14 @@ class NDArray { * \return NDArray in new shape */ NDArray Reshape(const TShape &shape) const; + /*! + * \brief Return a copy of this NDArray without autograd history + */ + NDArray Detach() const { + NDArray ret(*this); + ret.entry_ = autograd::AGNodeEntry{nullptr, 0, 0}; + return ret; + } /*! * \brief Allocate the space if it is delayed allocated. * This is an internal function used by system that normal user should not use diff --git a/nnvm b/nnvm index 93072dc8733a..7796ac76ccea 160000 --- a/nnvm +++ b/nnvm @@ -1 +1 @@ -Subproject commit 93072dc8733aa2a89459ecf16413d96ad0b998db +Subproject commit 7796ac76ccea1fba31afc32056c83f6da38b6c57 diff --git a/python/mxnet/_ctypes/common.py b/python/mxnet/_ctypes/common.py new file mode 100644 index 000000000000..5773a6a99c61 --- /dev/null +++ b/python/mxnet/_ctypes/common.py @@ -0,0 +1,30 @@ +# coding: utf-8 +"""Common code between symbolic and ndarray.""" +from __future__ import absolute_import as _abs + +import ctypes + +from ..base import _LIB +from ..base import c_array, c_str +from ..base import OpHandle, CachedOpHandle +from ..base import check_call + + +class CachedOp(object): + """Cached operator handle.""" + __slots__ = ["handle", "op"] + def __init__(self, op, num_input, **kwargs): + self.op = op + op_handle = OpHandle() + check_call(_LIB.NNGetOpHandle(c_str(op), ctypes.byref(op_handle))) + self.handle = CachedOpHandle() + check_call(_LIB.MXCachedCreateOp( + op_handle, + ctypes.c_int(num_input), + ctypes.c_int(len(kwargs)), + c_array(ctypes.c_char_p, [c_str(key) for key in kwargs.keys()]), + c_array(ctypes.c_char_p, [c_str(str(val)) for val in kwargs.values()]), + ctypes.byref(self.handle))) + + def __del__(self): + check_call(_LIB.MXCachedFree(self.handle)) diff --git a/python/mxnet/_ctypes/ndarray.py b/python/mxnet/_ctypes/ndarray.py index 37879e955f7d..a678e1726f02 100644 --- a/python/mxnet/_ctypes/ndarray.py +++ b/python/mxnet/_ctypes/ndarray.py @@ -1,7 +1,7 @@ # coding: utf-8 # pylint: disable=invalid-name, protected-access, too-many-arguments # pylint: disable=global-statement, unused-import -"""Symbolic configuration API.""" +"""NDArray configuration API.""" from __future__ import absolute_import as _abs import ctypes @@ -13,6 +13,7 @@ from ..base import NDArrayHandle, OpHandle from ..base import check_call from ..ndarray_doc import _build_doc +from .common import CachedOp class NDArrayBase(object): @@ -78,3 +79,33 @@ def _imperative_invoke(handle, ndargs, keys, vals, out): else: return [_ndarray_cls(ctypes.cast(output_vars[i], NDArrayHandle)) for i in range(num_output.value)] + + +def invoke(cached_op, args, out=None, name=None): # pylint: disable=unused-argument + """ctypes implementation of imperative invoke wrapper""" + if out is not None: + original_output = out + if isinstance(out, NDArrayBase): + out = (out,) + num_output = ctypes.c_int(len(out)) + output_vars = c_array(NDArrayHandle, [i.handle for i in out]) + output_vars = ctypes.cast(output_vars, ctypes.POINTER(NDArrayHandle)) + else: + original_output = None + output_vars = ctypes.POINTER(NDArrayHandle)() + num_output = ctypes.c_int(0) + + check_call(_LIB.MXCachedInvoke( + cached_op.handle, + ctypes.c_int(len(args)), + c_array(NDArrayHandle, [arr.handle for arr in args]), + ctypes.byref(num_output), + ctypes.byref(output_vars))) + + if original_output is not None: + return original_output + if num_output.value == 1: + return _ndarray_cls(ctypes.cast(output_vars[0], NDArrayHandle)) + else: + return [_ndarray_cls(ctypes.cast(output_vars[i], NDArrayHandle)) + for i in range(num_output.value)] diff --git a/python/mxnet/_ctypes/symbol.py b/python/mxnet/_ctypes/symbol.py index 00d935d4b0be..2ffa1a933540 100644 --- a/python/mxnet/_ctypes/symbol.py +++ b/python/mxnet/_ctypes/symbol.py @@ -4,15 +4,12 @@ from __future__ import absolute_import as _abs import ctypes -import sys -import numpy as _numpy from ..base import _LIB -from ..base import c_array, c_str, mx_uint, py_str -from ..base import SymbolHandle, OpHandle +from ..base import c_array, c_str, mx_uint +from ..base import SymbolHandle from ..base import check_call -from ..symbol_doc import _build_doc from ..name import NameManager -from ..attribute import AttrScope +from .common import CachedOp # pylint: disable=unused-import _symbol_cls = None @@ -105,122 +102,38 @@ def _set_symbol_class(cls): _symbol_cls = cls -def _make_atomic_symbol_function(handle, name): - """Create an atomic symbol function by handle and funciton name.""" - real_name = ctypes.c_char_p() - desc = ctypes.c_char_p() - num_args = mx_uint() - arg_names = ctypes.POINTER(ctypes.c_char_p)() - arg_types = ctypes.POINTER(ctypes.c_char_p)() - arg_descs = ctypes.POINTER(ctypes.c_char_p)() - key_var_num_args = ctypes.c_char_p() - ret_type = ctypes.c_char_p() - - check_call(_LIB.MXSymbolGetAtomicSymbolInfo( - handle, ctypes.byref(real_name), ctypes.byref(desc), - ctypes.byref(num_args), - ctypes.byref(arg_names), - ctypes.byref(arg_types), - ctypes.byref(arg_descs), - ctypes.byref(key_var_num_args), - ctypes.byref(ret_type))) - narg = int(num_args.value) - func_name = name - key_var_num_args = py_str(key_var_num_args.value) - ret_type = py_str(ret_type.value) if ret_type.value is not None else '' - doc_str = _build_doc(func_name, - py_str(desc.value), - [py_str(arg_names[i]) for i in range(narg)], - [py_str(arg_types[i]) for i in range(narg)], - [py_str(arg_descs[i]) for i in range(narg)], - key_var_num_args, - ret_type) - - def creator(*args, **kwargs): - """Activation Operator of Neural Net. - The parameters listed below can be passed in as keyword arguments. - - Parameters - ---------- - name : string, required. - Name of the resulting symbol. - - Returns - ------- - symbol: Symbol - the resulting symbol - """ - param_keys = [] - param_vals = [] - symbol_kwargs = {} - - attr = kwargs.pop('attr', None) - kwargs.update(AttrScope.current.get(attr)) - name = kwargs.pop('name', None) - if 'dtype' in kwargs: - kwargs['dtype'] = _numpy.dtype(kwargs['dtype']).name - - if key_var_num_args and key_var_num_args not in kwargs: - param_keys.append(c_str(key_var_num_args)) - param_vals.append(c_str(str(len(args)))) - - for k, v in kwargs.items(): - if isinstance(v, SymbolBase): - symbol_kwargs[k] = v - else: - param_keys.append(c_str(k)) - param_vals.append(c_str(str(v))) - # create atomic symbol - param_keys = c_array(ctypes.c_char_p, param_keys) - param_vals = c_array(ctypes.c_char_p, param_vals) - sym_handle = SymbolHandle() - check_call(_LIB.MXSymbolCreateAtomicSymbol( - handle, - mx_uint(len(param_keys)), - param_keys, param_vals, - ctypes.byref(sym_handle))) - - if len(args) != 0 and len(symbol_kwargs) != 0: - raise TypeError( - '%s can only accept input' - 'Symbols either as positional or keyword arguments, not both' % func_name) - s = _symbol_cls(sym_handle) - - hint = func_name.lower() - name = NameManager.current.get(name, hint) - s._compose(*args, name=name, **symbol_kwargs) - return s - - creator.__name__ = func_name - creator.__doc__ = doc_str - creator.__module__ = 'mxnet.symbol' - return creator - - -def _init_symbol_module(symbol_class, root_namespace): - """List and add all the atomic symbol functions to current module.""" - _set_symbol_class(symbol_class) - plist = ctypes.POINTER(ctypes.c_char_p)() - size = ctypes.c_uint() - - check_call(_LIB.MXListAllOpNames(ctypes.byref(size), - ctypes.byref(plist))) - op_names = [] - for i in range(size.value): - op_names.append(py_str(plist[i])) - - module_obj = sys.modules["%s.symbol" % root_namespace] - module_internal = sys.modules["%s._symbol_internal" % root_namespace] - module_contrib = sys.modules["%s.contrib.symbol" % root_namespace] - for name in op_names: - hdl = OpHandle() - check_call(_LIB.NNGetOpHandle(c_str(name), ctypes.byref(hdl))) - function = _make_atomic_symbol_function(hdl, name) - if function.__name__.startswith('_contrib_'): - function.__name__ = function.__name__[9:] - function.__module__ = 'mxnet.contrib.symbol' - setattr(module_contrib, function.__name__, function) - elif function.__name__.startswith('_'): - setattr(module_internal, function.__name__, function) - else: - setattr(module_obj, function.__name__, function) +def invoke(cached_op, args, name=None): + """Call cached symbolic operator""" + ret = SymbolHandle() + hint = cached_op.op.lower() + name = c_str(NameManager.current.get(name, hint)) + check_call(_LIB.MXCachedCreateSymbol( + cached_op.handle, + name, + mx_uint(len(args)), + c_array(SymbolHandle, [s.handle for s in args]), + ctypes.byref(ret))) + return _symbol_cls(ret) + + +def _symbol_creator(handle, args, kwargs, keys, vals, name): + sym_handle = SymbolHandle() + check_call(_LIB.MXSymbolCreateAtomicSymbol( + ctypes.c_void_p(handle), + mx_uint(len(keys)), + c_array(ctypes.c_char_p, [c_str(i) for i in keys]), + c_array(ctypes.c_char_p, [c_str(str(i)) for i in vals]), + ctypes.byref(sym_handle))) + + if args and kwargs: + raise TypeError( + 'Operators with variable length input can only accept input' + 'Symbols either as positional or keyword arguments, not both') + s = _symbol_cls(sym_handle) + if args: + s._compose(*args, name=name) + elif kwargs: + s._compose(name=name, **kwargs) + else: + s._compose(name=name) + return s diff --git a/python/mxnet/base.py b/python/mxnet/base.py index 83d06e5b9bb6..aeb7ef8ecfd1 100644 --- a/python/mxnet/base.py +++ b/python/mxnet/base.py @@ -59,6 +59,7 @@ def _load_lib(): NDArrayHandle = ctypes.c_void_p FunctionHandle = ctypes.c_void_p OpHandle = ctypes.c_void_p +CachedOpHandle = ctypes.c_void_p SymbolHandle = ctypes.c_void_p ExecutorHandle = ctypes.c_void_p DataIterCreatorHandle = ctypes.c_void_p diff --git a/python/mxnet/contrib/autograd.py b/python/mxnet/contrib/autograd.py index 40ab289c8f4c..e56361efdb1f 100644 --- a/python/mxnet/contrib/autograd.py +++ b/python/mxnet/contrib/autograd.py @@ -104,24 +104,48 @@ def mark_variables(variables, gradients, grad_reqs='write'): c_array(mx_uint, grad_reqs), c_array(NDArrayHandle, gradient_handles))) -def compute_gradient(outputs): + +def backward(outputs, out_grads=None, retain_graph=False): """Compute the gradients of outputs w.r.t variables. Parameters ---------- outputs: list of NDArray - - Returns - ------- - gradients: list of NDArray + out_grads: list of NDArray or None """ + assert isinstance(outputs, (list, tuple)), \ + "outputs must be a list or tuple of NDArrays" output_handles = [] for arr in outputs: output_handles.append(arr.handle) - check_call(_LIB.MXAutogradComputeGradient( + if out_grads is None: + check_call(_LIB.MXAutogradBackward( + len(output_handles), + c_array(NDArrayHandle, output_handles), + ctypes.c_void_p(0), + ctypes.c_int(retain_graph))) + return + + ograd_handles = [] + for arr in out_grads: + if arr is not None: + ograd_handles.append(arr.handle) + else: + ograd_handles.append(NDArrayHandle(0)) + assert len(ograd_handles) == len(output_handles), \ + "outputs and out_grads must have the same length" + + check_call(_LIB.MXAutogradBackward( len(output_handles), - c_array(NDArrayHandle, output_handles))) + c_array(NDArrayHandle, output_handles), + c_array(NDArrayHandle, ograd_handles), + ctypes.c_int(retain_graph))) + + +def compute_gradient(outputs): + """Deprecated. Please use backward""" + backward(outputs) def grad_and_loss(func, argnum=None): diff --git a/python/mxnet/cython/base.pyi b/python/mxnet/cython/base.pyi index a60aaef375d0..651258135ef3 100644 --- a/python/mxnet/cython/base.pyi +++ b/python/mxnet/cython/base.pyi @@ -7,6 +7,7 @@ from cpython.version cimport PY_MAJOR_VERSION ctypedef void* SymbolHandle ctypedef void* NDArrayHandle ctypedef void* OpHandle +ctypedef void* CachedOpHandle ctypedef unsigned nn_uint cdef py_str(const char* x): @@ -98,3 +99,75 @@ cdef extern from "mxnet/c_api.h": const char **param_keys, const char **param_vals); int MXNDArrayFree(NDArrayHandle handle); + int MXCachedCreateOp(OpHandle creator, + int num_inputs, + int num_params, + const char **param_keys, + const char **param_vals, + CachedOpHandle *out); + int MXCachedFree(CachedOpHandle handle); + int MXCachedInvoke(CachedOpHandle handle, + int num_inputs, + NDArrayHandle *inputs, + int *num_outputs, + NDArrayHandle **outputs); + int MXCachedCreateSymbol(CachedOpHandle handle, + const char* name, + unsigned num_args, + SymbolHandle* args, + SymbolHandle* out); + + +cdef class CachedOp: + """Cached operator handle.""" + cdef CachedOpHandle chandle + cdef string cop + + cdef _set_handle(self, handle): + cdef unsigned long long ptr + if handle is None: + self.chandle = NULL + else: + ptr = handle.value + self.chandle = (ptr) + + property handle: + def __get__(self): + if self.chandle == NULL: + return None + else: + return _ctypes.cast(self.chandle, _ctypes.c_void_p) + def __set__(self, value): + self._set_handle(value) + + property op: + def __get__(self): + return py_str(self.cop.c_str()) + def __set__(self, value): + self.cop = c_str(value) + + def __init__(self, op, num_input, **kwargs): + cdef OpHandle op_handle + cdef vector[string] ckeys + cdef vector[string] cvals + + self.op = op + CALL(NNGetOpHandle(self.cop.c_str(), &op_handle)) + + for k, v in kwargs.items(): + ckeys.push_back(c_str(k)) + cvals.push_back(c_str(str(v))) + + cdef vector[const char*] param_keys = SVec2Ptr(ckeys) + cdef vector[const char*] param_vals = SVec2Ptr(cvals) + + CALL(MXCachedCreateOp( + op_handle, + num_input, + len(kwargs), + CBeginPtr(param_keys), + CBeginPtr(param_vals), + &self.chandle)) + + def __del__(self): + CALL(MXCachedFree(self.chandle)) diff --git a/python/mxnet/cython/ndarray.pyx b/python/mxnet/cython/ndarray.pyx index 6071f7964df8..24e37b54c7be 100644 --- a/python/mxnet/cython/ndarray.pyx +++ b/python/mxnet/cython/ndarray.pyx @@ -60,9 +60,52 @@ cdef NewArray(NDArrayHandle handle): (nd).cwritable = True return nd + +def invoke(cached_op, args, out=None, name=None): + """ctypes implementation of imperative invoke wrapper""" + cdef vector[NDArrayHandle] ndvars + cdef vector[NDArrayHandle] output_vars + cdef NDArrayHandle* p_output_vars + cdef NDArrayHandle ret_handle + cdef int num_output + + for i in args: + ndvars.push_back((i).chandle) + + original_output = None + if out is not None: + original_output = out + if isinstance(out, NDArrayBase): + output_vars.push_back((out).chandle) + else: + for i in out: + output_vars.push_back((i).chandle) + + num_output = output_vars.size() + if output_vars.size() == 0: + output_vars.resize(1) + p_output_vars = NULL + else: + p_output_vars = &output_vars[0] + + CALL(MXCachedInvoke( + (cached_op).chandle, + len(args), + &ndvars[0] if ndvars.size() != 0 else NULL, + &num_output, + &p_output_vars)) + + if original_output is not None: + return original_output + if num_output == 1: + return NewArray(p_output_vars[0]) + else: + return tuple(NewArray(p_output_vars[i]) for i in range(num_output)) + + def _imperative_invoke(handle, ndargs, keys, vals, out): """cython implementation of imperative invoke wrapper""" - cdef int64_t ihandle = handle + cdef unsigned long long ihandle = handle cdef OpHandle chandle = ihandle cdef vector[string] ckeys cdef vector[string] cvals diff --git a/python/mxnet/cython/symbol.pyx b/python/mxnet/cython/symbol.pyx index 40184f623812..e8787fba77a3 100644 --- a/python/mxnet/cython/symbol.pyx +++ b/python/mxnet/cython/symbol.pyx @@ -68,7 +68,7 @@ cdef SymbolSetAttr(SymbolHandle handle, dict kwargs): _symbol_cls = SymbolBase -cdef _set_symbol_class(cls): +def _set_symbol_class(cls): global _symbol_cls _symbol_cls = cls @@ -78,129 +78,68 @@ cdef NewSymbol(SymbolHandle handle): (sym).chandle = handle return sym -cdef _make_atomic_symbol_function(OpHandle handle, string name): - """Create an atomic symbol function by handle and funciton name.""" - cdef const char *real_name - cdef const char *desc - cdef nn_uint num_args - cdef const char** arg_names - cdef const char** arg_types - cdef const char** arg_descs - cdef const char* return_type - cdef const char* key_var_num_args - - CALL(MXSymbolGetAtomicSymbolInfo( - handle, &real_name, &desc, - &num_args, &arg_names, - &arg_types, &arg_descs, - &key_var_num_args, &return_type)) - func_name = py_str(name.c_str()) - - key_vargs = py_str(key_var_num_args) - num_args = int(num_args) - doc_str = _build_doc(func_name, - py_str(desc), - [py_str(arg_names[i]) for i in range(num_args)], - [py_str(arg_types[i]) for i in range(num_args)], - [py_str(arg_descs[i]) for i in range(num_args)], - key_vargs, - py_str(return_type) if return_type != NULL else '') - - func_hint = func_name.lower() - - def creator(*args, **kwargs): - cdef vector[string] sparam_keys - cdef vector[string] sparam_vals - cdef vector[SymbolHandle] symbol_args - cdef vector[string] ssymbol_keys - cdef SymbolHandle ret_handle - attr = kwargs.pop("attr", None) - kwargs.update(AttrScope.current.get(attr)) - name = kwargs.pop("name", None) - - if key_vargs: - if key_vargs not in kwargs: - sparam_keys.push_back(c_str(key_vargs)) - sparam_vals.push_back(c_str(str(len(args)))) - - if len(kwargs) != 0: - for k, v in kwargs.items(): - if isinstance(v, SymbolBase): - ssymbol_keys.push_back(c_str(k)) - symbol_args.push_back((v).chandle) - elif k == 'dtype': - sparam_keys.push_back(c_str(k)) - sparam_vals.push_back(c_str(_numpy.dtype(v).name)) - else: - sparam_keys.push_back(c_str(k)) - sparam_vals.push_back(c_str(str(v))) - - if len(args) != 0: - if symbol_args.size() != 0: - raise TypeError("compose only accept input Symbols\ - either as positional or keyword arguments, not both") - for v in args: - if not isinstance(v, SymbolBase): - raise TypeError('Compose expect `Symbol` as arguments') - symbol_args.push_back((v).chandle) - - cdef vector[const char*] param_keys = SVec2Ptr(sparam_keys) - cdef vector[const char*] param_vals = SVec2Ptr(sparam_vals) - cdef vector[const char*] symbol_keys = SVec2Ptr(ssymbol_keys) - - CALL(MXSymbolCreateAtomicSymbol( - handle, - param_keys.size(), - CBeginPtr(param_keys), - CBeginPtr(param_vals), - &ret_handle)) - num_args = (symbol_args.size()) - - name = NameManager.current.get(name, func_hint) - - cdef const char* c_name = NULL - - if name: - name = c_str(name) - c_name = name - - CALL(NNSymbolCompose( - ret_handle, - c_name, - num_args, - &symbol_keys[0] if symbol_keys.size() != 0 else NULL, - &symbol_args[0] if symbol_args.size() != 0 else NULL)) - return NewSymbol(ret_handle) - - creator.__name__ = func_name - creator.__doc__ = doc_str - creator.__module__ = 'mxnet.symbol' - return creator - - -def _init_symbol_module(symbol_class, root_namespace): - """List and add all the atomic symbol functions to current module.""" - cdef const char** op_name_ptrs - cdef nn_uint size - cdef vector[string] op_names - cdef OpHandle handle - - _set_symbol_class(symbol_class) - CALL(MXListAllOpNames(&size, &op_name_ptrs)) - for i in range(size): - op_names.push_back(string(op_name_ptrs[i])) - - module_obj = _sys.modules["%s.symbol" % root_namespace] - module_internal = _sys.modules["%s._symbol_internal" % root_namespace] - module_contrib = _sys.modules["%s.contrib.symbol" % root_namespace] - for i in range(op_names.size()): - CALL(NNGetOpHandle(op_names[i].c_str(), &handle)) - function = _make_atomic_symbol_function(handle, op_names[i]) - if function.__name__.startswith('_contrib_'): - function.__name__ = function.__name__[9:] - function.__module__ = 'mxnet.contrib.symbol' - setattr(module_contrib, function.__name__, function) - elif function.__name__.startswith('_'): - setattr(module_internal, function.__name__, function) - else: - setattr(module_obj, function.__name__, function) + +def invoke(cached_op, args, name=None): + cdef SymbolHandle ret + cdef vector[SymbolHandle] sym_args + hint = cached_op.op.lower() + cdef string cname = c_str(NameManager.current.get(name, hint)) + for i in args: + sym_args.push_back((i).chandle) + CALL(MXCachedCreateSymbol( + (cached_op).chandle, + cname.c_str(), + len(args), + &sym_args[0] if sym_args.size() != 0 else NULL, + &ret)) + return NewSymbol(ret) + + +def _symbol_creator(handle, args, kwargs, keys, vals, name): + cdef unsigned long long ihandle = handle + cdef OpHandle chandle = ihandle + cdef vector[string] ckeys + cdef vector[string] cvals + cdef vector[string] sym_keys + cdef vector[SymbolHandle] sym_args + cdef SymbolHandle ret_handle + cdef string cname = c_str(name) + + for i in keys: + ckeys.push_back(c_str(i)) + for i in vals: + cvals.push_back(c_str(str(i))) + + cdef vector[const char*] param_keys = SVec2Ptr(ckeys) + cdef vector[const char*] param_vals = SVec2Ptr(cvals) + + CALL(MXSymbolCreateAtomicSymbol( + chandle, + param_keys.size(), + CBeginPtr(param_keys), + CBeginPtr(param_vals), + &ret_handle)) + + if args and kwargs: + raise TypeError( + 'Operators with variable length input can only accept input' + 'Symbols either as positional or keyword arguments, not both') + + if args: + for i in args: + sym_args.push_back((i).chandle) + elif kwargs: + for k, v in kwargs.items(): + sym_keys.push_back(c_str(k)) + sym_args.push_back((v).chandle) + + cdef vector[const char*] csym_keys = SVec2Ptr(sym_keys) + + CALL(NNSymbolCompose( + ret_handle, + cname.c_str(), + sym_args.size(), + &csym_keys[0] if csym_keys.size() != 0 else NULL, + &sym_args[0] if sym_args.size() != 0 else NULL)) + + return NewSymbol(ret_handle) diff --git a/python/mxnet/ndarray.py b/python/mxnet/ndarray.py index f86404eb9853..c5d6754555d1 100644 --- a/python/mxnet/ndarray.py +++ b/python/mxnet/ndarray.py @@ -31,15 +31,19 @@ # pylint: disable=unused-import try: if int(_os.environ.get("MXNET_ENABLE_CYTHON", True)) == 0: - from ._ctypes.ndarray import NDArrayBase, _set_ndarray_class, _imperative_invoke + from ._ctypes.ndarray import NDArrayBase, _set_ndarray_class + from ._ctypes.ndarray import invoke, CachedOp, _imperative_invoke elif _sys.version_info >= (3, 0): from ._cy3.ndarray import NDArrayBase, _set_ndarray_class, _imperative_invoke + from ._cy3.ndarray import invoke, CachedOp, _imperative_invoke else: from ._cy2.ndarray import NDArrayBase, _set_ndarray_class, _imperative_invoke + from ._cy2.ndarray import invoke, CachedOp, _imperative_invoke except ImportError: if int(_os.environ.get("MXNET_ENFORCE_CYTHON", False)) != 0: raise ImportError("Cython Module cannot be loaded but MXNET_ENFORCE_CYTHON=1") from ._ctypes.ndarray import NDArrayBase, _set_ndarray_class, _imperative_invoke + from ._ctypes.ndarray import invoke, CachedOp, _imperative_invoke # pylint: enable=unused-import # pylint: disable= no-member @@ -749,6 +753,24 @@ def T(self): return transpose(self) # pylint: enable= invalid-name, undefined-variable + @property + def _fresh_grad(self): + """Whether this array's corresponding gradient array + (registered via `autograd.mark_variables`) has been + updated by `autograd.backward` since last reset. + + `_fresh_grad` need to be manually set to False + after consuming gradient (usually after updating this + array). + """ + out = ctypes.c_int() + check_call(_LIB.MXNDArrayGetGradState(self.handle, ctypes.byref(out))) + return out.value + + @_fresh_grad.setter + def _fresh_grad(self, state): + check_call(_LIB.MXNDArraySetGradState(self.handle, ctypes.c_int(state))) + def asnumpy(self): """Returns a ``numpy.ndarray`` object with value copied from this array. @@ -910,7 +932,7 @@ def detach(self): check_call(_LIB.MXNDArrayDetach(self.handle, ctypes.byref(hdl))) return NDArray(hdl) - def backward(self, out_grad=None): + def backward(self, out_grad=None, retain_graph=False): """Compute the gradients of this NDArray w.r.t variables. Parameters @@ -924,7 +946,8 @@ def backward(self, out_grad=None): check_call(_LIB.MXAutogradBackward( 1, c_array(NDArrayHandle, [self.handle]), - c_array(NDArrayHandle, ograd_handles))) + c_array(NDArrayHandle, ograd_handles), + ctypes.c_int(retain_graph))) def onehot_encode(indices, out): @@ -2327,7 +2350,7 @@ def %s(*%s, **kwargs):"""%(func_name, arr_name)) for i in {}: assert isinstance(i, NDArrayBase), \\ "Positional arguments must have NDArray type, " \\ - "but got %s"%str(type(i)) + "but got %s"%str(i) ndargs.append(i)""".format(arr_name)) if dtype_name is not None: code.append(""" @@ -2335,10 +2358,7 @@ def %s(*%s, **kwargs):"""%(func_name, arr_name)) kwargs['%s'] = np.dtype(kwargs['%s']).name"""%( dtype_name, dtype_name, dtype_name)) code.append(""" - try: - kwargs.pop('name') - except: - pass + _ = kwargs.pop('name', None) out = kwargs.pop('out', None) keys = list(kwargs.keys()) vals = list(kwargs.values())""") @@ -2353,7 +2373,7 @@ def %s(%s): code.append(""" if {name} is not None: assert isinstance({name}, NDArrayBase), \\ - "Argument {name} must have NDArray type, but got %s"%str(type({name})) + "Argument {name} must have NDArray type, but got %s"%str({name}) ndargs.append({name})""".format(name=name)) # kwargs for name in kwarg_names: diff --git a/python/mxnet/symbol.py b/python/mxnet/symbol.py index d09de16facd3..16cbeae36531 100644 --- a/python/mxnet/symbol.py +++ b/python/mxnet/symbol.py @@ -14,27 +14,33 @@ from .base import _LIB, numeric_types from .base import c_array, c_str, mx_uint, py_str, string_types, mx_real_t -from .base import NDArrayHandle, ExecutorHandle, SymbolHandle -from .base import check_call, MXNetError +from .base import NDArrayHandle, ExecutorHandle, SymbolHandle, OpHandle +from .base import check_call, MXNetError, _Null # pylint: disable=unused-import from .context import Context, cpu from .ndarray import NDArray, zeros as _nd_zeros, _DTYPE_NP_TO_MX, _DTYPE_MX_TO_NP +from .name import NameManager # pylint: disable=unused-import from .executor import Executor from . import _symbol_internal as _internal from .attribute import AttrScope +from .symbol_doc import _build_doc # Use different version of SymbolBase # When possible, use cython to speedup part of computation. try: if int(_os.environ.get("MXNET_ENABLE_CYTHON", True)) == 0: - from ._ctypes.symbol import SymbolBase, _init_symbol_module + from ._ctypes.symbol import SymbolBase, _set_symbol_class + from ._ctypes.symbol import CachedOp, invoke, _symbol_creator # pylint: disable=unused-import elif _sys.version_info >= (3, 0): - from ._cy3.symbol import SymbolBase, _init_symbol_module + from ._cy3.symbol import SymbolBase, _set_symbol_class + from ._cy3.symbol import CachedOp, invoke, _symbol_creator # pylint: disable=unused-import else: - from ._cy2.symbol import SymbolBase, _init_symbol_module + from ._cy2.symbol import SymbolBase, _set_symbol_class + from ._cy2.symbol import CachedOp, invoke, _symbol_creator # pylint: disable=unused-import except ImportError: if int(_os.environ.get("MXNET_ENFORCE_CYTHON", False)) != 0: raise ImportError("Cython Module cannot be loaded but MXNET_ENFORCE_CYTHON=1") - from ._ctypes.symbol import SymbolBase, _init_symbol_module + from ._ctypes.symbol import SymbolBase, _set_symbol_class + from ._ctypes.symbol import CachedOp, invoke, _symbol_creator # pylint: disable=unused-import _GRAD_REQ_MAP = {'null': 0, 'write': 1, 'add': 3} @@ -1651,9 +1657,6 @@ def load_json(json_str): return Symbol(handle) -# Initialize the atomic symbol in startups -_init_symbol_module(Symbol, "mxnet") - # pylint: disable=no-member # pylint: disable=redefined-builtin def pow(base, exp): @@ -1901,3 +1904,189 @@ def arange(start, stop=None, step=1.0, repeat=1, name=None, dtype=None): dtype = _numpy.float32 return _internal._arange(start=start, stop=stop, step=step, repeat=repeat, name=name, dtype=dtype) + + +def _make_atomic_symbol_function(handle, name): + """Create an atomic symbol function by handle and funciton name.""" + real_name = ctypes.c_char_p() + desc = ctypes.c_char_p() + num_args = mx_uint() + arg_names = ctypes.POINTER(ctypes.c_char_p)() + arg_types = ctypes.POINTER(ctypes.c_char_p)() + arg_descs = ctypes.POINTER(ctypes.c_char_p)() + key_var_num_args = ctypes.c_char_p() + ret_type = ctypes.c_char_p() + + check_call(_LIB.MXSymbolGetAtomicSymbolInfo( + handle, ctypes.byref(real_name), ctypes.byref(desc), + ctypes.byref(num_args), + ctypes.byref(arg_names), + ctypes.byref(arg_types), + ctypes.byref(arg_descs), + ctypes.byref(key_var_num_args), + ctypes.byref(ret_type))) + narg = int(num_args.value) + arg_names = [py_str(arg_names[i]) for i in range(narg)] + arg_types = [py_str(arg_types[i]) for i in range(narg)] + func_name = name + key_var_num_args = py_str(key_var_num_args.value) + ret_type = py_str(ret_type.value) if ret_type.value is not None else '' + doc_str = _build_doc(func_name, + py_str(desc.value), + arg_names, + arg_types, + [py_str(arg_descs[i]) for i in range(narg)], + key_var_num_args, + ret_type) + + dtype_name = None + arr_name = None + ndsignature = [] + signature = [] + ndarg_names = [] + kwarg_names = [] + for i in range(narg): + name, atype = arg_names[i], arg_types[i] + if name == 'dtype': + dtype_name = name + signature.append('%s=_Null'%name) + elif atype.startswith('NDArray') or atype.startswith('Symbol'): + assert not arr_name, \ + "Op can only have one argument with variable " \ + "size and it must be the last argument." + if atype.endswith('[]'): + ndsignature.append('*%s'%name) + arr_name = name + else: + ndsignature.append('%s=None'%name) + ndarg_names.append(name) + else: + signature.append('%s=_Null'%name) + kwarg_names.append(name) + #signature.append('is_train=False') + signature.append('name=None') + signature.append('attr=None') + signature.append('out=None') + signature.append('**kwargs') + signature = ndsignature + signature + + code = [] + if arr_name: + code.append(""" +def %s(*%s, **kwargs):"""%(func_name, arr_name)) + code.append(""" + sym_args = [] + for i in {}: + assert isinstance(i, SymbolBase), \\ + "Positional arguments must be Symbol instances, " \\ + "but got %s"%str(i) + sym_args.append(i)""".format(arr_name)) + if dtype_name is not None: + code.append(""" + if '%s' in kwargs: + kwargs['%s'] = _numpy.dtype(kwargs['%s']).name"""%( + dtype_name, dtype_name, dtype_name)) + code.append(""" + attr = kwargs.pop('attr', None) + kwargs.update(AttrScope.current.get(attr)) + name = kwargs.pop('name', None) + name = NameManager.current.get(name, '%s') + _ = kwargs.pop('out', None) + keys = [] + vals = [] + sym_kwargs = dict() + for k, v in kwargs.items(): + if isinstance(v, SymbolBase): + sym_kwargs[k] = v + else: + keys.append(k) + vals.append(v)"""%(func_name.lower())) + if key_var_num_args: + code.append(""" + if '%s' not in kwargs: + keys.append('%s') + vals.append(len(sym_args) + len(sym_kwargs))"""%( + key_var_num_args, key_var_num_args)) + + code.append(""" + return _symbol_creator(%d, sym_args, sym_kwargs, keys, vals, name)"""%( + handle.value)) + else: + code.append(""" +def %s(%s): + kwargs.update(AttrScope.current.get(attr)) + sym_kwargs = dict() + keys = [] + vals = []"""%(func_name, ', '.join(signature))) + code.append(""" + for k, v in kwargs.items(): + if isinstance(v, SymbolBase): + sym_kwargs[k] = v + else: + keys.append(k) + vals.append(v)""") + # NDArray args + for name in ndarg_names: + code.append(""" + if {name} is not None: + assert isinstance({name}, SymbolBase), \\ + "Argument {name} must be Symbol instances, but got %s"%str({name}) + sym_kwargs['{name}'] = {name}""".format(name=name)) + # kwargs + for name in kwarg_names: + code.append(""" + if %s is not _Null: + keys.append('%s') + vals.append(%s)"""%(name, name, name)) + # dtype + if dtype_name is not None: + code.append(""" + if %s is not _Null: + keys.append('%s') + vals.append(_numpy.dtype(%s).name)"""%(dtype_name, dtype_name, dtype_name)) + + code.append(""" + name = NameManager.current.get(name, '%s') + return _symbol_creator(%d, None, sym_kwargs, keys, vals, name)"""%( + func_name.lower(), handle.value)) + + local = {} + exec(''.join(code), None, local) # pylint: disable=exec-used + symbol_function = local[func_name] + symbol_function.__name__ = func_name + symbol_function.__doc__ = doc_str + symbol_function.__module__ = 'mxnet.symbol' + return symbol_function + + +def _init_symbol_module(symbol_class, root_namespace): + """List and add all the atomic symbol functions to current module.""" + _set_symbol_class(symbol_class) + plist = ctypes.POINTER(ctypes.c_char_p)() + size = ctypes.c_uint() + + check_call(_LIB.MXListAllOpNames(ctypes.byref(size), + ctypes.byref(plist))) + op_names = [] + for i in range(size.value): + op_names.append(py_str(plist[i])) + + module_obj = _sys.modules["%s.symbol" % root_namespace] + module_internal = _sys.modules["%s._symbol_internal" % root_namespace] + module_contrib = _sys.modules["%s.contrib.symbol" % root_namespace] + for name in op_names: + hdl = OpHandle() + check_call(_LIB.NNGetOpHandle(c_str(name), ctypes.byref(hdl))) + function = _make_atomic_symbol_function(hdl, name) + if function.__name__.startswith('_contrib_'): + function.__name__ = function.__name__[9:] + function.__module__ = 'mxnet.contrib.symbol' + setattr(module_contrib, function.__name__, function) + elif function.__name__.startswith('_'): + setattr(module_internal, function.__name__, function) + else: + setattr(module_obj, function.__name__, function) + + +# Initialize the atomic symbol in startups +_init_symbol_module(Symbol, "mxnet") diff --git a/src/c_api/c_api.cc b/src/c_api/c_api.cc index 41986a0d577b..9d60c8615027 100644 --- a/src/c_api/c_api.cc +++ b/src/c_api/c_api.cc @@ -398,6 +398,27 @@ int MXNDArrayGetContext(NDArrayHandle handle, API_END(); } +int MXNDArrayDetach(NDArrayHandle handle, NDArrayHandle *out) { + API_BEGIN(); + NDArray *arr = static_cast(handle); + *out = new NDArray(arr->Detach()); + API_END(); +} + +int MXNDArraySetGradState(NDArrayHandle handle, int state) { + API_BEGIN(); + NDArray *arr = static_cast(handle); + arr->set_fresh_out_grad(static_cast(state)); + API_END(); +} + +int MXNDArrayGetGradState(NDArrayHandle handle, int *out) { + API_BEGIN(); + NDArray *arr = static_cast(handle); + *out = arr->fresh_out_grad(); + API_END(); +} + int MXListFunctions(mx_uint *out_size, FunctionHandle **out_array) { API_BEGIN(); diff --git a/src/c_api/c_api_ndarray.cc b/src/c_api/c_api_ndarray.cc index 66a237a4bd36..0be1d3574dd9 100644 --- a/src/c_api/c_api_ndarray.cc +++ b/src/c_api/c_api_ndarray.cc @@ -87,7 +87,8 @@ void SetNDInputsOutputs(const nnvm::Op* op, ndoutputs.resize(infered_num_outputs); } else { CHECK(!AutogradRuntime::Get()->IsTraining()) - << "Cannot assign to NDArray or specify 'out' when training with autograd"; + << "Inplace operations (+=, -=, op(..., out=x) etc.) and assignment are " + << "not supported when you are inside a train_section using autograd."; CHECK(*num_outputs == infered_num_outputs || *num_outputs == num_visible_outputs) << "Expecting " << infered_num_outputs << " (all) or " << num_visible_outputs << " (visible only) outputs, got " @@ -321,26 +322,18 @@ void PushOperator(std::shared_ptr opr, 0, PROFILER_MESSAGE(op->name.c_str())); } -int MXImperativeInvoke(AtomicSymbolCreator creator, - int num_inputs, - NDArrayHandle *inputs, - int *num_outputs, - NDArrayHandle **outputs, - int num_params, - const char **param_keys, - const char **param_vals) { +void ImperativeInvokeImpl(const nnvm::NodeAttrs& attrs, + int num_inputs, + NDArrayHandle *inputs, + int *num_outputs, + NDArrayHandle **outputs) { static auto& fcpu = nnvm::Op::GetAttr("FCompute"); static auto& fgpu = nnvm::Op::GetAttr("FCompute"); static auto& ndfunc = nnvm::Op::GetAttr("FNDArrayFunction"); static auto& createop = nnvm::Op::GetAttr("FCreateLayerOp"); - const nnvm::Op* op = static_cast(creator); - NDArray** outarray = *reinterpret_cast(outputs); MXAPIThreadLocalEntry *ret = MXAPIThreadLocalStore::Get(); - - API_BEGIN(); - nnvm::NodeAttrs attrs; - SetOpAttrs(op, &attrs, - num_inputs, num_params, param_keys, param_vals); + NDArray** outarray = *reinterpret_cast(outputs); + const nnvm::Op *op = attrs.op; int infered_num_outputs; int num_visible_outputs; @@ -408,6 +401,57 @@ int MXImperativeInvoke(AtomicSymbolCreator creator, *outarray[i] = std::move(ndoutputs[i]); } } +} + +int MXImperativeInvoke(AtomicSymbolCreator creator, + int num_inputs, + NDArrayHandle *inputs, + int *num_outputs, + NDArrayHandle **outputs, + int num_params, + const char **param_keys, + const char **param_vals) { + const nnvm::Op* op = static_cast(creator); + + API_BEGIN(); + nnvm::NodeAttrs attrs; + SetOpAttrs(op, &attrs, num_inputs, num_params, param_keys, param_vals); + ImperativeInvokeImpl(attrs, num_inputs, inputs, num_outputs, outputs); + API_END(); +} + +int MXCachedCreateOp(AtomicSymbolCreator creator, + int num_inputs, + int num_params, + const char **param_keys, + const char **param_vals, + CachedOpHandle *out) { + const nnvm::Op* op = static_cast(creator); + + API_BEGIN(); + nnvm::NodeAttrs *attrs = new nnvm::NodeAttrs; + SetOpAttrs(op, attrs, num_inputs, num_params, param_keys, param_vals); + *out = attrs; + API_END(); +} + +int MXCachedFree(CachedOpHandle handle) { + nnvm::NodeAttrs *attrs = static_cast(handle); + + API_BEGIN(); + delete attrs; + API_END(); +} + +int MXCachedInvoke(CachedOpHandle handle, + int num_inputs, + NDArrayHandle *inputs, + int *num_outputs, + NDArrayHandle **outputs) { + nnvm::NodeAttrs *attrs = static_cast(handle); + + API_BEGIN(); + ImperativeInvokeImpl(*attrs, num_inputs, inputs, num_outputs, outputs); API_END(); } @@ -438,16 +482,31 @@ int MXAutogradMarkVariables(mx_uint num_var, int MXAutogradComputeGradient(mx_uint num_output, NDArrayHandle *output_handles) { + return MXAutogradBackward(num_output, output_handles, nullptr, 0); +} + +int MXAutogradBackward(mx_uint num_output, + NDArrayHandle *output_handles, + NDArrayHandle *ograd_handles, + int retain_graph) { API_BEGIN(); MXAPIThreadLocalEntry *ret = MXAPIThreadLocalStore::Get(); - std::vector outputs; + std::vector outputs, ograds; outputs.reserve(num_output); for (mx_uint i = 0; i < num_output; ++i) { outputs.emplace_back(*static_cast(output_handles[i])); } - AutogradRuntime::Get()->ComputeGradient(outputs); + ograds.reserve(num_output); + for (mx_uint i = 0; i < num_output; ++i) { + if (ograd_handles != nullptr && ograd_handles[i] != nullptr) { + ograds.emplace_back(*static_cast(ograd_handles[i])); + } else { + ograds.emplace_back(); + } + } + AutogradRuntime::Get()->ComputeGradient(outputs, ograds, retain_graph); API_END(); } diff --git a/src/c_api/c_api_symbolic.cc b/src/c_api/c_api_symbolic.cc index fdf095b09361..27df5b2de1f3 100644 --- a/src/c_api/c_api_symbolic.cc +++ b/src/c_api/c_api_symbolic.cc @@ -124,6 +124,22 @@ int MXSymbolCreateAtomicSymbol(AtomicSymbolCreator creator, API_END_HANDLE_ERROR(delete s;); } +int MXCachedCreateSymbol(CachedOpHandle handle, + const char* name, + mx_uint num_args, + SymbolHandle* args, + SymbolHandle* out) { + nnvm::Symbol *s = new nnvm::Symbol(); + const nnvm::NodeAttrs *attrs = static_cast(handle); + API_BEGIN(); + *s = nnvm::Symbol::CreateFunctor(*attrs); + nnvm::array_view parg( + (nnvm::Symbol**)args, (nnvm::Symbol**)args + num_args); // NOLINT(*) + s->Compose(parg, std::unordered_map(), name); + *out = s; + API_END_HANDLE_ERROR(delete s;) +} + int MXSymbolCreateVariable(const char *name, SymbolHandle *out) { return NNSymbolCreateVariable(name, out); } diff --git a/src/ndarray/autograd.cc b/src/ndarray/autograd.cc index e7b57956a39b..ce1b98f095d8 100644 --- a/src/ndarray/autograd.cc +++ b/src/ndarray/autograd.cc @@ -49,6 +49,10 @@ nnvm::NodeEntry AGNodeEntry::nn_entry() const { return nnvm::NodeEntry{ag_node->nn_node, index, version}; } +bool AGNodeEntry::is_none() const { + return ag_node == nullptr || ag_node->outputs.empty(); +} + AutogradRuntime::AutogradRuntime() {} void AutogradRuntime::MarkVariables( @@ -56,13 +60,21 @@ void AutogradRuntime::MarkVariables( const std::vector& grad_reqs, const std::vector& gradients) { for (uint32_t i = 0; i < variables.size(); ++i) { + std::string str_c(std::to_string(variable_count_++)); + AGNodeEntry e{AGNode::Create(Node::Create()), 0, 0}; variables[i]->entry_.clear(); - e.ag_node->outputs.push_back(*variables[i]); + e.ag_node->outputs.emplace_back(*variables[i]); + + AGNodeEntry ge{AGNode::Create(Node::Create()), 0, 0}; gradients[i]->entry_.clear(); - e.ag_node->out_grads.push_back(*gradients[i]); + ge.ag_node->outputs.emplace_back(*gradients[i]); + ge.ag_node->nn_node->attrs.name = "grad" + str_c; + gradients[i]->entry_ = std::move(ge); + e.ag_node->out_grads.emplace_back(*gradients[i]); + e.ag_node->grad_req = static_cast(grad_reqs[i]); - e.ag_node->nn_node->attrs.name = "agvar" + std::to_string(variable_count_++); + e.ag_node->nn_node->attrs.name = "var" + str_c; variables[i]->entry_ = std::move(e); // assign last to prevent cyclic reference } } @@ -102,30 +114,28 @@ AGNodePtr AutogradRuntime::RecordOp(const nnvm::Op* op, NodePtr nn_node = Node::Create(); nn_node->attrs = attrs; - nn_node->attrs.name = "agnode_" + std::to_string(node_count_++); + nn_node->attrs.name = "node_" + std::to_string(node_count_++); AGNodePtr ag_node = AGNode::Create(nn_node); ag_node->opr = opr; for (uint32_t i = 0; i < outputs.size(); ++i) { - if (outputs[i].entry_.ag_node == nullptr || - !outputs[i].entry_.ag_node->out_grads.size()) { - outputs[i].entry_.clear(); - ag_node->outputs.push_back(outputs[i]); - outputs[i].entry_ = AGNodeEntry{ag_node, i, 0}; - } else { - NDArray copy = outputs[i]; - copy.entry_.clear(); - ag_node->outputs.push_back(copy); - } + CHECK(outputs[i].entry_.is_none()) + << "Output NDArray is non-empty and already in another computation graph. " + << "Assigning to it will cause undefined behavior when evaluating gradients. " + << "Please call backward first to clear the graph or do this out side of " + << "a train section. "; + outputs[i].entry_.clear(); + ag_node->outputs.push_back(outputs[i]); + outputs[i].entry_ = AGNodeEntry{ag_node, i, 0}; } for (size_t i = 0; i < inputs.size(); ++i) { - if (inputs[i].entry_.ag_node.get() == nullptr) { + if (inputs[i].entry_.is_none()) { AGNodeEntry e{AGNode::Create(Node::Create()), 0, 0}; e.ag_node->outputs.emplace_back(inputs[i]); e.ag_node->out_grads.emplace_back(); - e.ag_node->nn_node->attrs.name = "agvar_" + std::to_string(variable_count_++); + e.ag_node->nn_node->attrs.name = "var_" + std::to_string(variable_count_++); inputs[i].entry_ = std::move(e); // assign last to prevent cyclic reference } nn_node->inputs.push_back(inputs[i].entry_.nn_entry()); @@ -135,15 +145,19 @@ AGNodePtr AutogradRuntime::RecordOp(const nnvm::Op* op, return ag_node; } -void AutogradRuntime::ComputeGradient(const std::vector& outputs) { +void AutogradRuntime::ComputeGradient(const std::vector& outputs, + const std::vector& ograds, + bool retain_graph) { static auto& fmutate_inputs = nnvm::Op::GetAttr("FMutateInputs"); std::vector heads; Symbol sym; NodeEntryMap feed_dict; for (const auto& i : outputs) { - CHECK(i.entry_.ag_node.get() != nullptr) - << "Cannot differentiate node because it doesn't have " - << "computation history. Did you forget to set is_training?"; + CHECK(!i.entry_.is_none()) + << "Cannot differentiate node because it is not in a computational graph. " + << "You need to set is_training to true or use a train_section to save " + << "computational graphs for backward. If you want to differentiate the same " + << "graph twice, you need to pass retain_graph=True to backward."; heads.emplace_back(i.entry_); sym.outputs.emplace_back(i.entry_.nn_entry()); } @@ -176,6 +190,9 @@ void AutogradRuntime::ComputeGradient(const std::vector& outputs) { if (mutable_set.count(n.get())) { aux_states.push_back(n->outputs[0]); } else { + if (n->grad_req != kNullOp) { + n->fresh_out_grad = true; + } args.push_back(n->outputs[0]); args_grad.push_back(n->out_grads[0]); grad_reqs.push_back(n->grad_req); @@ -193,19 +210,27 @@ void AutogradRuntime::ComputeGradient(const std::vector& outputs) { std::vector head_grads; head_grads.reserve(exec->head_grad_array_.size()); - - for (size_t i = 0; i < exec->output_arrays_.size(); ++i) { - NDArray grad(exec->output_arrays_[i].shape(), exec->output_arrays_[i].ctx()); - grad = static_cast(1.0); - head_grads.push_back(grad); + CHECK_EQ(ograds.size(), exec->output_arrays_.size()); + + for (size_t i = 0; i < ograds.size(); ++i) { + if (ograds[i].is_none()) { + head_grads.emplace_back( + exec->output_arrays_[i].shape(), exec->output_arrays_[i].ctx(), + false, exec->output_arrays_[i].dtype()); + head_grads.back() = static_cast(1.0); + } else { + head_grads.emplace_back(ograds[i]); + } } exec->Backward(head_grads); delete exec; } - for (auto& i : heads) { - i.ag_node->clear_history(); + if (!retain_graph) { + for (auto& i : heads) { + i.ag_node->clear_history(); + } } } diff --git a/src/ndarray/autograd.h b/src/ndarray/autograd.h index 3603b0a111d0..e6868064ca0d 100644 --- a/src/ndarray/autograd.h +++ b/src/ndarray/autograd.h @@ -20,6 +20,7 @@ namespace mxnet { namespace autograd { + class AGNode { public: OpReqType grad_req; @@ -28,9 +29,10 @@ class AGNode { std::vector inputs; std::vector outputs; std::vector out_grads; + bool fresh_out_grad; explicit AGNode(const nnvm::NodePtr& nn_node_) : - grad_req(kNullOp), nn_node(nn_node_) {} + grad_req(kNullOp), nn_node(nn_node_), fresh_out_grad(false) {} static AGNodePtr Create(const nnvm::NodePtr& nn_node_) { return std::make_shared(nn_node_); @@ -77,7 +79,9 @@ class AutogradRuntime { std::vector* p_inputs, std::vector* p_outputs); /*! \brief compute the gradient of outputs w.r.t variables. */ - void ComputeGradient(const std::vector& outputs); + void ComputeGradient(const std::vector& outputs, + const std::vector& ograds, + bool retain_graph); /*! \return AutogradRuntime singleton */ static AutogradRuntime* Get(); /*! \brief Get shared pointer reference to AutogradRuntime singleton. diff --git a/src/ndarray/ndarray.cc b/src/ndarray/ndarray.cc index 717ba170aaf7..025624c923f5 100644 --- a/src/ndarray/ndarray.cc +++ b/src/ndarray/ndarray.cc @@ -26,11 +26,12 @@ namespace mxnet { NDArray NDArray::Reshape(const TShape &shape) const { using namespace autograd; - CHECK_GE(shape_.Size(), shape.Size()) - << "NDArray.Reshape: target shape size is different from current shape"; - NDArray ret = *this; - ret.shape_ = shape; if (AutogradRuntime::Get()->IsTraining()) { + CHECK_GE(shape_.Size(), shape.Size()) + << "NDArray.Reshape: target shape must have must have the same size as " + << "current shape when in train_section."; + NDArray ret = *this; + ret.shape_ = shape; // fake a Reshape op ret.entry_.clear(); const nnvm::Op* op = nnvm::Op::Get("Reshape"); @@ -47,6 +48,10 @@ NDArray NDArray::Reshape(const TShape &shape) const { op, attrs, &inputs, &outputs); return outputs[0]; } else { + CHECK_GE(shape_.Size(), shape.Size()) + << "NDArray.Reshape: target shape size is larger current shape"; + NDArray ret = *this; + ret.shape_ = shape; return ret; } } @@ -91,6 +96,20 @@ NDArray NDArray::At(index_t idx) const { } } + +bool NDArray::fresh_out_grad() const { + if (entry_.ag_node != nullptr) return entry_.ag_node->fresh_out_grad; + return false; +} + + +void NDArray::set_fresh_out_grad(bool state) const { + CHECK(entry_.ag_node != nullptr) + << "NDArray has not been marked as a variable and does not have gradient state"; + entry_.ag_node->fresh_out_grad = state; +} + + /*! * \brief run a ternary operation * \param lhs left operand diff --git a/src/operator/custom/custom.cc b/src/operator/custom/custom.cc index 29f624ead2ad..8fb324c1f5c2 100644 --- a/src/operator/custom/custom.cc +++ b/src/operator/custom/custom.cc @@ -193,10 +193,10 @@ The custom operator must be registered before it can be used. Please check the tutorial here: http://mxnet.io/how_to/new_op.html. )code") +.add_argument("data", "NDArray-or-Symbol[]", "Input data for the custom operator.") .add_argument("op_type", "string", "Name of the custom operator. " "This is the name that is passed to `mx.operator.register` " - "to register the operator.") -.add_argument("data", "NDArray-or-Symbol", "Input data for the custom operator."); + "to register the operator."); } // namespace op diff --git a/src/operator/custom/native_op.cc b/src/operator/custom/native_op.cc index 7ab0614a041c..2ccd286e8cd3 100644 --- a/src/operator/custom/native_op.cc +++ b/src/operator/custom/native_op.cc @@ -21,6 +21,7 @@ DMLC_REGISTER_PARAMETER(NativeOpParam); MXNET_REGISTER_OP_PROPERTY(_Native, NativeOpProp) .describe("Stub for implementing an operator implemented in native frontend language.") +.add_argument("data", "NDArray-or-Symbol[]", "Input data for the custom operator.") .add_arguments(NativeOpParam::__FIELDS__()); } // namespace op diff --git a/src/operator/custom/ndarray_op.cc b/src/operator/custom/ndarray_op.cc index 773fe7753930..9815f888a98b 100644 --- a/src/operator/custom/ndarray_op.cc +++ b/src/operator/custom/ndarray_op.cc @@ -126,6 +126,7 @@ DMLC_REGISTER_PARAMETER(NDArrayOpParam); MXNET_REGISTER_OP_PROPERTY(_NDArray, NDArrayOpProp) .describe("Stub for implementing an operator implemented in native frontend language with ndarray.") +.add_argument("data", "NDArray-or-Symbol[]", "Input data for the custom operator.") .add_arguments(NDArrayOpParam::__FIELDS__()); } // namespace op diff --git a/src/operator/tensor/broadcast_reduce_op.h b/src/operator/tensor/broadcast_reduce_op.h index 2288224a2197..13f112b6f59d 100644 --- a/src/operator/tensor/broadcast_reduce_op.h +++ b/src/operator/tensor/broadcast_reduce_op.h @@ -20,6 +20,7 @@ namespace op { struct ReduceAxesParam : public dmlc::Parameter { TShape axis; bool keepdims; + bool exclude; DMLC_DECLARE_PARAMETER(ReduceAxesParam) { DMLC_DECLARE_FIELD(axis).set_default(TShape()) .describe(R"code(The axis or axes along which to perform the reduction. @@ -30,10 +31,15 @@ struct ReduceAxesParam : public dmlc::Parameter { If `axis` is int, a reduction is performed on a particular axis. If `axis` is a tuple of ints, a reduction is performed on all the axes - specified in the tuple.)code"); + specified in the tuple. + + If `exclude` is true, reduction will be performed on the axes that are + NOT in axis instead.)code"); DMLC_DECLARE_FIELD(keepdims).set_default(false) .describe("If this is set to `True`, the reduced axes are left " "in the result as dimension with size one."); + DMLC_DECLARE_FIELD(exclude).set_default(false) + .describe("Whether to perform reduction on axis that are NOT in axis instead."); } }; @@ -150,42 +156,68 @@ inline bool ReduceAxisShape(const nnvm::NodeAttrs& attrs, return true; } -inline bool ReduceAxesShape(const nnvm::NodeAttrs& attrs, - std::vector *in_attrs, - std::vector *out_attrs) { - CHECK_EQ(in_attrs->size(), 1U); - CHECK_EQ(out_attrs->size(), 1U); - if ((*in_attrs)[0].ndim() == 0) return false; - const ReduceAxesParam& param = nnvm::get(attrs.parsed); - TShape &ishape = (*in_attrs)[0]; - TShape oshape; - if (param.axis.ndim() == 0) { - if (param.keepdims) { - oshape = TShape(ishape.ndim()); +inline TShape ReduceAxesShapeImpl(const TShape& ishape, const TShape& axis, + bool keepdims, bool exclude) { + if (axis.ndim() == 0) { + if (keepdims) { + return TShape(ishape.ndim()); } else { - oshape = TShape(1); + return TShape(1); } - } else { - if (param.keepdims) { - oshape = ishape; - for (index_t i = 0; i < param.axis.ndim(); ++i) { - oshape[param.axis[i]] = 1; - } - } else { - CHECK_LT(param.axis[param.axis.ndim()-1], ishape.ndim()) - << "Reduction axis " << param.axis[param.axis.ndim()-1] - << " Exceeds input dimensions " << ishape; - oshape = TShape(std::max(1, ishape.ndim() - param.axis.ndim())); - for (index_t i = 0, j = 0, k = 0; i < ishape.ndim(); ++i) { - if (j < param.axis.ndim() && i == param.axis[j]) { + } + + CHECK_LT(axis[axis.ndim()-1], ishape.ndim()) + << "Reduction axis " << axis[axis.ndim()-1] + << " Exceeds input dimensions " << ishape; + + if (keepdims) { + TShape oshape(ishape); + if (exclude) { + for (index_t i = 0, j = 0; i < ishape.ndim(); ++i) { + if (j < axis.ndim() && i == axis[j]) { ++j; continue; } - oshape[k++] = ishape[i]; + oshape[i] = 1; } + return oshape; + } + + for (index_t i = 0; i < axis.ndim(); ++i) { + oshape[axis[i]] = 1; } + return oshape; } - SHAPE_ASSIGN_CHECK(*out_attrs, 0, oshape); + + if (exclude) { + TShape oshape = TShape(axis.ndim()); + for (index_t i = 0; i < axis.ndim(); ++i) { + oshape[i] = ishape[axis[i]]; + } + return oshape; + } + + TShape oshape = TShape(std::max(1, ishape.ndim() - axis.ndim())); + for (index_t i = 0, j = 0, k = 0; i < ishape.ndim(); ++i) { + if (j < axis.ndim() && i == axis[j]) { + ++j; + continue; + } + oshape[k++] = ishape[i]; + } + return oshape; +} + +inline bool ReduceAxesShape(const nnvm::NodeAttrs& attrs, + std::vector *in_attrs, + std::vector *out_attrs) { + CHECK_EQ(in_attrs->size(), 1U); + CHECK_EQ(out_attrs->size(), 1U); + if ((*in_attrs)[0].ndim() == 0) return false; + const ReduceAxesParam& param = nnvm::get(attrs.parsed); + SHAPE_ASSIGN_CHECK(*out_attrs, 0, + ReduceAxesShapeImpl((*in_attrs)[0], param.axis, + param.keepdims, param.exclude)); return true; } @@ -332,20 +364,12 @@ void ReduceAxesCompute(const nnvm::NodeAttrs& attrs, const std::vector& inputs, const std::vector& req, const std::vector& outputs) { - // using namespace mshadow; - // using namespace mshadow::expr; const ReduceAxesParam& param = nnvm::get(attrs.parsed); TShape small; - if (!param.keepdims) { - if (param.axis.ndim() == 0) { - small = TShape(inputs[0].shape_.ndim()); - } else { - small = inputs[0].shape_; - for (index_t i = 0; i < param.axis.ndim(); ++i) - small[param.axis[i]] = 1; - } - } else { + if (param.keepdims) { small = outputs[0].shape_; + } else { + small = ReduceAxesShapeImpl(inputs[0].shape_, param.axis, true, param.exclude); } ReduceAxesComputeImpl(attrs, ctx, inputs, req, outputs, small); @@ -362,12 +386,10 @@ void ReduceAxesBackwardUseInOut(const nnvm::NodeAttrs& attrs, using namespace mshadow::expr; const ReduceAxesParam& param = nnvm::get(attrs.parsed); TShape small; - if (param.axis.ndim() == 0) { - small = TShape(outputs[0].shape_.ndim()); + if (param.keepdims) { + small = inputs[0].shape_; } else { - small = outputs[0].shape_; - for (index_t i = 0; i < param.axis.ndim(); ++i) - small[param.axis[i]] = 1; + small = ReduceAxesShapeImpl(outputs[0].shape_, param.axis, true, param.exclude); } TShape src_shape, dst_shape; @@ -452,13 +474,12 @@ inline void ReduceAxesBackwardUseNone(const nnvm::NodeAttrs& attrs, using namespace mshadow::expr; const ReduceAxesParam& param = nnvm::get(attrs.parsed); TShape small; - if (param.axis.ndim() == 0) { - small = TShape(outputs[0].shape_.ndim()); + if (param.keepdims) { + small = inputs[0].shape_; } else { - small = outputs[0].shape_; - for (index_t i = 0; i < param.axis.ndim(); ++i) - small[param.axis[i]] = 1; + small = ReduceAxesShapeImpl(outputs[0].shape_, param.axis, true, param.exclude); } + BroadcastComputeImpl(attrs, ctx, inputs, req, outputs, small); if (normalize) { Stream *s = ctx.get_stream(); diff --git a/src/operator/tensor/elemwise_unary_op.cc b/src/operator/tensor/elemwise_unary_op.cc index ce29a2fdb308..073bbe16d491 100644 --- a/src/operator/tensor/elemwise_unary_op.cc +++ b/src/operator/tensor/elemwise_unary_op.cc @@ -113,6 +113,10 @@ MXNET_OPERATOR_REGISTER_UNARY(make_loss) // identity output as first input, but attributes are constrainted to be like rhs NNVM_REGISTER_OP(_identity_with_attr_like_rhs) .set_num_inputs(2) +.set_attr("FListInputNames", + [](const NodeAttrs& attrs) { + return std::vector{"lhs", "rhs"}; + }) .set_attr( "FInplaceOption", [](const NodeAttrs& attrs) { return std::vector >{{0, 0}}; @@ -131,7 +135,9 @@ NNVM_REGISTER_OP(_identity_with_attr_like_rhs) {n->inputs[1]}, nullptr, &n); lhs.push_back(nnvm::NodeEntry{ng, 0, 0}); return lhs; - }); + }) +.add_argument("lhs", "NDArray-or-Symbol", "First input.") +.add_argument("rhs", "NDArray-or-Symbol", "Second input."); DMLC_REGISTER_PARAMETER(CastParam); NNVM_REGISTER_OP(Cast) diff --git a/tests/python/unittest/test_autograd.py b/tests/python/unittest/test_autograd.py index c84438d72363..24b417afc233 100644 --- a/tests/python/unittest/test_autograd.py +++ b/tests/python/unittest/test_autograd.py @@ -84,10 +84,84 @@ def test_training(): assert (y.asnumpy() == x.asnumpy()).all() +def test_out_grads(): + x = nd.ones((3, 5)) + dx = nd.zeros_like(x) + mark_variables([x], [dx]) + da = None + db = nd.array([1,2,3,4,5]) + dc = nd.array([5,4,3,2,1]) + + with train_section(): + a, b, c = nd.split(x, axis=0, num_outputs=3, squeeze_axis=True) + backward([a, b, c], [da, db, dc]) + + assert (dx.asnumpy() == np.array( + [[1,1,1,1,1], + [1,2,3,4,5], + [5,4,3,2,1]])).all() + + +def test_detach_updated_grad(): + x = nd.ones((2, 2)) + dx = nd.zeros_like(x) + y = nd.ones_like(x) + dy = nd.zeros_like(x) + mark_variables([x, y], [dx, dy]) + assert x._fresh_grad == False + assert y._fresh_grad == False + + with train_section(): + x2 = x + 2 + y2 = x2 + y + y2.backward() + assert (dx.asnumpy() == 1).all() + assert x._fresh_grad == True + assert y._fresh_grad == True + + dx[:] = 0 + x._fresh_grad = False + y._fresh_grad = False + assert x._fresh_grad == False + assert y._fresh_grad == False + with train_section(): + x2 = x + 2 + x2 = x2.detach() + y2 = x2 + y + y2.backward() + assert (dx.asnumpy() == 0).all() + assert y._fresh_grad == True + assert x._fresh_grad == False + + +def test_retain_grad(): + x = mx.nd.ones((2, 2)) + dx = mx.nd.zeros((2, 2)) + mark_variables([x], [dx], grad_reqs='add') + with train_section(): + y = x + 1 + y.backward(retain_graph=False) + assert (dx.asnumpy() == 1).all() + + dx[:] = 0 + with train_section(): + y = x + 1 + y.backward(retain_graph=True) + y.backward(retain_graph=False) + assert (dx.asnumpy() == 2).all() + + try: + with train_section(): + y = x + 1 + y.backward() + y.backward() + except Exception: + return + + raise AssertionError( + "differentiating the same graph twice without retain_graph should fail") + if __name__ == "__main__": - test_training() - test_unary_func() - test_binary_func() - test_operator_with_state() - test_argnum() + import nose + nose.runmodule() diff --git a/tests/python/unittest/test_init.py b/tests/python/unittest/test_init.py index 372ad3553c06..79862269795d 100644 --- a/tests/python/unittest/test_init.py +++ b/tests/python/unittest/test_init.py @@ -29,6 +29,6 @@ def test_aux_init(): if __name__ == '__main__': - test_default_init() test_variable_init() + test_default_init() test_aux_init() diff --git a/tests/python/unittest/test_ndarray.py b/tests/python/unittest/test_ndarray.py index fcc7d70f20fe..2be95a9766af 100644 --- a/tests/python/unittest/test_ndarray.py +++ b/tests/python/unittest/test_ndarray.py @@ -627,6 +627,17 @@ def test_iter(): assert same(y[i].asnumpy(), x[i].asnumpy()) +def test_cached(): + op = mx.nd.CachedOp('Convolution', 3, kernel=(3, 3), num_filter=10) + data = mx.nd.ones((3, 4, 10, 10)) + weight = mx.nd.ones((10, 4, 3, 3)) + bias = mx.nd.ones((10,)) + o1 = mx.nd.invoke(op, [data, weight, bias]) + bias[:] = 2 + o2 = mx.nd.invoke(op, [data, weight, bias]) + assert_almost_equal(o2.asnumpy(), o1.asnumpy()+1) + + if __name__ == '__main__': import nose nose.runmodule() diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py index 82c20cdb17df..f0c4ea6bb376 100644 --- a/tests/python/unittest/test_operator.py +++ b/tests/python/unittest/test_operator.py @@ -1217,7 +1217,7 @@ def test_reshape_new(src_shape, shape_args, reverse, dst_shape): assert_allclose(exe.grad_arrays[0].asnumpy(), out_grad_npy.reshape((5, 4, 3, 7))) def test_reduce(): - sample_num = 200 + sample_num = 500 def test_reduce_inner(numpy_reduce_func, numpy_reduce_grad_func, mx_reduce_sym, nan_prob = 0): for i in range(sample_num): # Generate random data that has ndim between 1-7 and all the shape dims between 1-5 @@ -1226,6 +1226,7 @@ def test_reduce_inner(numpy_reduce_func, numpy_reduce_grad_func, mx_reduce_sym, shape = np.random.randint(1, 6, size=(ndim,)) axis_num = np.random.randint(0, ndim, size=1) axis_flags = np.random.randint(0, 2, size=ndim) + exclude = np.random.randint(0, 2) axes = [] for (axis, flag) in enumerate(axis_flags): if flag: @@ -1240,6 +1241,9 @@ def test_reduce_inner(numpy_reduce_func, numpy_reduce_grad_func, mx_reduce_sym, a = mx.symbol.Variable('a') if axes is None: b = mx_reduce_sym(a, keepdims=keepdims) + elif exclude and isinstance(axes, tuple) and len(axes) < ndim: + naxes = [i for i in range(ndim) if i not in axes] + b = mx_reduce_sym(a, axis=naxes, keepdims=keepdims, exclude=True) else: b = mx_reduce_sym(a, axis=axes, keepdims=keepdims) dat_npy = np.random.rand(*shape) @@ -1267,6 +1271,7 @@ def test_reduce_inner(numpy_reduce_func, numpy_reduce_grad_func, mx_reduce_sym, bc_grad_groundtruth = np.broadcast_to(grad_groundtruth, grad_nd.shape) equal_backward = almost_equal_ignore_nan(grad_nd.asnumpy(), bc_grad_groundtruth, 1E-4, 1E-4) assert equal_backward + test_reduce_inner(lambda data, axis, keepdims:np_reduce(data, axis, keepdims, np.sum), lambda outgrad, data, outdata, axis, keepdims, keepdim_shape: outgrad.reshape(keepdim_shape), @@ -3012,7 +3017,7 @@ def test_pick_helper(index_type=np.int32): test_pick_helper(np.int32) test_pick_helper(np.float32) - + def check_ctc_loss(acts, labels, loss_truth): in_var = mx.sym.Variable('input') labels_var = mx.sym.Variable('labels') @@ -3053,7 +3058,7 @@ def test_ctc_loss(): true_loss = np.array([7.3557, 5.4091], dtype=np.float32) # from Torch check_ctc_loss(acts2, labels2, true_loss) - + def test_quantization_op(): min0 = mx.nd.array([0.0]) max0 = mx.nd.array([1.0]) @@ -3110,71 +3115,5 @@ def create_operator(self, ctx, shapes, dtypes): if __name__ == '__main__': - test_custom_op() - test_log_softmax() - test_new_softmax() - test_pick() - test_l2_normalization() - test_sequence_mask() - test_roipooling() - test_batchnorm_training() - test_order() - test_grid_generator() - test_dot() - test_cast() - test_clip() - test_index2d() - test_scalarop() - test_reduce() - test_init() - test_expand_dims() - test_slice_axis() - test_softmax() - test_broadcast_binary_op() - test_flip() - test_crop() - test_transpose() - test_convolution_grouping() - test_nearest_upsampling() - test_binary_op_duplicate_input() - test_elementwise_sum() - test_concat() - test_slice_channel() - test_regression() - test_python_op() - test_swapaxes() - test_scalar_pow() - test_symbol_pow() - test_pow_fn() - test_embedding() - test_rsqrt_cos_sin() - test_maximum_minimum() - test_maximum_minimum_scalar() - test_abs() - test_round_ceil_floor() - test_deconvolution() - check_softmax_with_ignore_label(default_context()) - test_convolution_dilated_impulse_response() - test_reshape() - test_broadcast() - test_stn() - test_batch_dot() - test_correlation() - test_support_vector_machine_l1_svm() - test_support_vector_machine_l2_svm() - test_pad() - test_instance_normalization() - test_mathematical() - test_special_functions_using_scipy() - test_blockgrad() - test_take() - test_bilinear_sampler() - test_binary_logic() - test_repeat() - test_tile() - test_one_hot() - test_where() - test_ctc_loss() - test_quantization_op() - test_relu() - test_sigmoid() + import nose + nose.runmodule() diff --git a/tests/python/unittest/test_symbol.py b/tests/python/unittest/test_symbol.py index ab25f48eeb52..28fc8a4fc77b 100644 --- a/tests/python/unittest/test_symbol.py +++ b/tests/python/unittest/test_symbol.py @@ -224,17 +224,20 @@ def test_zero_prop2(): assert False + +def test_cached(): + op = mx.sym.CachedOp('Convolution', 3, kernel=(3, 3), num_filter=10) + data = mx.sym.var('data') + weight = mx.sym.var('weight') + bias = mx.sym.var('bias') + out = mx.sym.invoke(op, [data, weight, bias], 'conv') + assert out.list_arguments() == ['data', 'weight', 'bias'] + assert out.list_outputs() == ['conv_output'] + with mx.name.Prefix('test_'): + assert mx.sym.invoke(op, [data, weight, bias]).name == 'test_convolution0' + assert mx.sym.invoke(op, [data, weight, bias]).name == 'test_convolution1' + + if __name__ == '__main__': - test_zero_prop2() - test_zero_prop() - test_blockgrad() - test_symbol_children() - test_load_000800() - test_symbol_infer_shape_var() - test_symbol_infer_shape() - test_symbol_infer_type() - test_symbol_internal() - test_symbol_basic() - test_symbol_compose() - test_symbol_saveload() - test_symbol_pickle() + import nose + nose.runmodule() From 2a09f161cf99e3f3130742e1bf744f126cd18c68 Mon Sep 17 00:00:00 2001 From: "Qiang Kou (KK)" Date: Tue, 30 May 2017 21:32:16 -0700 Subject: [PATCH 009/834] Revert "[R][DOC] fix R tutorials (#6472)" (#6499) This reverts commit 215ae4a0dd1a96ce75e1c451809dbfade286cfcd. --- docs/tutorials/index.md | 17 ----- docs/tutorials/r/CallbackFunctionTutorial.md | 20 +++-- ...tsDogsFinetune.md => CatsDogsFinetune.rmd} | 74 ++++++++++--------- docs/tutorials/r/CustomIteratorTutorial.md | 33 ++++----- docs/tutorials/r/CustomLossFunction.md | 62 ++++++++++++++++ docs/tutorials/r/charRnnModel.md | 5 +- .../r/classifyRealImageWithPretrainedModel.md | 22 +++--- docs/tutorials/r/fiveMinutesNeuralNetwork.md | 72 ++++++++++++++++-- docs/tutorials/r/mnistCompetition.md | 62 +++++++++++----- docs/tutorials/r/ndarray.md | 10 ++- docs/tutorials/r/symbol.md | 10 ++- 11 files changed, 264 insertions(+), 123 deletions(-) rename docs/tutorials/r/{CatsDogsFinetune.md => CatsDogsFinetune.rmd} (88%) create mode 100644 docs/tutorials/r/CustomLossFunction.md diff --git a/docs/tutorials/index.md b/docs/tutorials/index.md index ce00b74ed1bd..aed11a4bebf1 100644 --- a/docs/tutorials/index.md +++ b/docs/tutorials/index.md @@ -27,23 +27,6 @@ These tutorials introduce a few fundamental concepts in deep learning and how to python/predict_image vision/large_scale_classification ``` - -## R - -```eval_rst -.. toctree:: - :maxdepth: 1 - - r/ndarray - r/symbol - r/fiveMinutesNeuralNetwork - r/classifyRealImageWithPretrainedModel - r/CatsDogsFinetune - r/mnistCompetition - r/charRnnModel - r/CallbackFunctionTutorial - r/CustomIteratorTutorial -```
More tutorials and examples are available in the GitHub [repository](https://github.com/dmlc/mxnet/tree/master/example). diff --git a/docs/tutorials/r/CallbackFunctionTutorial.md b/docs/tutorials/r/CallbackFunctionTutorial.md index 3290831b46c2..103352dd2907 100644 --- a/docs/tutorials/r/CallbackFunctionTutorial.md +++ b/docs/tutorials/r/CallbackFunctionTutorial.md @@ -1,11 +1,11 @@ -Callback function Tutorial +Callback Function ====================================== This tutorial provides guidelines for using and writing callback functions, which can very useful in model training. -## Model Training Example - +Model Training Example +---------- Let's begin with a small example. We can build and train a model with the following code: @@ -56,8 +56,8 @@ Let's begin with a small example. We can build and train a model with the follow We also provide two optional parameters, `batch.end.callback` and `epoch.end.callback`, which can provide great flexibility in model training. -## How to Use Callback Functions - +How to Use Callback Functions +--------- This package provides two callback functions: @@ -148,8 +148,8 @@ You also can save the training and evaluation errors for later use by passing a ## [1] 12.715069 14.810532 15.840361 10.898733 9.349706 9.363087 ``` -## How to Write Your Own Callback Functions - +How to Write Your Own Callback Functions +---------- You can find the source code for the two callback functions on [GitHub](https://github.com/dmlc/mxnet/blob/master/R-package/R/callback.R) and use it as a template: @@ -247,4 +247,8 @@ Yes! You can stop the training early with `return(FALSE)`. See the following exa When the validation metric dips below the threshold we set, the training process stops. - +## Next Steps +* [Neural Networks with MXNet in Five Minutes](http://mxnet.io/tutorials/r/fiveMinutesNeuralNetwork.html) +* [Classify Real-World Images with a Pretrained Model](http://mxnet.io/tutorials/r/classifyRealImageWithPretrainedModel.html) +* [Handwritten Digits Classification Competition](http://mxnet.io/tutorials/r/mnistCompetition.html) +* [Character Language Model Using RNN](http://mxnet.io/tutorials/r/charRnnModel.html) diff --git a/docs/tutorials/r/CatsDogsFinetune.md b/docs/tutorials/r/CatsDogsFinetune.rmd similarity index 88% rename from docs/tutorials/r/CatsDogsFinetune.md rename to docs/tutorials/r/CatsDogsFinetune.rmd index 18cd3bb8164a..a99e7042804e 100644 --- a/docs/tutorials/r/CatsDogsFinetune.md +++ b/docs/tutorials/r/CatsDogsFinetune.rmd @@ -1,27 +1,31 @@ -Dogs vs. Cats classification with MXNet and R -====================================== +--- +title: "Dogs vs. Cats classification with mxnet and R" +author: "Andrey Ogurtsov (https://github.com/statist-bhfz/)" +date: "February 25, 2017" +--- -## Packages and prerequisites +## 1. Packages and prerequisites -Ubuntu 16.04, **mxnet** (compiled with GPU support), **imager** for image processind, -**abind** for manipulations with arrays. It is almost end-to-end R solution for Kaggle -competition , -we will use Python only for creating .rec-files. +Ubuntu 16, **mxnet** 0.9.4 (compiled with GPU support), **imager** for image processind, **abind** for manipulations with arrays. It is almost end-to-end R solution for Kaggle competition https://www.kaggle.com/c/dogs-vs-cats-redux-kernels-edition/, we will use Python only for creating .rec-files. -Thanks to [jeremiedb](https://github.com/jeremiedb), my code for fine-tuning is -largely based on his [answers](https://github.com/dmlc/mxnet/issues/4817). +Thanks to [jeremiedb](https://github.com/jeremiedb), my code for fine-tuning is largely based on his [answers](https://github.com/dmlc/mxnet/issues/4817). -```r +```{r} +knitr::opts_chunk$set(eval = FALSE) +``` + +```{r} library(imager) library(mxnet) library(abind) ``` -## Image processing -### Renaming train files +## 2. Image processing + +### 2.1. Renaming train files -```r +```{r} files <- list.files("train") old_names <- sapply(files, strsplit, split = ".", fixed = TRUE) max_length <- max(sapply(old_names, function(x) nchar(x[[2]]))) @@ -46,9 +50,9 @@ dir.create("./train/dog") Map(function(x, y) file.rename(from = x, to = y), files, new_names) ``` -### Train images: 224x224, padded with empty space +### 2.2. Train images: 224x224, padded with empty space -```r +```{r} files <- list.files("train", recursive = TRUE) new_names <- paste0("train_pad_224x224/", files) files <- paste0("./train/", files) @@ -72,9 +76,9 @@ Map(function(x, y) { }, x = files, y = new_names) ``` -### Renaming test files +### 2.3. Renaming test files -```r +```{r} files <- list.files("test") max_length <- max(sapply(files, nchar)) zeros <- max_length - sapply(files, nchar) @@ -87,9 +91,9 @@ Map(function(x, y) file.rename(from = x, to = y), files, newnames) ``` -### Test images: 224x224, padded with empty space +### 2.4. Test images: 224x224, padded with empty space -```r +```{r} files <- list.files("test") new_names <- paste0("test_pad_224x224/", files) files <- paste0("./test/", files) @@ -102,18 +106,18 @@ Map(function(x, y) { }, x = files, y = new_names) ``` -### Creating .rec files +### 2.5. Creating .rec files -``` +```{bash, eval = FALSE} python ~/mxnet/tools/im2rec.py --list=1 --recursive=1 --train-ratio=0.8 cats_dogs train_pad_224x224 python ~/mxnet/tools/im2rec.py --num-thread=4 --pass-through=1 cats_dogs_train.lst train_pad_224x224 python ~/mxnet/tools/im2rec.py --num-thread=4 --pass-through=1 cats_dogs_val.lst train_pad_224x224 ``` -## Iterators +## 3. Iterators -```r +```{r} get_iterator <- function(data_shape, train_data, val_data, @@ -138,7 +142,7 @@ get_iterator <- function(data_shape, ``` -```r +```{r} data <- get_iterator(data_shape = c(224, 224, 3), train_data = "/media/andrey/Data/KAGGLE/cats_dogs/cats_dogs_train.rec", val_data = "/media/andrey/Data/KAGGLE/cats_dogs/cats_dogs_val.rec", @@ -148,13 +152,13 @@ val <- data$val ``` -## Load pretrained model +## 4. Load pretrained model Model from http://data.dmlc.ml/models/imagenet/ Last fully connected layes for 1000 classes replaced with new layer for 2 classes. -```r +```{r} inception_bn <- mx.model.load("models/inception_bn/Inception-BN", iteration = 126) @@ -189,9 +193,9 @@ arg_params_new[["fc1_bias"]] <- fc1_bias_new ``` -## Train (fine-tune) model +## 5. Train (fine-tune) model -```r +```{r} model <- mx.model.FeedForward.create( symbol = new_soft, X = train, @@ -213,13 +217,13 @@ model <- mx.model.FeedForward.create( ) ``` -```r +```{r} model <- mx.model.load("inception_bn", 1) ``` Continue training with decreased speed (`learning.rate = 0.03`): -```r +```{r} model <- mx.model.FeedForward.create( symbol = model$symbol, X = train, @@ -241,16 +245,16 @@ model <- mx.model.FeedForward.create( ) ``` -```r +```{r} model <- mx.model.load("inception_bn", 1) ``` My R session crashed after each iteration, so I made some iterations manually. -## Make predictions +## 6. Make predictions -```r +```{r} preprocImage<- function(src, # URL or file location height = 224, width = 224, @@ -275,7 +279,7 @@ preprocImage<- function(src, # URL or file location } ``` -```r +```{r} files <- list.files("test_pad_224x224/") files <- paste0("./test_pad_224x224/", files) @@ -299,5 +303,3 @@ probs <- t(do.call(cbind, probs)) preds <- data.frame(id = 1:12500, label = probs[, 2]) write.csv(preds, "subm.csv", row.names = FALSE, quote = FALSE) ``` - - diff --git a/docs/tutorials/r/CustomIteratorTutorial.md b/docs/tutorials/r/CustomIteratorTutorial.md index b35b8926f359..1ad634bcd669 100644 --- a/docs/tutorials/r/CustomIteratorTutorial.md +++ b/docs/tutorials/r/CustomIteratorTutorial.md @@ -1,26 +1,23 @@ Custom Iterator Tutorial ====================================== -This tutorial provides a guideline on how to use and write custom iterators, -which can very useful when having a dataset that does not fit into memory. +This tutorial provides a guideline on how to use and write custom iterators, which can very useful when having a dataset that does not fit into memory. -## Getting the data - - -The data we are going to use is the [MNIST dataset](http://yann.lecun.com/exdb/mnist/) in -CSV format, the data can be found in this [web](http://pjreddie.com/projects/mnist-in-csv/). +Getting the data +---------- +The data we are going to use is the [MNIST dataset](http://yann.lecun.com/exdb/mnist/) in CSV format, the data can be found in this [web](http://pjreddie.com/projects/mnist-in-csv/). To download the data: -``` +```bash wget http://pjreddie.com/media/files/mnist_train.csv wget http://pjreddie.com/media/files/mnist_test.csv ``` You'll get two files, `mnist_train.csv` that contains 60.000 examples of hand written numbers and `mxnist_test.csv` that contains 10.000 examples. The first element of each line in the CSV is the label, which is a number between 0 and 9. The rest of the line are 784 numbers between 0 and 255, corresponding to the levels of grey of a matrix of 28x28. Therefore, each line contains an image of 28x28 pixels of a hand written number and its true label. -## Custom CSV Iterator - +Custom CSV Iterator +---------- Next we are going to create a custom CSV Iterator based on the [C++ CSVIterator class](https://github.com/dmlc/mxnet/blob/master/src/io/iter_csv.cc). For that we are going to use the R function `mx.io.CSVIter` as a base class. This class has as parameters `data.csv, data.shape, batch.size` and two main functions, `iter.next()` that calls the iterator in the next batch of data and `value()` that returns the train data and the label. @@ -135,8 +132,8 @@ batch.size <- 100 train.iter <- CustomCSVIter$new(iter = NULL, data.csv = "mnist_train.csv", data.shape = 28, batch.size = batch.size) ``` -## CNN Model - +CNN Model +---------- For this tutorial we are going to use the known LeNet architecture: @@ -159,8 +156,8 @@ lenet.model <- function(){ network <- lenet.model() ``` -## Training with the Custom Iterator - +Training with the Custom Iterator +---------- Finally, we can directly add the custom iterator as the training data source. ```r @@ -179,7 +176,7 @@ model <- mx.model.FeedForward.create(symbol=network, The last 2 iterations with a K80 GPU looks like this: -``` +```bash [8] Train-accuracy=0.998866666666667 Batch [100] Speed: 15413.0104454713 samples/sec Train-accuracy=0.999 Batch [200] Speed: 16629.3412459049 samples/sec Train-accuracy=0.99935 @@ -197,11 +194,9 @@ Batch [600] Speed: 13818.7899518255 samples/sec Train-accuracy=0.99975 [10] Train-accuracy=0.99975 ``` -## Conclusion - +Conclusion +---------- We have shown how to create a custom CSV Iterator by extending the class `mx.io.CSVIter`. In our class, we iteratively read from a CSV file a batch of data that will be transformed and then processed in the stochastic gradient descent optimization. That way, we are able to manage CSV files that are bigger than the memory of the machine we are using. Based of this custom iterator, we can also create data loaders that internally transform or expand the data, allowing to manage files of any size. - - diff --git a/docs/tutorials/r/CustomLossFunction.md b/docs/tutorials/r/CustomLossFunction.md new file mode 100644 index 000000000000..a7104803cacb --- /dev/null +++ b/docs/tutorials/r/CustomLossFunction.md @@ -0,0 +1,62 @@ +Customized loss function +====================================== + +This tutorial provides guidelines for using customized loss function in network construction. + + +Model Training Example +---------- + +Let's begin with a small regression example. We can build and train a regression model with the following code: + + + ```r + library(mxnet) + data(BostonHousing, package="mlbench") + train.ind = seq(1, 506, 3) + train.x = data.matrix(BostonHousing[train.ind, -14]) + train.y = BostonHousing[train.ind, 14] + test.x = data.matrix(BostonHousing[-train.ind, -14]) + test.y = BostonHousing[-train.ind, 14] + data <- mx.symbol.Variable("data") + fc1 <- mx.symbol.FullyConnected(data, num_hidden=1) + lro <- mx.symbol.LinearRegressionOutput(fc1) + mx.set.seed(0) + model <- mx.model.FeedForward.create( + lro, X=train.x, y=train.y, + eval.data=list(data=test.x, label=test.y), + ctx=mx.cpu(), num.round=10, array.batch.size=20, + learning.rate=2e-6, momentum=0.9, eval.metric=mx.metric.rmse) + ``` + +Besides the `LinearRegressionOutput`, we also provide `LogisticRegressionOutput` and `MAERegressionOutput`. +However, this might not be enough for real-world models. You can provide your own loss function +by using `mx.symbol.MakeLoss` when constructing the network. + + +How to Use Your Own Loss Function +--------- + +We still use our previous example. + + ```r + library(mxnet) + data <- mx.symbol.Variable("data") + fc1 <- mx.symbol.FullyConnected(data, num_hidden=1) + lro <- mx.symbol.MakeLoss(mx.symbol.square(mx.symbol.Reshape(fc1, shape = 0) - label)) + ``` + +In the last line of network definition, we do not use the predefined loss function. We define the loss +by ourselves, which is `(pred-label)^2`. + +We have provided many operations on the symbols, so you can also define `|pred-label|` using the line below. + + ```r + lro <- mx.symbol.MakeLoss(mx.symbol.abs(mx.symbol.Reshape(fc1, shape = 0) - label)) + ``` + +## Next Steps +* [Neural Networks with MXNet in Five Minutes](http://mxnet.io/tutorials/r/fiveMinutesNeuralNetwork.html) +* [Classify Real-World Images with a PreTrained Model](http://mxnet.io/tutorials/r/classifyRealImageWithPretrainedModel.html) +* [Handwritten Digits Classification Competition](http://mxnet.io/tutorials/r/mnistCompetition.html) +* [Character Language Model Using RNN](http://mxnet.io/tutorials/r/charRnnModel.html) diff --git a/docs/tutorials/r/charRnnModel.md b/docs/tutorials/r/charRnnModel.md index e4d6eb354b96..82e10a11f8d5 100644 --- a/docs/tutorials/r/charRnnModel.md +++ b/docs/tutorials/r/charRnnModel.md @@ -1,4 +1,4 @@ -Character language model using RNN +Char RNN Example ============================================= This tutorial shows how to use an LSTM model to build a char-level language model, and generate text from it. For demonstration purposes, we use a Shakespearean text. You can find the data on [GitHub](https://github.com/dmlc/web-data/tree/master/mxnet/tinyshakespeare). @@ -310,4 +310,5 @@ In `mxnet`, other RNN models, like custom RNN and GRU, are also provided: - For a custom RNN model, you can replace `mx.lstm` with `mx.rnn` to train an RNN model. You can replace `mx.lstm.inference` and `mx.lstm.forward` with `mx.rnn.inference` and `mx.rnn.forward` to build inference from an RNN model and get the forward result from the inference model. - For a GRU model, you can replace `mx.lstm` with `mx.gru` to train a GRU model. You can replace `mx.lstm.inference` and `mx.lstm.forward` with `mx.gru.inference` and `mx.gru.forward` to build inference from a GRU model and get the forward result from the inference model. - +## Next Steps +* [MXNet tutorials index](http://mxnet.io/tutorials/index.html) diff --git a/docs/tutorials/r/classifyRealImageWithPretrainedModel.md b/docs/tutorials/r/classifyRealImageWithPretrainedModel.md index 1272d611fb4a..4276fdeef282 100644 --- a/docs/tutorials/r/classifyRealImageWithPretrainedModel.md +++ b/docs/tutorials/r/classifyRealImageWithPretrainedModel.md @@ -9,8 +9,8 @@ image. For information about the network architecture, see [1]. The pre-trained Inception-BatchNorm network is able to be downloaded from [this link](http://data.mxnet.io/mxnet/data/Inception.zip) This model gives the recent state-of-art prediction accuracy on image net dataset. -## Load the MXNet Package - +Load the MXNet Package +--------------- To get started, load the mxnet package: ```r @@ -60,8 +60,8 @@ Now load the imager package to load and preprocess the images in R: ## save.image ``` -## Load the PreTrained Model - +Load the PreTrained Model +------------------------- Make sure you unzip the pre-trained model in the current folder. Use the model loading function to load the model into R: @@ -76,8 +76,8 @@ Load in the mean image, which is used for preprocessing using: mean.img = as.array(mx.nd.load("Inception/mean_224.nd")[["mean_img"]]) ``` -## Load and Preprocess the Image - +Load and Preprocess the Image +----------------------------- Now, we are ready to classify a real image. In this example, we simply take the parrots image from the imager package. You can use another image, if you prefer. @@ -89,7 +89,7 @@ Load and plot the image: plot(im) ``` -![](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/knitr/classifyRealImageWithPretrainedModel-unnamed-chunk-5-1.png) +![plot of chunk unnamed-chunk-5](../../web-data/mxnet/knitr/classifyRealImageWithPretrainedModel-unnamed-chunk-5-1.png) Before feeding the image to the deep network, we need to perform some preprocessing to make the image meet the deep network input requirements. Preprocessing @@ -125,8 +125,8 @@ Use the defined preprocessing function to get the normalized image: normed <- preproc.image(im, mean.img) ``` -## Classify the Image - +Classify the Image +------------------ Now we are ready to classify the image! Use the ```predict``` function to get the probability over classes: @@ -179,4 +179,6 @@ Reference --------- [1] Ioffe, Sergey, and Christian Szegedy. "Batch normalization: Accelerating deep network training by reducing internal covariate shift." arXiv preprint arXiv:1502.03167 (2015). - +## Next Steps +* [Handwritten Digits Classification Competition](http://mxnet.io/tutorials/r/mnistCompetition.html) +* [Character Language Model using RNN](http://mxnet.io/tutorials/r/charRnnModel.html) diff --git a/docs/tutorials/r/fiveMinutesNeuralNetwork.md b/docs/tutorials/r/fiveMinutesNeuralNetwork.md index d328d95e0f37..9104e8f05c2f 100644 --- a/docs/tutorials/r/fiveMinutesNeuralNetwork.md +++ b/docs/tutorials/r/fiveMinutesNeuralNetwork.md @@ -1,4 +1,4 @@ -Neural networks with MXNet in five minutes +Develop a Neural Network with MXNet in Five Minutes ============================================= This tutorial is designed for new users of the `mxnet` package for R. It shows how to construct a neural network to do regression in 5 minutes. It shows how to perform classification and regression tasks, respectively. The data we use is in the `mlbench` package. @@ -169,9 +169,35 @@ Next, make prediction with this structure and other parameters with `mx.model.Fe ## [8] Train-rmse=9.10463850277417 ## [9] Train-rmse=9.03977049028532 ## [10] Train-rmse=8.96870685004475 - - ............ - + ## [11] Train-rmse=8.93113287361574 + ## [12] Train-rmse=8.89937257821847 + ## [13] Train-rmse=8.87182096922953 + ## [14] Train-rmse=8.84476075083586 + ## [15] Train-rmse=8.81464673014974 + ## [16] Train-rmse=8.78672567900196 + ## [17] Train-rmse=8.76265872846474 + ## [18] Train-rmse=8.73946101419974 + ## [19] Train-rmse=8.71651926303267 + ## [20] Train-rmse=8.69457600919277 + ## [21] Train-rmse=8.67354928674563 + ## [22] Train-rmse=8.65328755392436 + ## [23] Train-rmse=8.63378039680078 + ## [24] Train-rmse=8.61488162586984 + ## [25] Train-rmse=8.5965105183022 + ## [26] Train-rmse=8.57868133563275 + ## [27] Train-rmse=8.56135851937663 + ## [28] Train-rmse=8.5444819772098 + ## [29] Train-rmse=8.52802114610432 + ## [30] Train-rmse=8.5119504512622 + ## [31] Train-rmse=8.49624261719241 + ## [32] Train-rmse=8.48087453238701 + ## [33] Train-rmse=8.46582689119887 + ## [34] Train-rmse=8.45107881002491 + ## [35] Train-rmse=8.43661331401712 + ## [36] Train-rmse=8.42241575909639 + ## [37] Train-rmse=8.40847217331365 + ## [38] Train-rmse=8.39476931796395 + ## [39] Train-rmse=8.38129658373974 ## [40] Train-rmse=8.36804269059018 ## [41] Train-rmse=8.35499817678397 ## [42] Train-rmse=8.34215505742154 @@ -237,9 +263,35 @@ This is an example of the mean absolute error metric. Simply plug it into the tr ## [8] Train-mae=7.02742733055105 ## [9] Train-mae=7.00618194618469 ## [10] Train-mae=6.92541576984028 - - ............ - + ## [11] Train-mae=6.87530243690643 + ## [12] Train-mae=6.84757369098564 + ## [13] Train-mae=6.82966501611388 + ## [14] Train-mae=6.81151759574811 + ## [15] Train-mae=6.78394182841811 + ## [16] Train-mae=6.75914719419347 + ## [17] Train-mae=6.74180388773481 + ## [18] Train-mae=6.725853071279 + ## [19] Train-mae=6.70932178215848 + ## [20] Train-mae=6.6928868798746 + ## [21] Train-mae=6.6769521329138 + ## [22] Train-mae=6.66184809505939 + ## [23] Train-mae=6.64754504809777 + ## [24] Train-mae=6.63358514060577 + ## [25] Train-mae=6.62027640889088 + ## [26] Train-mae=6.60738245232238 + ## [27] Train-mae=6.59505546771818 + ## [28] Train-mae=6.58346195800437 + ## [29] Train-mae=6.57285477783945 + ## [30] Train-mae=6.56259003960424 + ## [31] Train-mae=6.5527790788975 + ## [32] Train-mae=6.54353428422991 + ## [33] Train-mae=6.5344172368447 + ## [34] Train-mae=6.52557652526432 + ## [35] Train-mae=6.51697905850079 + ## [36] Train-mae=6.50847898812758 + ## [37] Train-mae=6.50014844106303 + ## [38] Train-mae=6.49207674844397 + ## [39] Train-mae=6.48412070125341 ## [40] Train-mae=6.47650500999557 ## [41] Train-mae=6.46893867486053 ## [42] Train-mae=6.46142131653097 @@ -255,4 +307,8 @@ This is an example of the mean absolute error metric. Simply plug it into the tr Congratulations! You've learned the basics for using MXNet in R. To learn how to use MXNet's advanced features, see the other tutorials. - + +## Next Steps +* [Classify Real-World Images with Pre-trained Model](http://mxnet.io/tutorials/r/classifyRealImageWithPretrainedModel.html) +* [Handwritten Digits Classification Competition](http://mxnet.io/tutorials/r/mnistCompetition.html) +* [Character Language Model using RNN](http://mxnet.io/tutorials/r/charRnnModel.html) diff --git a/docs/tutorials/r/mnistCompetition.md b/docs/tutorials/r/mnistCompetition.md index cfac8316ac82..04e68cf41ce2 100644 --- a/docs/tutorials/r/mnistCompetition.md +++ b/docs/tutorials/r/mnistCompetition.md @@ -1,4 +1,4 @@ -Handwritten digits classification competition +Handwritten Digits Classification Competition ============================================= [MNIST](http://yann.lecun.com/exdb/mnist/) is a handwritten digits image data set created by Yann LeCun. Every digit is represented by a 28 x 28 pixel image. It's become a standard data set for testing classifiers on simple image input. A neural network is a strong model for image classification tasks. There's a [long-term hosted competition](https://www.kaggle.com/c/digit-recognizer) on Kaggle using this data set. @@ -93,7 +93,7 @@ We assign CPU to `mxnet`. Now, you can run the following command to train the ne ctx=devices, num.round=10, array.batch.size=100, learning.rate=0.07, momentum=0.9, eval.metric=mx.metric.accuracy, initializer=mx.init.uniform(0.07), - epoch.end.callback=mx.callback.log.train.metric(100)) + epoch.end.callback=mx.callback.log.train.metric(100)) ``` ``` @@ -108,9 +108,35 @@ We assign CPU to `mxnet`. Now, you can run the following command to train the ne ## Batch [300] Train-accuracy=0.955866666666666 ## Batch [400] Train-accuracy=0.957525000000001 ## [2] Train-accuracy=0.958309523809525 - - ............ - + ## Batch [100] Train-accuracy=0.968 + ## Batch [200] Train-accuracy=0.9677 + ## Batch [300] Train-accuracy=0.9696 + ## Batch [400] Train-accuracy=0.970650000000002 + ## [3] Train-accuracy=0.970809523809526 + ## Batch [100] Train-accuracy=0.973 + ## Batch [200] Train-accuracy=0.974249999999999 + ## Batch [300] Train-accuracy=0.976 + ## Batch [400] Train-accuracy=0.977100000000003 + ## [4] Train-accuracy=0.977452380952384 + ## Batch [100] Train-accuracy=0.9834 + ## Batch [200] Train-accuracy=0.981949999999999 + ## Batch [300] Train-accuracy=0.981900000000001 + ## Batch [400] Train-accuracy=0.982600000000003 + ## [5] Train-accuracy=0.983000000000003 + ## Batch [100] Train-accuracy=0.983399999999999 + ## Batch [200] Train-accuracy=0.98405 + ## Batch [300] Train-accuracy=0.985000000000001 + ## Batch [400] Train-accuracy=0.985725000000003 + ## [6] Train-accuracy=0.985952380952384 + ## Batch [100] Train-accuracy=0.988999999999999 + ## Batch [200] Train-accuracy=0.9876 + ## Batch [300] Train-accuracy=0.988100000000001 + ## Batch [400] Train-accuracy=0.988750000000003 + ## [7] Train-accuracy=0.988880952380955 + ## Batch [100] Train-accuracy=0.991999999999999 + ## Batch [200] Train-accuracy=0.9912 + ## Batch [300] Train-accuracy=0.990066666666668 + ## Batch [400] Train-accuracy=0.990275000000003 ## [8] Train-accuracy=0.990452380952384 ## Batch [100] Train-accuracy=0.9937 ## Batch [200] Train-accuracy=0.99235 @@ -174,12 +200,12 @@ data <- mx.symbol.Variable('data') conv1 <- mx.symbol.Convolution(data=data, kernel=c(5,5), num_filter=20) tanh1 <- mx.symbol.Activation(data=conv1, act_type="tanh") pool1 <- mx.symbol.Pooling(data=tanh1, pool_type="max", - kernel=c(2,2), stride=c(2,2)) + kernel=c(2,2), stride=c(2,2)) # second conv conv2 <- mx.symbol.Convolution(data=pool1, kernel=c(5,5), num_filter=50) tanh2 <- mx.symbol.Activation(data=conv2, act_type="tanh") pool2 <- mx.symbol.Pooling(data=tanh2, pool_type="max", - kernel=c(2,2), stride=c(2,2)) + kernel=c(2,2), stride=c(2,2)) # first fullc flatten <- mx.symbol.Flatten(data=pool2) fc1 <- mx.symbol.FullyConnected(data=flatten, num_hidden=500) @@ -221,10 +247,10 @@ Start by training on the CPU first. Because this takes a bit time, we run it for mx.set.seed(0) tic <- proc.time() model <- mx.model.FeedForward.create(lenet, X=train.array, y=train.y, - ctx=device.cpu, num.round=1, array.batch.size=100, - learning.rate=0.05, momentum=0.9, wd=0.00001, - eval.metric=mx.metric.accuracy, - epoch.end.callback=mx.callback.log.train.metric(100)) + ctx=device.cpu, num.round=1, array.batch.size=100, + learning.rate=0.05, momentum=0.9, wd=0.00001, + eval.metric=mx.metric.accuracy, + epoch.end.callback=mx.callback.log.train.metric(100)) ``` ``` @@ -252,10 +278,10 @@ Train on a GPU: mx.set.seed(0) tic <- proc.time() model <- mx.model.FeedForward.create(lenet, X=train.array, y=train.y, - ctx=device.gpu, num.round=5, array.batch.size=100, - learning.rate=0.05, momentum=0.9, wd=0.00001, - eval.metric=mx.metric.accuracy, - epoch.end.callback=mx.callback.log.train.metric(100)) + ctx=device.gpu, num.round=5, array.batch.size=100, + learning.rate=0.05, momentum=0.9, wd=0.00001, + eval.metric=mx.metric.accuracy, + epoch.end.callback=mx.callback.log.train.metric(100)) ``` ``` @@ -307,7 +333,7 @@ Now, we can submit the result to Kaggle to see the improvement of our ranking! write.csv(submission, file='submission.csv', row.names=FALSE, quote=FALSE) ``` -![](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/knitr/mnistCompetition-kaggle-submission.png) - +![](../../web-data/mxnet/knitr/mnistCompetition-kaggle-submission.png) - +## Next Steps +* [Character Language Model using RNN](http://mxnet.io/tutorials/r/charRnnModel.html) diff --git a/docs/tutorials/r/ndarray.md b/docs/tutorials/r/ndarray.md index 4d5379ffa3b3..e00f9470688b 100644 --- a/docs/tutorials/r/ndarray.md +++ b/docs/tutorials/r/ndarray.md @@ -1,4 +1,4 @@ -# NDArray - Imperative tensor operations on CPU/GPU +# NDArray: Vectorized Tensor Computations on CPUs and GPUs `NDArray` is the basic vectorized operation unit in MXNet for matrix and tensor computations. Users can perform usual calculations as on an R"s array, but with two additional features: @@ -197,4 +197,10 @@ parallel. The actual computations are finished, allowing us to copy the results someplace else, such as `as.array(a)` or `mx.nd.save(a, "temp.dat")`. To write highly parallelized codes, we only need to postpone when we need the results. - +## Next Steps +* [Symbol](http://mxnet.io/tutorials/r/symbol.html) +* [Write and use callback functions](http://mxnet.io/tutorials/r/CallbackFunctionTutorial.html) +* [Neural Networks with MXNet in Five Minutes](http://mxnet.io/tutorials/r/fiveMinutesNeuralNetwork.html) +* [Classify Real-World Images with Pre-trained Model](http://mxnet.io/tutorials/r/classifyRealImageWithPretrainedModel.html) +* [Handwritten Digits Classification Competition](http://mxnet.io/tutorials/r/mnistCompetition.html) +* [Character Language Model using RNN](http://mxnet.io/tutorials/r/charRnnModel.html) \ No newline at end of file diff --git a/docs/tutorials/r/symbol.md b/docs/tutorials/r/symbol.md index 1a8f46b9f70f..8333265e1273 100644 --- a/docs/tutorials/r/symbol.md +++ b/docs/tutorials/r/symbol.md @@ -1,5 +1,4 @@ -Symbol - Neural network graphs and auto-differentiation -====================================== +# Symbol and Automatic Differentiation The computational unit `NDArray` requires a way to construct neural networks. MXNet provides a symbolic interface, named Symbol, to do this. Symbol combines both flexibility and efficiency. @@ -123,4 +122,9 @@ composition. Because MXNet does more in-place memory allocation, it can be more memory efficient than CXXNet and gets to the same runtime with greater flexibility. - +## Next Steps +* [Write and use callback functions](http://mxnet.io/tutorials/r/CallbackFunctionTutorial.html) +* [Neural Networks with MXNet in Five Minutes](http://mxnet.io/tutorials/r/fiveMinutesNeuralNetwork.html) +* [Classify Real-World Images with Pre-trained Model](http://mxnet.io/tutorials/r/classifyRealImageWithPretrainedModel.html) +* [Handwritten Digits Classification Competition](http://mxnet.io/tutorials/r/mnistCompetition.html) +* [Character Language Model using RNN](http://mxnet.io/tutorials/r/charRnnModel.html) From 0f2c2cd05d22fabaa24e176e38d14b156264a693 Mon Sep 17 00:00:00 2001 From: Andrei Paleyes Date: Wed, 31 May 2017 20:41:27 +0100 Subject: [PATCH 010/834] Fixed few typos in the Python tutorials (#6509) --- docs/tutorials/python/linear-regression.md | 4 ++-- docs/tutorials/python/predict_image.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/tutorials/python/linear-regression.md b/docs/tutorials/python/linear-regression.md index f67c1c2fb919..eb99642179a6 100644 --- a/docs/tutorials/python/linear-regression.md +++ b/docs/tutorials/python/linear-regression.md @@ -1,6 +1,6 @@ # Linear Regression -In this tutorial we'll walk though how one can implement *linear regression* using MXNet APIs. +In this tutorial we'll walk through how one can implement *linear regression* using MXNet APIs. The function we are trying to learn is: *y = x1 + 2x2*, where *(x1,x2)* are input features and *y* is the corresponding label. @@ -71,7 +71,7 @@ and make up various components of the model. Symbols are used to define: One such example is the `FullyConnected` symbol which specifies a fully connected layer of a neural network. 3. **Outputs:** Output symbols are MXNet's way of defining a loss. They are - suffixed with the word "Output" (eg. the `SoftmaxOutput` layer. You can also + suffixed with the word "Output" (eg. the `SoftmaxOutput` layer). You can also [create your own loss function](https://github.com/dmlc/mxnet/blob/master/docs/tutorials/r/CustomLossFunction.md#how-to-use-your-own-loss-function). Some examples of existing losses are: `LinearRegressionOutput`, which computes the l2-loss between it's input symbol and the labels provided to it; diff --git a/docs/tutorials/python/predict_image.md b/docs/tutorials/python/predict_image.md index 90db3896e54a..f473bdb1d6f5 100644 --- a/docs/tutorials/python/predict_image.md +++ b/docs/tutorials/python/predict_image.md @@ -100,7 +100,7 @@ all_layers = sym.get_internals() all_layers.list_outputs()[-10:] ``` -A often used layer for feature extraction is the one before the last fully +An often used layer for feature extraction is the one before the last fully connected layer. For ResNet, and also Inception, it is the flatten layer with name `flatten0` which reshapes the 4-D convolutional layer output into 2-D for the fully connected layer. The following source code extracts a new Symbol which From 36520dc6300b4510aaf73b902c7492311cd32520 Mon Sep 17 00:00:00 2001 From: Mu Li Date: Wed, 31 May 2017 12:43:39 -0700 Subject: [PATCH 011/834] Remove BUILD_TAG in DOCKER_IMG_NAME in ci_build.sh (#6513) To avoid generating too many images when run `docker images` --- tests/ci_build/ci_build.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/ci_build/ci_build.sh b/tests/ci_build/ci_build.sh index 9041fab73d9e..fa2d37ee718f 100755 --- a/tests/ci_build/ci_build.sh +++ b/tests/ci_build/ci_build.sh @@ -69,13 +69,12 @@ function upsearch () { cd .. && upsearch "$1" } -# Set up WORKSPACE and BUILD_TAG. Jenkins will set them for you or we pick +# Set up WORKSPACE. Jenkins will set them for you or we pick # reasonable defaults if you run it outside of Jenkins. WORKSPACE="${WORKSPACE:-${SCRIPT_DIR}/../../}" -BUILD_TAG="${BUILD_TAG:-mx-ci}" # Determine the docker image name -DOCKER_IMG_NAME="${BUILD_TAG}.${CONTAINER_TYPE}" +DOCKER_IMG_NAME="mx-ci.${CONTAINER_TYPE}" # Under Jenkins matrix build, the build tag may contain characters such as # commas (,) and equal signs (=), which are not valid inside docker image names. @@ -98,6 +97,7 @@ echo "CI_DOCKER_EXTRA_PARAMS: ${CI_DOCKER_EXTRA_PARAMS[@]}" echo "COMMAND: ${COMMAND[@]}" echo "CONTAINER_TYPE: ${CONTAINER_TYPE}" echo "BUILD_TAG: ${BUILD_TAG}" +echo "NODE_NAME: ${NODE_NAME}" echo "DOCKER CONTAINER NAME: ${DOCKER_IMG_NAME}" echo "PRE_COMMAND: ${PRE_COMMAND}" echo "" From d73f2dc9437865f53268d33840b240d67d87719d Mon Sep 17 00:00:00 2001 From: ziheng Date: Wed, 31 May 2017 12:52:34 -0700 Subject: [PATCH 012/834] Fix for build (#6511) --- plugin/torch/torch_base.h | 14 +++++++------- plugin/warpctc/warpctc-inl.h | 12 ++++++------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/plugin/torch/torch_base.h b/plugin/torch/torch_base.h index 0a7d22f8fc05..acfefe7ac382 100644 --- a/plugin/torch/torch_base.h +++ b/plugin/torch/torch_base.h @@ -125,11 +125,11 @@ class TorchTensor { } static const char* TensorType(TBlob data) { - return TensorType(data.dev_mask_); + return TensorType(data.dev_mask()); } static const char* ModuleType(TBlob data) { - return TensorType(data.dev_mask_); + return TensorType(data.dev_mask()); } static THGeneralTensor TBlobToTHTensor(TorchState* torchState, TBlob data) { @@ -140,7 +140,7 @@ class TorchTensor { THLongStorage_set(thshape, i, data.shape_[i]); } CHECK_EQ(data.type_flag_, mshadow::kFloat32) << "Torch Interface only support float32"; - switch (data.dev_mask_) { + switch (data.dev_mask()) { case cpu::kDevMask: { THFloatStorage* storage = THFloatStorage_newWithData(static_cast(data.dptr_), size); @@ -191,7 +191,7 @@ class TorchTensor { static void SetInternal(TorchState* torchState, THGeneralTensor tensor, const TBlob& blob) { size_t size = blob.Size(); - switch (blob.dev_mask_) { + switch (blob.dev_mask()) { case cpu::kDevMask: { THFloatStorage* storage = THFloatStorage_newWithData(static_cast(blob.dptr_), size); @@ -216,7 +216,7 @@ class TorchTensor { } #endif default: - LOG(FATAL) << "Unknown device type " << blob.dev_mask_; + LOG(FATAL) << "Unknown device type " << blob.dev_mask(); } } @@ -249,7 +249,7 @@ class TorchTensor { static void CopyIfDifferent(TorchState* torchState, TBlob dst, THGeneralTensor th_dst) { lua_State* L = torchState->L; if (luaT_isudata(L, -1, TorchTensor::TensorType(cpu::kDevMask))) { - CHECK_EQ(dst.dev_mask_, cpu::kDevMask) << "Device type mismatch."; + CHECK_EQ(dst.dev_mask(), cpu::kDevMask) << "Device type mismatch."; THFloatTensor* src = static_cast( luaT_toudata(L, -1, TorchTensor::TensorType(cpu::kDevMask))); if (src->storage != static_cast(th_dst)->storage) { @@ -257,7 +257,7 @@ class TorchTensor { } #if MXNET_USE_CUDA } else if (luaT_isudata(L, -1, TorchTensor::TensorType(gpu::kDevMask))) { - CHECK_EQ(dst.dev_mask_, gpu::kDevMask) << "Device type mismatch."; + CHECK_EQ(dst.dev_mask(), gpu::kDevMask) << "Device type mismatch."; THCudaTensor* src = static_cast( luaT_toudata(L, -1, TorchTensor::TensorType(gpu::kDevMask))); if (src->storage != static_cast(th_dst)->storage) { diff --git a/plugin/warpctc/warpctc-inl.h b/plugin/warpctc/warpctc-inl.h index a5b570b76cf9..328c08b2db02 100644 --- a/plugin/warpctc/warpctc-inl.h +++ b/plugin/warpctc/warpctc-inl.h @@ -121,16 +121,16 @@ class WarpCTCOp : public Operator { TBlob label = in_data[warpctc_enum::kLabel]; CHECK_EQ(data.shape_.ndim(), 2) << "input data shape should be 2 (t*n, p)"; ctcOptions info; //please updated to latest baidu/warp-ctc NOLINT(*) - if (data.dev_mask_ == cpu::kDevMask) { + if (data.dev_mask() == cpu::kDevMask) { info.loc = CTC_CPU; info.num_threads = 1; - } else if (data.dev_mask_ == gpu::kDevMask) { + } else if (data.dev_mask() == gpu::kDevMask) { #if MXNET_USE_CUDA info.loc = CTC_GPU; info.stream = ctx.get_stream()->stream_; } else { #endif - LOG(FATAL) << "Unknown device type " << data.dev_mask_; + LOG(FATAL) << "Unknown device type " << data.dev_mask(); } info.blank_label = 0; @@ -149,7 +149,7 @@ class WarpCTCOp : public Operator { int* flat_labels = static_cast(label.dptr_); int* cpu_raw_labels = flat_labels; float* grads = static_cast(in_grad[warpctc_enum::kData].dptr_); - if (data.dev_mask_ == gpu::kDevMask) { + if (data.dev_mask() == gpu::kDevMask) { #if MXNET_USE_CUDA cpu_raw_labels = reinterpret_cast(malloc(sizeof(int) * label.Size())); cuda_status = cudaMemcpyAsync(cpu_raw_labels, flat_labels, @@ -193,9 +193,9 @@ class WarpCTCOp : public Operator { info), "Error: compute_ctc_loss"); - if (data.dev_mask_ == cpu::kDevMask) { + if (data.dev_mask() == cpu::kDevMask) { free(cpu_labels); - } else if (data.dev_mask_ == gpu::kDevMask) { + } else if (data.dev_mask() == gpu::kDevMask) { #if MXNET_USE_CUDA free(cpu_raw_labels); free(cpu_labels); From bc83786a6b7d1f283376b366e1d51479c33b7d2d Mon Sep 17 00:00:00 2001 From: lxn2 Date: Wed, 31 May 2017 13:54:32 -0700 Subject: [PATCH 013/834] Test install.md for macOS (#6231) @szha I'm going to merge this PR, its goal is to test install.md. Once mxnet-distro moved to mxnet, we can run the test for mxnet-distro first. --- .travis.yml | 5 +- docs/get_started/install.md | 12 +- tests/jenkins/run_test_installation_docs.sh | 239 +++++++++++++------- tests/travis/run_test.sh | 17 ++ tests/travis/setup.sh | 2 +- 5 files changed, 184 insertions(+), 91 deletions(-) diff --git a/.travis.yml b/.travis.yml index c8ba0b1e645b..a5f28fa5d115 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,9 @@ env: - TASK=cpp_test # run tests/python - TASK=python_test - - TASK=r_test + #- TASK=r_test + - TASK=installation_packaged_test + - TASK=installation_source_test # - TASK=julia JULIA_VER=0.4 # - TASK=scala_test @@ -78,7 +80,6 @@ addons: before_install: - export NVCC_PREFIX=${HOME} - source dmlc-core/scripts/travis/travis_setup_env.sh - - export PYTHONPATH=${PYTHONPATH}:${PWD}/python - export MAVEN_SKIP_RC=true - export MAVEN_OPTS="-Xmx512m -XX:MaxPermSize=256m -XX:-UseGCOverheadLimit -XX:+CMSClassUnloadingEnabled -XX:+UseConcMarkSweepGC" diff --git a/docs/get_started/install.md b/docs/get_started/install.md index f81307833029..4930a59d615d 100644 --- a/docs/get_started/install.md +++ b/docs/get_started/install.md @@ -462,14 +462,14 @@ If not already installed, [download and install Xcode](https://developer.apple.c
**Step 1** Install prerequisites - Homebrew, python development tools. - + ```bash # Install Homebrew -$ /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" +$ /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" /dev/null || brew install python ``` **Step 2** Install virtualenv for macOS. @@ -523,14 +523,14 @@ Install *MXNet* with OpenBLAS acceleration.
**Step 1** Install prerequisites - Homebrew, python development tools. - + ```bash # Install Homebrew -$ /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" +$ /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" /dev/null || brew install python ``` **Step 2** Install MXNet with OpenBLAS acceleration. diff --git a/tests/jenkins/run_test_installation_docs.sh b/tests/jenkins/run_test_installation_docs.sh index 921c89a16605..09ebf28f7f67 100755 --- a/tests/jenkins/run_test_installation_docs.sh +++ b/tests/jenkins/run_test_installation_docs.sh @@ -75,7 +75,7 @@ function retrieve_closest_index() { cur_num=${arr[${i}]} if [[ ${cur_num} -eq ${number} || ${cur_num} -gt ${number} ]] then - echo ${i} + echo "${i}" return fi done @@ -124,7 +124,7 @@ function retrieve_commands() { fi done done - echo ${commands} + echo "${commands}" } # Sorts array of numbers. @@ -139,26 +139,32 @@ function retrieve_commands() { function sort() { declare -a lineno_array=("${!1}") size=${#lineno_array[@]} - for (( i=1; i<=$(( $size-1)); i++ )) + for((i=1;i 0 && ${lineno_array[$j-1]} > ${lineno_array[$j]} )); do - x=${lineno_array[$j-1]} - lineno_array[$j-1]=${lineno_array[$j]} - lineno_array[$j]=$x - j=$j-1 - done + temp=${lineno_array[i]} + j=$((i-1)) + while [ $temp -lt ${lineno_array[j]} ] + do + lineno_array[j+1]=${lineno_array[j]} + j=$(( $j-1 )) + if [ $j == -1 ] + then + break + fi + done + lineno_array[j+1]=$temp done printf "${lineno_array[*]}" } -if (( $# < 1 )); then +if (( $# < 2 )); then echo "" - echo "Usage: $(basename $0) FILE" + echo "Usage: $(basename $0) FILE ENV" echo "" exit 1 fi FILE=${1} +TASK=${2} # get all line numbers with "```" signifying start or end of source section and put them in an array SOURCE_REGEX="\`\`\`" @@ -170,6 +176,10 @@ PIP_LINENO_ALL=($(grep -n "
" "${FILE}" | cut -d : -f 1)) DOCKER_LINENO_ALL=($(grep -n "
" "${FILE}" | cut -d : -f 1)) BUILDFROMSOURCE_LINENO_ALL=($(grep -n "
" "${FILE}" | cut -d : -f 1)) +# validation instructions +PYTHON_GPU_VALIDATION="import mxnet as mx; a = mx.nd.ones((2, 3), mx.gpu()); b = a * 2 + 1; b.asnumpy()" +PYTHON_CPU_VALIDATION="import mxnet as mx; a = mx.nd.ones((2, 3)); b = a * 2 + 1; b.asnumpy()" + # Given two line numbers, collects instruction sets for installing via Virtualenv, Pip, Docker, and source within the # two lines assuming there is one of each. # @@ -232,74 +242,139 @@ function set_instruction_set() { ${sorted_indexes[$end_buildfromsource_command_index]}) } +if [[ "${TASK}" == "linux" ]] +then + + ########################LINUX-PYTHON-CPU############################ + echo + echo + echo "### Testing LINUX-PYTHON-CPU ###" + echo + # range of all lines inside Linux-Python-CPU instructions + LINUX_PYTHON_CPU_START_LINENO=$(grep -n "START - Linux Python CPU Installation Instructions" "${FILE}" | cut -d : -f 1) + LINUX_PYTHON_CPU_END_LINENO=$(grep -n "END - Linux Python CPU Installation Instructions" "${FILE}" | cut -d : -f 1) + + set_instruction_set ${LINUX_PYTHON_CPU_START_LINENO} ${LINUX_PYTHON_CPU_END_LINENO} + + virtualenv_commands="${virtualenv_commands} python -c \"${PYTHON_CPU_VALIDATION}\"" + echo + echo "### Testing Virtualenv ###" + echo "${virtualenv_commands}" + echo + docker run --rm ubuntu:14.04 bash -c "${virtualenv_commands}" + + pip_commands="${pip_commands} python -c \"${PYTHON_CPU_VALIDATION}\"" + echo + echo "### Testing Pip ###" + echo "${pip_commands}" + echo + docker run --rm ubuntu:14.04 bash -c "${pip_commands}" + + docker_img=$(echo "$docker_commands" | sed 's/.*docker pull \(.*\)/\1/' | sed 's/;.*//') + docker_commands="${docker_commands} docker run ${docker_img} python -c \"${PYTHON_CPU_VALIDATION}\"" + echo + echo "### Testing Docker ###" + echo "${docker_commands}" + echo + eval "${docker_commands}" + + buildfromsource_commands="${buildfromsource_commands} python -c \"${PYTHON_CPU_VALIDATION}\"" + echo + echo "### Testing Build From Source ###" + echo "${buildfromsource_commands}" + echo + docker run --rm ubuntu:14.04 bash -c "${buildfromsource_commands}" + + #########################LINUX-PYTHON-GPU########################### + + echo + echo + echo "### Testing LINUX-PYTHON-GPU ###" + echo + # range of all lines inside Linux-Python-GPU instructions + LINUX_PYTHON_GPU_START_LINENO=$(grep -n "START - Linux Python GPU Installation Instructions" "${FILE}" | cut -d : -f 1) + LINUX_PYTHON_GPU_END_LINENO=$(grep -n "END - Linux Python GPU Installation Instructions" "${FILE}" | cut -d : -f 1) + + set_instruction_set ${LINUX_PYTHON_GPU_START_LINENO} ${LINUX_PYTHON_GPU_END_LINENO} + + virtualenv_commands="${virtualenv_commands} python -c \"${PYTHON_GPU_VALIDATION}\"" + echo + echo "### Testing Virtualenv ###" + echo "${virtualenv_commands}" + echo + nvidia-docker run --rm nvidia/cuda:8.0-cudnn5-devel-ubuntu14.04 bash -c "${virtualenv_commands}" + + pip_commands="${pip_commands} python -c \"${PYTHON_GPU_VALIDATION}\"" + echo + echo "### Testing Pip ###" + echo "${pip_commands}" + echo + nvidia-docker run --rm nvidia/cuda:8.0-cudnn5-devel-ubuntu14.04 bash -c "${pip_commands}" + + docker_img=$(echo "$docker_commands" | sed 's/.*docker pull \(.*\)/\1/' | sed 's/;.*//') + docker_commands="${docker_commands} nvidia-docker run ${docker_img} python -c \"${PYTHON_GPU_VALIDATION}\"" + echo + echo "### Testing Docker ###" + echo "${docker_commands}" + echo + eval "${docker_commands}" + + buildfromsource_commands="${buildfromsource_commands} python -c \"${PYTHON_GPU_VALIDATION}\"" + echo + echo "### Testing Build From Source ###" + echo "${buildfromsource_commands}" + echo + nvidia-docker run --rm nvidia/cuda:8.0-cudnn5-devel-ubuntu14.04 bash -c "${buildfromsource_commands}" + +else + + #########################MACOS-PYTHON-CPU########################### + # Currently this section is invoked in ../travis/run_test.sh so this test can run on MacOS. + echo + echo + echo "### Testing MACOS-PYTHON-CPU ###" + echo + # range of all lines inside MacOS-Python-CPU instructions + MAC_PYTHON_CPU_START_LINENO=$(grep -n "START - MacOS Python CPU Installation Instructions" "${FILE}" | cut -d : -f 1) + MAC_PYTHON_CPU_END_LINENO=$(grep -n "END - Mac OS Python CPU Installation Instructions" "${FILE}" | cut -d : -f 1) + + set_instruction_set ${MAC_PYTHON_CPU_START_LINENO} ${MAC_PYTHON_CPU_END_LINENO} + + if [[ "${TASK}" == "installation_packaged_test" ]] + then + virtualenv_commands="${virtualenv_commands} python -c \"import sys; print hasattr(sys, 'real_prefix'); ${PYTHON_CPU_VALIDATION}\"; deactivate;" + echo + echo "### Testing Virtualenv ###" + echo "${virtualenv_commands}" + echo + eval "${virtualenv_commands}" + + pip_commands="${pip_commands} python -c \"${PYTHON_CPU_VALIDATION}\"" + echo + echo "### Testing Pip ###" + echo "${pip_commands}" + echo + eval "${pip_commands}" + + exit + fi + + ###COMMENTING THIS OUT FOR NOW AS TRAVIS DOES NOT SUPPORT DOCKER FOR MAC +# echo +# echo "### Testing Docker ###" +# echo "${docker_commands}" +# echo +# eval ${docker_commands} -########################LINUX-PYTHON-CPU############################ -echo -echo -echo "### Testing LINUX-PYTHON-CPU ###" -echo -# range of all lines inside Linux-Python-CPU instructions -LINUX_PYTHON_CPU_START_LINENO=$(grep -n "START - Linux Python CPU Installation Instructions" "${FILE}" | cut -d : -f 1) -LINUX_PYTHON_CPU_END_LINENO=$(grep -n "END - Linux Python CPU Installation Instructions" "${FILE}" | cut -d : -f 1) - -set_instruction_set ${LINUX_PYTHON_CPU_START_LINENO} ${LINUX_PYTHON_CPU_END_LINENO} - -echo -echo "### Testing Virtualenv ###" -echo "${virtualenv_commands}" -echo -docker run --rm ubuntu:14.04 bash -c "${virtualenv_commands}" - -echo -echo "### Testing Pip ###" -echo "${pip_commands}" -echo -docker run --rm ubuntu:14.04 bash -c "${pip_commands}" - -echo -echo "### Testing Docker ###" -echo "${docker_commands}" -echo -eval ${docker_commands} - -echo -echo "### Testing Build From Source ###" -echo "${buildfromsource_commands}" -echo -docker run --rm ubuntu:14.04 bash -c "${buildfromsource_commands}" - -#########################LINUX-PYTHON-GPU########################### - -echo -echo -echo "### Testing LINUX-PYTHON-GPU ###" -echo -# range of all lines inside Linux-Python-GPU instructions -LINUX_PYTHON_GPU_START_LINENO=$(grep -n "START - Linux Python GPU Installation Instructions" "${FILE}" | cut -d : -f 1) -LINUX_PYTHON_GPU_END_LINENO=$(grep -n "END - Linux Python GPU Installation Instructions" "${FILE}" | cut -d : -f 1) - -set_instruction_set ${LINUX_PYTHON_GPU_START_LINENO} ${LINUX_PYTHON_GPU_END_LINENO} - -echo -echo "### Testing Virtualenv ###" -echo "${virtualenv_commands}" -echo -nvidia-docker run --rm nvidia/cuda:7.5-cudnn5-devel bash -c "${virtualenv_commands}" - -echo -echo "### Testing Pip ###" -echo "${pip_commands}" -echo -nvidia-docker run --rm nvidia/cuda:7.5-cudnn5-devel bash -c "${pip_commands}" - -echo -echo "### Testing Docker ###" -echo "${docker_commands}" -echo -eval ${docker_commands} - -echo -echo "### Testing Build From Source ###" -echo "${buildfromsource_commands}" -echo -nvidia-docker run --rm nvidia/cuda:7.5-cudnn5-devel bash -c "${buildfromsource_commands}" + if [[ "${TASK}" == "installation_source_test" ]] + then + buildfromsource_commands="${buildfromsource_commands} python -c \"${PYTHON_CPU_VALIDATION}\"" + echo + echo "### Testing Build From Source ###" + echo "${buildfromsource_commands}" + echo + eval "${buildfromsource_commands}" + + exit + fi +fi diff --git a/tests/travis/run_test.sh b/tests/travis/run_test.sh index cff4196b6043..540d013f1f16 100755 --- a/tests/travis/run_test.sh +++ b/tests/travis/run_test.sh @@ -5,6 +5,22 @@ then exit 0 fi +if [[ ${TASK} == *"installation"* ]]; then + git remote add main https://github.com/dmlc/mxnet.git + git fetch main master + echo "File changes compared to origin/master:" + echo "**********************************" + git diff --name-only remotes/main/master + echo "**********************************" + + if [[ ! $(git diff --name-only remotes/main/master | grep install.md) ]]; then + echo "No changes to install.md. Skipping installation tasks..." + exit 0 + fi + ./tests/jenkins/run_test_installation_docs.sh docs/get_started/install.md ${TASK} + exit $? +fi + if [ ${TASK} == "lint" ]; then make lint || exit -1 echo "Check documentations of c++ code..." @@ -93,6 +109,7 @@ if [ ${TASK} == "r_test" ]; then fi if [ ${TASK} == "python_test" ]; then + export PYTHONPATH=${PYTHONPATH}:${PWD}/python make all || exit -1 # use cached dir for storing data rm -rf ${PWD}/data diff --git a/tests/travis/setup.sh b/tests/travis/setup.sh index ec071009bda5..e1027b0db05e 100755 --- a/tests/travis/setup.sh +++ b/tests/travis/setup.sh @@ -5,7 +5,7 @@ then exit 0 fi -if [ ${TRAVIS_OS_NAME} == "osx" ]; then +if [[ ${TRAVIS_OS_NAME} == "osx" && ${TASK} != *"installation"* ]]; then brew update brew tap homebrew/science brew install opencv From fa61571eaf6608d57f291465f0b34c4d5844367f Mon Sep 17 00:00:00 2001 From: lxn2 Date: Wed, 31 May 2017 17:36:08 -0700 Subject: [PATCH 014/834] Revert "Test install.md for macOS (#6231)" (#6520) This reverts commit bc83786a6b7d1f283376b366e1d51479c33b7d2d. --- .travis.yml | 5 +- docs/get_started/install.md | 12 +- tests/jenkins/run_test_installation_docs.sh | 239 +++++++------------- tests/travis/run_test.sh | 17 -- tests/travis/setup.sh | 2 +- 5 files changed, 91 insertions(+), 184 deletions(-) diff --git a/.travis.yml b/.travis.yml index a5f28fa5d115..c8ba0b1e645b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,9 +17,7 @@ env: - TASK=cpp_test # run tests/python - TASK=python_test - #- TASK=r_test - - TASK=installation_packaged_test - - TASK=installation_source_test + - TASK=r_test # - TASK=julia JULIA_VER=0.4 # - TASK=scala_test @@ -80,6 +78,7 @@ addons: before_install: - export NVCC_PREFIX=${HOME} - source dmlc-core/scripts/travis/travis_setup_env.sh + - export PYTHONPATH=${PYTHONPATH}:${PWD}/python - export MAVEN_SKIP_RC=true - export MAVEN_OPTS="-Xmx512m -XX:MaxPermSize=256m -XX:-UseGCOverheadLimit -XX:+CMSClassUnloadingEnabled -XX:+UseConcMarkSweepGC" diff --git a/docs/get_started/install.md b/docs/get_started/install.md index 4930a59d615d..f81307833029 100644 --- a/docs/get_started/install.md +++ b/docs/get_started/install.md @@ -462,14 +462,14 @@ If not already installed, [download and install Xcode](https://developer.apple.c
**Step 1** Install prerequisites - Homebrew, python development tools. - + ```bash # Install Homebrew -$ /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" /dev/null || brew install python +$ brew install python ``` **Step 2** Install virtualenv for macOS. @@ -523,14 +523,14 @@ Install *MXNet* with OpenBLAS acceleration.
**Step 1** Install prerequisites - Homebrew, python development tools. - + ```bash # Install Homebrew -$ /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" /dev/null || brew install python +$ brew install python ``` **Step 2** Install MXNet with OpenBLAS acceleration. diff --git a/tests/jenkins/run_test_installation_docs.sh b/tests/jenkins/run_test_installation_docs.sh index 09ebf28f7f67..921c89a16605 100755 --- a/tests/jenkins/run_test_installation_docs.sh +++ b/tests/jenkins/run_test_installation_docs.sh @@ -75,7 +75,7 @@ function retrieve_closest_index() { cur_num=${arr[${i}]} if [[ ${cur_num} -eq ${number} || ${cur_num} -gt ${number} ]] then - echo "${i}" + echo ${i} return fi done @@ -124,7 +124,7 @@ function retrieve_commands() { fi done done - echo "${commands}" + echo ${commands} } # Sorts array of numbers. @@ -139,32 +139,26 @@ function retrieve_commands() { function sort() { declare -a lineno_array=("${!1}") size=${#lineno_array[@]} - for((i=1;i 0 && ${lineno_array[$j-1]} > ${lineno_array[$j]} )); do + x=${lineno_array[$j-1]} + lineno_array[$j-1]=${lineno_array[$j]} + lineno_array[$j]=$x + j=$j-1 + done done printf "${lineno_array[*]}" } -if (( $# < 2 )); then +if (( $# < 1 )); then echo "" - echo "Usage: $(basename $0) FILE ENV" + echo "Usage: $(basename $0) FILE" echo "" exit 1 fi FILE=${1} -TASK=${2} # get all line numbers with "```" signifying start or end of source section and put them in an array SOURCE_REGEX="\`\`\`" @@ -176,10 +170,6 @@ PIP_LINENO_ALL=($(grep -n "
" "${FILE}" | cut -d : -f 1)) DOCKER_LINENO_ALL=($(grep -n "
" "${FILE}" | cut -d : -f 1)) BUILDFROMSOURCE_LINENO_ALL=($(grep -n "
" "${FILE}" | cut -d : -f 1)) -# validation instructions -PYTHON_GPU_VALIDATION="import mxnet as mx; a = mx.nd.ones((2, 3), mx.gpu()); b = a * 2 + 1; b.asnumpy()" -PYTHON_CPU_VALIDATION="import mxnet as mx; a = mx.nd.ones((2, 3)); b = a * 2 + 1; b.asnumpy()" - # Given two line numbers, collects instruction sets for installing via Virtualenv, Pip, Docker, and source within the # two lines assuming there is one of each. # @@ -242,139 +232,74 @@ function set_instruction_set() { ${sorted_indexes[$end_buildfromsource_command_index]}) } -if [[ "${TASK}" == "linux" ]] -then - - ########################LINUX-PYTHON-CPU############################ - echo - echo - echo "### Testing LINUX-PYTHON-CPU ###" - echo - # range of all lines inside Linux-Python-CPU instructions - LINUX_PYTHON_CPU_START_LINENO=$(grep -n "START - Linux Python CPU Installation Instructions" "${FILE}" | cut -d : -f 1) - LINUX_PYTHON_CPU_END_LINENO=$(grep -n "END - Linux Python CPU Installation Instructions" "${FILE}" | cut -d : -f 1) - - set_instruction_set ${LINUX_PYTHON_CPU_START_LINENO} ${LINUX_PYTHON_CPU_END_LINENO} - - virtualenv_commands="${virtualenv_commands} python -c \"${PYTHON_CPU_VALIDATION}\"" - echo - echo "### Testing Virtualenv ###" - echo "${virtualenv_commands}" - echo - docker run --rm ubuntu:14.04 bash -c "${virtualenv_commands}" - - pip_commands="${pip_commands} python -c \"${PYTHON_CPU_VALIDATION}\"" - echo - echo "### Testing Pip ###" - echo "${pip_commands}" - echo - docker run --rm ubuntu:14.04 bash -c "${pip_commands}" - - docker_img=$(echo "$docker_commands" | sed 's/.*docker pull \(.*\)/\1/' | sed 's/;.*//') - docker_commands="${docker_commands} docker run ${docker_img} python -c \"${PYTHON_CPU_VALIDATION}\"" - echo - echo "### Testing Docker ###" - echo "${docker_commands}" - echo - eval "${docker_commands}" - - buildfromsource_commands="${buildfromsource_commands} python -c \"${PYTHON_CPU_VALIDATION}\"" - echo - echo "### Testing Build From Source ###" - echo "${buildfromsource_commands}" - echo - docker run --rm ubuntu:14.04 bash -c "${buildfromsource_commands}" - - #########################LINUX-PYTHON-GPU########################### - - echo - echo - echo "### Testing LINUX-PYTHON-GPU ###" - echo - # range of all lines inside Linux-Python-GPU instructions - LINUX_PYTHON_GPU_START_LINENO=$(grep -n "START - Linux Python GPU Installation Instructions" "${FILE}" | cut -d : -f 1) - LINUX_PYTHON_GPU_END_LINENO=$(grep -n "END - Linux Python GPU Installation Instructions" "${FILE}" | cut -d : -f 1) - - set_instruction_set ${LINUX_PYTHON_GPU_START_LINENO} ${LINUX_PYTHON_GPU_END_LINENO} - - virtualenv_commands="${virtualenv_commands} python -c \"${PYTHON_GPU_VALIDATION}\"" - echo - echo "### Testing Virtualenv ###" - echo "${virtualenv_commands}" - echo - nvidia-docker run --rm nvidia/cuda:8.0-cudnn5-devel-ubuntu14.04 bash -c "${virtualenv_commands}" - - pip_commands="${pip_commands} python -c \"${PYTHON_GPU_VALIDATION}\"" - echo - echo "### Testing Pip ###" - echo "${pip_commands}" - echo - nvidia-docker run --rm nvidia/cuda:8.0-cudnn5-devel-ubuntu14.04 bash -c "${pip_commands}" - - docker_img=$(echo "$docker_commands" | sed 's/.*docker pull \(.*\)/\1/' | sed 's/;.*//') - docker_commands="${docker_commands} nvidia-docker run ${docker_img} python -c \"${PYTHON_GPU_VALIDATION}\"" - echo - echo "### Testing Docker ###" - echo "${docker_commands}" - echo - eval "${docker_commands}" - - buildfromsource_commands="${buildfromsource_commands} python -c \"${PYTHON_GPU_VALIDATION}\"" - echo - echo "### Testing Build From Source ###" - echo "${buildfromsource_commands}" - echo - nvidia-docker run --rm nvidia/cuda:8.0-cudnn5-devel-ubuntu14.04 bash -c "${buildfromsource_commands}" - -else - - #########################MACOS-PYTHON-CPU########################### - # Currently this section is invoked in ../travis/run_test.sh so this test can run on MacOS. - echo - echo - echo "### Testing MACOS-PYTHON-CPU ###" - echo - # range of all lines inside MacOS-Python-CPU instructions - MAC_PYTHON_CPU_START_LINENO=$(grep -n "START - MacOS Python CPU Installation Instructions" "${FILE}" | cut -d : -f 1) - MAC_PYTHON_CPU_END_LINENO=$(grep -n "END - Mac OS Python CPU Installation Instructions" "${FILE}" | cut -d : -f 1) - - set_instruction_set ${MAC_PYTHON_CPU_START_LINENO} ${MAC_PYTHON_CPU_END_LINENO} - - if [[ "${TASK}" == "installation_packaged_test" ]] - then - virtualenv_commands="${virtualenv_commands} python -c \"import sys; print hasattr(sys, 'real_prefix'); ${PYTHON_CPU_VALIDATION}\"; deactivate;" - echo - echo "### Testing Virtualenv ###" - echo "${virtualenv_commands}" - echo - eval "${virtualenv_commands}" - - pip_commands="${pip_commands} python -c \"${PYTHON_CPU_VALIDATION}\"" - echo - echo "### Testing Pip ###" - echo "${pip_commands}" - echo - eval "${pip_commands}" - - exit - fi - - ###COMMENTING THIS OUT FOR NOW AS TRAVIS DOES NOT SUPPORT DOCKER FOR MAC -# echo -# echo "### Testing Docker ###" -# echo "${docker_commands}" -# echo -# eval ${docker_commands} - if [[ "${TASK}" == "installation_source_test" ]] - then - buildfromsource_commands="${buildfromsource_commands} python -c \"${PYTHON_CPU_VALIDATION}\"" - echo - echo "### Testing Build From Source ###" - echo "${buildfromsource_commands}" - echo - eval "${buildfromsource_commands}" - - exit - fi -fi +########################LINUX-PYTHON-CPU############################ +echo +echo +echo "### Testing LINUX-PYTHON-CPU ###" +echo +# range of all lines inside Linux-Python-CPU instructions +LINUX_PYTHON_CPU_START_LINENO=$(grep -n "START - Linux Python CPU Installation Instructions" "${FILE}" | cut -d : -f 1) +LINUX_PYTHON_CPU_END_LINENO=$(grep -n "END - Linux Python CPU Installation Instructions" "${FILE}" | cut -d : -f 1) + +set_instruction_set ${LINUX_PYTHON_CPU_START_LINENO} ${LINUX_PYTHON_CPU_END_LINENO} + +echo +echo "### Testing Virtualenv ###" +echo "${virtualenv_commands}" +echo +docker run --rm ubuntu:14.04 bash -c "${virtualenv_commands}" + +echo +echo "### Testing Pip ###" +echo "${pip_commands}" +echo +docker run --rm ubuntu:14.04 bash -c "${pip_commands}" + +echo +echo "### Testing Docker ###" +echo "${docker_commands}" +echo +eval ${docker_commands} + +echo +echo "### Testing Build From Source ###" +echo "${buildfromsource_commands}" +echo +docker run --rm ubuntu:14.04 bash -c "${buildfromsource_commands}" + +#########################LINUX-PYTHON-GPU########################### + +echo +echo +echo "### Testing LINUX-PYTHON-GPU ###" +echo +# range of all lines inside Linux-Python-GPU instructions +LINUX_PYTHON_GPU_START_LINENO=$(grep -n "START - Linux Python GPU Installation Instructions" "${FILE}" | cut -d : -f 1) +LINUX_PYTHON_GPU_END_LINENO=$(grep -n "END - Linux Python GPU Installation Instructions" "${FILE}" | cut -d : -f 1) + +set_instruction_set ${LINUX_PYTHON_GPU_START_LINENO} ${LINUX_PYTHON_GPU_END_LINENO} + +echo +echo "### Testing Virtualenv ###" +echo "${virtualenv_commands}" +echo +nvidia-docker run --rm nvidia/cuda:7.5-cudnn5-devel bash -c "${virtualenv_commands}" + +echo +echo "### Testing Pip ###" +echo "${pip_commands}" +echo +nvidia-docker run --rm nvidia/cuda:7.5-cudnn5-devel bash -c "${pip_commands}" + +echo +echo "### Testing Docker ###" +echo "${docker_commands}" +echo +eval ${docker_commands} + +echo +echo "### Testing Build From Source ###" +echo "${buildfromsource_commands}" +echo +nvidia-docker run --rm nvidia/cuda:7.5-cudnn5-devel bash -c "${buildfromsource_commands}" diff --git a/tests/travis/run_test.sh b/tests/travis/run_test.sh index 540d013f1f16..cff4196b6043 100755 --- a/tests/travis/run_test.sh +++ b/tests/travis/run_test.sh @@ -5,22 +5,6 @@ then exit 0 fi -if [[ ${TASK} == *"installation"* ]]; then - git remote add main https://github.com/dmlc/mxnet.git - git fetch main master - echo "File changes compared to origin/master:" - echo "**********************************" - git diff --name-only remotes/main/master - echo "**********************************" - - if [[ ! $(git diff --name-only remotes/main/master | grep install.md) ]]; then - echo "No changes to install.md. Skipping installation tasks..." - exit 0 - fi - ./tests/jenkins/run_test_installation_docs.sh docs/get_started/install.md ${TASK} - exit $? -fi - if [ ${TASK} == "lint" ]; then make lint || exit -1 echo "Check documentations of c++ code..." @@ -109,7 +93,6 @@ if [ ${TASK} == "r_test" ]; then fi if [ ${TASK} == "python_test" ]; then - export PYTHONPATH=${PYTHONPATH}:${PWD}/python make all || exit -1 # use cached dir for storing data rm -rf ${PWD}/data diff --git a/tests/travis/setup.sh b/tests/travis/setup.sh index e1027b0db05e..ec071009bda5 100755 --- a/tests/travis/setup.sh +++ b/tests/travis/setup.sh @@ -5,7 +5,7 @@ then exit 0 fi -if [[ ${TRAVIS_OS_NAME} == "osx" && ${TASK} != *"installation"* ]]; then +if [ ${TRAVIS_OS_NAME} == "osx" ]; then brew update brew tap homebrew/science brew install opencv From 98cb4e135875a6f2e84270dd3873f4ab6f045cae Mon Sep 17 00:00:00 2001 From: Yao Wang Date: Wed, 31 May 2017 17:36:24 -0700 Subject: [PATCH 015/834] Fix Search Description (#6521) --- docs/_static/searchtools_custom.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_static/searchtools_custom.js b/docs/_static/searchtools_custom.js index b2a17af4bcc2..c6fd37f72233 100644 --- a/docs/_static/searchtools_custom.js +++ b/docs/_static/searchtools_custom.js @@ -492,7 +492,7 @@ var Search = { displayNextItem(); }); } else if (DOCUMENTATION_OPTIONS.HAS_SOURCE) { - $.ajax({url: DOCUMENTATION_OPTIONS.URL_ROOT + '_sources/' + item[0] + '.md.txt', + $.ajax({url: DOCUMENTATION_OPTIONS.URL_ROOT + '_sources/' + item[0] + '.txt', dataType: "text", complete: function(jqxhr, textstatus) { var data = jqxhr.responseText; From 26b1cb9ad0bcde9206863a6f847455ff3ec3c266 Mon Sep 17 00:00:00 2001 From: Madan Jampani Date: Wed, 31 May 2017 17:37:06 -0700 Subject: [PATCH 016/834] Remove duplicate new operator how-to (#6519) --- docs/how_to/index.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/how_to/index.md b/docs/how_to/index.md index c6a8cf67ad1d..d7e42eaa2094 100644 --- a/docs/how_to/index.md +++ b/docs/how_to/index.md @@ -11,8 +11,6 @@ and full working examples, visit the [tutorials section](../tutorials/index.md). * [How do I work with variable-length input in MXNet (bucketing)?](http://mxnet.io/how_to/bucketing.html) -* [How do I create new operators with MXNet?](new_op.md) - * [How do I visualize neural networks as computation graphs?](http://mxnet.io/how_to/visualize_graph.html) From 6a00ba42c1418f17131482a363dbb74d3037398c Mon Sep 17 00:00:00 2001 From: Lyken Syu Date: Thu, 1 Jun 2017 10:25:10 -0700 Subject: [PATCH 017/834] hide _move_var _move_mean _beta _gamma for visualization (#6523) --- python/mxnet/visualization.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python/mxnet/visualization.py b/python/mxnet/visualization.py index 6e56dfb612ec..97b6bfa25b1b 100644 --- a/python/mxnet/visualization.py +++ b/python/mxnet/visualization.py @@ -240,6 +240,9 @@ def looks_like_weight(name): return True if name.endswith("_bias"): return True + if name.endswith("_beta") or name.endswith("_gamma") or \ + name.endswith("_moving_var") or name.endswith("_moving_mean"): + return True return False # make nodes From 1f7a7cd4ae3ba04a5ccd5cdc9b4abed7a0ace869 Mon Sep 17 00:00:00 2001 From: Roshani Nagmote Date: Thu, 1 Jun 2017 13:59:29 -0700 Subject: [PATCH 018/834] add link to style guide (#6529) --- docs/community/contribute.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/community/contribute.md b/docs/community/contribute.md index 6c8d5629fe14..3a39743af33d 100644 --- a/docs/community/contribute.md +++ b/docs/community/contribute.md @@ -8,7 +8,7 @@ After your patch has been merged, remember to add your name to [CONTRIBUTORS.md] ### Core Library -- Follow the Google C++ Style Guide for C++ code. +- Follow the [Google C++ Style Guide](https://google.github.io/styleguide/cppguide.html) for C++ code. - Use doxygen to document all of the interface code. - To reproduce the linter checks, type ```make lint```. From 2cbab7bc6ff9fecf32bdc144c881fb804ce72dea Mon Sep 17 00:00:00 2001 From: Roshani Nagmote Date: Thu, 1 Jun 2017 21:50:52 -0700 Subject: [PATCH 019/834] Module tutorial improvements and metric API doc linked (#6532) * Module tutorial improvements * prerequisite section added * Metric API md file created and linked from index page * link to fit function added * section rearranged * fixes after review * fixes after review --- docs/api/python/index.md | 1 + docs/api/python/metric.md | 28 +++++++ docs/tutorials/basic/module.md | 135 +++++++++++++++++++++++---------- 3 files changed, 122 insertions(+), 42 deletions(-) create mode 100644 docs/api/python/metric.md diff --git a/docs/api/python/index.md b/docs/api/python/index.md index 19fe822d94c4..6051c0e858c3 100644 --- a/docs/api/python/index.md +++ b/docs/api/python/index.md @@ -33,4 +33,5 @@ imported by running: io optimization callback + metric ``` diff --git a/docs/api/python/metric.md b/docs/api/python/metric.md new file mode 100644 index 000000000000..50a4a9be4550 --- /dev/null +++ b/docs/api/python/metric.md @@ -0,0 +1,28 @@ +# Evaluation Metric API + +```eval_rst +.. currentmodule:: mxnet.metric +``` + +## Overview + +This document lists all the evaluation metrics available to evaluate +the performance of a learned model. + +```eval_rst +.. autosummary:: + :nosignatures: + + mxnet.metric +``` + +## API Reference + + + +```eval_rst +.. automodule:: mxnet.metric + :members: +``` + + diff --git a/docs/tutorials/basic/module.md b/docs/tutorials/basic/module.md index b42993aec22f..6f28bdeda182 100644 --- a/docs/tutorials/basic/module.md +++ b/docs/tutorials/basic/module.md @@ -8,14 +8,24 @@ steps. All this can be quite daunting to both newcomers as well as experienced developers. Luckily, MXNet modularizes commonly used code for training and inference in -the `module` (`mod` for short) package. `module` provides both a -high-level and intermediate-level interfaces for executing predefined networks. +the `module` (`mod` for short) package. `Module` provides both high-level and +intermediate-level interfaces for executing predefined networks. One can use +both interfaces interchangeably. We will show the usage of both interfaces in +this tutorial. + +## Prerequisites + +To complete this tutorial, we need: + +- MXNet. See the instructions for your operating system in [Setup and Installation](http://mxnet.io/get_started/install.html) +- [Python](https://www.python.org/downloads/) ## Preliminary In this tutorial we will demonstrate `module` usage by training a [Multilayer Perceptron](https://en.wikipedia.org/wiki/Multilayer_perceptron) (MLP) -on the [UCI letter recognition](https://archive.ics.uci.edu/ml/datasets/letter+recognition) dataset. +on the [UCI letter recognition](https://archive.ics.uci.edu/ml/datasets/letter+recognition) +dataset. The following code downloads the dataset and creates an 80:20 train:test split. It also initializes a training data iterator to return a batch of 32 @@ -48,9 +58,7 @@ net = mx.sym.SoftmaxOutput(net, name='softmax') mx.viz.plot_network(net) ``` -## High-level Interface - -### Creating a Module +## Creating a Module Now we are ready to introduce module. The commonly used module class is `Module`. We can construct a module by specifying the following parameters: @@ -70,12 +78,69 @@ mod = mx.mod.Module(symbol=net, label_names=['softmax_label']) ``` -### Train, Predict, and Evaluate +## Intermediate-level Interface + +We have created module. Now let us see how to run training and inference using module's intermediate-level APIs. These APIs give developers flexibility to do step-by-step +computation by running `forward` and `backward` passes. It's also useful for debugging. + +To train a module, we need to perform following steps: + +- `bind` : Prepares environment for the computation by allocating memory. +- `init_params` : Assigns and initializes parameters. +- `init_optimizer` : Initializes optimizers. Defaults to `sgd`. +- `metric.create` : Creates evaluation metric from input metric name. +- `forward` : Forward computation. +- `update_metric` : Evaluates and accumulates evaluation metric on outputs of the last forward computation. +- `backward` : Backward computation. +- `update` : Updates parameters according to the installed optimizer and the gradients computed in the previous forward-backward batch. + +This can be used as follows: + +```python +# allocate memory given the input data and label shapes +mod.bind(data_shapes=train_iter.provide_data, label_shapes=train_iter.provide_label) +# initialize parameters by uniform random numbers +mod.init_params(initializer=mx.init.Uniform(scale=.1)) +# use SGD with learning rate 0.1 to train +mod.init_optimizer(optimizer='sgd', optimizer_params=(('learning_rate', 0.1), )) +# use accuracy as the metric +metric = mx.metric.create('acc') +# train 5 epochs, i.e. going over the data iter one pass +for epoch in range(5): + train_iter.reset() + metric.reset() + for batch in train_iter: + mod.forward(batch, is_train=True) # compute predictions + mod.update_metric(metric, batch.label) # accumulate prediction accuracy + mod.backward() # compute gradients + mod.update() # update parameters + print('Epoch %d, Training %s' % (epoch, metric.get())) +``` + +To learn more about these APIs, visit [Module API](http://mxnet.io/api/python/module.html). + +## High-level Interface + +### Train + +Module also provides high-level APIs for training, predicting and evaluating for +user convenience. Instead of doing all the steps mentioned in the above section, +one can simply call [fit API](http://mxnet.io/api/python/module.html#mxnet.module.BaseModule.fit) +and it internally executes the same steps. -Module provides high-level APIs for training, predicting and evaluating. -To fit a module, simply call the `fit` function. +To fit a module, call the `fit` function as follows: ```python +# reset train_iter to the beginning +train_iter.reset() + +# create a module +mod = mx.mod.Module(symbol=net, + context=mx.cpu(), + data_names=['data'], + label_names=['softmax_label']) + +# fit the module mod.fit(train_iter, eval_data=val_iter, optimizer='sgd', @@ -84,7 +149,12 @@ mod.fit(train_iter, num_epoch=8) ``` -To predict with module, simply call `predict()`. It will collect and +By default, `fit` function has `eval_metric` set to `accuracy`, `optimizer` to `sgd` +and optimizer_params to `(('learning_rate', 0.01),)`. + +### Predict and Evaluate + +To predict with module, we can call `predict()`. It will collect and return all the prediction results. ```python @@ -93,12 +163,23 @@ assert y.shape == (4000, 26) ``` If we do not need the prediction outputs, but just need to evaluate on a test -set, we can call the `score()` function: +set, we can call the `score()` function. It runs prediction in the input validation +dataset and evaluates the performance according to the given input metric. + +It can be used as follows: ```python -mod.score(val_iter, ['mse', 'acc']) +score = mod.score(val_iter, ['mse', 'acc']) +print "Accuracy score is ", score ``` +Some of the other metrics which can be used are `top_k_acc`(top-k-accuracy), +`F1`, `RMSE`, `MSE`, `MAE`, `ce`(CrossEntropy). To learn more about the metrics, +visit [Evaluation metric](http://mxnet.io/api/python/metric.html). + +One can vary number of epochs, learning_rate, optimizer parameters to change the score +and tune these parameters to get best score. + ### Save and Load We can save the module parameters after each training epoch by using a checkpoint callback. @@ -139,34 +220,4 @@ mod.fit(train_iter, begin_epoch=3) ``` -## Intermediate-level Interface - -We already saw how to use module for basic training and inference. Now we are -going to see a more flexible usage of module. Instead of calling -the high-level `fit` and `predict` APIs, one can write a training program with the intermediate-level APIs -`forward` and `backward`. - -```python -# create module -mod = mx.mod.Module(symbol=net) -# allocate memory by given the input data and label shapes -mod.bind(data_shapes=train_iter.provide_data, label_shapes=train_iter.provide_label) -# initialize parameters by uniform random numbers -mod.init_params(initializer=mx.init.Uniform(scale=.1)) -# use SGD with learning rate 0.1 to train -mod.init_optimizer(optimizer='sgd', optimizer_params=(('learning_rate', 0.1), )) -# use accuracy as the metric -metric = mx.metric.create('acc') -# train 5 epochs, i.e. going over the data iter one pass -for epoch in range(5): - train_iter.reset() - metric.reset() - for batch in train_iter: - mod.forward(batch, is_train=True) # compute predictions - mod.update_metric(metric, batch.label) # accumulate prediction accuracy - mod.backward() # compute gradients - mod.update() # update parameters - print('Epoch %d, Training %s' % (epoch, metric.get())) -``` - From fd91cded595f190acdaa10a20c0a55d889871eee Mon Sep 17 00:00:00 2001 From: Naveen Swamy Date: Thu, 1 Jun 2017 22:21:37 -0700 Subject: [PATCH 020/834] add requests, graphviz used test_utils and visualization to python setup (#6533) --- python/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/setup.py b/python/setup.py index 4078a203a0b9..8a8693038b3c 100644 --- a/python/setup.py +++ b/python/setup.py @@ -11,7 +11,7 @@ else: from setuptools import setup from setuptools.extension import Extension - kwargs = {'install_requires': ['numpy'], 'zip_safe': False} + kwargs = {'install_requires': ['numpy', 'requests', 'graphviz'], 'zip_safe': False} with_cython = False if '--with-cython' in sys.argv: From 4c8aa7929d6d0a1f4420adb4023ccd84fd63cd6b Mon Sep 17 00:00:00 2001 From: "Qiang Kou (KK)" Date: Fri, 2 Jun 2017 05:23:32 +0000 Subject: [PATCH 021/834] [R] compatibility with R 3.2.0. close #6525 (#6534) --- R-package/R/model.R | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/R-package/R/model.R b/R-package/R/model.R index 2b4893b73eda..53aa78ac0b76 100644 --- a/R-package/R/model.R +++ b/R-package/R/model.R @@ -106,7 +106,7 @@ mx.model.train <- function(symbol, ctx, input.shape, output.shape, sliceinfo <- mx.model.slice.shape(input.shape, ndevice) sliceinfo2 <- mx.model.slice.shape(output.shape, ndevice) arg_names <- arguments(symbol) - label_name <- arg_names[endsWith(arg_names, "label")] + label_name <- arg_names[mx.util.str.endswith(arg_names, "label")] train.execs <- lapply(1:ndevice, function(i) { arg_lst <- list(symbol = symbol, ctx = ctx[[i]], grad.req = "write", data=sliceinfo[[i]]$shape) @@ -268,7 +268,7 @@ mx.model.train <- function(symbol, ctx, input.shape, output.shape, mx.model.init.params <- function(symbol, input.shape, output.shape, initializer, ctx) { if (!is.MXSymbol(symbol)) stop("symbol need to be MXSymbol") arg_names <- arguments(symbol) - label_name <- arg_names[endsWith(arg_names, "label")] + label_name <- arg_names[mx.util.str.endswith(arg_names, "label")] arg_lst <- list(symbol = symbol, data=input.shape) arg_lst[[label_name]] = output.shape @@ -497,7 +497,7 @@ predict.MXFeedForwardModel <- function(model, X, ctx=NULL, array.batch.size=128, if (!X$iter.next()) stop("Cannot predict on empty iterator") dlist = X$value() arg_names <- arguments(model$symbol) - label_name <- arg_names[endsWith(arg_names, "label")] + label_name <- arg_names[mx.util.str.endswith(arg_names, "label")] arg_lst <- list(symbol = model$symbol, ctx = ctx, data = dim(dlist$data), grad.req="null") arg_lst[[label_name]] <- dim(dlist$label) From 19f842cff49a98e3922c66b820642dda67022e93 Mon Sep 17 00:00:00 2001 From: Yao Wang Date: Fri, 2 Jun 2017 14:20:46 -0700 Subject: [PATCH 022/834] Fix Nightly Tutorial Test (#6544) --- tests/nightly/test_tutorial.py | 2 +- tests/nightly/test_tutorial_config.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/nightly/test_tutorial.py b/tests/nightly/test_tutorial.py index b55c809898af..5adb149d03db 100644 --- a/tests/nightly/test_tutorial.py +++ b/tests/nightly/test_tutorial.py @@ -48,7 +48,7 @@ def test_tutorial_nb(file_path): """ tutorial_name = os.path.basename(file_path) notebook = nbformat.read(file_path + '_python.ipynb', as_version=4) - eprocessor = ExecutePreprocessor(timeout=900) + eprocessor = ExecutePreprocessor(timeout=1800) try: eprocessor.preprocess(notebook, {'metadata': {}}) except Exception as err: diff --git a/tests/nightly/test_tutorial_config.txt b/tests/nightly/test_tutorial_config.txt index 629857a1cb78..428309b84c8c 100644 --- a/tests/nightly/test_tutorial_config.txt +++ b/tests/nightly/test_tutorial_config.txt @@ -4,4 +4,4 @@ basic/module basic/data python/linear-regression python/mnist -python/predict_imag +python/predict_image From e41ab865a2cee02fad63380b85d63278601ea439 Mon Sep 17 00:00:00 2001 From: "Qiang Kou (KK)" Date: Fri, 2 Jun 2017 21:21:37 +0000 Subject: [PATCH 023/834] [R] more tweaking for R 3.2.0 (#6542) --- R-package/DESCRIPTION | 4 +++- R-package/R/model.R | 15 ++++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/R-package/DESCRIPTION b/R-package/DESCRIPTION index 2c8c8aa04d8d..c879c737ca0c 100644 --- a/R-package/DESCRIPTION +++ b/R-package/DESCRIPTION @@ -2,7 +2,7 @@ Package: mxnet Type: Package Title: MXNet Version: 0.10.1 -Date: 2015-12-23 +Date: 2017-06-02 Author: Tianqi Chen, Qiang Kou, Tong He Maintainer: Qiang Kou Repository: DMLC @@ -27,6 +27,8 @@ Suggests: knitr, rmarkdown, imager +Depends: + R (>= 3.2.0) LinkingTo: Rcpp VignetteBuilder: knitr RoxygenNote: 5.0.1 diff --git a/R-package/R/model.R b/R-package/R/model.R index 53aa78ac0b76..2e9a555a3477 100644 --- a/R-package/R/model.R +++ b/R-package/R/model.R @@ -106,7 +106,10 @@ mx.model.train <- function(symbol, ctx, input.shape, output.shape, sliceinfo <- mx.model.slice.shape(input.shape, ndevice) sliceinfo2 <- mx.model.slice.shape(output.shape, ndevice) arg_names <- arguments(symbol) - label_name <- arg_names[mx.util.str.endswith(arg_names, "label")] + tmp <- unlist(lapply(arg_names, function(a) { + mxnet:::mx.util.str.endswith(a, "label") + })) + label_name <- arg_names[tmp] train.execs <- lapply(1:ndevice, function(i) { arg_lst <- list(symbol = symbol, ctx = ctx[[i]], grad.req = "write", data=sliceinfo[[i]]$shape) @@ -268,7 +271,10 @@ mx.model.train <- function(symbol, ctx, input.shape, output.shape, mx.model.init.params <- function(symbol, input.shape, output.shape, initializer, ctx) { if (!is.MXSymbol(symbol)) stop("symbol need to be MXSymbol") arg_names <- arguments(symbol) - label_name <- arg_names[mx.util.str.endswith(arg_names, "label")] + tmp <- unlist(lapply(arg_names, function(a) { + mxnet:::mx.util.str.endswith(a, "label") + })) + label_name <- arg_names[tmp] arg_lst <- list(symbol = symbol, data=input.shape) arg_lst[[label_name]] = output.shape @@ -497,7 +503,10 @@ predict.MXFeedForwardModel <- function(model, X, ctx=NULL, array.batch.size=128, if (!X$iter.next()) stop("Cannot predict on empty iterator") dlist = X$value() arg_names <- arguments(model$symbol) - label_name <- arg_names[mx.util.str.endswith(arg_names, "label")] + tmp <- unlist(lapply(arg_names, function(a) { + mxnet:::mx.util.str.endswith(a, "label") + })) + label_name <- arg_names[tmp] arg_lst <- list(symbol = model$symbol, ctx = ctx, data = dim(dlist$data), grad.req="null") arg_lst[[label_name]] <- dim(dlist$label) From 551996ae683b417bc9ae46fd0ad1dcc26effe479 Mon Sep 17 00:00:00 2001 From: Danlu Chen Date: Fri, 2 Jun 2017 17:22:11 -0400 Subject: [PATCH 024/834] fix typo on BatchNorm for mirroring (#6541) * fix bug on BatchNorm for mirroring * keep CuDNNBN --- src/executor/graph_executor.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/executor/graph_executor.cc b/src/executor/graph_executor.cc index cdbb129304b1..6f8f820e02dc 100644 --- a/src/executor/graph_executor.cc +++ b/src/executor/graph_executor.cc @@ -221,6 +221,7 @@ nnvm::Graph GraphExecutor::InitFullGraph( if (type == "FullyConnected") return false; if (type == "Concat") return false; if (type == "SoftmaxOutput") return false; + if (type == "BatchNorm") return false; if (type == "CuDNNBatchNorm") return false; return true; }; From 3959d1e725ce1edb819a5a0df48dfb9f1f29af58 Mon Sep 17 00:00:00 2001 From: Roshani Nagmote Date: Fri, 2 Jun 2017 19:25:49 -0700 Subject: [PATCH 025/834] dlpack directory added (#6550) --- amalgamation/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/amalgamation/Makefile b/amalgamation/Makefile index 2446667c1e9e..23a9c318fe31 100644 --- a/amalgamation/Makefile +++ b/amalgamation/Makefile @@ -61,8 +61,9 @@ dmlc.d: dmlc-minimum0.cc mxnet_predict0.d: mxnet_predict0.cc nnvm.d dmlc.d ${CXX} ${CFLAGS} -M -MT mxnet_predict0.o \ - -I ${MXNET_ROOT}/ -I ${MXNET_ROOT}/mshadow/ -I ${MXNET_ROOT}/dmlc-core/include \ + -I ${MXNET_ROOT}/ -I ${MXNET_ROOT}/mshadow/ -I ${MXNET_ROOT}/dmlc-core/include -I ${MXNET_ROOT}/dmlc-core/src \ -I ${MXNET_ROOT}/nnvm/include \ + -I ${MXNET_ROOT}/dlpack/include \ -I ${MXNET_ROOT}/include \ -D__MIN__=$(MIN) mxnet_predict0.cc > mxnet_predict0.d cat dmlc.d >> mxnet_predict0.d From 728d6326a1a820949eb467683405eaa7fa0c4abe Mon Sep 17 00:00:00 2001 From: jeremiedb Date: Sat, 3 Jun 2017 01:21:37 -0400 Subject: [PATCH 026/834] [R] CGAN R demo scripts. close #6040 (#6551) * add cgan R demo scripts --- example/gan/CGAN_mnist_R/CGAN_mnist_setup.R | 104 +++++++++++ example/gan/CGAN_mnist_R/CGAN_train.R | 182 ++++++++++++++++++++ example/gan/CGAN_mnist_R/iterators.R | 62 +++++++ 3 files changed, 348 insertions(+) create mode 100644 example/gan/CGAN_mnist_R/CGAN_mnist_setup.R create mode 100644 example/gan/CGAN_mnist_R/CGAN_train.R create mode 100644 example/gan/CGAN_mnist_R/iterators.R diff --git a/example/gan/CGAN_mnist_R/CGAN_mnist_setup.R b/example/gan/CGAN_mnist_R/CGAN_mnist_setup.R new file mode 100644 index 000000000000..f3ebf0fe1718 --- /dev/null +++ b/example/gan/CGAN_mnist_R/CGAN_mnist_setup.R @@ -0,0 +1,104 @@ +require("imager") +require("dplyr") +require("readr") +require("mxnet") + +source("iterators.R") + +###################################################### +### Data import and preperation +### First download MNIST train data at Kaggle: +### https://www.kaggle.com/c/digit-recognizer/data +###################################################### +train <- read_csv('data/train.csv') +train<- data.matrix(train) + +train_data <- train[,-1] +train_data <- t(train_data/255*2-1) +train_label <- as.integer(train[,1]) + +dim(train_data) <- c(28, 28, 1, ncol(train_data)) + +################################################## +#### Model parameters +################################################## +random_dim<- 96 +gen_features<- 96 +dis_features<- 32 +image_depth = 1 +fix_gamma<- T +no_bias<- T +eps<- 1e-5 + 1e-12 +batch_size<- 64 + + +################################################## +#### Generator Symbol +################################################## +data = mx.symbol.Variable('data') + +gen_rand<- mx.symbol.normal(loc=0, scale=1, shape=c(1, 1, random_dim, batch_size), name="gen_rand") +gen_concat<- mx.symbol.Concat(data = list(data, gen_rand), num.args = 2, name="gen_concat") + +g1 = mx.symbol.Deconvolution(gen_concat, name='g1', kernel=c(4,4), num_filter=gen_features*4, no_bias=T) +gbn1 = mx.symbol.BatchNorm(g1, name='gbn1', fix_gamma=fix_gamma, eps=eps) +gact1 = mx.symbol.Activation(gbn1, name='gact1', act_type='relu') + +g2 = mx.symbol.Deconvolution(gact1, name='g2', kernel=c(3,3), stride=c(2,2), pad=c(1,1), num_filter=gen_features*2, no_bias=no_bias) +gbn2 = mx.symbol.BatchNorm(g2, name='gbn2', fix_gamma=fix_gamma, eps=eps) +gact2 = mx.symbol.Activation(gbn2, name='gact2', act_type='relu') + +g3 = mx.symbol.Deconvolution(gact2, name='g3', kernel=c(4,4), stride=c(2,2), pad=c(1,1), num_filter=gen_features, no_bias=no_bias) +gbn3 = mx.symbol.BatchNorm(g3, name='gbn3', fix_gamma=fix_gamma, eps=eps) +gact3 = mx.symbol.Activation(gbn3, name='gact3', act_type='relu') + +g4 = mx.symbol.Deconvolution(gact3, name='g4', kernel=c(4,4), stride=c(2,2), pad=c(1,1), num_filter=image_depth, no_bias=no_bias) +G_sym = mx.symbol.Activation(g4, name='G_sym', act_type='tanh') + + +################################################## +#### Discriminator Symbol +################################################## +data = mx.symbol.Variable('data') +dis_digit = mx.symbol.Variable('digit') +label = mx.symbol.Variable('label') + +dis_digit<- mx.symbol.Reshape(data=dis_digit, shape=c(1,1,10,batch_size), name="digit_reshape") +dis_digit<- mx.symbol.broadcast_to(data=dis_digit, shape=c(28,28,10, batch_size), name="digit_broadcast") + +data_concat <- mx.symbol.Concat(list(data, dis_digit), num.args = 2, dim = 1, name='dflat_concat') + +d1 = mx.symbol.Convolution(data=data_concat, name='d1', kernel=c(3,3), stride=c(1,1), pad=c(0,0), num_filter=24, no_bias=no_bias) +dbn1 = mx.symbol.BatchNorm(d1, name='dbn1', fix_gamma=fix_gamma, eps=eps) +dact1 = mx.symbol.LeakyReLU(dbn1, name='dact1', act_type='elu', slope=0.25) +pool1 <- mx.symbol.Pooling(data=dact1, name="pool1", pool_type="max", kernel=c(2,2), stride=c(2,2), pad=c(0,0)) + +d2 = mx.symbol.Convolution(pool1, name='d2', kernel=c(3,3), stride=c(2,2), pad=c(0,0), num_filter=32, no_bias=no_bias) +dbn2 = mx.symbol.BatchNorm(d2, name='dbn2', fix_gamma=fix_gamma, eps=eps) +dact2 = mx.symbol.LeakyReLU(dbn2, name='dact2', act_type='elu', slope=0.25) + +d3 = mx.symbol.Convolution(dact2, name='d3', kernel=c(3,3), stride=c(1,1), pad=c(0,0), num_filter=64, no_bias=no_bias) +dbn3 = mx.symbol.BatchNorm(d3, name='dbn3', fix_gamma=fix_gamma, eps=eps) +dact3 = mx.symbol.LeakyReLU(dbn3, name='dact3', act_type='elu', slope=0.25) + +d4 = mx.symbol.Convolution(dact2, name='d3', kernel=c(4,4), stride=c(1,1), pad=c(0,0), num_filter=64, no_bias=no_bias) +dbn4 = mx.symbol.BatchNorm(d4, name='dbn4', fix_gamma=fix_gamma, eps=eps) +dact4 = mx.symbol.LeakyReLU(dbn4, name='dact4', act_type='elu', slope=0.25) + +# pool4 <- mx.symbol.Pooling(data=dact3, name="pool4", pool_type="avg", kernel=c(4,4), stride=c(1,1), pad=c(0,0)) + +dflat = mx.symbol.Flatten(dact4, name="dflat") + +dfc <- mx.symbol.FullyConnected(data=dflat, name="dfc", num_hidden=1, no_bias=F) +D_sym = mx.symbol.LogisticRegressionOutput(data=dfc, label=label, name='D_sym') + + +######################## +### Graph +######################## +input_shape_G<- c(1, 1, 10, batch_size) +input_shape_D<- c(28, 28, 1, batch_size) + +graph.viz(G_sym, type = "graph", direction = "LR") +graph.viz(D_sym, type = "graph", direction = "LR") + diff --git a/example/gan/CGAN_mnist_R/CGAN_train.R b/example/gan/CGAN_mnist_R/CGAN_train.R new file mode 100644 index 000000000000..6778d6b9c2b1 --- /dev/null +++ b/example/gan/CGAN_mnist_R/CGAN_train.R @@ -0,0 +1,182 @@ +##################################################### +### Training module for GAN +##################################################### + +devices<- mx.cpu() + +data_shape_G<- c(1, 1, 10, batch_size) +data_shape_D<- c(28, 28, 1, batch_size) +digit_shape_D<- c(10, batch_size) + +mx.metric.binacc <- mx.metric.custom("binacc", function(label, pred) { + res <- mean(label==round(pred)) + return(res) +}) + +mx.metric.logloss <- mx.metric.custom("logloss", function(label, pred) { + res <- mean(label*log(pred)+(1-label)*log(1-pred)) + return(res) +}) + +############################################## +### Define iterators +iter_G<- G_iterator(batch_size = batch_size) +iter_D<- D_iterator(batch_size = batch_size) + +exec_G<- mx.simple.bind(symbol = G_sym, data=data_shape_G, ctx = devices, grad.req = "write") +exec_D<- mx.simple.bind(symbol = D_sym, data=data_shape_D, digit=digit_shape_D, ctx = devices, grad.req = "write") + +### initialize parameters - To Do - personalise each layer +initializer<- mx.init.Xavier(rnd_type = "gaussian", factor_type = "avg", magnitude = 3) + +arg_param_ini_G<- mx.init.create(initializer = initializer, shape.array = mx.symbol.infer.shape(G_sym, data=data_shape_G)$arg.shapes, ctx = mx.cpu()) +aux_param_ini_G<- mx.init.create(initializer = initializer, shape.array = mx.symbol.infer.shape(G_sym, data=data_shape_G)$aux.shapes, ctx = mx.cpu()) + +arg_param_ini_D<- mx.init.create(initializer = initializer, shape.array = mx.symbol.infer.shape(D_sym, data=data_shape_D, digit=digit_shape_D)$arg.shapes, ctx = mx.cpu()) +aux_param_ini_D<- mx.init.create(initializer = initializer, shape.array = mx.symbol.infer.shape(D_sym, data=data_shape_D, digit=digit_shape_D)$aux.shapes, ctx = mx.cpu()) + +mx.exec.update.arg.arrays(exec_G, arg_param_ini_G, match.name=TRUE) +mx.exec.update.aux.arrays(exec_G, aux_param_ini_G, match.name=TRUE) + +mx.exec.update.arg.arrays(exec_D, arg_param_ini_D, match.name=TRUE) +mx.exec.update.aux.arrays(exec_D, aux_param_ini_D, match.name=TRUE) + +input_names_G <- mxnet:::mx.model.check.arguments(G_sym) +input_names_D <- mxnet:::mx.model.check.arguments(D_sym) + + +################################################### +#initialize optimizers +optimizer_G<-mx.opt.create(name = "adadelta", + rho=0.92, + epsilon = 1e-6, + wd=0, + rescale.grad=1/batch_size, + clip_gradient=1) + +updater_G<- mx.opt.get.updater(optimizer = optimizer_G, weights = exec_G$ref.arg.arrays) + +optimizer_D<-mx.opt.create(name = "adadelta", + rho=0.92, + epsilon = 1e-6, + wd=0, + rescale.grad=1/batch_size, + clip_gradient=1) +updater_D<- mx.opt.get.updater(optimizer = optimizer_D, weights = exec_D$ref.arg.arrays) + +#################################### +#initialize metric +metric_G<- mx.metric.binacc +metric_G_value<- metric_G$init() + +metric_D<- mx.metric.binacc +metric_D_value<- metric_D$init() + +iteration<- 1 +iter_G$reset() +iter_D$reset() + + +for (iteration in 1:2400) { + + iter_G$iter.next() + iter_D$iter.next() + + ### Random input to Generator to produce fake sample + G_values <- iter_G$value() + G_data <- G_values[input_names_G] + mx.exec.update.arg.arrays(exec_G, arg.arrays = G_data, match.name=TRUE) + mx.exec.forward(exec_G, is.train=T) + + ### Feed Discriminator with Concatenated Generator images and real images + ### Random input to Generator + D_data_fake <- exec_G$ref.outputs$G_sym_output + D_digit_fake <- G_values$data %>% mx.nd.Reshape(shape=c(-1, batch_size)) + + D_values <- iter_D$value() + D_data_real <- D_values$data + D_digit_real <- D_values$digit + + ### Train loop on fake + mx.exec.update.arg.arrays(exec_D, arg.arrays = list(data=D_data_fake, digit=D_digit_fake, label=mx.nd.array(rep(0, batch_size))), match.name=TRUE) + mx.exec.forward(exec_D, is.train=T) + mx.exec.backward(exec_D) + update_args_D<- updater_D(weight = exec_D$ref.arg.arrays, grad = exec_D$ref.grad.arrays) + mx.exec.update.arg.arrays(exec_D, update_args_D, skip.null=TRUE) + + metric_D_value <- metric_D$update(label = mx.nd.array(rep(0, batch_size)), exec_D$ref.outputs[["D_sym_output"]], metric_D_value) + + ### Train loop on real + mx.exec.update.arg.arrays(exec_D, arg.arrays = list(data=D_data_real, digit=D_digit_real, label=mx.nd.array(rep(1, batch_size))), match.name=TRUE) + mx.exec.forward(exec_D, is.train=T) + mx.exec.backward(exec_D) + update_args_D<- updater_D(weight = exec_D$ref.arg.arrays, grad = exec_D$ref.grad.arrays) + mx.exec.update.arg.arrays(exec_D, update_args_D, skip.null=TRUE) + + metric_D_value <- metric_D$update(mx.nd.array(rep(1, batch_size)), exec_D$ref.outputs[["D_sym_output"]], metric_D_value) + + ### Update Generator weights - use a seperate executor for writing data gradients + exec_D_back<- mxnet:::mx.symbol.bind(symbol = D_sym, arg.arrays = exec_D$arg.arrays, aux.arrays = exec_D$aux.arrays, grad.reqs = rep("write", length(exec_D$arg.arrays)), ctx = devices) + mx.exec.update.arg.arrays(exec_D_back, arg.arrays = list(data=D_data_fake, digit=D_digit_fake, label=mx.nd.array(rep(1, batch_size))), match.name=TRUE) + mx.exec.forward(exec_D_back, is.train=T) + mx.exec.backward(exec_D_back) + D_grads<- exec_D_back$ref.grad.arrays$data + mx.exec.backward(exec_G, out_grads=D_grads) + + update_args_G<- updater_G(weight = exec_G$ref.arg.arrays, grad = exec_G$ref.grad.arrays) + mx.exec.update.arg.arrays(exec_G, update_args_G, skip.null=TRUE) + + ### Update metrics + #metric_G_value <- metric_G$update(values[[label_name]], exec_G$ref.outputs[[output_name]], metric_G_value) + + if (iteration %% 25==0){ + D_metric_result <- metric_D$get(metric_D_value) + cat(paste0("[", iteration, "] ", D_metric_result$name, ": ", D_metric_result$value, "\n")) + } + + if (iteration==1 | iteration %% 100==0){ + + metric_D_value<- metric_D$init() + + par(mfrow=c(3,3), mar=c(0.1,0.1,0.1,0.1)) + for (i in 1:9) { + img <- as.array(exec_G$ref.outputs$G_sym_output)[,,,i] + plot(as.cimg(img), axes=F) + } + + print(as.numeric(as.array(G_values$digit))) + print(as.numeric(as.array(D_values$label))) + + } +} + +mx.symbol.save(D_sym, filename = "models/D_sym_model_v1.json") +mx.nd.save(exec_D$arg.arrays, filename = "models/D_aux_params_v1.params") +mx.nd.save(exec_D$aux.arrays, filename = "models/D_aux_params_v1.params") + +mx.symbol.save(G_sym, filename = "models/G_sym_model_v1.json") +mx.nd.save(exec_G$arg.arrays, filename = "models/G_arg_params_v1.params") +mx.nd.save(exec_G$aux.arrays, filename = "models/G_aux_params_v1.params") + + +### Inference +G_sym<- mx.symbol.load("models/G_sym_model_v1.json") +G_arg_params<- mx.nd.load("models/G_arg_params_v1.params") +G_aux_params<- mx.nd.load("models/G_aux_params_v1.params") + +digit<- mx.nd.array(rep(9, times=batch_size)) +data<- mx.nd.one.hot(indices = digit, depth = 10) +data<- mx.nd.reshape(data = data, shape = c(1,1,-1, batch_size)) + +exec_G<- mx.simple.bind(symbol = G_sym, data=data_shape_G, ctx = devices, grad.req = "null") +mx.exec.update.arg.arrays(exec_G, G_arg_params, match.name=TRUE) +mx.exec.update.arg.arrays(exec_G, list(data=data), match.name=TRUE) +mx.exec.update.aux.arrays(exec_G, G_aux_params, match.name=TRUE) + +mx.exec.forward(exec_G, is.train=F) + +par(mfrow=c(3,3), mar=c(0.1,0.1,0.1,0.1)) +for (i in 1:9) { + img <- as.array(exec_G$ref.outputs$G_sym_output)[,,,i] + plot(as.cimg(img), axes=F) +} diff --git a/example/gan/CGAN_mnist_R/iterators.R b/example/gan/CGAN_mnist_R/iterators.R new file mode 100644 index 000000000000..fa113c554b75 --- /dev/null +++ b/example/gan/CGAN_mnist_R/iterators.R @@ -0,0 +1,62 @@ + +G_iterator<- function(batch_size){ + + batch<- 0 + batch_per_epoch<-5 + + reset<- function(){ + batch<<- 0 + } + + iter.next<- function(){ + batch<<- batch+1 + if (batch>batch_per_epoch) { + return(FALSE) + } else { + return(TRUE) + } + } + + value<- function(){ + set.seed(123+batch) + digit<- mx.nd.array(sample(0:9, size = batch_size, replace = T)) + data<- mx.nd.one.hot(indices = digit, depth = 10) + data<- mx.nd.reshape(data = data, shape = c(1,1,-1, batch_size)) + return(list(data=data, digit=digit)) + } + + return(list(reset=reset, iter.next=iter.next, value=value, batch_size=batch_size, batch=batch)) +} + +D_iterator<- function(batch_size){ + + batch<- 0 + batch_per_epoch<-5 + + reset<- function(){ + batch<<- 0 + } + + iter.next<- function(){ + batch<<- batch+1 + if (batch>batch_per_epoch) { + return(FALSE) + } else { + return(TRUE) + } + } + + value<- function(){ + set.seed(123+batch) + idx<- sample(length(train_label), size = batch_size, replace = T) + data<- train_data[,,,idx, drop=F] + label<- mx.nd.array(train_label[idx]) + digit<- mx.nd.one.hot(indices = label, depth = 10) + + return(list(data=mx.nd.array(data), digit=digit, label=label)) + } + + return(list(reset=reset, iter.next=iter.next, value=value, batch_size=batch_size, batch=batch)) +} + + From 13c146369b74e732091bce1fdac1702e3d1a4aec Mon Sep 17 00:00:00 2001 From: Yao Wang Date: Fri, 2 Jun 2017 22:32:34 -0700 Subject: [PATCH 027/834] Small fix (#6552) --- tests/nightly/test_tutorial.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/nightly/test_tutorial.py b/tests/nightly/test_tutorial.py index 5adb149d03db..56f6ecd8afc5 100644 --- a/tests/nightly/test_tutorial.py +++ b/tests/nightly/test_tutorial.py @@ -79,4 +79,5 @@ def test_tutorial_nb(file_path): print "Test Summary End" print "Stats start" print "[Passed: %d of %d]" % (success_num, len(tutorial_list)) + print "Stats end" From 673a6db0b66a12f464b67bbc1befbbea72b44dfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=A2=81=E5=BE=B7=E6=BE=8E?= Date: Sat, 3 Jun 2017 13:34:22 +0800 Subject: [PATCH 028/834] [Scala] hide _move_var _move_mean _beta _gamma for visualization (#6555) * [Scala] hide _move_var _move_mean _beta _gamma for visualization * fix code style --- .../core/src/main/scala/ml/dmlc/mxnet/Visualization.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scala-package/core/src/main/scala/ml/dmlc/mxnet/Visualization.scala b/scala-package/core/src/main/scala/ml/dmlc/mxnet/Visualization.scala index b4dcb93e1856..49c66a9f8aed 100644 --- a/scala-package/core/src/main/scala/ml/dmlc/mxnet/Visualization.scala +++ b/scala-package/core/src/main/scala/ml/dmlc/mxnet/Visualization.scala @@ -216,8 +216,9 @@ object Visualization { // Internal helper to figure out if node should be hidden with hide_weights def looksLikeWeight(name: String): Boolean = { - if (name.endsWith("_weight") || name.endsWith("_bias")) true - else false + if (name.endsWith("_weight") || name.endsWith("_bias") + || name.endsWith("_beta") || name.endsWith("_gamma") + || name.endsWith("_moving_var") || name.endsWith("_moving_mean")) { true } else { false } } // make nodes From 10af1c7329cde92099f3e3a99dc0909467114cb8 Mon Sep 17 00:00:00 2001 From: reminisce Date: Fri, 2 Jun 2017 23:58:22 -0700 Subject: [PATCH 029/834] Improve symbol bindings (#5870) * Initial checkin Add init functions for simple bind in graph_executor Add simple_bind c_api Add simple bind c-api Assign zeros to in_args, arg_grads, and aux_states Add simple_bind2 python interface Fix python interface bugs Interface changes Fix Fix core dump Add bind_ith_exec c_api Change simple_bind2 Fix seg fault Finish simple_bind Change _bind_ith_exec Refactor simple_bind initialization flow for bind Consolidate bind and simple_bind graph init flow Fix bug Clean up Add comments Clean up Clean up Minor correction Rename APIs in graph executor Refactor Rebase Delete deprecated functions Move more front-end work to backend Bug fix Fix failed tests Minor fix Fix lint Fix lint Revert unnecessary changes Revert Revert Clean up Fix lint Fix bind_ith_exec calling simple_bind Fix bugs for _bind_ith_exec * Add unit test (#1) * Add unit test * Fix * Small fix * Fix lint * Fix lint * Fix bugs of missing ndarrays in shared_buffer * Fix lint * Simple bind (#3) * Add bucketing test * Skip pylint * Use cpu to train * Fix bug * Remove merge message * Fix lint * Add logging to test_bucketing.py * Reduce model size (#4) * Add checks for shape/type inferences * Add printing error messages for shape/type inference failure --- include/mxnet/c_api.h | 32 ++ include/mxnet/executor.h | 32 ++ python/mxnet/module/executor_group.py | 82 +--- python/mxnet/symbol.py | 248 +++++++--- python/mxnet/test_utils.py | 23 + src/c_api/c_api_executor.cc | 309 ++++++++++++ src/c_api/c_api_symbolic.cc | 1 - src/executor/graph_executor.cc | 636 +++++++++++++++++++++---- src/executor/graph_executor.h | 82 +++- tests/python/train/test_bucketing.py | 115 +++++ tests/python/unittest/test_executor.py | 2 +- tests/python/unittest/test_module.py | 122 +++++ 12 files changed, 1438 insertions(+), 246 deletions(-) create mode 100644 tests/python/train/test_bucketing.py diff --git a/include/mxnet/c_api.h b/include/mxnet/c_api.h index d2efdf585e88..90270f776456 100644 --- a/include/mxnet/c_api.h +++ b/include/mxnet/c_api.h @@ -1149,6 +1149,38 @@ MXNET_DLL int MXExecutorBindEX(SymbolHandle symbol_handle, NDArrayHandle *aux_states, ExecutorHandle shared_exec, ExecutorHandle *out); + +MXNET_DLL int MXExecutorSimpleBind(SymbolHandle symbol_handle, + int dev_type, + int dev_id, + const mx_uint num_g2c_keys, + const char** g2c_keys, + const int* g2c_dev_types, + const int* g2c_dev_ids, + const mx_uint provided_grad_req_list_len, + const char** provided_grad_req_names, + const char** provided_grad_req_types, + const mx_uint num_provided_arg_shapes, + const char** provided_arg_shape_names, + const mx_uint* provided_arg_shape_data, + const mx_uint* provided_arg_shape_idx, + const mx_uint num_provided_arg_dtypes, + const char** provided_arg_dtype_names, + const int* provided_arg_dtypes, + const mx_uint num_shared_arg_names, + const char** shared_arg_name_list, + int* shared_buffer_len, + const char** shared_buffer_name_list, + NDArrayHandle* shared_buffer_handle_list, + const char*** updated_shared_buffer_name_list, + NDArrayHandle** updated_shared_buffer_handle_list, + mx_uint* num_in_args, + NDArrayHandle** in_args, + NDArrayHandle** arg_grads, + mx_uint* num_aux_states, + NDArrayHandle** aux_states, + ExecutorHandle shared_exec_handle, + ExecutorHandle* out); /*! * \brief set a call back to notify the completion of operation */ diff --git a/include/mxnet/executor.h b/include/mxnet/executor.h index cf71666826ab..40bd60f5f405 100644 --- a/include/mxnet/executor.h +++ b/include/mxnet/executor.h @@ -69,6 +69,21 @@ class Executor { * \return array of outputs in the executor. */ virtual const std::vector &outputs() const = 0; + /*! + * \brief get input argument map, key is arg name, value is arg's NDArray. + * \return input argument map in the executor. + */ + virtual const std::unordered_map& in_arg_map() const = 0; + /*! + * \brief get input argument graident map, key is arg name, value is gradient's NDArray. + * \return input argument gradient map in the executor. + */ + virtual const std::unordered_map& arg_grad_map() const = 0; + /*! + * \brief get aux state map, key is arg name, value is aux state's NDArray. + * \return aux state map in the executor. + */ + virtual const std::unordered_map& aux_state_map() const = 0; /*! * \brief Create an operator by bind symbol with context and arguments. * If user do not want to compute the gradients of i-th argument, grad_req_type[i] can be kNullOp. @@ -91,6 +106,23 @@ class Executor { const std::vector &grad_req_type, const std::vector &aux_states, Executor* shared_exec = NULL); + + static Executor* SimpleBind(nnvm::Symbol symbol, + const Context& default_ctx, + const std::map& group2ctx, + const std::vector& in_arg_ctxes, + const std::vector& arg_grad_ctxes, + const std::vector& aux_state_ctxes, + const std::unordered_map& arg_shape_map, + const std::unordered_map& arg_dtype_map, + const std::vector& grad_req_types, + const std::unordered_set& param_names, + std::vector* in_args, + std::vector* arg_grads, + std::vector* aux_states, + std::unordered_map* + shared_data_arrays = nullptr, + Executor* shared_exec = nullptr); /*! * \brief the prototype of user-defined monitor callback */ diff --git a/python/mxnet/module/executor_group.py b/python/mxnet/module/executor_group.py index 74640df97f16..ce71fa3ad4e9 100755 --- a/python/mxnet/module/executor_group.py +++ b/python/mxnet/module/executor_group.py @@ -5,8 +5,6 @@ import logging from collections import OrderedDict -import numpy as np - from .. import context as ctx from .. import ndarray as nd from ..io import DataDesc @@ -564,6 +562,7 @@ def update_metric(self, eval_metric, labels): def _bind_ith_exec(self, i, data_shapes, label_shapes, shared_group): """Internal utility function to bind the i-th executor. + This function utilizes simple_bind python interface. """ shared_exec = None if shared_group is None else shared_group.execs[i] context = self.contexts[i] @@ -573,85 +572,14 @@ def _bind_ith_exec(self, i, data_shapes, label_shapes, shared_group): if label_shapes is not None: input_shapes.update(dict(label_shapes)) - arg_shapes, _, aux_shapes = self.symbol.infer_shape(**input_shapes) - assert arg_shapes is not None, "shape inference failed" - input_types = {x.name: x.dtype for x in data_shapes} if label_shapes is not None: input_types.update({x.name: x.dtype for x in label_shapes}) - arg_types, _, aux_types = self.symbol.infer_type(**input_types) - assert arg_types is not None, "type inference failed" - - arg_arrays = [] - grad_arrays = {} if self.for_training else None - - def _get_or_reshape(name, shared_data_arrays, arg_shape, arg_type, context, logger): - """Internal helper to get a memory block or re-use by re-shaping.""" - if name in shared_data_arrays: - arg_arr = shared_data_arrays[name] - if np.prod(arg_arr.shape) >= np.prod(arg_shape): - # nice, we can directly re-use this data blob - assert arg_arr.dtype == arg_type - arg_arr = arg_arr.reshape(arg_shape) - else: - logger.warning(('bucketing: data "%s" has a shape %s' % (name, arg_shape)) + - (', which is larger than already allocated ') + - ('shape %s' % (arg_arr.shape,)) + - ('. Need to re-allocate. Consider putting ') + - ('default_bucket_key to') + - (' be the bucket taking the largest input for better ') + - ('memory sharing.')) - arg_arr = nd.zeros(arg_shape, context, dtype=arg_type) - - # replace existing shared array because the new one is bigger - shared_data_arrays[name] = arg_arr - else: - arg_arr = nd.zeros(arg_shape, context, dtype=arg_type) - shared_data_arrays[name] = arg_arr - - return arg_arr - - # create or borrow arguments and gradients - for j in range(len(self.arg_names)): - name = self.arg_names[j] - if name in self.param_names: # model parameters - if shared_exec is None: - arg_arr = nd.zeros(arg_shapes[j], context, dtype=arg_types[j]) - if self.grad_req[name] != 'null': - grad_arr = nd.zeros(arg_shapes[j], context, dtype=arg_types[j]) - grad_arrays[name] = grad_arr - else: - arg_arr = shared_exec.arg_dict[name] - assert arg_arr.shape == arg_shapes[j] - assert arg_arr.dtype == arg_types[j] - if self.grad_req[name] != 'null': - grad_arrays[name] = shared_exec.grad_dict[name] - else: # data, label, or states - arg_arr = _get_or_reshape(name, shared_data_arrays, arg_shapes[j], arg_types[j], - context, self.logger) - - # data might also need grad if inputs_need_grad is True - if self.grad_req[name] != 'null': - grad_arrays[name] = _get_or_reshape('grad of ' + name, shared_data_arrays, - arg_shapes[j], arg_types[j], context, - self.logger) - - arg_arrays.append(arg_arr) - - # create or borrow aux variables - if shared_exec is None: - aux_arrays = [nd.zeros(s, context, dtype=t) for s, t in zip(aux_shapes, aux_types)] - else: - for j, arr in enumerate(shared_exec.aux_arrays): - assert aux_shapes[j] == arr.shape - assert aux_types[j] == arr.dtype - aux_arrays = shared_exec.aux_arrays[:] - - executor = self.symbol.bind(ctx=context, args=arg_arrays, - args_grad=grad_arrays, aux_states=aux_arrays, - grad_req=self.grad_req, shared_exec=shared_exec) - # Get the total bytes allocated for this executor + executor = self.symbol.simple_bind(ctx=context, grad_req=self.grad_req, + type_dict=input_types, shared_arg_names=self.param_names, + shared_exec=shared_exec, + shared_buffer=shared_data_arrays, **input_shapes) self._total_exec_bytes += int(executor.debug_str().split('\n')[-3].split()[1]) return executor diff --git a/python/mxnet/symbol.py b/python/mxnet/symbol.py index 16cbeae36531..d1f52b4b48f5 100644 --- a/python/mxnet/symbol.py +++ b/python/mxnet/symbol.py @@ -13,11 +13,11 @@ import numpy as _numpy from .base import _LIB, numeric_types -from .base import c_array, c_str, mx_uint, py_str, string_types, mx_real_t +from .base import c_array, c_str, mx_uint, py_str, string_types from .base import NDArrayHandle, ExecutorHandle, SymbolHandle, OpHandle -from .base import check_call, MXNetError, _Null # pylint: disable=unused-import +from .base import check_call, MXNetError, _Null # pylint: disable=unused-import from .context import Context, cpu -from .ndarray import NDArray, zeros as _nd_zeros, _DTYPE_NP_TO_MX, _DTYPE_MX_TO_NP +from .ndarray import NDArray, _DTYPE_NP_TO_MX, _DTYPE_MX_TO_NP from .name import NameManager # pylint: disable=unused-import from .executor import Executor from . import _symbol_internal as _internal @@ -526,7 +526,7 @@ def list_attr(self, recursive=False): pairs = ctypes.POINTER(ctypes.c_char_p)() f_handle = _LIB.MXSymbolListAttrShallow check_call(f_handle(self.handle, ctypes.byref(size), ctypes.byref(pairs))) - return {py_str(pairs[i*2]): py_str(pairs[i*2+1]) for i in range(size.value)} + return {py_str(pairs[i * 2]): py_str(pairs[i * 2 + 1]) for i in range(size.value)} def attr_dict(self): """Recursively gets all attributes from the symbol and its children. @@ -552,8 +552,8 @@ def attr_dict(self): check_call(f_handle(self.handle, ctypes.byref(size), ctypes.byref(pairs))) ret = {} for i in range(size.value): - name, key = py_str(pairs[i*2]).split('$') - val = py_str(pairs[i*2+1]) + name, key = py_str(pairs[i * 2]).split('$') + val = py_str(pairs[i * 2 + 1]) if name not in ret: ret[name] = {} ret[name][key] = val @@ -776,7 +776,7 @@ def infer_type(self, *args, **kwargs): if s is not None: s = _numpy.dtype(s).type if s not in _DTYPE_NP_TO_MX: - raise TypeError('Argument need to be one of '+str(_DTYPE_NP_TO_MX)) + raise TypeError('Argument need to be one of ' + str(_DTYPE_NP_TO_MX)) sdata.append(_DTYPE_NP_TO_MX[s]) else: sdata.append(-1) @@ -885,7 +885,7 @@ def infer_shape(self, *args, **kwargs): if len(unknowns) >= 10: unknowns.append('...') break - unknowns.append('%s: %s'%(name, str(shape))) + unknowns.append('%s: %s' % (name, str(shape))) warnings.warn( "Cannot decide shape for the following arguments " + "(0s in shape means unknown dimensions). " + @@ -1012,7 +1012,7 @@ def _infer_shape_impl(self, partial, *args, **kwargs): return (arg_shapes, out_shapes, aux_shapes) else: return (None, None, None) - # pylint: enable=too-many-locals + # pylint: enable=too-many-locals def debug_str(self): """Gets a debug string of symbol. @@ -1160,12 +1160,10 @@ def _get_ndarray_inputs(arg_key, args, arg_names, allow_missing): raise TypeError('Only accept list of NDArrays or dict of str to NDArray') return c_array(NDArrayHandle, arg_handles), arg_arrays - def simple_bind(self, ctx, - grad_req='write', - type_dict=None, - group2ctx=None, - **kwargs): - """Binds current symbol to get an executor, allocate all the arguments needed. + def simple_bind(self, ctx, grad_req='write', type_dict=None, group2ctx=None, + shared_arg_names=None, shared_exec=None, shared_buffer=None, **kwargs): + """Bind current symbol to get an executor, allocate all the arguments needed. + Allows specifying data types. This function simplifies the binding procedure. You need to specify only input data shapes. Before binding the executor, the function allocates arguments and auxiliary states @@ -1175,7 +1173,7 @@ def simple_bind(self, ctx, ---------- >>> x = mx.sym.Variable('x') >>> y = mx.sym.FullyConnected(x, num_hidden=4) - >>> exe = y.simple_bind(mx.cpu(), x=(5,4), grad_req=[]) + >>> exe = y.simple_bind(mx.cpu(), x=(5,4), grad_req='null') >>> exe.forward() [] >>> exe.outputs[0].asnumpy() @@ -1208,6 +1206,19 @@ def simple_bind(self, ctx, group2ctx : Dict of string to mx.Context The dict mapping the `ctx_group` attribute to the context assignment. + shared_arg_names : List of string + The argument names whose `NDArray` of shared_exec can be reused for initializing + the current executor. + + shared_exec : Executor + The executor whose arg_arrays, arg_arrays, grad_arrays, and aux_arrays can be + reused for initializing the current executor. + + shared_buffer : Dict of string to `NDArray` + The dict mapping argument names to the `NDArray` that can be reused for initializing + the current executor. This buffer will be checked for reuse if one argument name + of the current executor is not found in `shared_arg_names`. + kwargs : Dict of str->shape Input shape dictionary, name->shape @@ -1216,47 +1227,172 @@ def simple_bind(self, ctx, executor : mxnet.Executor The generated executor """ - # pylint: disable=too-many-locals - if type_dict is None: - attrs = self.attr_dict() - type_dict = {k: mx_real_t for k in self.list_arguments() - if k not in attrs or '__dtype__' not in attrs[k]} - arg_shapes, _, aux_shapes = self.infer_shape(**kwargs) - arg_types, _, aux_types = self.infer_type(**type_dict) - - if arg_shapes is None or arg_types is None: - raise ValueError("Input node is not complete") - + num_provided_arg_types = 0 + provided_arg_type_names = ctypes.POINTER(ctypes.c_char_p)() # provided type argument names + provided_arg_type_data = ctypes.POINTER(mx_uint)() # provided types + if type_dict is not None: + provided_arg_type_names = [] + provided_arg_type_data = [] + for k, v in type_dict.items(): + v = _numpy.dtype(v).type + if v in _DTYPE_NP_TO_MX: + provided_arg_type_names.append(c_str(k)) + provided_arg_type_data.append(ctypes.c_int(_DTYPE_NP_TO_MX[v])) + num_provided_arg_types = mx_uint(len(provided_arg_type_names)) + provided_arg_type_names = c_array(ctypes.c_char_p, provided_arg_type_names) + provided_arg_type_data = c_array(ctypes.c_int, provided_arg_type_data) + + provided_arg_shape_data = [] # shape data + # argument shape index in sdata, + # e.g. [sdata[indptr[0]], sdata[indptr[1]]) is the shape of the first arg + provided_arg_shape_idx = [0] + provided_arg_shape_names = [] # provided argument names + for k, v in kwargs.items(): + # if k not in listed_arguments and k not in listed_aux_states: + # raise ValueError('arg name %s is not valid', k) + if isinstance(v, tuple): + provided_arg_shape_names.append(c_str(k)) + provided_arg_shape_data.extend(v) + provided_arg_shape_idx.append(len(provided_arg_shape_data)) + + provided_req_type_list_len = 0 + provided_grad_req_types = ctypes.POINTER(ctypes.c_char_p)() + provided_grad_req_names = ctypes.POINTER(ctypes.c_char_p)() + if grad_req is not None: + if isinstance(grad_req, string_types): + # use provided_req_type_list_len = 0 to indicate this situation + provided_req_type_list_len = 0 + provided_grad_req_types = [c_str(grad_req)] + elif isinstance(grad_req, list): + if len(grad_req) == 0: + raise RuntimeError('grad_req in simple_bind cannot be an empty list') + provided_grad_req_types = [c_str(item) for item in grad_req] + provided_req_type_list_len = len(provided_grad_req_types) + elif isinstance(grad_req, dict): + if len(grad_req) == 0: + raise RuntimeError('grad_req in simple_bind cannot be an empty dict') + provided_grad_req_names = [] + provided_grad_req_types = [] + for k, v in grad_req.items(): + provided_grad_req_names.append(c_str(k)) + provided_grad_req_types.append(c_str(v)) + provided_grad_req_names = c_array(ctypes.c_char_p, provided_grad_req_names) + provided_req_type_list_len = len(provided_grad_req_types) + provided_grad_req_types = c_array(ctypes.c_char_p, provided_grad_req_types) + + num_ctx_map_keys = mx_uint(0) + ctx_map_keys = ctypes.POINTER(ctypes.c_char_p)() + ctx_map_dev_types = ctypes.POINTER(ctypes.c_int)() + ctx_map_dev_ids = ctypes.POINTER(ctypes.c_int)() if group2ctx is not None: - attr_dict = self.attr_dict() - arg_ctx = [group2ctx.get(attr_dict[name]['__ctx_group__'], ctx) \ - if name in attr_dict and '__ctx_group__' in attr_dict[name] \ - else ctx for name in self.list_arguments()] - aux_ctx = [group2ctx.get(attr_dict[name]['__ctx_group__'], ctx) \ - if name in attr_dict and '__ctx_group__' in attr_dict[name] \ - else ctx for name in self.list_auxiliary_states()] - else: - arg_ctx = [ctx] * len(arg_shapes) - aux_ctx = [ctx] * len(aux_shapes) - - # alloc space - arg_ndarrays = [ - _nd_zeros(shape, dev, dtype=dtype) - for dtype, dev, shape in zip(arg_types, arg_ctx, arg_shapes)] - if grad_req != 'null': - grad_ndarrays = {} - for name, shape, dev, dtype in zip( - self.list_arguments(), arg_shapes, arg_ctx, arg_types): - if not isinstance(grad_req, dict) or grad_req[name] != 'null': - grad_ndarrays[name] = _nd_zeros(shape, dev, dtype=dtype) + ctx_map_keys = [] + ctx_map_dev_types = [] + ctx_map_dev_ids = [] + for key, val in group2ctx.items(): + ctx_map_keys.append(c_str(key)) + ctx_map_dev_types.append(ctypes.c_int(val.device_typeid)) + ctx_map_dev_ids.append(ctypes.c_int(val.device_id)) + num_ctx_map_keys = mx_uint(len(ctx_map_keys)) + ctx_map_keys = c_array(ctypes.c_char_p, ctx_map_keys) + ctx_map_dev_types = c_array(ctypes.c_int, ctx_map_dev_types) + ctx_map_dev_ids = c_array(ctypes.c_int, ctx_map_dev_ids) + + # prepare param names + shared_arg_name_list = [] + if shared_arg_names is not None: + if not isinstance(shared_arg_names, list): + raise ValueError('shared_arg_names in simple_bind must be a list or None') + shared_arg_name_list = [c_str(name) for name in shared_arg_names] + + # prepare shared_buffer + if shared_buffer is None: + shared_buffer_len = ctypes.c_int(-1) + shared_buffer_names = ctypes.POINTER(ctypes.c_char_p)() + shared_buffer_handles = ctypes.POINTER(NDArrayHandle)() else: - grad_ndarrays = None + if not isinstance(shared_buffer, dict): + raise ValueError('shared_buffer in simple_bind must be dict or None') + shared_buffer_names = [] + shared_buffer_handles = [] + for k, v in shared_buffer.items(): + shared_buffer_names.append(c_str(k)) + shared_buffer_handles.append(v.handle) + shared_buffer_names = c_array(ctypes.c_char_p, shared_buffer_names) + shared_buffer_len = ctypes.c_int(len(shared_buffer_handles)) + shared_buffer_handles = c_array(NDArrayHandle, shared_buffer_handles) + updated_shared_buffer_names = ctypes.POINTER(ctypes.c_char_p)() + updated_shared_buffer_handles = ctypes.POINTER(NDArrayHandle)() + + # prepare shared_exec_handle + shared_exec_handle = shared_exec.handle if shared_exec is not None else ExecutorHandle() + + # prepare current executor handle + exe_handle = ExecutorHandle() + + # prepare current executor's in_args, arg_grads, and aux_states + num_in_args = ctypes.c_uint() + in_arg_handles = ctypes.POINTER(NDArrayHandle)() + arg_grad_handles = ctypes.POINTER(NDArrayHandle)() + num_aux_states = ctypes.c_uint() + aux_state_handles = ctypes.POINTER(NDArrayHandle)() - aux_ndarrays = [_nd_zeros(shape, dev, dtype=dtype) - for shape, dev, dtype in zip(aux_shapes, aux_ctx, aux_types)] - executor = self.bind(ctx, arg_ndarrays, - grad_ndarrays, grad_req, aux_ndarrays, - group2ctx=group2ctx) + try: + check_call(_LIB.MXExecutorSimpleBind(self.handle, + ctypes.c_int(ctx.device_typeid), + ctypes.c_int(ctx.device_id), + num_ctx_map_keys, + ctx_map_keys, + ctx_map_dev_types, + ctx_map_dev_ids, + mx_uint(provided_req_type_list_len), + provided_grad_req_names, + provided_grad_req_types, + mx_uint(len(provided_arg_shape_names)), + c_array(ctypes.c_char_p, provided_arg_shape_names), + c_array(mx_uint, provided_arg_shape_data), + c_array(mx_uint, provided_arg_shape_idx), + num_provided_arg_types, + provided_arg_type_names, + provided_arg_type_data, + mx_uint(len(shared_arg_name_list)), + c_array(ctypes.c_char_p, shared_arg_name_list), + ctypes.byref(shared_buffer_len), + shared_buffer_names, + shared_buffer_handles, + ctypes.byref(updated_shared_buffer_names), + ctypes.byref(updated_shared_buffer_handles), + ctypes.byref(num_in_args), + ctypes.byref(in_arg_handles), + ctypes.byref(arg_grad_handles), + ctypes.byref(num_aux_states), + ctypes.byref(aux_state_handles), + shared_exec_handle, + ctypes.byref(exe_handle))) + except MXNetError: + print("simple_bind error. Arguments:") + for k, v in kwargs.items(): + print(" %s: %s" % (k, v)) + raise RuntimeError('simple_bind failed') + + # update shared_buffer + if shared_buffer is not None: + for i in range(shared_buffer_len.value): + k = py_str(updated_shared_buffer_names[i]) + v = NDArray(NDArrayHandle(updated_shared_buffer_handles[i])) + shared_buffer[k] = v + + # create in_args, arg_grads, and aux_states for the current executor + arg_arrays = [NDArray(NDArrayHandle(in_arg_handles[i])) for i in range(num_in_args.value)] + grad_arrays = [NDArray(NDArrayHandle(arg_grad_handles[i])) + if arg_grad_handles[i] is not None + else None for i in range(num_in_args.value)] + aux_arrays = [NDArray(NDArrayHandle(aux_state_handles[i])) + for i in range(num_aux_states.value)] + + executor = Executor(exe_handle, self, ctx, grad_req, group2ctx) + executor.arg_arrays = arg_arrays + executor.grad_arrays = grad_arrays + executor.aux_arrays = aux_arrays return executor def bind(self, ctx, args, args_grad=None, grad_req='write', @@ -1441,6 +1577,7 @@ def grad(self, wrt): c_wrt, ctypes.byref(handle))) return Symbol(handle) + # pylint: enable= no-member def eval(self, ctx=cpu(), **kwargs): @@ -1500,7 +1637,6 @@ def reshape(self, shape): """ return reshape(self, shape=shape) - def var(name, attr=None, shape=None, lr_mult=None, wd_mult=None, dtype=None, init=None, **kwargs): """Creates a symbolic variable with specified name. @@ -1565,9 +1701,11 @@ def var(name, attr=None, shape=None, lr_mult=None, wd_mult=None, dtype=None, ini ret._set_attr(**attr) return ret + # for back compatibility Variable = var + def Group(symbols): """Creates a symbol that contains a collection of other symbols, grouped together. diff --git a/python/mxnet/test_utils.py b/python/mxnet/test_utils.py index 6089edae5a56..3ab44d0917a1 100644 --- a/python/mxnet/test_utils.py +++ b/python/mxnet/test_utils.py @@ -1020,3 +1020,26 @@ def set_env_var(key, val, default_val=""): prev_val = os.environ.get(key, default_val) os.environ[key] = val return prev_val + +def same_array(array1, array2): + """Check whether two NDArrays sharing the same memory block + + Parameters + ---------- + + array1 : NDArray + First NDArray to be checked + array2 : NDArray + Second NDArray to be checked + + Returns + ------- + bool + Whether two NDArrays share the same memory + """ + array1[:] += 1 + if not same(array1.asnumpy(), array2.asnumpy()): + array1[:] -= 1 + return False + array1[:] -= 1 + return same(array1.asnumpy(), array2.asnumpy()) diff --git a/src/c_api/c_api_executor.cc b/src/c_api/c_api_executor.cc index ce765acd77bf..ca49402ecf7e 100644 --- a/src/c_api/c_api_executor.cc +++ b/src/c_api/c_api_executor.cc @@ -154,6 +154,315 @@ int MXExecutorBindEX(SymbolHandle symbol_handle, API_END_HANDLE_ERROR(delete exec); } +/*! + * \brief + * \param symbol_handle symbol handle + * \param dev_type default device type + * \param dev_id default device id + * \param num_g2c_keys number of group2ctx keys + * \param g2c_keys key list of group2ctx + * \param g2c_dev_types device type list of group2ctx + * \param g2c_dev_ids id list of group2ctx + * \param provided_grad_req_list_len grad_req length provided by users in front-end + * \param provided_grad_req_names grad_req names provided by users in front-end + * \param provided_grad_req_types req types provided by users in front-end + * \param num_provided_arg_shapes number of user provided in_arg and aux_state shapes + * \param provided_arg_shape_names name list of provided shapes + * \param provided_arg_shape_data provided shape data + * \param provided_arg_shape_idx provided shape data index + * \param num_provided_arg_dtypes number of user provided in_arg and axu_state dtypes + * \param provided_arg_dtype_names argument name list of provided dtypes + * \param provided_arg_dtypes data of provided dtypes + * \param num_shared_arg_names number of parameter names passed from _bind_ith_exec + * \param shared_arg_name_list parameter name list passed from _bind_ith_exec + * \param shared_buffer_len number of shared data arrays passed from _bind_ith_exec + * \param shared_buffer_name_list shared data array names passed from _bind_ith_exec + * \param shared_buffer_handle_list shared data array handles passed from _bind_ith_exec + * \param updated_shared_buffer_name_list updated shared data array names after binding + * \param updated_shared_buffer_handle_list updated shared data arrays after binding + * \param num_in_args number of input arguments of this sym + * \param in_args list_arguments associated with the current executor + * \param arg_grads list of gradients of in_args associated with the current executor + * \param num_aux_states number of aux states of this sym + * \param aux_states list_auxiliary_states associated with the current executor + * \param shared_exec_handle shared excutor handle passed from _bind_ith_exec + * \param out the handle of the executor to be created + */ +int MXExecutorSimpleBind(SymbolHandle symbol_handle, + int dev_type, + int dev_id, + const mx_uint num_g2c_keys, + const char** g2c_keys, + const int* g2c_dev_types, + const int* g2c_dev_ids, + const mx_uint provided_grad_req_list_len, + const char** provided_grad_req_names, + const char** provided_grad_req_types, + const mx_uint num_provided_arg_shapes, + const char** provided_arg_shape_names, + const mx_uint* provided_arg_shape_data, + const mx_uint* provided_arg_shape_idx, + const mx_uint num_provided_arg_dtypes, + const char** provided_arg_dtype_names, + const int* provided_arg_dtypes, + const mx_uint num_shared_arg_names, + const char** shared_arg_name_list, + int* shared_buffer_len, + const char** shared_buffer_name_list, + NDArrayHandle* shared_buffer_handle_list, + const char*** updated_shared_buffer_name_list, + NDArrayHandle** updated_shared_buffer_handle_list, + mx_uint* num_in_args, + NDArrayHandle** in_args, + NDArrayHandle** arg_grads, + mx_uint* num_aux_states, + NDArrayHandle** aux_states, + ExecutorHandle shared_exec_handle, + ExecutorHandle* out) { + MXAPIThreadLocalEntry *ret = MXAPIThreadLocalStore::Get(); + API_BEGIN(); + nnvm::Symbol *sym = static_cast(symbol_handle); + + // get in_arg names + std::vector in_arg_names = sym->ListInputNames(nnvm::Symbol::kReadOnlyArgs); + std::vector aux_state_names = sym->ListInputNames(nnvm::Symbol::kAuxiliaryStates); + + // attr_dict for setting up type_dict and arg/aux ctx + std::unordered_map> attr_dict; + if (nullptr == provided_arg_dtypes || nullptr != g2c_keys) { + std::vector> attrs = + sym->ListAttrsRecursive(); + attr_dict.reserve(attrs.size()); + for (const auto& tp : attrs) { + attr_dict[std::get<0>(tp)][std::get<1>(tp)] = std::get<2>(tp); + } + } + + // setup arg_dtype_map + std::unordered_map arg_dtype_map; + if (nullptr == provided_arg_dtypes) { // use attr_dict + for (const auto& arg_name : in_arg_names) { + const auto it = attr_dict.find(arg_name); + if (it == attr_dict.end() || !it->second.count("__dtype__")) { + arg_dtype_map[arg_name] = mshadow::kFloat32; + } + } + } else { // use user input type_dict + // create dtype map for in_args and aux_states + arg_dtype_map.reserve(num_provided_arg_dtypes); + for (mx_uint i = 0; i < num_provided_arg_dtypes; ++i) { + arg_dtype_map[provided_arg_dtype_names[i]] = provided_arg_dtypes[i]; + } + } + + // create default ctx + Context ctx = Context::Create(static_cast(dev_type), dev_id); + // create ctx map + std::map ctx_map; + std::vector in_arg_ctx_vec(in_arg_names.size(), ctx); + std::vector aux_state_ctx_vec(aux_state_names.size(), ctx); + if (nullptr != g2c_keys) { // use user input group2ctx dict + for (mx_uint i = 0; i < num_g2c_keys; ++i) { + ctx_map[g2c_keys[i]] = Context::Create( + static_cast(g2c_dev_types[i]), g2c_dev_ids[i]); + } + + // initialize in_arg_ctx_vec using group2ctx if there are any + for (size_t i = 0; i < in_arg_ctx_vec.size(); ++i) { + const auto it1 = attr_dict.find(in_arg_names[i]); + if (it1 != attr_dict.end()) { + const auto it2 = it1->second.find("__ctx_group__"); + if (it2 != it1->second.end()) { + const auto it3 = ctx_map.find(it2->second); + if (it3 != ctx_map.end()) { + in_arg_ctx_vec[i] = it3->second; + } + } + } + } + + // initialize aux_state_ctx_vec using group2ctx if there are any + for (size_t i = 0; i < aux_state_ctx_vec.size(); ++i) { + const auto it1 = attr_dict.find(aux_state_names[i]); + if (it1 != attr_dict.end()) { + const auto it2 = it1->second.find("__ctx_group__"); + if (it2 != it1->second.end()) { + const auto it3 = ctx_map.find(it2->second); + if (it3 != ctx_map.end()) { + aux_state_ctx_vec[i] = it3->second; + } + } + } + } + } + + // create provided_grad_req_map + const std::map req_map = + {{"null", kNullOp}, {"write", kWriteTo}, {"add", kAddTo}}; + std::unordered_map provided_grad_req_map; + std::string grad_req_type; + if (0 == provided_grad_req_list_len + && nullptr == provided_grad_req_names + && nullptr != provided_grad_req_types) { // string, grad_req='write' + CHECK_EQ(req_map.count(provided_grad_req_types[0]), 1U) + << "grad_req=" << provided_grad_req_types[0] << " is not a valid input in simple_bind; " + "only \'null\', \'write\', and \'add\' are supported"; + grad_req_type = "string"; + } else if (provided_grad_req_list_len > 0 + && nullptr == provided_grad_req_names + && nullptr != provided_grad_req_types) { // list, grad_req=['null', 'write'] + grad_req_type = "list"; + CHECK_EQ(provided_grad_req_list_len, in_arg_names.size()) + << "The length of grad_req list does not match the number of input arguments in simple_bind, " + "expected " << in_arg_names.size() << ", provided " << provided_grad_req_list_len; + } else if (provided_grad_req_list_len > 0 + && nullptr != provided_grad_req_names + && nullptr != provided_grad_req_types) { // dict, grad_req=['lhs': 'null', 'rhs': 'write'] + grad_req_type = "dict"; + provided_grad_req_map.reserve(provided_grad_req_list_len); + for (mx_uint i = 0; i < provided_grad_req_list_len; ++i) { + CHECK_EQ(req_map.count(provided_grad_req_types[i]), 1U) + << "grad_req=" << provided_grad_req_types[i] << " is not a valid input in simple_bind; " + "only \'null\', \'write\', and \'add\' are supported"; + provided_grad_req_map[provided_grad_req_names[i]] = provided_grad_req_types[i]; + } + } else { // grad_req is None + grad_req_type = "none"; + } + + // initialize arg_grad_ctx_vec and grad_req_type_vec + std::vector arg_grad_ctx_vec(in_arg_names.size(), ctx); + std::vector grad_req_type_vec(in_arg_names.size(), kNullOp); + if ("none" != grad_req_type) { + for (size_t i = 0; i < in_arg_names.size(); ++i) { + OpReqType cur_req = kNullOp; + if ("string" == grad_req_type) { + cur_req = req_map.at(provided_grad_req_types[0]); + } else if ("list" == grad_req_type) { + CHECK_EQ(req_map.count(provided_grad_req_types[i]), 1U) + << "grad_req=" << provided_grad_req_types[i] << " is not a valid input in simple_bind; " + "only \'null\', \'write\', and \'add\' are supported"; + cur_req = req_map.at(provided_grad_req_types[i]); + } else if ("dict" == grad_req_type) { + const auto it = provided_grad_req_map.find(in_arg_names[i]); + if (it != provided_grad_req_map.end()) { + cur_req = req_map.at(it->second); + } + } + if (kNullOp != cur_req) { + arg_grad_ctx_vec[i] = in_arg_ctx_vec[i]; + grad_req_type_vec[i] = static_cast(cur_req); + } + } + } + + // create shape map for in_args and aux_states + std::unordered_map arg_shape_map(num_provided_arg_shapes); + for (mx_uint i = 0; i < num_provided_arg_shapes; ++i) { + auto p = arg_shape_map.emplace(provided_arg_shape_names[i], + TShape(provided_arg_shape_data+provided_arg_shape_idx[i], + provided_arg_shape_data+provided_arg_shape_idx[i+1])); + CHECK(p.second) << "Duplicate shapes are provided for argument " + << provided_arg_shape_names[i] << " in simple_bind"; + } + + // create para name set for sharing data array memory + std::unordered_set shared_arg_name_set(num_shared_arg_names); + for (mx_uint i = 0; i < num_shared_arg_names; ++i) { + shared_arg_name_set.insert(shared_arg_name_list[i]); + } + + // create shared_buffer_map + std::unordered_map shared_buffer_map; + std::vector shared_exec_in_args; + std::vector shared_exec_arg_grads; + std::vector shared_exec_aux_states; + bool use_shared_buffer = (*shared_buffer_len >= 0); + if (*shared_buffer_len > 0) { + // create shared_buffer_map + shared_buffer_map.reserve(*shared_buffer_len); + NDArray** shared_buffer_ptrs = + reinterpret_cast(shared_buffer_handle_list); + for (int i = 0; i < *shared_buffer_len; ++i) { + shared_buffer_map[shared_buffer_name_list[i]] = *(shared_buffer_ptrs[i]); + } + } + + // create temporary place holders for the initialized NDArrays + // to be passed back to front end + std::vector in_arg_vec; + std::vector arg_grad_vec; + std::vector aux_state_vec; + + *out = Executor::SimpleBind(*sym, ctx, ctx_map, in_arg_ctx_vec, arg_grad_ctx_vec, + aux_state_ctx_vec, arg_shape_map, arg_dtype_map, grad_req_type_vec, + shared_arg_name_set, &in_arg_vec, &arg_grad_vec, &aux_state_vec, + use_shared_buffer? &shared_buffer_map : nullptr, + reinterpret_cast(shared_exec_handle)); + + // copy ndarray ptrs to ret->handles so that front end + // can access them + ret->ret_handles.clear(); + ret->ret_handles.reserve(in_arg_vec.size()+arg_grad_vec.size()+aux_state_vec.size() + +shared_buffer_map.size()); + size_t nd_idx = 0; + for (const auto& nd : in_arg_vec) { + if (nd.is_none()) { + LOG(FATAL) << "Input argument NDArray cannot be un-allocated"; + } + ret->ret_handles.push_back(new NDArray(nd)); + } + if (in_arg_vec.size() > 0) { + *num_in_args = in_arg_vec.size(); + *in_args = &(ret->ret_handles[nd_idx]); + nd_idx = ret->ret_handles.size(); + } + + for (const auto& nd : arg_grad_vec) { + if (nd.is_none()) { + ret->ret_handles.push_back(nullptr); + } else { + ret->ret_handles.push_back(new NDArray(nd)); + } + } + if (arg_grad_vec.size() > 0) { + *arg_grads = &(ret->ret_handles[nd_idx]); + nd_idx = ret->ret_handles.size(); + } + + for (const auto& nd : aux_state_vec) { + if (nd.is_none()) { + LOG(FATAL) << "Auxiliary argument NDArray cannot be un-allocated"; + } + ret->ret_handles.push_back(new NDArray(nd)); + } + if (aux_state_vec.size() > 0) { + *num_aux_states = aux_state_vec.size(); + *aux_states = &(ret->ret_handles[nd_idx]); + nd_idx = ret->ret_handles.size(); + } + + if (use_shared_buffer) { + ret->ret_vec_str.clear(); + ret->ret_vec_str.reserve(shared_buffer_map.size()); + ret->ret_vec_charp.clear(); + ret->ret_vec_charp.reserve(shared_buffer_map.size()); + for (const auto& kv : shared_buffer_map) { + if (kv.second.is_none()) { + LOG(FATAL) << "Shared data NDArray cannot be un-allocated"; + } + ret->ret_handles.push_back(new NDArray(kv.second)); + ret->ret_vec_str.emplace_back(kv.first); + ret->ret_vec_charp.push_back(ret->ret_vec_str.back().c_str()); + } + *shared_buffer_len = shared_buffer_map.size(); + *updated_shared_buffer_handle_list = &(ret->ret_handles[nd_idx]); + *updated_shared_buffer_name_list = &(ret->ret_vec_charp[0]); + } + + API_END(); +} + int MXExecutorSetMonitorCallback(ExecutorHandle handle, ExecutorMonitorCallback callback, void* callback_handle) { diff --git a/src/c_api/c_api_symbolic.cc b/src/c_api/c_api_symbolic.cc index 27df5b2de1f3..cad9e604df60 100644 --- a/src/c_api/c_api_symbolic.cc +++ b/src/c_api/c_api_symbolic.cc @@ -379,7 +379,6 @@ int MXSymbolSaveToJSON(SymbolHandle symbol, const char **out_json) { API_END(); } - namespace mxnet { template diff --git a/src/executor/graph_executor.cc b/src/executor/graph_executor.cc index 6f8f820e02dc..b41d1734d946 100644 --- a/src/executor/graph_executor.cc +++ b/src/executor/graph_executor.cc @@ -78,6 +78,18 @@ const std::vector& GraphExecutor::outputs() const { return output_arrays_; } +const std::unordered_map& GraphExecutor::in_arg_map() const { + return in_arg_map_; +} + +const std::unordered_map& GraphExecutor::arg_grad_map() const { + return arg_grad_map_; +} + +const std::unordered_map& GraphExecutor::aux_state_map() const { + return aux_state_map_; +} + nnvm::NodeEntry AttrHint(nnvm::NodeEntry src, nnvm::NodeEntry like) { static const Op* id_like = Op::Get("_identity_with_attr_like_rhs"); nnvm::NodePtr n = nnvm::Node::Create(); @@ -178,10 +190,12 @@ inline ValueType get_node_attr( } } -nnvm::Graph GraphExecutor::InitFullGraph( - nnvm::Symbol symbol, - const std::vector& grad_req_type, - const std::vector& arg_grad_store) { +/*! + * \brief Create the graph for backward pass. + * This is triggered by both simple_bind and bind flows. + */ +nnvm::Graph GraphExecutor::InitFullGraph(nnvm::Symbol symbol, + const std::vector& grad_req_types) { using nnvm::NodePtr; using nnvm::NodeEntry; // initial information @@ -191,7 +205,7 @@ nnvm::Graph GraphExecutor::InitFullGraph( nnvm::Graph g; g.outputs = symbol.outputs; bool need_grad = false; - for (OpReqType req : grad_req_type) { + for (OpReqType req : grad_req_types) { if (req != kNullOp) need_grad = true; } if (!need_grad) return g; @@ -202,10 +216,8 @@ nnvm::Graph GraphExecutor::InitFullGraph( } std::vector args = symbol.ListInputs(nnvm::Symbol::kReadOnlyArgs); std::vector xs; - for (size_t i = 0; i < grad_req_type.size(); ++i) { - if (grad_req_type[i] != kNullOp) { - grad_store_.emplace_back( - std::make_pair(grad_req_type[i], arg_grad_store[i])); + for (size_t i = 0; i < grad_req_types.size(); ++i) { + if (grad_req_types[i] != kNullOp) { xs.emplace_back(NodeEntry{args[i], 0, 0}); } } @@ -242,13 +254,16 @@ nnvm::Graph GraphExecutor::InitFullGraph( return g; } -// pass to assign context to the graph +/*! + * \brief Assign context to the graph. + * This is triggered by both simple_bind and bind flows. + */ Graph AssignContext(Graph g, const Context& default_ctx, const std::map& ctx_map, - const std::vector& in_args, - const std::vector >& grad_store, - const std::vector& aux_states, + const std::vector& in_arg_ctxes, + const std::vector& arg_grad_ctxes, + const std::vector& aux_state_ctxes, size_t num_forward_inputs, size_t num_forward_outputs) { const auto& idx = g.indexed_graph(); @@ -257,56 +272,65 @@ Graph AssignContext(Graph g, if (ctx_map.size() == 0) { g.attrs["context"] = std::make_shared( ContextVector(idx.num_nodes(), default_ctx)); - for (const auto& x : in_args) { - CHECK(x.ctx() == default_ctx) - << "Input array is in " << x.ctx() << " while binding with ctx=" << default_ctx + for (const auto& x : in_arg_ctxes) { + CHECK(x == default_ctx) + << "Input array is in " << x << " while binding with ctx=" << default_ctx << ". All arguments must be in global context (" << default_ctx << ") unless group2ctx is specified for cross-device graph."; } - for (const auto& x : grad_store) { - CHECK(x.second.ctx() == default_ctx) - << "Gradient array is in " << x.second.ctx() << " while binding with ctx=" + for (const auto& x : arg_grad_ctxes) { + CHECK(x == default_ctx) + << "Gradient array is in " << x << " while binding with ctx=" << default_ctx << ". All gradients must be in global context (" << default_ctx << ") unless group2ctx is specified for cross-device graph."; } return g; } + // otherwise, use context assignment. - std::map ctx2id; - std::vector ctx_list; - nnvm::DeviceVector device(idx.num_nodes(), -1); - nnvm::DeviceAssignMap device_map; + std::map ctx2id; // map ctx to device id + std::vector ctx_list; // index is device id + nnvm::DeviceVector device(idx.num_nodes(), -1); // index is node id + nnvm::DeviceAssignMap device_map; // map arg name to device id + // loop through the user input ctx_map and + // populate maps and lists for (auto &kv : ctx_map) { - if (ctx2id.count(kv.second) == 0) { - ctx2id[kv.second] = static_cast(ctx_list.size()); - ctx_list.push_back(kv.second); + if (ctx2id.count(kv.second) == 0) { // if context has no device id, create one + ctx2id[kv.second] = static_cast(ctx_list.size()); // assign device id to ctx + ctx_list.push_back(kv.second); // save ctx to the list } + // assign device id to to the arg name with the corresponding ctx device_map[kv.first] = ctx2id.at(kv.second); } + // loop through all the rest of input nodes not specified + // in the ctx_map and populate maps and lists size_t arg_top = 0, aux_top = 0; for (size_t i = 0; i < num_forward_inputs; ++i) { const uint32_t nid = idx.input_nodes().at(i); Context ctx; - if (mutable_nodes.count(nid)) { - CHECK_LT(aux_top, aux_states.size()); - ctx = aux_states[aux_top].ctx(); + if (mutable_nodes.count(nid)) { // aux node is mutable + CHECK_LT(aux_top, aux_state_ctxes.size()); + ctx = aux_state_ctxes[aux_top]; ++aux_top; - } else { - CHECK_LT(arg_top, in_args.size()); - ctx = in_args[arg_top].ctx(); + } else { // regular input node is immutable + CHECK_LT(arg_top, in_arg_ctxes.size()); + ctx = in_arg_ctxes[arg_top]; ++arg_top; } - if (ctx2id.count(ctx) == 0) { - ctx2id[ctx] = static_cast(ctx_list.size()); - ctx_list.push_back(ctx); + if (ctx2id.count(ctx) == 0) { // if the current ctx is not in the map of ctx and device id + ctx2id[ctx] = static_cast(ctx_list.size()); // assign the current ctx with device id + ctx_list.push_back(ctx); // save the current ctx in the list } - device[nid] = ctx2id.at(ctx); + device[nid] = ctx2id.at(ctx); // assign device id to the current node } + + // loop through backward input nodes and populate maps and lists + // the backward input nodes is the gradient of the loss wrt the output for (size_t i = num_forward_outputs; i < g.outputs.size(); ++i) { const uint32_t nid = idx.outputs()[i].node_id; - Context ctx = grad_store[i - num_forward_outputs].second.ctx(); + Context ctx = arg_grad_ctxes[i - num_forward_outputs]; if (ctx2id.count(ctx) == 0) { ctx2id[ctx] = static_cast(ctx_list.size()); ctx_list.push_back(ctx); @@ -318,6 +342,7 @@ Graph AssignContext(Graph g, device[nid] = devid; } } + g.attrs["device"] = std::make_shared(std::move(device)); g = nnvm::pass::PlaceDevice(g, "__ctx_group__", device_map, "_CrossDeviceCopy"); const auto& assigned_device = g.GetAttr("device"); @@ -334,27 +359,366 @@ Graph AssignContext(Graph g, return g; } +void HandleInferShapeError(const size_t num_forward_inputs, + const nnvm::IndexedGraph& idx, + const nnvm::ShapeVector& inferred_shapes) { + int cnt = 10; + std::ostringstream oss; + for (size_t i = 0; i < num_forward_inputs; ++i) { + const uint32_t nid = idx.input_nodes().at(i); + const uint32_t eid = idx.entry_id(nid, 0); + const TShape& inferred_shape = inferred_shapes[eid]; + if (inferred_shape.ndim() == 0 || inferred_shape.Size() == 0U) { + const std::string& arg_name = idx[nid].source->attrs.name; + oss << arg_name << ": " << inferred_shape << ", "; + if (--cnt == 0) { + oss << "..."; + break; + } + } + } + LOG(FATAL) << "InferShape pass cannot decide shapes for the following arguments " + "(0s in shapes mean unknown dimension size). Please consider " + "providing them as inputs:\n" + << oss.str(); +} + +void HandleInferTypeError(const size_t num_forward_inputs, + const nnvm::IndexedGraph& idx, + const nnvm::DTypeVector& inferred_dtypes) { + int cnt = 10; + std::ostringstream oss; + for (size_t i = 0; i < num_forward_inputs; ++i) { + const uint32_t nid = idx.input_nodes().at(i); + const uint32_t eid = idx.entry_id(nid, 0); + const int inferred_dtype = inferred_dtypes[eid]; + if (inferred_dtype == -1) { + const std::string& arg_name = idx[nid].source->attrs.name; + oss << arg_name << ": " << inferred_dtype << ", "; + if (--cnt == 0) { + oss << "..."; + break; + } + } + } + LOG(FATAL) << "InferType pass cannot decide dtypes for the following arguments " + "(-1 means unknown dtype). Please consider providing them as inputs:\n" + << oss.str(); +} + +/*! + * \brief GraphExecutor initializer for regular bind flow in which + * input arguments and gradients are provided by users. This initializer + * uses the user provided NDArrays to populate data entries of the graph. + */ void GraphExecutor::Init(nnvm::Symbol symbol, const Context& default_ctx, const std::map& ctx_map, const std::vector& in_args, const std::vector& arg_grad_store, - const std::vector& grad_req_type, + const std::vector& grad_req_types, const std::vector& aux_states, Executor* shared_exec, const nnvm::NodeEntryMap& feed_dict) { - nnvm::Graph g = InitGraph(symbol, default_ctx, - ctx_map, in_args, arg_grad_store, - grad_req_type, aux_states, feed_dict); + // create in_arg_ctxes, arg_grad_ctxes, aux_state_ctxes + auto get_ctx1 = [](const NDArray& nd) { return nd.ctx(); }; + auto get_ctx2 = [default_ctx](const NDArray& nd) -> Context { + if (nd.is_none()) return default_ctx; + return nd.ctx(); + }; + std::vector in_arg_ctxes(in_args.size()); + std::transform(in_args.begin(), in_args.end(), in_arg_ctxes.begin(), get_ctx1); + std::vector arg_grad_ctxes(arg_grad_store.size()); + std::transform(arg_grad_store.begin(), arg_grad_store.end(), arg_grad_ctxes.begin(), get_ctx2); + std::vector aux_state_ctxes(aux_states.size()); + std::transform(aux_states.begin(), aux_states.end(), aux_state_ctxes.begin(), get_ctx1); + + nnvm::Graph g = InitGraph(symbol, default_ctx, ctx_map, in_arg_ctxes, + arg_grad_ctxes, aux_state_ctxes, grad_req_types); + + // create arg_shapes and arg_dtypes for shape and type inferences + const auto& idx = g.indexed_graph(); + const auto& mutable_nodes = idx.mutable_input_nodes(); + size_t arg_top = 0, aux_top = 0; + data_entry_.resize(idx.num_node_entries()); + nnvm::ShapeVector arg_shapes; + nnvm::DTypeVector arg_dtypes; + for (size_t i = 0; i < num_forward_inputs_; ++i) { + const uint32_t nid = idx.input_nodes().at(i); + const std::string& arg_name = idx[nid].source->attrs.name; + if (mutable_nodes.count(nid)) { + CHECK_LT(aux_top, aux_states.size()); + data_entry_[idx.entry_id(nid, 0)] = aux_states[aux_top]; + arg_shapes.push_back(aux_states[aux_top].shape()); + arg_dtypes.push_back(aux_states[aux_top].dtype()); + aux_state_map_.emplace(arg_name, aux_states[aux_top]); + ++aux_top; + } else { + CHECK_LT(arg_top, in_args.size()); + data_entry_[idx.entry_id(nid, 0)] = in_args[arg_top]; + arg_shapes.push_back(in_args[arg_top].shape()); + arg_dtypes.push_back(in_args[arg_top].dtype()); + in_arg_map_.emplace(arg_name, in_args[arg_top]); + if (kNullOp != grad_req_types[arg_top]) { + grad_store_.emplace_back(grad_req_types[arg_top], arg_grad_store[arg_top]); + arg_grad_map_.emplace(arg_name, arg_grad_store[arg_top]); + } + ++arg_top; + } + } + + // expand arg_shapes and arg_dtypes to contain backward inputs + arg_shapes.resize(idx.input_nodes().size(), TShape()); + g = nnvm::pass::InferShape(g, arg_shapes, "__shape__"); + if (g.GetAttr("shape_num_unknown_nodes") != 0U) { + HandleInferShapeError(num_forward_inputs_, g.indexed_graph(), + g.GetAttr("shape")); + } + + arg_dtypes.resize(idx.input_nodes().size(), -1); + g = nnvm::pass::InferType(g, arg_dtypes, "__dtype__"); + if (g.GetAttr("dtype_num_unknown_nodes") != 0U) { + HandleInferTypeError(num_forward_inputs_, g.indexed_graph(), + g.GetAttr("dtype")); + } + + // Initialize the rest attributes of the graph. + // This function can be called by regular bind + // operation flow as well. + FinishInitGraph(symbol, g, shared_exec, feed_dict); +} + +/*! + * \brief Initialize in_args, arg_grads, and aux_states + * and their data_entry_ of the executor. This function + * is called for regular simple_bind flow, i.e. no + * shared data arrays are provided. + */ +void GraphExecutor::InitArguments(const nnvm::IndexedGraph& idx, + const nnvm::ShapeVector& inferred_shapes, + const nnvm::DTypeVector& inferred_dtypes, + const std::vector& in_arg_ctxes, + const std::vector& arg_grad_ctxes, + const std::vector& aux_state_ctxes, + const std::vector& grad_req_types, + std::vector* in_arg_vec, + std::vector* arg_grad_vec, + std::vector* aux_state_vec) { + // initialize in_args, arg_grads, and aux_states + // populate grad_store_ + data_entry_.resize(idx.num_node_entries()); + size_t arg_top = 0, aux_top = 0; + const auto& mutable_nodes = idx.mutable_input_nodes(); + for (size_t i = 0; i < num_forward_inputs_; ++i) { + const uint32_t nid = idx.input_nodes().at(i); + const uint32_t eid = idx.entry_id(nid, 0); + const TShape& inferred_shape = inferred_shapes[eid]; + const int inferred_dtype = inferred_dtypes[eid]; + const std::string& arg_name = idx[nid].source->attrs.name; + if (mutable_nodes.count(nid)) { // aux_states + aux_state_vec->emplace_back(inferred_shape, aux_state_ctxes[aux_top], false, inferred_dtype); + aux_state_vec->back() = 0; + data_entry_[eid] = aux_state_vec->back(); + aux_state_map_.emplace(arg_name, aux_state_vec->back()); + ++aux_top; + } else { // in_args + in_arg_vec->emplace_back(inferred_shape, in_arg_ctxes[arg_top], false, inferred_dtype); + in_arg_vec->back() = 0; + data_entry_[eid] = in_arg_vec->back(); + if (kNullOp == grad_req_types[arg_top]) { + arg_grad_vec->emplace_back(); + } else { + arg_grad_vec->emplace_back(inferred_shape, arg_grad_ctxes[arg_top], false, inferred_dtype); + arg_grad_vec->back() = 0; + grad_store_.emplace_back(grad_req_types[arg_top], arg_grad_vec->back()); + arg_grad_map_.emplace(arg_name, arg_grad_vec->back()); + } + in_arg_map_.emplace(arg_name, in_arg_vec->back()); + ++arg_top; + } + } +} + +/*! + * \brief If the requested ndarray's shape size is less than + * the corresponding shared_data_array's shape size, reuse + * the memory allocation; otherwise, create a zero ndarray. + */ +NDArray ReshapeOrCreate(const std::string& name, + const TShape& dest_arg_shape, + const int dest_arg_dtype, + const Context& ctx, + std::unordered_map* shared_buffer) { + auto it = shared_buffer->find(name); + if (it != shared_buffer->end()) { + if (it->second.shape().Size() >= dest_arg_shape.Size()) { // memory can be reused + CHECK_EQ(it->second.dtype(), dest_arg_dtype) + << "Requested arg array's dtype does not match the reusable ndarray"; + return it->second.Reshape(dest_arg_shape); + } else { + LOG(WARNING) << "Bucketing: data " << name << " has a shape " << dest_arg_shape + << ", which is larger than already allocated shape " << it->second.shape() + << ". Need to re-allocate. Consider putting default bucket key to be " + << "the bucket taking the largest input for better memory sharing."; + it->second = NDArray(dest_arg_shape, ctx, false, dest_arg_dtype); + it->second = 0; + return it->second; + } // arg_array.shape().Size() >= arg_shape.Size() + } else { + auto p = shared_buffer->emplace(name, NDArray(dest_arg_shape, ctx, false, dest_arg_dtype)); + p.first->second = 0; + return p.first->second; + } // if (it != shared_buffer->end()) +} + +/*! + * \brief Initialize in_args, arg_grads, and aux_states + * and their data_entry_ of the executor using + * shared_buffer from DataParallelExecutorGroup + * and shared_exec if available. + */ +void GraphExecutor::InitArguments(const nnvm::IndexedGraph& idx, + const nnvm::ShapeVector& inferred_shapes, + const nnvm::DTypeVector& inferred_dtypes, + const std::vector& in_arg_ctxes, + const std::vector& arg_grad_ctxes, + const std::vector& aux_state_ctxes, + const std::vector& grad_req_types, + const std::unordered_set& shared_arg_names, + const Executor* shared_exec, + std::unordered_map* shared_buffer, + std::vector* in_arg_vec, + std::vector* arg_grad_vec, + std::vector* aux_state_vec) { + // initialize in_args, arg_grads, and aux_states and populate grad_store_ + data_entry_.resize(idx.num_node_entries()); + size_t arg_top = 0, aux_top = 0; + const auto& mutable_nodes = idx.mutable_input_nodes(); + for (size_t i = 0; i < num_forward_inputs_; ++i) { + const uint32_t nid = idx.input_nodes().at(i); + const uint32_t eid = idx.entry_id(nid, 0); + const TShape& inferred_shape = inferred_shapes[eid]; + const int inferred_dtype = inferred_dtypes[eid]; + const std::string& arg_name = idx[nid].source->attrs.name; + if (mutable_nodes.count(nid)) { // aux_states + if (nullptr != shared_exec) { + const NDArray& aux_nd = shared_exec->aux_state_map().at(arg_name); + CHECK_EQ(inferred_shape, aux_nd.shape()) + << "Inferred shape does not match shared_exec.aux_array's shape." + " Therefore, the allocated memory for shared_exec.aux_array cannot" + " be resued for creating auxilliary NDArray of the argument" + << arg_name << " for the current executor"; + CHECK_EQ(inferred_dtype, aux_nd.dtype()) + << "Inferred dtype does not match shared_exec.aux_array's dtype." + " Therefore, the allocated memory for shared_exec.aux_array cannot" + " be resued for creating auxilliary NDArray of the argument" + << arg_name << " for the current executor"; + aux_state_vec->emplace_back(aux_nd); + } else { + aux_state_vec->emplace_back(inferred_shape, aux_state_ctxes[aux_top], + false, inferred_dtype); + aux_state_vec->back() = 0; + } // if (has_shared_exec) + data_entry_[eid] = aux_state_vec->back(); + aux_state_map_.emplace(arg_name, aux_state_vec->back()); + ++aux_top; + } else { // in_args + if (shared_arg_names.count(arg_name)) { // model parameter + if (nullptr != shared_exec) { + const NDArray& in_arg_nd = shared_exec->in_arg_map().at(arg_name); + CHECK_EQ(inferred_shape, in_arg_nd.shape()) + << "Inferred shape does not match shared_exec.arg_array's shape" + " Therefore, the allocated memory for shared_exec.arg_array cannot" + " be resued for creating NDArray of the argument" + << arg_name << " for the current executor"; + CHECK_EQ(inferred_dtype, in_arg_nd.dtype()) + << "Inferred dtype does not match shared_exec.arg_array's dtype" + " Therefore, the allocated memory for shared_exec.arg_array cannot" + " be resued for creating NDArray of the argument" + << arg_name << " for the current executor"; + in_arg_vec->emplace_back(in_arg_nd); + if (kNullOp == grad_req_types[arg_top]) { + arg_grad_vec->emplace_back(); + } else { + arg_grad_vec->emplace_back(shared_exec->arg_grad_map().at(arg_name)); + grad_store_.emplace_back(grad_req_types[arg_top], arg_grad_vec->back()); + } // if (kNullOp == grad_req_types[arg_top]) + } else { // !has shared_exec + in_arg_vec->emplace_back(inferred_shape, in_arg_ctxes[arg_top], false, inferred_dtype); + in_arg_vec->back() = 0; + if (kNullOp == grad_req_types[arg_top]) { + arg_grad_vec->emplace_back(); + } else { + arg_grad_vec->emplace_back(inferred_shape, arg_grad_ctxes[arg_top], + false, inferred_dtype); + arg_grad_vec->back() = 0; + grad_store_.emplace_back(grad_req_types[arg_top], arg_grad_vec->back()); + } // if (kNullOp == grad_req_types[arg_top]) + } // if (has_shared_exec) + } else { // !shared_arg_names.count(arg_name) + in_arg_vec->emplace_back(ReshapeOrCreate(arg_name, inferred_shape, inferred_dtype, + in_arg_ctxes[arg_top], shared_buffer)); + if (kNullOp == grad_req_types[arg_top]) { + arg_grad_vec->emplace_back(); + } else { + arg_grad_vec->emplace_back(ReshapeOrCreate("grad of " + arg_name, inferred_shape, + inferred_dtype, arg_grad_ctxes[arg_top], + shared_buffer)); + grad_store_.emplace_back(grad_req_types[arg_top], arg_grad_vec->back()); + } // if (kNullOp == grad_req_types[arg_top]) + } // if (shared_arg_names.count(arg_name)) + in_arg_map_.emplace(arg_name, in_arg_vec->back()); + if (!arg_grad_vec->back().is_none()) { + arg_grad_map_.emplace(arg_name, arg_grad_vec->back()); + } + data_entry_[eid] = in_arg_vec->back(); + ++arg_top; + } + } +} + +/*! + * \brief Finish graph initialization after shape and dtype inferences. + * This function is used by both simple_bind and bind flows. + */ +void GraphExecutor::FinishInitGraph(nnvm::Symbol symbol, + nnvm::Graph g, + Executor* shared_exec, + const nnvm::NodeEntryMap& feed_dict) { + const auto& idx = g.indexed_graph(); + for (size_t j = num_forward_outputs_; j < idx.outputs().size(); ++j) { + data_entry_[idx.entry_id(idx.outputs()[j])] = grad_store_[j - num_forward_outputs_].second; + } + + { + // memory allocator + const int kBadStorageID = -1; + const int kExternalStorageID = -2; + nnvm::StorageVector arg_storage_id(idx.num_node_entries(), kBadStorageID); + for (size_t j = num_forward_outputs_; j < idx.outputs().size(); ++j) { + arg_storage_id[idx.entry_id(idx.outputs()[j])] = kExternalStorageID; + } + for (const auto& kv : feed_dict) { + uint32_t eid = idx.entry_id(kv.first); + data_entry_[eid] = kv.second; + arg_storage_id[eid] = kExternalStorageID; + } + g.attrs["storage"] = std::make_shared(std::move(arg_storage_id)); + g = nnvm::ApplyPass(g, "PlanMemory"); + } + g = DetectInplaceAddTo(g); + g.attrs["saved_opr"] = std::make_shared(std::move(saved_opr_)); g = AttachOpExecs(g); g = AttachOpResources(g); graph_ = std::move(g); + if (shared_exec != nullptr) { this->InitDataEntryMemory(&(dynamic_cast(shared_exec)->data_pool_)); } else { this->InitDataEntryMemory(nullptr); } + { // initialize output arrays auto& idx = graph_.indexed_graph(); @@ -374,22 +738,120 @@ void GraphExecutor::Init(nnvm::Symbol symbol, this->InitOpSegs(); } +/*! + * \brief GraphExecutor initializer for simple bind flow in + * which only certain input shapes and dtypes are provided by users. + * The initializer uses these shapes and dtypes to perform + * shape and dtype inferences, and then create NDArrays + * to populate data entries of the graph. The created NDArrays + * for in_args, arg_grads and aux_states are passed to the + * front end to attach the created executor. + * In front end, if the simple_bind flow is trigger by + * _bind_ith_exec, the shared data arrays of DataParallelExecutorGroup + * and shared executor will be taken into account in creating + * NDArrays for in_args, arg_grads, and aux_states for resuing + * already allocated memory. + */ +void GraphExecutor::Init(nnvm::Symbol symbol, + const Context& default_ctx, + const std::map& ctx_map, + const std::vector& in_arg_ctxes, + const std::vector& arg_grad_ctxes, + const std::vector& aux_state_ctxes, + const std::unordered_map& arg_shape_map, + const std::unordered_map& arg_dtype_map, + const std::vector& grad_req_types, + const std::unordered_set& shared_arg_names, + std::vector* in_arg_vec, + std::vector* arg_grad_vec, + std::vector* aux_state_vec, + std::unordered_map* shared_buffer, + Executor* shared_exec, + const nnvm::NodeEntryMap& feed_dict) { + nnvm::Graph g = InitGraph(symbol, default_ctx, ctx_map, in_arg_ctxes, arg_grad_ctxes, + aux_state_ctxes, grad_req_types); + // The following code of shape and dtype inferences and argument + // initialization is for simple_bind only. Regular bind operation + // should do this differently. + + // Initialize arg_shapes and arg_dtypes for shape and type inferences. + // It contains all in_args and aux_states' shapes and types in a certain order. + const nnvm::IndexedGraph& idx = g.indexed_graph(); + nnvm::ShapeVector arg_shapes(idx.input_nodes().size(), TShape()); + nnvm::DTypeVector arg_dtypes(idx.input_nodes().size(), -1); + for (size_t i = 0; i < num_forward_inputs_; ++i) { + const uint32_t nid = idx.input_nodes().at(i); + const std::string& name = idx[nid].source->attrs.name; + auto it1 = arg_shape_map.find(name); + if (arg_shape_map.end() != it1) { + arg_shapes[i] = it1->second; + } + auto it2 = arg_dtype_map.find(name); + if (arg_dtype_map.end() != it2) { + arg_dtypes[i] = it2->second; + } + } + g = nnvm::pass::InferShape(g, arg_shapes, "__shape__"); + if (g.GetAttr("shape_num_unknown_nodes") != 0U) { + HandleInferShapeError(num_forward_inputs_, g.indexed_graph(), + g.GetAttr("shape")); + } + + g = nnvm::pass::InferType(g, arg_dtypes, "__dtype__"); + if (g.GetAttr("dtype_num_unknown_nodes") != 0U) { + HandleInferTypeError(num_forward_inputs_, g.indexed_graph(), + g.GetAttr("dtype")); + } + + // Create in_args, arg_grads, and aux_states using + // the inferred shapes and dtypes. + if (nullptr == shared_buffer) { // regular simple bind + InitArguments(idx, g.GetAttr("shape"), + g.GetAttr("dtype"), + in_arg_ctxes, arg_grad_ctxes, aux_state_ctxes, + grad_req_types, in_arg_vec, arg_grad_vec, aux_state_vec); + } else { // simple bind using shared data arrays and shared_exec + InitArguments(idx, g.GetAttr("shape"), + g.GetAttr("dtype"), + in_arg_ctxes, arg_grad_ctxes, aux_state_ctxes, + grad_req_types, shared_arg_names, shared_exec, + shared_buffer, in_arg_vec, arg_grad_vec, aux_state_vec); + } + // The above code of shape and dtype inferences and argument + // initialization is for simple_bind only. Regular bind operation + // should do this differently. + + // Initialize the rest attributes of the graph. + // This function can be called by regular bind + // operation flow as well. + FinishInitGraph(symbol, g, shared_exec, feed_dict); +} + +/*! + * \brief This function is triggered by both simple_bind + * and bind flows. + * Setup backward graph, create device and context + * attributes in the graph, and calculate the number + * of forward nodes. + */ Graph GraphExecutor::InitGraph(nnvm::Symbol symbol, const Context& default_ctx, const std::map& ctx_map, - const std::vector& in_args, - const std::vector& arg_grad_store, - const std::vector& grad_req_type, - const std::vector& aux_states, - const nnvm::NodeEntryMap& feed_dict) { + const std::vector& in_arg_ctxes, + const std::vector& arg_grad_ctxes, + const std::vector& aux_state_ctxes, + const std::vector& grad_req_types) { // setup gradient - nnvm::Graph g = InitFullGraph(symbol, grad_req_type, arg_grad_store); + nnvm::Graph g = InitFullGraph(symbol, grad_req_types); + + // create "device" and "context" attrs for the graph g = AssignContext(g, default_ctx, ctx_map, - in_args, - grad_store_, - aux_states, + in_arg_ctxes, + arg_grad_ctxes, + aux_state_ctxes, num_forward_inputs_, num_forward_outputs_); + const auto& idx = g.indexed_graph(); // get number of nodes used in forward pass num_forward_nodes_ = 0; @@ -397,55 +859,6 @@ Graph GraphExecutor::InitGraph(nnvm::Symbol symbol, num_forward_nodes_ = std::max( num_forward_nodes_, static_cast(idx.outputs()[i].node_id + 1)); } - // Setup data entry, shape and type. - data_entry_.resize(idx.num_node_entries()); - auto mutable_nodes = idx.mutable_input_nodes(); - nnvm::ShapeVector arg_shapes; - nnvm::DTypeVector arg_types; - size_t arg_top = 0, aux_top = 0; - for (size_t i = 0; i < num_forward_inputs_; ++i) { - const uint32_t nid = idx.input_nodes().at(i); - if (mutable_nodes.count(nid)) { - CHECK_LT(aux_top, aux_states.size()); - data_entry_[idx.entry_id(nid, 0)] = aux_states[aux_top]; - arg_shapes.push_back(aux_states[aux_top].shape()); - arg_types.push_back(aux_states[aux_top].dtype()); - ++aux_top; - } else { - CHECK_LT(arg_top, in_args.size()); - data_entry_[idx.entry_id(nid, 0)] = in_args[arg_top]; - arg_shapes.push_back(in_args[arg_top].shape()); - arg_types.push_back(in_args[arg_top].dtype()); - ++arg_top; - } - } - for (size_t j = num_forward_outputs_; j < idx.outputs().size(); ++j) { - data_entry_[idx.entry_id(idx.outputs()[j])] - = grad_store_[j - num_forward_outputs_].second; - } - arg_shapes.resize(idx.input_nodes().size(), TShape()); - arg_types.resize(idx.input_nodes().size(), -1); - // other initializations - g = nnvm::pass::InferShape(g, arg_shapes, "__shape__"); - g = nnvm::pass::InferType(g, arg_types, "__dtype__"); - - { - // memory allocator - const int kBadStorageID = -1; - const int kExternalStorageID = -2; - nnvm::StorageVector arg_storage_id(idx.num_node_entries(), kBadStorageID); - for (size_t j = num_forward_outputs_; j < idx.outputs().size(); ++j) { - arg_storage_id[idx.entry_id(idx.outputs()[j])] = kExternalStorageID; - } - for (const auto& kv : feed_dict) { - uint32_t eid = idx.entry_id(kv.first); - data_entry_[eid] = kv.second; - arg_storage_id[eid] = kExternalStorageID; - } - g.attrs["storage"] = std::make_shared(std::move(arg_storage_id)); - g = nnvm::ApplyPass(g, "PlanMemory"); - } - g = DetectInplaceAddTo(g); return g; } @@ -913,6 +1326,31 @@ GraphExecutor::CachedSegOpr GraphExecutor::CreateCachedSegOpr(size_t topo_start, } } // namespace exec +Executor *Executor::SimpleBind(nnvm::Symbol symbol, + const Context& default_ctx, + const std::map& group2ctx, + const std::vector& in_arg_ctxes, + const std::vector& arg_grad_ctxes, + const std::vector& aux_state_ctxes, + const std::unordered_map& arg_shape_map, + const std::unordered_map& arg_dtype_map, + const std::vector& grad_req_types, + const std::unordered_set& shared_arg_names, + std::vector* in_args, + std::vector* arg_grads, + std::vector* aux_states, + std::unordered_map* shared_buffer, + Executor* shared_exec) { + auto exec = new exec::GraphExecutor(); + exec->Init(symbol, default_ctx, group2ctx, + in_arg_ctxes, arg_grad_ctxes, aux_state_ctxes, + arg_shape_map, arg_dtype_map, + grad_req_types, shared_arg_names, + in_args, arg_grads, aux_states, + shared_buffer, shared_exec); + return exec; +} + Executor *Executor::Bind(nnvm::Symbol symbol, const Context& default_ctx, const std::map& group2ctx, diff --git a/src/executor/graph_executor.h b/src/executor/graph_executor.h index d9c3a3e6aa47..d5a4e8c3aa6c 100644 --- a/src/executor/graph_executor.h +++ b/src/executor/graph_executor.h @@ -49,19 +49,47 @@ class GraphExecutor : public Executor { void PartialForward(bool is_train, int step, int *step_left) override; void Backward(const std::vector &head_grads) override; const std::vector& outputs() const override; + const std::unordered_map& in_arg_map() const override; + const std::unordered_map& arg_grad_map() const override; + const std::unordered_map& aux_state_map() const override; void Print(std::ostream &os) const override; // NOLINT(*) void SetMonitorCallback(const MonitorCallback& callback) override; - // initialized the executor + // Initialize the rest of attributes + // after setting up arguments. + void FinishInitGraph(nnvm::Symbol symbol, nnvm::Graph g, + Executor* shared_exec = nullptr, + const nnvm::NodeEntryMap& feed_dict + = nnvm::NodeEntryMap()); + + // initialize executor for bind void Init(nnvm::Symbol symbol, const Context& default_ctx, const std::map& ctx_map, const std::vector& in_args, const std::vector& arg_grad_store, - const std::vector& grad_req_type, + const std::vector& grad_req_types, const std::vector& aux_states, Executor* shared_exec = nullptr, const nnvm::NodeEntryMap& feed_dict = nnvm::NodeEntryMap()); + // initialize executor for simple bind + void Init(nnvm::Symbol symbol, + const Context& default_ctx, + const std::map& ctx_map, + const std::vector& in_arg_ctxes, + const std::vector& arg_grad_ctxes, + const std::vector& aux_state_ctxes, + const std::unordered_map& arg_shape_map, + const std::unordered_map& arg_dtype_map, + const std::vector& grad_req_types, + const std::unordered_set& shared_arg_names, + std::vector* in_arg_vec, + std::vector* arg_grad_vec, + std::vector* aux_state_vec, + std::unordered_map* shared_buffer = nullptr, + Executor* shared_exec = nullptr, + const nnvm::NodeEntryMap& feed_dict + = nnvm::NodeEntryMap()); protected: // Information about operational node @@ -94,21 +122,43 @@ class GraphExecutor : public Executor { // list of op executors std::vector exec_list; }; - - // internal initialization of the graph. + // Initialize in_args, arg_grads, and aux_states + void InitArguments(const nnvm::IndexedGraph& idx, + const nnvm::ShapeVector& inferred_shapes, + const nnvm::DTypeVector& inferred_dtypes, + const std::vector& in_arg_ctxes, + const std::vector& arg_grad_ctxes, + const std::vector& aux_state_ctxes, + const std::vector& grad_req_types, + std::vector* in_arg_vec, + std::vector* arg_grad_vec, + std::vector* aux_state_vec); + // Initialize in_args, arg_grads and aux_states with + // shared_buffer and shared_exec + void InitArguments(const nnvm::IndexedGraph& idx, + const nnvm::ShapeVector& inferred_shapes, + const nnvm::DTypeVector& inferred_dtypes, + const std::vector& in_arg_ctxes, + const std::vector& arg_grad_ctxes, + const std::vector& aux_state_ctxes, + const std::vector& grad_req_types, + const std::unordered_set& shared_arg_names, + const Executor* shared_exec, + std::unordered_map* shared_buffer, + std::vector* in_arg_vec, + std::vector* arg_grad_vec, + std::vector* aux_state_vec); + // internal initialization of the graph for simple bind Graph InitGraph(nnvm::Symbol symbol, const Context& default_ctx, const std::map& ctx_map, - const std::vector& in_args, - const std::vector& arg_grad_store, - const std::vector& grad_req_type, - const std::vector& aux_states, - const nnvm::NodeEntryMap& feed_dict - = nnvm::NodeEntryMap()); - // initialize the full graph, including gradient. + const std::vector& in_arg_ctxes, + const std::vector& arg_grad_ctxes, + const std::vector& aux_state_ctxes, + const std::vector& grad_req_types); + // intialize the full graph for simple bind, including gradient Graph InitFullGraph(nnvm::Symbol symbol, - const std::vector& grad_req_type, - const std::vector& arg_grad_store); + const std::vector& grad_req_types); // initialize the cached operator void InitCachedOps(); // initialize the opr segments for bulk exec @@ -140,6 +190,12 @@ class GraphExecutor : public Executor { std::vector data_pool_; // output arrays std::vector output_arrays_; + // input argument map, key is arg name, value is arg's NDArray + std::unordered_map in_arg_map_; + // arg grad map, key is arg name, value is arg grad NDArray + std::unordered_map arg_grad_map_; + // aux state map, key is aux state name, value is aux state NDArray + std::unordered_map aux_state_map_; // gradient store std::vector > grad_store_; // array to hold head gradient. diff --git a/tests/python/train/test_bucketing.py b/tests/python/train/test_bucketing.py new file mode 100644 index 000000000000..85ea107c5ca2 --- /dev/null +++ b/tests/python/train/test_bucketing.py @@ -0,0 +1,115 @@ +# pylint: skip-file +import numpy as np +import mxnet as mx +import random +from random import randint + + +def test_bucket_module(): + import logging + head = '%(asctime)-15s %(message)s' + logging.basicConfig(level=logging.DEBUG, format=head) + console = logging.StreamHandler() + console.setLevel(logging.DEBUG) + logging.getLogger('').addHandler(console) + + class DummySentenceIter(mx.rnn.BucketSentenceIter): + """Dummy sentence iterator to output sentences the same as input. + """ + + def __init__(self, sentences, batch_size, buckets=None, invalid_label=-1, + data_name='data', label_name='l2_label', dtype='float32', + layout='NTC'): + super(DummySentenceIter, self).__init__(sentences, batch_size, + buckets=buckets, invalid_label=invalid_label, + data_name=data_name, label_name=label_name, + dtype=dtype, layout=layout) + + def reset(self): + """Resets the iterator to the beginning of the data.""" + self.curr_idx = 0 + random.shuffle(self.idx) + for buck in self.data: + np.random.shuffle(buck) + + self.nddata = [] + self.ndlabel = [] + for buck in self.data: + self.nddata.append(mx.nd.array(buck, dtype=self.dtype)) + self.ndlabel.append(mx.nd.array(buck, dtype=self.dtype)) + + batch_size = 128 + num_epochs = 5 + num_hidden = 25 + num_embed = 25 + num_layers = 2 + len_vocab = 50 + buckets = [10, 20, 30, 40] + + invalid_label = 0 + num_sentence = 1000 + + train_sent = [] + val_sent = [] + + for _ in range(num_sentence): + len_sentence = randint(1, max(buckets) + 10) + train_sentence = [] + val_sentence = [] + for _ in range(len_sentence): + train_sentence.append(randint(1, len_vocab)) + val_sentence.append(randint(1, len_vocab)) + train_sent.append(train_sentence) + val_sent.append(val_sentence) + + data_train = DummySentenceIter(train_sent, batch_size, buckets=buckets, + invalid_label=invalid_label) + data_val = DummySentenceIter(val_sent, batch_size, buckets=buckets, + invalid_label=invalid_label) + + stack = mx.rnn.SequentialRNNCell() + for i in range(num_layers): + stack.add(mx.rnn.LSTMCell(num_hidden=num_hidden, prefix='lstm_l%d_' % i)) + + def sym_gen(seq_len): + data = mx.sym.Variable('data') + label = mx.sym.Variable('l2_label') + embed = mx.sym.Embedding(data=data, input_dim=len_vocab, + output_dim=num_embed, name='embed') + + stack.reset() + outputs, states = stack.unroll(seq_len, inputs=embed, merge_outputs=True) + + pred = mx.sym.Reshape(outputs, shape=(-1, num_hidden)) + pred = mx.sym.FullyConnected(data=pred, num_hidden=1, name='pred') + pred = mx.sym.reshape(pred, shape=(batch_size, -1)) + loss = mx.sym.LinearRegressionOutput(pred, label, name='l2_loss') + + return loss, ('data',), ('l2_label',) + + contexts = mx.cpu(0) + + model = mx.mod.BucketingModule( + sym_gen=sym_gen, + default_bucket_key=data_train.default_bucket_key, + context=contexts) + + logging.info('Begin fit...') + model.fit( + train_data=data_train, + eval_data=data_val, + eval_metric=mx.metric.MSE(), + kvstore='device', + optimizer='sgd', + optimizer_params={'learning_rate': 0.01, + 'momentum': 0, + 'wd': 0.00001}, + initializer=mx.init.Xavier(factor_type="in", magnitude=2.34), + num_epoch=num_epochs, + batch_end_callback=mx.callback.Speedometer(batch_size, 50)) + logging.info('Finished fit...') + assert model.score(data_val, mx.metric.MSE())[0][1] < 350, "High mean square error." + + +if __name__ == "__main__": + test_bucket_module() diff --git a/tests/python/unittest/test_executor.py b/tests/python/unittest/test_executor.py index b190b2898843..c1cc013b81c0 100644 --- a/tests/python/unittest/test_executor.py +++ b/tests/python/unittest/test_executor.py @@ -121,7 +121,7 @@ def test_reshape(): x = mx.sym.Variable('x') y = mx.sym.FullyConnected(x, num_hidden=4) - exe = y.simple_bind(mx.cpu(), x=(5,4), grad_req=[]) + exe = y.simple_bind(mx.cpu(), x=(5,4), grad_req='null') exe.arg_arrays[0][:] = 1 exe.arg_arrays[1][:] = mx.nd.ones((4,4)) exe.arg_arrays[2][:] = 0 diff --git a/tests/python/unittest/test_module.py b/tests/python/unittest/test_module.py index 5508a37c9567..9f3cff8e1265 100644 --- a/tests/python/unittest/test_module.py +++ b/tests/python/unittest/test_module.py @@ -2,6 +2,8 @@ import mxnet.ndarray as nd import numpy as np from functools import reduce +from mxnet.module.executor_group import DataParallelExecutorGroup + def test_module_dtype(): dtype = np.float16 @@ -45,6 +47,7 @@ def test_module_input_grads(): assert np.all(b_grad == 2), b_grad assert np.all(c_grad == 3), c_grad + def test_module_layout(): sym = mx.sym.Variable('data') sym = mx.sym.Activation(data=sym, act_type='relu', __layout__='TNC') @@ -62,6 +65,7 @@ def test_module_layout(): for x in mod.get_outputs(merge_multi_context=False)[0]: assert x.shape == hdshape + def test_save_load(): def dict_equ(a, b): assert set(a) == set(b) @@ -101,6 +105,7 @@ def dict_equ(a, b): dict_equ(mod.get_params()[0], mod2.get_params()[0]) dict_equ(mod._kvstore._updater.states, mod2._updater.states) + def test_module_reshape(): data = mx.sym.Variable('data') sym = mx.sym.FullyConnected(data, num_hidden=20, name='fc') @@ -127,6 +132,7 @@ def test_module_reshape(): assert mod.get_outputs()[0].shape == dshape assert (mod.get_params()[0]['fc_bias'].asnumpy() == -3).all() + def test_module_states(): stack = mx.rnn.SequentialRNNCell() for i in range(2): @@ -153,6 +159,7 @@ def test_module_states(): for x1, x2 in zip(out1, out2): assert not mx.test_utils.almost_equal(x1.asnumpy(), x2.asnumpy(), rtol=1e-3) + def test_module_switch_bucket(): vocab_dim = 5000 num_hidden = 100 @@ -207,6 +214,7 @@ def create_bucketing_module(key): #the default bucket is expected to reuse the bytes allocated assert total_bytes_after == total_bytes_before + def test_monitor(): # data iter mx.random.seed(11) @@ -254,6 +262,119 @@ def mean_abs(x): break assert(mon_result_counts == [2, 2, 1, 6, 6, 4]) + +def test_executor_group(): + def get_rnn_sym(num_layers, num_words, num_hidden, num_embed, seq_len): + stack = mx.rnn.SequentialRNNCell() + for i in range(num_layers): + stack.add(mx.rnn.LSTMCell(num_hidden=num_hidden, prefix='lstm_l%d_' % i)) + data = mx.sym.Variable('data') + label = mx.sym.Variable('softmax_label') + embed = mx.sym.Embedding(data=data, input_dim=num_words, + output_dim=num_embed, name='embed') + + stack.reset() + outputs, states = stack.unroll(seq_len, inputs=embed, merge_outputs=True) + + pred = mx.sym.Reshape(outputs, shape=(-1, num_hidden)) + pred = mx.sym.FullyConnected(data=pred, num_hidden=num_words, name='pred') + + label = mx.sym.Reshape(label, shape=(-1,)) + pred = mx.sym.SoftmaxOutput(data=pred, label=label, name='softmax') + return pred + + def test_shared_exec_group(exec_grp_shared, exec_grp_created, shared_arg_names=None, extra_args=None): + # Test shared data arrays + for i in range(len(exec_grp_shared.execs)): + # test same shared_data_arrays for two exec groups + shared_data_array1 = exec_grp_shared.shared_data_arrays[i] + shared_data_array2 = exec_grp_created.shared_data_arrays[i] + if extra_args is not None: + assert len(shared_data_array1) == len(extra_args),\ + "exec_grp_shared.shared_data_arrays[%d] should have same number of args as extra_args" + assert len(shared_data_array1) == len(shared_data_array2),\ + "length of shared_data_array of the shared executor group not equal to the created executor group" + for k, v in shared_data_array1.items(): + if extra_args is not None: + assert k in extra_args, "arg %s is not in extra_args" % k + assert k in shared_data_array2,\ + "arg %s of the shared executor group not in the shared_data_array of the created executor group" % k + assert mx.test_utils.same_array(v, shared_data_array2[k]) + + for data_name, array in exec_grp_shared.shared_data_arrays[i].items(): + assert data_name in exec_grp_created.shared_data_arrays[i], \ + "Shared input data '%s' is not in " \ + "shared_data_arrays of created executor group." % (data_name) + assert mx.test_utils.same_array(array, exec_grp_created.shared_data_arrays[i][data_name]), \ + "Shared input data '%s' does not share memory." % (data_name) + + # Test shared argument arrays and gradient arrays + exec_shared = exec_grp_shared.execs[i] + exec_created = exec_grp_created.execs[i] + if shared_arg_names is not None: + # test shared arguments + for arg_name in shared_arg_names: + assert arg_name in exec_created.arg_dict, \ + "Shared argument '%s' is not in arg_dict of created executor group." % (arg_name) + assert mx.test_utils.same_array(exec_shared.arg_dict[arg_name], exec_created.arg_dict[arg_name]), \ + "Shared argument '%s' does not share memory." % (arg_name) + # test shared argument gradients + for arg_name in shared_arg_names: + assert arg_name in exec_created.grad_dict, \ + "Shared argument gradient '%s' is not in " \ + "grad_dict of created executor group." % (arg_name) + assert mx.test_utils.same_array(exec_shared.grad_dict[arg_name], exec_created.grad_dict[arg_name]), \ + "Shared argument gradient '%s' does not sharing memory." % (arg_name) + + for arg_name, grad in exec_grp_shared.grad_req.items(): + assert grad == exec_grp_created.grad_req[arg_name], \ + "Gradient requirements for shared argument '%s' are inconsistent. " \ + "Shared executor group requires '%s' while created executor group requires '%s'" \ + %(arg_name, grad, exec_grp_created.grad_req[arg_name]) + + contexts = [mx.cpu(0), mx.cpu(1)] + workload = [1] * len(contexts) + batch_size = 32 + max_bucket_size = 80 + num_words = 1000 + num_hidden = 100 + num_embed = 200 + data_shapes = [('data', (batch_size, max_bucket_size))] + label_shapes = [('softmax_label', (batch_size, max_bucket_size))] + + # generate an rnn sym with #layers=5 + sym = get_rnn_sym(num_layers=3, num_words=num_words, num_hidden=num_hidden, + num_embed=num_embed, seq_len=max_bucket_size) + arg_names1 = sym.list_arguments() + input_names = [name[0] for name in data_shapes] + [name[0] for name in label_shapes] + shared_arg_names = [name for name in arg_names1 if name not in input_names] + exec_group1 = DataParallelExecutorGroup(symbol=sym, contexts=contexts, + workload=workload, data_shapes=data_shapes, + label_shapes=label_shapes, param_names=shared_arg_names, + for_training=True, inputs_need_grad=False) + + # shared_data_arrays should only have input "data" and "softmax_label" arrays + for i in range(len(contexts)): + assert len(exec_group1.shared_data_arrays[i]) == len(input_names),\ + "exec_group1.shared_data_arrays[%d] should have the same number of names as in input_names" % i + for name in input_names: + assert name in exec_group1.shared_data_arrays[i],\ + "arg %s should be in exec_group1.shared_data_arrays[%d]" % (name, i) + + # generate an rnn sym with #layers=5 + sym = get_rnn_sym(num_layers=5, num_words=num_words, num_hidden=num_hidden, + num_embed=num_embed, seq_len=max_bucket_size) + arg_names2 = sym.list_arguments() + exec_group2 = DataParallelExecutorGroup(symbol=sym, contexts=contexts, + workload=workload, data_shapes=data_shapes, + label_shapes=label_shapes, param_names=shared_arg_names, + for_training=True, inputs_need_grad=False, + shared_group=exec_group1) + extra_args = [name for name in arg_names2 if name not in shared_arg_names] + test_shared_exec_group(exec_grp_shared=exec_group1, exec_grp_created=exec_group2, + shared_arg_names=shared_arg_names, extra_args=extra_args) + + if __name__ == '__main__': test_module_dtype() test_module_input_grads() @@ -263,3 +384,4 @@ def mean_abs(x): test_module_layout() test_module_switch_bucket() test_monitor() + test_executor_group() From f4e4731bae61c0d388f105086fa14016b94775a4 Mon Sep 17 00:00:00 2001 From: Eric Junyuan Xie Date: Fri, 2 Jun 2017 23:59:30 -0700 Subject: [PATCH 030/834] add random multinomial sample (#6553) * add random multinomial sample * Update initialize.cc * Update test_random.py --- .../{tensor => random}/multisample_op.cc | 22 +-- .../{tensor => random}/multisample_op.h | 6 +- src/operator/random/sample_multinomial_op.cc | 93 +++++++++ src/operator/random/sample_multinomial_op.cu | 33 ++++ src/operator/random/sample_multinomial_op.h | 176 ++++++++++++++++++ src/operator/{tensor => random}/sample_op.cc | 0 src/operator/{tensor => random}/sample_op.cu | 0 src/operator/{tensor => random}/sample_op.h | 8 +- tests/python/unittest/test_random.py | 24 +++ 9 files changed, 344 insertions(+), 18 deletions(-) rename src/operator/{tensor => random}/multisample_op.cc (96%) rename src/operator/{tensor => random}/multisample_op.h (97%) create mode 100644 src/operator/random/sample_multinomial_op.cc create mode 100644 src/operator/random/sample_multinomial_op.cu create mode 100644 src/operator/random/sample_multinomial_op.h rename src/operator/{tensor => random}/sample_op.cc (100%) rename src/operator/{tensor => random}/sample_op.cu (100%) rename src/operator/{tensor => random}/sample_op.h (98%) diff --git a/src/operator/tensor/multisample_op.cc b/src/operator/random/multisample_op.cc similarity index 96% rename from src/operator/tensor/multisample_op.cc rename to src/operator/random/multisample_op.cc index b5179b31426e..303d1d2f0086 100644 --- a/src/operator/tensor/multisample_op.cc +++ b/src/operator/random/multisample_op.cc @@ -138,8 +138,8 @@ DMLC_REGISTER_PARAMETER(MultiSampleParam); }) \ .set_attr("FCompute", MultiSampleOpForward) \ .set_attr("FGradient", MakeZeroGradNodes) \ - .add_arguments(MultiSampleParam::__FIELDS__()) \ - .add_argument(input_name_1, "NDArray-or-Symbol", input_desc_1) + .add_argument(input_name_1, "NDArray-or-Symbol", input_desc_1) \ + .add_arguments(MultiSampleParam::__FIELDS__()) #define MXNET_OPERATOR_REGISTER_SAMPLING1(distr, sampler, input_name, input_desc, \ description) \ @@ -153,7 +153,7 @@ DMLC_REGISTER_PARAMETER(MultiSampleParam); .add_argument(input_name_2, "NDArray-or-Symbol", input_desc_2); inline std::string uniform_desc() { - return std::string(R"code(Concurrent sampling from multiple + return std::string(R"code(Concurrent sampling from multiple uniform distributions on the intervals given by *[low,high)*. The parameters of the distributions are provided as input arrays. @@ -170,9 +170,9 @@ has the same shape as the input arrays. Examples:: low = [ 0.0, 2.5 ] - high = [ 1.0, 3.7 ] + high = [ 1.0, 3.7 ] - // Draw a single sample for each distribution + // Draw a single sample for each distribution sample_uniform(low, high) = [ 0.40451524, 3.18687344] // Draw a vector containing two samples for each distribution @@ -182,7 +182,7 @@ Examples:: } inline std::string normal_desc() { - return std::string(R"code(Concurrent sampling from multiple + return std::string(R"code(Concurrent sampling from multiple normal distributions with parameters *mu* (mean) and *sigma* (standard deviation). The parameters of the distributions are provided as input arrays. @@ -211,7 +211,7 @@ Examples:: } inline std::string gamma_desc() { - return std::string(R"code(Concurrent sampling from multiple + return std::string(R"code(Concurrent sampling from multiple gamma distributions with parameters *alpha* (shape) and *beta* (scale). The parameters of the distributions are provided as input arrays. @@ -240,7 +240,7 @@ Examples:: } inline std::string exponential_desc() { - return std::string(R"code(Concurrent sampling from multiple + return std::string(R"code(Concurrent sampling from multiple exponential distributions with parameters lambda (rate). The parameters of the distributions are provided as an input array. @@ -268,7 +268,7 @@ Examples:: } inline std::string poisson_desc() { - return std::string(R"code(Concurrent sampling from multiple + return std::string(R"code(Concurrent sampling from multiple Poisson distributions with parameters lambda (rate). The parameters of the distributions are provided as an input array. @@ -298,7 +298,7 @@ Examples:: } inline std::string negative_binomial_desc() { - return std::string(R"code(Concurrent sampling from multiple + return std::string(R"code(Concurrent sampling from multiple negative binomial distributions with parameters *k* (failure limit) and *p* (failure probability). The parameters of the distributions are provided as input arrays. @@ -329,7 +329,7 @@ Examples:: } inline std::string generalized_negative_binomial_desc() { - return std::string(R"code(Concurrent sampling from multiple + return std::string(R"code(Concurrent sampling from multiple generalized negative binomial distributions with parameters *mu* (mean) and *alpha* (dispersion). The parameters of the distributions are provided as input arrays. diff --git a/src/operator/tensor/multisample_op.h b/src/operator/random/multisample_op.h similarity index 97% rename from src/operator/tensor/multisample_op.h rename to src/operator/random/multisample_op.h index 6e84d1dab368..0b5b4cee6217 100644 --- a/src/operator/tensor/multisample_op.h +++ b/src/operator/random/multisample_op.h @@ -3,8 +3,8 @@ * \file sampling_op.h * \brief Function definitions of operators for sampling from multiple distributions */ -#ifndef MXNET_OPERATOR_TENSOR_MULTISAMPLE_OP_H_ -#define MXNET_OPERATOR_TENSOR_MULTISAMPLE_OP_H_ +#ifndef MXNET_OPERATOR_RANDOM_MULTISAMPLE_OP_H_ +#define MXNET_OPERATOR_RANDOM_MULTISAMPLE_OP_H_ #include #include @@ -174,4 +174,4 @@ void MultiSampleOpForward(const nnvm::NodeAttrs& attrs, } // namespace op } // namespace mxnet -#endif // MXNET_OPERATOR_TENSOR_MULTISAMPLE_OP_H_ +#endif // MXNET_OPERATOR_RANDOM_MULTISAMPLE_OP_H_ diff --git a/src/operator/random/sample_multinomial_op.cc b/src/operator/random/sample_multinomial_op.cc new file mode 100644 index 000000000000..9e6dbe99c045 --- /dev/null +++ b/src/operator/random/sample_multinomial_op.cc @@ -0,0 +1,93 @@ +/*! + * Copyright (c) 2017 by Contributors + * \file sample_multinomial_op.h + * \brief Operator for sampling from multinomial distributions + */ +#include "./sample_multinomial_op.h" + +namespace mxnet { +namespace op { + +DMLC_REGISTER_PARAMETER(SampleMultinomialParam); + + +NNVM_REGISTER_OP(sample_multinomial) +.describe(R"code(Concurrent sampling from multiple multinomial distributions. + +*data* is an *n* dimensional array whose last dimension has length *k*, where +*k* is the number of possible outcomes of each multinomial distribution. This +operator will draw *shape* samples from each distribution. If shape is empty +one sample will be drawn from each distribution. + +If *get_prob* is true, a second array containing log likelihood of the drawn +samples will also be returned. This is usually used for reinforcement learning +where you can provide reward as head gradient for this array to estimate +gradient. + +Note that the input distribution must be normalized, i.e. *data* must sum to +1 along its last axis. + +Examples:: + + probs = [[0, 0.1, 0.2, 0.3, 0.4], [0.4, 0.3, 0.2, 0.1, 0]] + + // Draw a single sample for each distribution + sample_multinomial(probs) = [3, 0] + + // Draw a vector containing two samples for each distribution + sample_multinomial(probs, shape=(2)) = [[4, 2], + [0, 0]] + + // requests log likelihood + sample_multinomial(probs, get_prob=True) = [2, 1], [0.2, 0.3] +)code") +.set_num_inputs(1) +.set_num_outputs([](const nnvm::NodeAttrs& attrs) { + const SampleMultinomialParam& param = nnvm::get(attrs.parsed); + return param.get_prob ? 2U : 1U; + }) +.set_attr_parser(ParamParser) +.set_attr("FInferShape", SampleMultinomialOpShape) +.set_attr("FInferType", SampleMultinomialOpType) +.set_attr("FResourceRequest", + [](const nnvm::NodeAttrs& attrs) { + return std::vector{ + ResourceRequest::kRandom, ResourceRequest::kTempSpace}; + }) +.set_attr("FGradient", + [](const nnvm::NodePtr& n, const std::vector& ograds) { + const SampleMultinomialParam& param = nnvm::get(n->attrs.parsed); + if (param.get_prob) { + return MakeGradNode("_backward_sample_multinomial", n, + {ograds[1], n->inputs[0], nnvm::NodeEntry{n, 0, 0}}, + std::unordered_map()); + } else { + return MakeZeroGradNodes(n, ograds); + } + }) +.set_attr("FCompute", SampleMultinomialForward) +.add_argument("data", "NDArray-or-Symbol", + "Distribution probabilities. Must sum to one on the last axis.") +.add_arguments(SampleMultinomialParam::__FIELDS__()); + + +struct SampleMultinomialBackwardCPUKernel { + template + MSHADOW_XINLINE static void Map(int i, index_t K, index_t M, + DType* ograd, DType* dist, IType* out, + DType* igrad) { + for (index_t j = 0; j < M; ++j) { + igrad[i*K + out[i*M + j]] += ograd[i*M + j] / dist[i*K + out[i*M + j]]; + } + } +}; + +NNVM_REGISTER_OP(_backward_sample_multinomial) +.set_num_inputs(3) +.set_num_outputs(1) +.set_attr("TIsBackward", true) +.set_attr("FCompute", + SampleMultinomialBackward); + +} // namespace op +} // namespace mxnet diff --git a/src/operator/random/sample_multinomial_op.cu b/src/operator/random/sample_multinomial_op.cu new file mode 100644 index 000000000000..434202d5e09b --- /dev/null +++ b/src/operator/random/sample_multinomial_op.cu @@ -0,0 +1,33 @@ +/*! + * Copyright (c) 2017 by Contributors + * \file sample_multinomial_op.h + * \brief Operator for sampling from multinomial distributions + */ +#include "./sample_multinomial_op.h" + +namespace mxnet { +namespace op { + +NNVM_REGISTER_OP(sample_multinomial) +.set_attr("FCompute", SampleMultinomialForward); + + +struct SampleMultinomialBackwardGPUKernel { + template + MSHADOW_XINLINE static void Map(int i, index_t K, index_t M, + DType* ograd, DType* dist, IType* out, + DType* igrad) { + for (index_t j = 0; j < M; ++j) { + atomicAdd(&igrad[i*K + out[i*M + j]], ograd[i*M + j] / dist[i*K + out[i*M + j]]); + } + } +}; + + +NNVM_REGISTER_OP(_backward_sample_multinomial) +.set_attr("FCompute", + SampleMultinomialBackward); + + +} // namespace op +} // namespace mxnet diff --git a/src/operator/random/sample_multinomial_op.h b/src/operator/random/sample_multinomial_op.h new file mode 100644 index 000000000000..ab73ebf0543e --- /dev/null +++ b/src/operator/random/sample_multinomial_op.h @@ -0,0 +1,176 @@ +/*! + * Copyright (c) 2017 by Contributors + * \file sample_multinomial_op.h + * \brief Operator for sampling from multinomial distributions + */ +#ifndef MXNET_OPERATOR_RANDOM_SAMPLE_MULTINOMIAL_OP_H_ +#define MXNET_OPERATOR_RANDOM_SAMPLE_MULTINOMIAL_OP_H_ + +#include +#include +#include "../mshadow_op.h" +#include "../mxnet_op.h" +#include "../operator_common.h" +#include "../elemwise_op_common.h" + +namespace mxnet { +namespace op { + +struct SampleMultinomialParam : public dmlc::Parameter { + TShape shape; + bool get_prob; + int dtype; + DMLC_DECLARE_PARAMETER(SampleMultinomialParam) { + DMLC_DECLARE_FIELD(shape) + .set_default(TShape()) + .describe("Shape to be sampled from each random distribution."); + DMLC_DECLARE_FIELD(get_prob) + .set_default(false) + .describe("Whether to also return the log probability of sampled " + "result. This is usually used for differentiating through " + "stochastic variables, e.g. in reinforcement learning."); + DMLC_DECLARE_FIELD(dtype) + .add_enum("int32", mshadow::kInt32) + .set_default(mshadow::kInt32) + .describe("DType of the output in case this can't be inferred. " + "Only support int32 for now."); + } +}; + + +inline bool SampleMultinomialOpShape(const nnvm::NodeAttrs& attrs, + std::vector* in_attrs, + std::vector* out_attrs) { + const SampleMultinomialParam& param = nnvm::get(attrs.parsed); + + CHECK_EQ(in_attrs->size(), 1U); + CHECK_EQ(out_attrs->size(), param.get_prob ? 2U : 1U); + const TShape& ishape = (*in_attrs)[0]; + if (!ishape.ndim()) return false; + + if (ishape.ndim() == 1) { + if (param.shape.ndim()) { + SHAPE_ASSIGN_CHECK(*out_attrs, 0, param.shape); + if (param.get_prob) SHAPE_ASSIGN_CHECK(*out_attrs, 0, param.shape); + } else { + SHAPE_ASSIGN_CHECK(*out_attrs, 0, TShape(1)); + if (param.get_prob) SHAPE_ASSIGN_CHECK(*out_attrs, 0, TShape(1)); + } + return true; + } + + TShape oshape(ishape.ndim() - 1 + param.shape.ndim()); + for (size_t i = 0; i < ishape.ndim() - 1; ++i) { + oshape[i] = ishape[i]; + } + for (size_t i = 0; i < param.shape.ndim(); ++i) { + oshape[i + ishape.ndim() - 1] = param.shape[i]; + } + SHAPE_ASSIGN_CHECK(*out_attrs, 0, oshape); + if (param.get_prob) SHAPE_ASSIGN_CHECK(*out_attrs, 1, oshape); + return true; +} + + +inline bool SampleMultinomialOpType(const nnvm::NodeAttrs& attrs, + std::vector* in_attrs, + std::vector* out_attrs) { + const SampleMultinomialParam& param = nnvm::get(attrs.parsed); + + CHECK_EQ(in_attrs->size(), 1U); + CHECK_EQ(out_attrs->size(), param.get_prob ? 2U : 1U); + int itype = (*in_attrs)[0]; + if (itype == -1) return false; + + TYPE_ASSIGN_CHECK(*out_attrs, 0, param.dtype); + if (param.get_prob) { + TYPE_ASSIGN_CHECK(*out_attrs, 1, itype); + } + return true; +} + +struct SampleMultinomialKernel { + template + MSHADOW_XINLINE static void Map(int i, index_t K, index_t M, + DType* dist, float* uniform, IType* out, + DType* prob) { + for (index_t j = 0; j < M; ++j) { + DType loc = static_cast(uniform[i*M + j]); + DType acc = 0; + bool found = false; + for (index_t k = 0; k < K; ++k) { + acc += dist[i*K + k]; + if (acc > loc) { + found = true; + out[i*M + j] = static_cast(k); + if (prob != nullptr) prob[i*M + j] = logf(dist[i*K + k]); + break; + } + } + if (!found) { + out[i*M + j] = static_cast(K-1); + if (prob != nullptr) prob[i*M + j] = logf(dist[i*K + K - 1]); + } + } + } +}; + + +template +void SampleMultinomialForward(const nnvm::NodeAttrs& attrs, + const OpContext& ctx, + const std::vector& inputs, + const std::vector& req, + const std::vector& outputs) { + using namespace mshadow; + using namespace mxnet_op; + const SampleMultinomialParam& param = nnvm::get(attrs.parsed); + + index_t K = inputs[0].shape_[inputs[0].ndim()-1]; + index_t N = inputs[0].Size()/K; + index_t M = outputs[0].Size()/N; + + Stream *s = ctx.get_stream(); + MSHADOW_REAL_TYPE_SWITCH(inputs[0].type_flag_, DType, { + Random *prnd = ctx.requested[0].get_random(s); + Tensor uniform = + ctx.requested[1].get_space_typed(Shape1(N*M), s); + prnd->SampleUniform(&uniform, 0, 1); + Kernel::Launch( + s, N, K, M, inputs[0].dptr(), uniform.dptr_, outputs[0].dptr(), + param.get_prob ? outputs[1].dptr() : nullptr); + }); +} + + +template +void SampleMultinomialBackward(const nnvm::NodeAttrs& attrs, + const OpContext& ctx, + const std::vector& inputs, + const std::vector& req, + const std::vector& outputs) { + using namespace mshadow; + using namespace mxnet_op; + if (req[0] == kNullOp) return; + + index_t K = outputs[0].shape_[outputs[0].ndim()-1]; + index_t N = outputs[0].Size()/K; + index_t M = inputs[0].Size()/N; + + Stream *s = ctx.get_stream(); + MSHADOW_REAL_TYPE_SWITCH(inputs[0].type_flag_, DType, { + if (req[0] != kAddTo) { + Tensor out = outputs[0].FlatTo1D(s); + out = 0; + } + Kernel::Launch( + s, N, K, M, inputs[0].dptr(), inputs[1].dptr(), + inputs[2].dptr(), outputs[0].dptr()); + }); +} + + +} // namespace op +} // namespace mxnet + +#endif // MXNET_OPERATOR_RANDOM_SAMPLE_MULTINOMIAL_OP_H_ diff --git a/src/operator/tensor/sample_op.cc b/src/operator/random/sample_op.cc similarity index 100% rename from src/operator/tensor/sample_op.cc rename to src/operator/random/sample_op.cc diff --git a/src/operator/tensor/sample_op.cu b/src/operator/random/sample_op.cu similarity index 100% rename from src/operator/tensor/sample_op.cu rename to src/operator/random/sample_op.cu diff --git a/src/operator/tensor/sample_op.h b/src/operator/random/sample_op.h similarity index 98% rename from src/operator/tensor/sample_op.h rename to src/operator/random/sample_op.h index b5f20bc57dd3..79655b3a3ba1 100644 --- a/src/operator/tensor/sample_op.h +++ b/src/operator/random/sample_op.h @@ -3,8 +3,8 @@ * \file sample_op.h * \brief Elementary sampling operators */ -#ifndef MXNET_OPERATOR_TENSOR_SAMPLE_OP_H_ -#define MXNET_OPERATOR_TENSOR_SAMPLE_OP_H_ +#ifndef MXNET_OPERATOR_RANDOM_SAMPLE_OP_H_ +#define MXNET_OPERATOR_RANDOM_SAMPLE_OP_H_ #include #include @@ -12,7 +12,7 @@ #include #include "../mshadow_op.h" #include "../elemwise_op_common.h" -#include "./init_op.h" +#include "../tensor/init_op.h" namespace mxnet { namespace op { @@ -386,4 +386,4 @@ inline std::vector SampleResource(const NodeAttrs& attrs) { } // namespace op } // namespace mxnet -#endif // MXNET_OPERATOR_TENSOR_SAMPLE_OP_H_ +#endif // MXNET_OPERATOR_RANDOM_SAMPLE_OP_H_ diff --git a/tests/python/unittest/test_random.py b/tests/python/unittest/test_random.py index a3f911cba358..0b5fd3a96dbf 100644 --- a/tests/python/unittest/test_random.py +++ b/tests/python/unittest/test_random.py @@ -170,6 +170,30 @@ def test_random(): check_with_device(mx.context.current_context(), 'float64') +def test_sample_multinomial(): + x = mx.nd.array([[0,1,2,3,4],[4,3,2,1,0]])/10.0 + dx = mx.nd.ones_like(x) + mx.contrib.autograd.mark_variables([x], [dx]) + with mx.contrib.autograd.train_section(): + y, prob = mx.nd.sample_multinomial(x, shape=1000, get_prob=True) + r = prob * 5 + r.backward() + + y = y.asnumpy() + x = x.asnumpy() + for i in range(x.shape[0]): + + freq = np.bincount(y[i], minlength=5)/1000.0*x[i].sum() + mx.test_utils.assert_almost_equal(freq, x[i], rtol=0.25) + rprob = x[i][y[i]]/x[i].sum() + mx.test_utils.assert_almost_equal(np.log(rprob), prob.asnumpy()[i]) + + real_dx = np.zeros((5,)) + for j in range(1000): + real_dx[y[i][j]] += 5.0 / rprob[j] + mx.test_utils.assert_almost_equal(real_dx, dx.asnumpy()[i]) + if __name__ == '__main__': test_random() + test_sample_multinomial() From 54261549cfa45dc6e6ca66641052ce03495e6096 Mon Sep 17 00:00:00 2001 From: "Qiang Kou (KK)" Date: Sat, 3 Jun 2017 19:35:57 +0000 Subject: [PATCH 031/834] [R] save/load MXNet model with RData format. close #362 (#6494) * [R] mx.serialize/mx.unserialize (close #362) --- R-package/R/model.R | 50 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/R-package/R/model.R b/R-package/R/model.R index 2e9a555a3477..80edbc804b06 100644 --- a/R-package/R/model.R +++ b/R-package/R/model.R @@ -489,6 +489,7 @@ function(symbol, X, y=NULL, ctx=NULL, begin.round=1, #' #' @export predict.MXFeedForwardModel <- function(model, X, ctx=NULL, array.batch.size=128, array.layout="auto") { + if (is.serialized(model)) model <- mx.unserialize(model) if (is.null(ctx)) ctx <- mx.ctx.default() if (is.array(X) || is.matrix(X)) { if (array.layout == "auto") { @@ -587,3 +588,52 @@ mx.model.save <- function(model, prefix, iteration) { mx.symbol.save(model$symbol, paste0(prefix, "-symbol.json")) mx.nd.save(save.dict, sprintf("%s-%04d.params", prefix, iteration)) } + +#' Check if the model has been serialized into RData-compatiable format. +#' +#' @return Logical indicator +#' +#' @export +is.serialized <- function(model) { + if (!is.null(model[['is.serialized']])) { + return(model[['is.serialized']]) + } else { + return(FALSE) + } +} + +#' Serialize MXNet model into RData-compatiable format. +#' +#' @param model The mxnet model +#' +#' @export +mx.serialize <- function(model) { + if (!is.serialized(model)) { + model_rdata <- list() + model_rdata[['symbol_json']] <- model$symbol$as.json() + model_rdata[['arg.params']] <- lapply(model$arg.params, as.array) + model_rdata[['aux.params']] <- lapply(model$aux.params, as.array) + model_rdata[['is.serialized']] <- TRUE + class(model_rdata) <- "MXFeedForwardModel" + return(model_rdata) + } else { + return(model) + } +} + +#' Unserialize MXNet model from Robject. +#' +#' @param model The mxnet model loaded from RData files. +#' +#' @export +mx.unserialize <- function(model) { + if (!is.serialized(model)) { + return(model) + } else { + symbol <- mx.symbol.load.json(model$symbol_json) + arg.params <- lapply(model$arg.params, mx.nd.array) + aux.params <- lapply(model$aux.params, mx.nd.array) + model <- list(symbol=symbol, arg.params=arg.params, aux.params=aux.params) + return(structure(model, class="MXFeedForwardModel")) + } +} From deff0131adad80febe4578a4346d5b5776054d96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=A2=81=E5=BE=B7=E6=BE=8E?= Date: Sun, 4 Jun 2017 20:44:18 +0800 Subject: [PATCH 032/834] [Scala] Optimizer support lr_mult and wd_mult (#6557) * [Scala] Optimizer support lr_mult and wd_mult --- .../main/scala/ml/dmlc/mxnet/LibInfo.scala | 6 + .../main/scala/ml/dmlc/mxnet/Optimizer.scala | 104 +++++++++++++++++- .../src/main/scala/ml/dmlc/mxnet/Symbol.scala | 33 ++++++ .../ml/dmlc/mxnet/optimizer/AdaGrad.scala | 2 +- .../scala/ml/dmlc/mxnet/optimizer/Adam.scala | 5 +- .../ml/dmlc/mxnet/optimizer/DCASGD.scala | 5 +- .../scala/ml/dmlc/mxnet/optimizer/NAG.scala | 5 +- .../ml/dmlc/mxnet/optimizer/RMSProp.scala | 2 +- .../scala/ml/dmlc/mxnet/optimizer/SGD.scala | 5 +- .../scala/ml/dmlc/mxnet/optimizer/SGLD.scala | 5 +- .../examples/scripts/customop/run_customop.sh | 2 +- .../scripts/customop/run_customopwithrtc.sh | 2 +- .../scripts/run_cnntextclassification.sh | 2 +- .../main/native/ml_dmlc_mxnet_native_c_api.cc | 46 ++++++++ 14 files changed, 203 insertions(+), 21 deletions(-) diff --git a/scala-package/core/src/main/scala/ml/dmlc/mxnet/LibInfo.scala b/scala-package/core/src/main/scala/ml/dmlc/mxnet/LibInfo.scala index f776117df8ed..97ba81528d30 100644 --- a/scala-package/core/src/main/scala/ml/dmlc/mxnet/LibInfo.scala +++ b/scala-package/core/src/main/scala/ml/dmlc/mxnet/LibInfo.scala @@ -186,6 +186,12 @@ private[mxnet] class LibInfo { paramVals: Array[String], symHandleRef: SymbolHandleRef): Int @native def mxSymbolSetAttr(handle: SymbolHandle, key: String, value: String): Int + @native def mxSymbolListAttrShallow(handle: SymbolHandle, + outSize: MXUintRef, + out: ArrayBuffer[String]): Int + @native def mxSymbolListAttr(handle: SymbolHandle, + outSize: MXUintRef, + out: ArrayBuffer[String]): Int @native def mxSymbolCompose(handle: SymbolHandle, name: String, keys: Array[String], diff --git a/scala-package/core/src/main/scala/ml/dmlc/mxnet/Optimizer.scala b/scala-package/core/src/main/scala/ml/dmlc/mxnet/Optimizer.scala index 128fc9b53296..27db5656d7d7 100644 --- a/scala-package/core/src/main/scala/ml/dmlc/mxnet/Optimizer.scala +++ b/scala-package/core/src/main/scala/ml/dmlc/mxnet/Optimizer.scala @@ -20,6 +20,7 @@ package ml.dmlc.mxnet import java.io._ import scala.collection.mutable +import scala.util.Either object Optimizer { def getUpdater(optimizer: Optimizer): MXKVStoreUpdater = { @@ -103,7 +104,10 @@ object Optimizer { } abstract class Optimizer extends Serializable { - protected var lrScale: mutable.Map[Int, Float] = mutable.HashMap.empty[Int, Float] + protected val lrMult: mutable.Map[Either[Int, String], Float] = + mutable.HashMap.empty[Either[Int, String], Float] + protected val wdMult: mutable.Map[Either[Int, String], Float] = + mutable.HashMap.empty[Either[Int, String], Float] protected var numUpdate: Int = 0 protected val indexUpdateCount: mutable.Map[Int, Int] = mutable.HashMap.empty[Int, Int] @@ -136,8 +140,62 @@ abstract class Optimizer extends Serializable { def deserializeState(bytes: Array[Byte]): AnyRef // Set individual learning rate scale for parameters - def setLrScale(lrScale: Map[Int, Float]) { - this.lrScale = mutable.Map(lrScale.toSeq: _*) + @deprecated("Use setLrMult instead.") + def setLrScale(lrScale: Map[Int, Float]): Unit = { + val argsLrScale: Map[Either[Int, String], Float] = lrScale.map { case (k, v) => Left(k) -> v } + setLrMult(argsLrScale) + } + + /** + * Sets an individual learning rate multiplier for each parameter. + * If you specify a learning rate multiplier for a parameter, then + * the learning rate for the parameter will be set as the product of + * the global learning rate and its multiplier. + * note:: The default learning rate multiplier of a `Variable` + * can be set with `lr_mult` argument in the constructor. + * @param argsLrMult: Map[Either[Int, String], Float] + * For each of its key-value entries, the learning rate multipler for the + * parameter specified in the key will be set as the given value. + * + * You can specify the parameter with either its name or its index. + * If you use the name, you should also call the `setSymbol` method first, + * and the name you specified in the key of `argsLrMult` should match + * the name of the parameter in the `sym` you pass to `setSymbol` method. + * If you use the index, it should correspond to the index of the parameter + * used in the `update` method. + * + * Specifying a parameter by its index is only supported for backward + * compatibility, and we recommend to use the name instead. + */ + def setLrMult(argsLrMult: Map[Either[Int, String], Float]): Unit = { + argsLrMult.foreach { case (k, v) => this.lrMult(k) = v } + } + + /** + * Sets an individual weight decay multiplier for each parameter. + * + * By default, the weight decay multipler is set as 0 for all + * parameters whose name don't end with ``_weight`` or ``_gamma``, if + * you call the `setIdx2Name` method to set idx2name. + * + * note:: The default weight decay multiplier for a `Variable` + * can be set with its `wd_mult` argument in the constructor. + * @param argsWdMult: Map[Either[Int, String], Float] + * For each of its key-value entries, the learning rate multipler for the + * parameter specified in the key will be set as the given value. + * + * You can specify the parameter with either its name or its index. + * If you use the name, you should also call the `setSymbol` method first, + * and the name you specified in the key of `argsWdMult` should match + * the name of the parameter in the `sym` you pass to `setSymbol` method. + * If you use the index, it should correspond to the index of the parameter + * used in the `update` method. + * + * Specifying a parameter by its index is only supported for backward + * compatibility, and we recommend to use the name instead. + */ + def setWdMult(argsWdMult: Map[Either[Int, String], Float]): Unit = { + argsWdMult.foreach { case (k, v) => this.wdMult(k) = v } } def setArgNames(argNames: Seq[String]): Unit = { @@ -160,14 +218,30 @@ abstract class Optimizer extends Serializable { this.rescaleGrad = rescaleGrad } - // TODO def setSymbol(sym: Symbol): Unit = { this.symbol = sym + if (this.symbol != null) { + val attr = this.symbol.attrMap + for (name <- this.symbol.listArguments()) { + if (attr.contains(name) && attr(name).contains("__lr_mult__")) { + this.lrMult(Right(name)) = attr(name)("__lr_mult__").toFloat + } + if (attr.contains(name) && attr(name).contains("__wd_mult__")) { + this.wdMult(Right(name)) = attr(name)("__wd_mult__").toFloat + } + } + } } - // TODO: Special treat weight decay in parameters. def setIdx2Name(paramIdx2Name: Map[Int, String]): Unit = { this.idx2name = paramIdx2Name + if (this.idx2name != null) { + for (n <- this.idx2name.values) { + if (!(n.endsWith("_weight") || n.endsWith("_gamma"))) { + this.wdMult(Right(n)) = 0f + } + } + } } /** @@ -180,8 +254,20 @@ abstract class Optimizer extends Serializable { numUpdate = Math.max(count, numUpdate) } + // Gets the learning rate given the index of the weight. + protected def getLr(index: Int, lr: Float): Float = { + var llr = lr + if (this.lrMult.contains(Left(index))) { + llr *= this.lrMult(Left(index)) + } else if (this.idx2name != null && this.idx2name.contains(index)) { + llr *= this.lrMult.getOrElse(Right(this.idx2name(index)), 1.0f) + } + llr + } + + // Gets weight decay for index. protected def getWd(index: Int, wd: Float): Float = { - if (specialized) { + var lwd = if (specialized) { if (this.weightSet.contains(index)) { wd } else { @@ -190,6 +276,12 @@ abstract class Optimizer extends Serializable { } else { wd } + if (this.wdMult.contains(Left(index))) { + lwd *= this.wdMult(Left(index)) + } else if (this.idx2name != null && this.idx2name.contains(index)) { + lwd *= this.wdMult.getOrElse(Right(this.idx2name(index)), 1.0f) + } + lwd } } diff --git a/scala-package/core/src/main/scala/ml/dmlc/mxnet/Symbol.scala b/scala-package/core/src/main/scala/ml/dmlc/mxnet/Symbol.scala index beb793a25713..41ae59c907ff 100644 --- a/scala-package/core/src/main/scala/ml/dmlc/mxnet/Symbol.scala +++ b/scala-package/core/src/main/scala/ml/dmlc/mxnet/Symbol.scala @@ -312,6 +312,39 @@ class Symbol private(private[mxnet] val handle: SymbolHandle) { } } + /** + * Gets all attributes from the symbol. + * @return Map[String, String], mapping attribute keys to values. + */ + def listAttr(): Map[String, String] = { + val outSize = new MXUintRef + val out = ArrayBuffer[String]() + checkCall(_LIB.mxSymbolListAttrShallow(handle, outSize, out)) + (0 until outSize.value).map(i => out(i * 2) -> out(i * 2 + 1)).toMap + } + + /** + * Recursively gets all attributes from the symbol and its children. + * @return Map[Map[String, String]], There is a key in the returned + * dict for every child with non-empty attribute set. For each symbol, + * the name of the symbol is its key in the dict and the correspond value + * is that symbol's attribute list (itself a dictionary). + */ + def attrMap(): Map[String, Map[String, String]] = { + val outSize = new MXUintRef + val out = ArrayBuffer[String]() + checkCall(_LIB.mxSymbolListAttr(handle, outSize, out)) + val result = { + val tmp = out.toArray.grouped(2).map{ strs => + val nk = strs(0).split('$') + (nk(0), nk(1), strs(1)) + }.toArray + val grouped = tmp.groupBy(_._1) + grouped.map { case (name, kvs) => name -> kvs.map(x => (x._2, x._3)).toMap } + } + result + } + /** * Save symbol into file. * You can also use pickle to do the job if you only work on python. diff --git a/scala-package/core/src/main/scala/ml/dmlc/mxnet/optimizer/AdaGrad.scala b/scala-package/core/src/main/scala/ml/dmlc/mxnet/optimizer/AdaGrad.scala index 759b9468f7d8..c13fe2ab1dba 100644 --- a/scala-package/core/src/main/scala/ml/dmlc/mxnet/optimizer/AdaGrad.scala +++ b/scala-package/core/src/main/scala/ml/dmlc/mxnet/optimizer/AdaGrad.scala @@ -42,7 +42,7 @@ class AdaGrad(val learningRate: Float = 0.05f, rescaleGradient: Float = 1.0f, * The auxiliary state used in optimization. */ override def update(index: Int, weight: NDArray, grad: NDArray, state: AnyRef): Unit = { - val lr = this.learningRate + val lr = getLr(index, this.learningRate) val resdGrad = rescaleGradient * grad val history = state.asInstanceOf[NDArray] diff --git a/scala-package/core/src/main/scala/ml/dmlc/mxnet/optimizer/Adam.scala b/scala-package/core/src/main/scala/ml/dmlc/mxnet/optimizer/Adam.scala index 10f90ae1e2ff..f611192c0905 100644 --- a/scala-package/core/src/main/scala/ml/dmlc/mxnet/optimizer/Adam.scala +++ b/scala-package/core/src/main/scala/ml/dmlc/mxnet/optimizer/Adam.scala @@ -57,14 +57,15 @@ class Adam(val learningRate: Float = 0.002f, beta1: Float = 0.9f, beta2: Float = * The auxiliary state used in optimization. */ override def update(index: Int, weight: NDArray, grad: NDArray, state: AnyRef): Unit = { - val lr = + var lr = (if (lrScheduler != null) { val scheduledLr = lrScheduler(numUpdate) updateCount(index) scheduledLr } else { this.learningRate - }) * lrScale.getOrElse(index, 1f) + }) + lr = getLr(index, lr) val (mean, variance) = state.asInstanceOf[(NDArray, NDArray)] diff --git a/scala-package/core/src/main/scala/ml/dmlc/mxnet/optimizer/DCASGD.scala b/scala-package/core/src/main/scala/ml/dmlc/mxnet/optimizer/DCASGD.scala index 763c0346482f..5af4caa2e634 100644 --- a/scala-package/core/src/main/scala/ml/dmlc/mxnet/optimizer/DCASGD.scala +++ b/scala-package/core/src/main/scala/ml/dmlc/mxnet/optimizer/DCASGD.scala @@ -43,14 +43,15 @@ class DCASGD(val learningRate: Float = 0.01f, momentum: Float = 0.0f, * The auxiliary state used in optimization. */ override def update(index: Int, weight: NDArray, grad: NDArray, state: AnyRef): Unit = { - val lr = + var lr = (if (lrScheduler != null) { val scheduledLr = lrScheduler(numUpdate) updateCount(index) scheduledLr } else { this.learningRate - }) * lrScale.getOrElse(index, 1f) + }) + lr = getLr(index, lr) val wd = getWd(index, this.wd) var resdGrad = grad * this.rescaleGrad diff --git a/scala-package/core/src/main/scala/ml/dmlc/mxnet/optimizer/NAG.scala b/scala-package/core/src/main/scala/ml/dmlc/mxnet/optimizer/NAG.scala index f1ee4cba637a..2b2ce5f461d5 100644 --- a/scala-package/core/src/main/scala/ml/dmlc/mxnet/optimizer/NAG.scala +++ b/scala-package/core/src/main/scala/ml/dmlc/mxnet/optimizer/NAG.scala @@ -49,14 +49,15 @@ class NAG(val learningRate: Float = 0.01f, momentum: Float = 0.0f, */ override def update(index: Int, weight: NDArray, grad: NDArray, state: AnyRef): Unit = { // TODO(bing) implement wd_bias, wd_gamma, wd_beta (copy from python package) - val lr = + var lr = (if (lrScheduler != null) { val scheduledLr = lrScheduler(numUpdate) updateCount(index) scheduledLr } else { this.learningRate - }) * lrScale.getOrElse(index, 1f) + }) + lr = getLr(index, lr) val wd = getWd(index, this.wd) var resdGrad = grad * this.rescaleGrad diff --git a/scala-package/core/src/main/scala/ml/dmlc/mxnet/optimizer/RMSProp.scala b/scala-package/core/src/main/scala/ml/dmlc/mxnet/optimizer/RMSProp.scala index a001eb05f496..b1b6e4004126 100644 --- a/scala-package/core/src/main/scala/ml/dmlc/mxnet/optimizer/RMSProp.scala +++ b/scala-package/core/src/main/scala/ml/dmlc/mxnet/optimizer/RMSProp.scala @@ -46,7 +46,7 @@ class RMSProp(val learningRate: Float = 0.002f, rescaleGradient: Float = 1.0f, * The auxiliary state used in optimization. */ override def update(index: Int, weight: NDArray, grad: NDArray, state: AnyRef): Unit = { - val lr = this.learningRate * lrScale.getOrElse(index, 1f) + val lr = getLr(index, this.learningRate) val (n, g, delta) = state.asInstanceOf[(NDArray, NDArray, NDArray)] val wd = getWd(index, this.wd) diff --git a/scala-package/core/src/main/scala/ml/dmlc/mxnet/optimizer/SGD.scala b/scala-package/core/src/main/scala/ml/dmlc/mxnet/optimizer/SGD.scala index e77d519ca29d..d3099d53f063 100644 --- a/scala-package/core/src/main/scala/ml/dmlc/mxnet/optimizer/SGD.scala +++ b/scala-package/core/src/main/scala/ml/dmlc/mxnet/optimizer/SGD.scala @@ -41,14 +41,15 @@ class SGD(val learningRate: Float = 0.01f, momentum: Float = 0.0f, */ override def update(index: Int, weight: NDArray, grad: NDArray, state: AnyRef): Unit = { // TODO(bing) implement wd_bias, wd_gamma, wd_beta (copy from python package) - val lr = + var lr = (if (lrScheduler != null) { val scheduledLr = lrScheduler(numUpdate) updateCount(index) scheduledLr } else { this.learningRate - }) * lrScale.getOrElse(index, 1f) + }) + lr = getLr(index, lr) val wd = getWd(index, this.wd) var resdGrad = grad * this.rescaleGrad diff --git a/scala-package/core/src/main/scala/ml/dmlc/mxnet/optimizer/SGLD.scala b/scala-package/core/src/main/scala/ml/dmlc/mxnet/optimizer/SGLD.scala index 8a1d8dcecd7c..cb509f4a062f 100644 --- a/scala-package/core/src/main/scala/ml/dmlc/mxnet/optimizer/SGLD.scala +++ b/scala-package/core/src/main/scala/ml/dmlc/mxnet/optimizer/SGLD.scala @@ -47,14 +47,15 @@ class SGLD(val learningRate: Float = 0.01f, rescaleGradient: Float = 1.0f, * The auxiliary state used in optimization. */ override def update(index: Int, weight: NDArray, grad: NDArray, state: AnyRef): Unit = { - val lr = + var lr = (if (lrScheduler != null) { val scheduledLr = lrScheduler(numUpdate) updateCount(index) scheduledLr } else { this.learningRate - }) * lrScale.getOrElse(index, 1f) + }) + lr = getLr(index, lr) val wd = getWd(index, this.wd) var resdGrad = grad * this.rescaleGrad diff --git a/scala-package/examples/scripts/customop/run_customop.sh b/scala-package/examples/scripts/customop/run_customop.sh index 44c8ef6d50d9..bd425edd5801 100644 --- a/scala-package/examples/scripts/customop/run_customop.sh +++ b/scala-package/examples/scripts/customop/run_customop.sh @@ -13,4 +13,4 @@ DATA_PATH=$2 java -Xmx4G -cp $CLASS_PATH \ ml.dmlc.mxnetexamples.customop.ExampleCustomOp \ --data-path $DATA_PATH \ - --gpu $GPU \ + --gpu $GPU diff --git a/scala-package/examples/scripts/customop/run_customopwithrtc.sh b/scala-package/examples/scripts/customop/run_customopwithrtc.sh index 2d1391054bf4..6009193c8ebb 100644 --- a/scala-package/examples/scripts/customop/run_customopwithrtc.sh +++ b/scala-package/examples/scripts/customop/run_customopwithrtc.sh @@ -13,4 +13,4 @@ DATA_PATH=$1 java -Xmx4G -cp $CLASS_PATH \ ml.dmlc.mxnetexamples.customop.ExampleCustomOpWithRtc \ --data-path $DATA_PATH \ - --gpu $GPU \ + --gpu $GPU diff --git a/scala-package/examples/scripts/run_cnntextclassification.sh b/scala-package/examples/scripts/run_cnntextclassification.sh index 8ace6ff22c29..a7cf7c0a6395 100644 --- a/scala-package/examples/scripts/run_cnntextclassification.sh +++ b/scala-package/examples/scripts/run_cnntextclassification.sh @@ -15,7 +15,7 @@ BATCH_SIZE=$5 SAVE_MODEL_PATH=$6 java -Xmx8G -cp $CLASS_PATH \ - ml.dmlc.mxnetexamples.cnnclassification.CNNTextClassification \ + ml.dmlc.mxnetexamples.cnntextclassification.CNNTextClassification \ --gpu $GPU \ --mr-dataset-path $MR_DATASET_PATH \ --w2v-file-path $W2V_FILE_PATH \ diff --git a/scala-package/native/src/main/native/ml_dmlc_mxnet_native_c_api.cc b/scala-package/native/src/main/native/ml_dmlc_mxnet_native_c_api.cc index 3accefcbffe6..65bf2b77579b 100644 --- a/scala-package/native/src/main/native/ml_dmlc_mxnet_native_c_api.cc +++ b/scala-package/native/src/main/native/ml_dmlc_mxnet_native_c_api.cc @@ -1114,6 +1114,52 @@ JNIEXPORT jint JNICALL Java_ml_dmlc_mxnet_LibInfo_mxSymbolSetAttr return ret; } +JNIEXPORT jint JNICALL Java_ml_dmlc_mxnet_LibInfo_mxSymbolListAttrShallow + (JNIEnv *env, jobject obj, jlong symbolPtr, jobject joutSize, jobject jout) { + mx_uint outSize; + const char** out; + + int ret = MXSymbolListAttrShallow(reinterpret_cast(symbolPtr), &outSize, &out); + + jclass refIntClass = env->FindClass("ml/dmlc/mxnet/Base$RefInt"); + jfieldID valueInt = env->GetFieldID(refIntClass, "value", "I"); + env->SetIntField(joutSize, valueInt, static_cast(outSize)); + + jclass arrayClass = env->FindClass("scala/collection/mutable/ArrayBuffer"); + jmethodID arrayAppend = env->GetMethodID(arrayClass, + "$plus$eq", "(Ljava/lang/Object;)Lscala/collection/mutable/ArrayBuffer;"); + for (size_t i = 0; i < outSize * 2; ++i) { + jstring jtmp = env->NewStringUTF(out[i]); + env->CallObjectMethod(jout, arrayAppend, jtmp); + env->DeleteLocalRef(jtmp); + } + + return ret; +} + +JNIEXPORT jint JNICALL Java_ml_dmlc_mxnet_LibInfo_mxSymbolListAttr + (JNIEnv *env, jobject obj, jlong symbolPtr, jobject joutSize, jobject jout) { + mx_uint outSize; + const char** out; + + int ret = MXSymbolListAttr(reinterpret_cast(symbolPtr), &outSize, &out); + + jclass refIntClass = env->FindClass("ml/dmlc/mxnet/Base$RefInt"); + jfieldID valueInt = env->GetFieldID(refIntClass, "value", "I"); + env->SetIntField(joutSize, valueInt, static_cast(outSize)); + + jclass arrayClass = env->FindClass("scala/collection/mutable/ArrayBuffer"); + jmethodID arrayAppend = env->GetMethodID(arrayClass, + "$plus$eq", "(Ljava/lang/Object;)Lscala/collection/mutable/ArrayBuffer;"); + for (size_t i = 0; i < outSize * 2; ++i) { + jstring jtmp = env->NewStringUTF(out[i]); + env->CallObjectMethod(jout, arrayAppend, jtmp); + env->DeleteLocalRef(jtmp); + } + + return ret; +} + JNIEXPORT jint JNICALL Java_ml_dmlc_mxnet_LibInfo_mxSymbolCompose (JNIEnv *env, jobject obj, jlong symbolPtr, jstring jname, jobjectArray jkeys, jlongArray jargs) { From da08c9203ecd1d8e3cd6a29ecf1a9238521f2351 Mon Sep 17 00:00:00 2001 From: ziheng Date: Sun, 4 Jun 2017 17:43:47 -0700 Subject: [PATCH 033/834] Some Changes to NDArray Interface (#6561) * Add SetTBlob * Remove raw_data and change offset_ to byte_offset_ * Access TBlob by friend class declaration * Fix MKL --- include/mxnet/ndarray.h | 47 ++++++++++++------------------------- include/mxnet/tensor_blob.h | 24 ++++--------------- src/ndarray/ndarray.cc | 4 +++- 3 files changed, 23 insertions(+), 52 deletions(-) diff --git a/include/mxnet/ndarray.h b/include/mxnet/ndarray.h index f30b09a054f0..504fd5e7676e 100644 --- a/include/mxnet/ndarray.h +++ b/include/mxnet/ndarray.h @@ -74,7 +74,7 @@ class NDArray { NDArray(const TShape &shape, Context ctx, bool delay_alloc = false, int dtype = mshadow::default_type_flag) : ptr_(std::make_shared(shape.Size(), ctx, delay_alloc, dtype)), - shape_(shape), offset_(0), dtype_(dtype), entry_({nullptr, 0, 0}) { + shape_(shape), dtype_(dtype), entry_({nullptr, 0, 0}) { #if MKL_EXPERIMENTAL == 1 Mkl_mem_ = std::make_shared(); #endif @@ -87,7 +87,7 @@ class NDArray { * \param dev_id the device id this tensor sits at */ NDArray(const TBlob &data, int dev_id) - : ptr_(std::make_shared(data, dev_id)), shape_(data.shape_), offset_(0), + : ptr_(std::make_shared(data, dev_id)), shape_(data.shape_), dtype_(data.type_flag_), entry_({nullptr, 0, 0}) { #if MKL_EXPERIMENTAL == 1 Mkl_mem_ = std::make_shared(); @@ -104,36 +104,9 @@ class NDArray { */ inline const TBlob& data() const { CheckAndAlloc(); -#if MKL_EXPERIMENTAL == 1 - MSHADOW_TYPE_SWITCH(dtype_, DType, { - tblob_ = TBlob(static_cast(ptr_->shandle.dptr) + offset_, - shape_, ptr_->shandle.ctx.dev_mask(), ptr_->shandle.ctx.dev_id, Mkl_mem_); - }); -#else - MSHADOW_TYPE_SWITCH(dtype_, DType, { - tblob_ = TBlob(static_cast(ptr_->shandle.dptr) + offset_, - shape_, ptr_->shandle.ctx.dev_mask(), ptr_->shandle.ctx.dev_id); - }); -#endif + SetTBlob(); return tblob_; } - /*! - * \return a chunk of raw data in TBlob - */ - inline TBlob raw_data(index_t offset, index_t length) const { - CheckAndAlloc(); - TBlob res; - TShape raw_shape(1); - raw_shape[0] = length; - MSHADOW_TYPE_SWITCH(dtype_, DType, { - res = TBlob(static_cast(ptr_->shandle.dptr) + offset_ + offset, - raw_shape, ptr_->shandle.ctx.dev_mask(), ptr_->shandle.ctx.dev_id); - }); -#if MKL_EXPERIMENTAL == 1 - res.Mkl_mem_ = Mkl_mem_; -#endif - return res; - } /*! * \return the context of NDArray, this function is only valid when the NDArray is not empty */ @@ -421,6 +394,16 @@ class NDArray { } }; + void SetTBlob() const { + tblob_.dptr_ = static_cast(ptr_->shandle.dptr) + byte_offset_; + tblob_.shape_ = shape_; + tblob_.type_flag_ = dtype_; + tblob_.SetDLTensor(ptr_->shandle.ctx.dev_mask(), ptr_->shandle.ctx.dev_id); +#if MKL_EXPERIMENTAL == 1 + tblob_.Mkl_mem_ = Mkl_mem_; +#endif + } + #if MKL_EXPERIMENTAL == 1 std::shared_ptr Mkl_mem_; #endif @@ -428,8 +411,8 @@ class NDArray { std::shared_ptr ptr_; /*! \brief shape of current NDArray */ TShape shape_; - /*! \brief offset in chunk */ - size_t offset_; + /*! \brief byte offset in chunk */ + size_t byte_offset_ = 0; /*! \brief type of data */ int dtype_ = -1; /*! \brief node entry for autograd */ diff --git a/include/mxnet/tensor_blob.h b/include/mxnet/tensor_blob.h index d142c20aa30a..9a9774acf14d 100755 --- a/include/mxnet/tensor_blob.h +++ b/include/mxnet/tensor_blob.h @@ -22,6 +22,9 @@ #endif namespace mxnet { +/* Forward declaration for friend declaration in TBlob */ +class NDArray; + /*! * \brief tensor blob class that can be used to hold tensor of any dimension, * any device and any data type, @@ -35,6 +38,7 @@ namespace mxnet { * and wait for further processing */ class TBlob { + friend class NDArray; public: /*! \brief pointer to the data */ void *dptr_; @@ -72,24 +76,6 @@ class TBlob { #endif SetDLTensor(dev_mask, dev_id); } -#if MKL_EXPERIMENTAL == 1 - /*! - * \brief constructor that construct TBlob from contiguous memory - * \param dptr the pointer to the memory - * \param shape the shape of the data - * \param dev_mask the device mask, can be cpu::kDevMask or gpu::kDevMask - * \param dev_id the device id - * \param Mkl_mem the mkl memory - */ - template - TBlob(DType *dptr, const TShape &shape, int dev_mask, int dev_id, - std::shared_ptr Mkl_mem) - : dptr_(dptr), shape_(shape), - type_flag_(mshadow::DataType::kFlag), - Mkl_mem_(Mkl_mem) { - SetDLTensor(dev_mask, dev_id); - } -#endif /*! * \brief constructor that construct TBlob from contiguous memory * \param dptr the pointer to the memory @@ -231,7 +217,7 @@ class TBlob { * \brief return the corresponding DLTensor * \return the address of internal DLTensor */ - inline const DLTensor& dltensor() { + inline const DLTensor& dltensor() const { return dltensor_; } diff --git a/src/ndarray/ndarray.cc b/src/ndarray/ndarray.cc index 025624c923f5..6f1795d6f368 100644 --- a/src/ndarray/ndarray.cc +++ b/src/ndarray/ndarray.cc @@ -63,7 +63,9 @@ NDArray NDArray::Slice(index_t begin, index_t end) const { CHECK(!is_none()) << "NDArray is not initialized"; CHECK_GE(shape_[0], end) << "Slice end index out of range"; size_t length = shape_.ProdShape(1, shape_.ndim()); - ret.offset_ += begin * length; + MSHADOW_TYPE_SWITCH(ret.dtype(), DType, { + ret.byte_offset_ += begin * length * sizeof(DType); + }); ret.shape_[0] = end - begin; if (AutogradRuntime::Get()->IsTraining()) { // fake a slice_axis op From b2e7c3aad0eee3af8aab22fc0ec24fbf530a5bd2 Mon Sep 17 00:00:00 2001 From: Eric Junyuan Xie Date: Mon, 5 Jun 2017 10:12:31 -0700 Subject: [PATCH 034/834] Update multibox_detection.cc --- src/operator/contrib/multibox_detection.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/operator/contrib/multibox_detection.cc b/src/operator/contrib/multibox_detection.cc index 6a4bfdd189b1..2bf49f34d1ff 100644 --- a/src/operator/contrib/multibox_detection.cc +++ b/src/operator/contrib/multibox_detection.cc @@ -176,7 +176,7 @@ MXNET_REGISTER_OP_PROPERTY(_contrib_MultiBoxDetection, MultiBoxDetectionProp) .describe("Convert multibox detection predictions.") .add_argument("cls_prob", "NDArray-or-Symbol", "Class probabilities.") .add_argument("loc_pred", "NDArray-or-Symbol", "Location regression predictions.") -.add_argument("anchors", "NDArray-or-Symbol", "Multibox prior anchor boxes") +.add_argument("anchor", "NDArray-or-Symbol", "Multibox prior anchor boxes") .add_arguments(MultiBoxDetectionParam::__FIELDS__()); } // namespace op } // namespace mxnet From 788c280251de4aa093819b67e6f3d036cbccaec1 Mon Sep 17 00:00:00 2001 From: Chris Olivier Date: Mon, 5 Jun 2017 14:54:16 -0700 Subject: [PATCH 035/834] batchnorm specify channel axis and performance optimizations for batchnorm (#6411) * Add channel_axis to batch norm, performance improvements * rearrange tests a bit * rearrange tests a bit * CR changes * cpp package link issue * Fix: MSVC wants all parallel omp to be int * CR comments, expand legal negative axes * lint * lint * Fix download link (#6431) * Fix Download Button * Small fix * Add release note (#6434) * Fixing tutorials. (#6436) Most of the fixes should be self evident. For tutorial on pre-trained models, one of the images doesn't exist anymore so selected a new one. Long-term, we should put such images on web-data repo but alas, some other day. For Handwritten digit tutorial, we are missing couple of imports in the test_utils.py that was recently created. Note that: for pre-trained model tutorial, we get a softmax_label warning and the probability scores are not really probabilities. Will deal with that issue in another PR. Testing: I've tried to test all the notebooks with this change and things look fine. * Formatting fixes (#6433) * Formatting fixes * lint fixed * fix * doc bash 2-5, for pack, unpack, pack_img and unpack_img (#6140) * doc bash for pack, unpack, pack_img and unpack_img * Add comments for labels could be 1d list * Update recordio.py * Update recordio.py * Update recordio.py fixing text * Update recordio.py fixing text * remove empty line * Improve style (#6445) * Correction (#6444) * CSVIter example correction * fix * Update documentation for MXNetDataIter in io.py (#6000) (#6113) * Update documentation for MXNetDataIter in io.py (#6000) * [DOC] Respond to feedback (#6113) * Fix minor issues with api pages. (#6410) 1. In the notes section for ndarray, references did not seem clear enough to be referring to mxnet.ndarray or numpy.ndarray. Added the package names as prefixes to make it more obvious. 2. "share the same C++ operator source codes" => "share the same code". Since we don't really need to throw in more details than required. 3. Other relatively minor language changes which will be obvious from the diff. Note that I'm relatively not sure about the need for 1/ since it makes things more verbose. Let me know if it unnecessary and I'll remove it. * fixing the early stop for maximize = T (#5915) close #4587 * Update documentation for mxnet.ndarray.GridGenerator. (#6430) * Update documentation for mxnet.ndarray.GridGenerator. Thanks @Lyken17 for https://github.com/dmlc/mxnet/issues/6147 * Fix lint error. * Minor fix. * Remove the example. * Update documentation for deconvolution operation. (#6184) * Update documentation for deconvolution operation. * Add examples. * Remove the example. * skip lines that have %matplotlib (#6451) * Fixing some more broken links before v0.10 release (#6449) * close #4838 (#6452) * Fix linear regression (#6432) * Fix Linear Regression Tutorial * Small fix * Pre-trained model tutorial fixes. (#6453) Before the change on running the tutorial for the first time: "UserWarning: Data provided by label_shapes don't match names specified by label_names ([] vs. ['softmax_label'])". It also showed probability of >>1 due to incorrect usage of np.argsort(). * Nightly test tutorial (#6447) * Add tutorial test * Fix pylint * Small fix * fix member variable name: make them end with underline (#6438) * [R] captcha example (#6443) * skip lines that have %matplotlib (#6459) * Fix cudnn_deconv not guarding no_bias (#6456) * Fixing up issues in install guide (#6463) * Fixing copy code functionality for bash command (#6465) * Residual unroll (#6397) * residual unroll * unroll for residual cell * merge_outputs fix * Linear regression Tutorial link (#6468) * Fixing a link in the linear regression tutorial. The link was initally going to mxnet-test.readthedocs.io. Changed it to mxnet.io/api. * More appropriate language. * bump up version number for release (#6462) * bump up version number for release * update version for scala/R/backend * [R][DOC] update R installation guide (#6457) * Use sphinx==1.3.5 in Dockerfile.doc (#6470) changed PR name * Add 0.10 release info to README.md and NEWS.md (#6471) @nswamy wants to merge it immediately, so i'm going to do it now. I also changed the PR title. * fix batchNorm cpp example (#6454) * Update im2rec.py (#6473) Updated Line 107 of 'im2rec.py'. Read an image as binary. * Change Interface of NDArray & TBlob for DLPack Compatible (#6345) * Change Interface of NDArray & TBlob for DLPack Compatible Fix for cudnn operator Fix cpp tests * Update nnvm * Fix for MKL mem * Fix for windows macro * Bump up version number to 0.10.1 * Update NDArray Save&Load * trigger update * Add test for legacy data load * Use LegacyTShapeLoad * trigger update * Update tensor_blob.h * change 'channel_axis' parameter to 'axis' * Change DEFAULT_CHANNEL_AXIS to DEFAULT_AXIS * wait for dlpack PR to go through * Trigger build --- CMakeLists.txt | 6 +- cpp-package/example/CMakeLists.txt | 3 +- src/common/cuda_utils.h | 6 +- src/operator/batch_norm-inl.h | 153 ++++++++- src/operator/batch_norm.cc | 329 ++++++-------------- src/operator/batch_norm.cu | 269 ++++++++-------- tests/CMakeLists.txt | 3 +- tests/cpp/operator/batchnorm_test.cc | 411 ++++++++++++++++++++++++- tests/python/unittest/test_operator.py | 33 ++ 9 files changed, 838 insertions(+), 375 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d0835300edaa..f6878b80e7d5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -144,6 +144,11 @@ include_directories("nnvm/include") include_directories("dmlc-core/include") include_directories("dlpack/include") +# commented out until PR goes through +#if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/dlpack) +# add_subdirectory(dlpack) +#endif() + if(NOT MSVC) set(BEGIN_WHOLE_ARCHIVE -Wl,--whole-archive) set(END_WHOLE_ARCHIVE -Wl,--no-whole-archive) @@ -426,7 +431,6 @@ if(USE_PROFILER) add_definitions(-DMXNET_USE_PROFILER) endif() -# Do tests after chrpath so that we use the "real" cuda driver add_subdirectory(tests) # AUTO_INSTALL_DIR -> Optional: specify post-build install direcory diff --git a/cpp-package/example/CMakeLists.txt b/cpp-package/example/CMakeLists.txt index 66e3dd8964b5..7083dfd014e9 100644 --- a/cpp-package/example/CMakeLists.txt +++ b/cpp-package/example/CMakeLists.txt @@ -3,7 +3,8 @@ if(NOT MSVC) endif() set(CPP_EXAMPLE_LIBS - ${BEGIN_WHOLE_ARCHIVE} mxnet ${END_WHOLE_ARCHIVE} + ${BEGIN_WHOLE_ARCHIVE} mxnet_static ${END_WHOLE_ARCHIVE} + ${BEGIN_WHOLE_ARCHIVE} dmlc ${END_WHOLE_ARCHIVE} ${mxnet_LINKER_LIBS} ) diff --git a/src/common/cuda_utils.h b/src/common/cuda_utils.h index bb0afb819cf2..d0defc30ffa6 100644 --- a/src/common/cuda_utils.h +++ b/src/common/cuda_utils.h @@ -9,8 +9,6 @@ #include #include -#if MXNET_USE_CUDA - /*! \brief Macros/inlines to assist CLion to parse Cuda files (*.cu, *.cuh) */ #ifdef __JETBRAINS_IDE__ #define __CUDACC__ 1 @@ -22,12 +20,14 @@ inline void __syncthreads() {} inline void __threadfence_block() {} template inline T __clz(const T val) { return val; } -struct __cuda_fake_struct { int x; int y; }; +struct __cuda_fake_struct { int x; int y; int z; }; extern __cuda_fake_struct blockDim; extern __cuda_fake_struct threadIdx; extern __cuda_fake_struct blockIdx; #endif +#if MXNET_USE_CUDA + #include #include #include diff --git a/src/operator/batch_norm-inl.h b/src/operator/batch_norm-inl.h index f93f64a2ebfc..1c735c4abff8 100755 --- a/src/operator/batch_norm-inl.h +++ b/src/operator/batch_norm-inl.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -30,6 +31,9 @@ namespace batchnorm { enum BatchNormOpInputs {kData, kGamma, kBeta}; // kGamma: weights, kBeta: biases enum BatchNormOpOutputs {kOut, kMean, kVar}; // req, out_data enum BatchNormOpAuxiliary {kMovingMean, kMovingVar}; // aux_states + +/*! \brief Default channel axis if none specified int he params */ +constexpr int DEFAULT_AXIS = 1; } // namespace batchnorm /*! \brief Parameters for BatchNoram operator */ @@ -39,6 +43,7 @@ struct BatchNormParam : public dmlc::Parameter { bool fix_gamma; bool use_global_stats; bool output_mean_var; + int axis; bool cudnn_off; DMLC_DECLARE_PARAMETER(BatchNormParam) { DMLC_DECLARE_FIELD(eps).set_default(1e-3f) @@ -54,6 +59,8 @@ struct BatchNormParam : public dmlc::Parameter { "This will force change batch-norm into a scale shift operator."); DMLC_DECLARE_FIELD(output_mean_var).set_default(false) .describe("Output All,normal mean and var"); + DMLC_DECLARE_FIELD(axis).set_default(mxnet::op::batchnorm::DEFAULT_AXIS) + .describe("Specify which shape axis the channel is specified"); DMLC_DECLARE_FIELD(cudnn_off).set_default(false) .describe("Do not select CUDNN operator, if available"); } @@ -187,7 +194,7 @@ class BatchNormOp : public Operator { }; // class BatchNormOp template -Operator *CreateOp(const BatchNormParam& param, const int dtype, const TShape& shape); +Operator *CreateOp(BatchNormParam param, const int dtype, const TShape& shape); #if DMLC_USE_CXX11 class BatchNormProp : public OperatorProperty { @@ -207,21 +214,28 @@ class BatchNormProp : public OperatorProperty { CHECK_EQ(in_shape->size(), 3U) << "Input:[data, gamma, beta]"; const TShape &dshape = in_shape->at(0); + const size_t channelAxis = static_cast(param_.axis < 0 + ? static_cast(dshape.ndim()) + param_.axis + : param_.axis); + CHECK_LT(channelAxis, dshape.ndim()) << "Channel axis out of range: " << param_.axis; + + const int channelCount = dshape[channelAxis]; + if (dshape.ndim() == 0) { return false; } - in_shape->at(1) = TShape(Shape1(dshape[1])); - in_shape->at(2) = TShape(Shape1(dshape[1])); + in_shape->at(1) = TShape(Shape1(channelCount)); + in_shape->at(2) = TShape(Shape1(channelCount)); out_shape->clear(); - out_shape->push_back(dshape); // kOut - out_shape->push_back(Shape1(dshape[1])); // kMean - out_shape->push_back(Shape1(dshape[1])); // kVar + out_shape->push_back(dshape); // kOut + out_shape->push_back(Shape1(channelCount)); // kMean + out_shape->push_back(Shape1(channelCount)); // kVar aux_shape->clear(); - aux_shape->push_back(Shape1(dshape[1])); // kMovingMean - aux_shape->push_back(Shape1(dshape[1])); // kMovingVar + aux_shape->push_back(Shape1(channelCount)); // kMovingMean + aux_shape->push_back(Shape1(channelCount)); // kMovingVar return true; } @@ -329,6 +343,129 @@ class BatchNormProp : public OperatorProperty { BatchNormParam param_; }; // class BatchNormProp +namespace batchnorm { + +template +class BNTensor3 { + enum { OUTER, CHANNEL, INNER, COUNT }; + + public: + inline BNTensor3(const TBlob& blob, const int indexOfChannel) + : dptr_(blob.dptr()) + , indexOfChannel_(static_cast(indexOfChannel < 0 + ? (static_cast(blob.shape_.ndim()) + indexOfChannel) + : indexOfChannel)) { + shape_[OUTER] = 1; + for (size_t i = 0; i < indexOfChannel_; ++i) { + shape_[OUTER] *= blob.shape_[i]; + } + shape_[CHANNEL] = blob.shape_[indexOfChannel_]; + shape_[INNER] = 1; + for (size_t i = indexOfChannel_ + 1, n = blob.shape_.ndim(); i < n; ++i) { + shape_[INNER] *= blob.shape_[i]; + } + } + + inline BNTensor3(DType *p, const TShape& shape, const int indexOfChannel) + : dptr_(p) + , indexOfChannel_(static_cast(indexOfChannel < 0 + ? (static_cast(shape.ndim()) + indexOfChannel) + : indexOfChannel)) { + shape_[OUTER] = 1; + for (size_t i = 0; i < indexOfChannel_; ++i) { + shape_[OUTER] *= shape[i]; + } + shape_[CHANNEL] = shape[indexOfChannel_]; + shape_[INNER] = 1; + for (size_t i = indexOfChannel_ + 1, n = shape.ndim(); i < n; ++i) { + shape_[INNER] *= shape[i]; + } + } + + MSHADOW_FORCE_INLINE bool IsEmpty() const { + return dptr_ == nullptr; + } + + MSHADOW_XINLINE size_t Size() const { + size_t n = 1; + for (int i = 0; i < COUNT; ++i) { + n *= shape_[i]; + } + return n; + } + + MSHADOW_XINLINE size_t ChannelCount() const { + return shape_[CHANNEL]; + } + + MSHADOW_XINLINE size_t OuterSize() const { + return shape_[OUTER]; + } + + MSHADOW_XINLINE size_t InnerSize() const { + return shape_[INNER]; + } + + /*! \brief start of a given channel's spatial data */ + MSHADOW_XINLINE size_t StartOffset(const size_t channel) const { + return channel * InnerSize(); + } + + /*! \brief This is the amount to skip to next same-channel data + * This is the number of bytes to skip from one past the end of the current spatial data + * to the next start of the same channel's "spatial data" + * It is assume that the pointer being calculated points just beyond the + * end of the last blobk of spatial data + * i.e. RGBRGB <-- 2 + * RRGGBB <-- 4 + **/ + MSHADOW_XINLINE size_t SkipLengthToNextSameChannelData() const { + return (ChannelCount() - 1) * InnerSize(); + } + + MSHADOW_XINLINE size_t offset(const size_t outer, + const size_t channel, + const size_t i) const { + const size_t spatial_size = InnerSize(); + const size_t skip_length = SkipLengthToNextSameChannelData(); + size_t off = StartOffset(channel); + off += outer * shape_[CHANNEL] * shape_[INNER]; + const size_t skips = i / spatial_size; + off += (1 + skip_length) * skips; + off += i % spatial_size; + return off; + } + + MSHADOW_XINLINE DType& get_ref(const size_t batch, + const size_t channel, + const size_t i) { + const size_t off = offset(batch, channel, i); + return dptr_[off]; + } + + MSHADOW_XINLINE const DType& get_ref(const size_t batch, + const size_t channel, + const size_t i) const { + const size_t off = offset(batch, channel, i); + return dptr_[off]; + } + + DType *dptr_; + size_t indexOfChannel_; + size_t shape_[COUNT]; +}; + +inline int GetRealAxis(const TShape& shape, int axis) { + if (axis < 0) { + axis += shape.ndim(); + } + return axis; +} + +extern volatile bool disable_mkl; + +} // namespace batchnorm + #endif // DMLC_USE_CXX11 } // namespace op } // namespace mxnet diff --git a/src/operator/batch_norm.cc b/src/operator/batch_norm.cc index 169cb608aa6e..0ef5733f9f8c 100644 --- a/src/operator/batch_norm.cc +++ b/src/operator/batch_norm.cc @@ -21,209 +21,48 @@ namespace mxnet { namespace op { namespace batchnorm { -template -class DeviceTensor3 { - DeviceTensor3(const DeviceTensor3&) = delete; - - public: - inline DeviceTensor3(const TBlob& blob, const size_t indexOfChannel) - : dptr_(blob.dptr()) - , indexOfChannel_(indexOfChannel) - , shape_(3) { - if (indexOfChannel) { - shape_[0] = 1; - for (size_t i = 0; i < indexOfChannel_; ++i) { - shape_[0] *= blob.shape_[i]; - } - } else { - shape_[0] = 0; - } - shape_[1] = blob.shape_[indexOfChannel_]; - shape_[2] = 1; - for (size_t i = indexOfChannel_ + 1, n = blob.shape_.ndim(); i < n; ++i) { - shape_[2] *= blob.shape_[i]; - } - } - - inline size_t Size() const { - size_t n = 1; - for (int i = 0; i < 3; ++i) { - n *= shape_[i]; - } - return n; - } - - inline size_t ChannelCount() const { - return shape_[1]; - } - - inline size_t BatchSize() const { - return shape_[0]; - } - - inline size_t SpatialSize() const { - return shape_[2]; - } - - DType *dptr_; - size_t indexOfChannel_; - TShape shape_; -}; - -/*! \brief offset, given indices such as bn, channel, depth, row, column */ -static inline index_t offset(const TShape& shape, - const size_t *indices, - const size_t indicesSize) { - const size_t dim = shape.ndim(); - size_t offset = 0; - for (size_t i = 0; i < dim; ++i) { - offset *= shape[i]; - if (indicesSize > i) { - offset += indices[i]; - } - } - return offset; -} +/*! \brief Global disable of batchnorm mkl operator for unit testing */ +volatile bool disable_mkl = false; /*! \brief Fast-foreach when you don't care about the position other than channel */ template -static inline void ForEachFast(const DeviceTensor3 &tensor, +static inline void ForEachFast(const BNTensor3 &tensor, const size_t channel, OnData onData) { - const size_t num = tensor.BatchSize(); - const size_t matrixSize = tensor.SpatialSize(); - - size_t indices[2] = {0, channel}; + const size_t num = tensor.OuterSize(); + const size_t matrixSize = tensor.InnerSize(); + const size_t skipLength = tensor.SkipLengthToNextSameChannelData(); + const size_t startOffset = tensor.StartOffset(channel); + DType *data = tensor.dptr_ + startOffset; - for (size_t batchItem = 0; batchItem < num; ++batchItem) { - indices[0] = batchItem; - DType *data = tensor.dptr_ + offset(tensor.shape_, &indices[0], - sizeof(indices)/sizeof(indices[0])); + for (size_t outer = 0; outer < num; ++outer) { for (size_t i = 0; i < matrixSize; ++i) { onData(data++); } + data += skipLength; } } /*! \brief Fast-foreach when you don't care about the position other than channel */ template -static inline void ForEachFast(const DeviceTensor3 &in_data, - const DeviceTensor3 &out_data, +static inline void ForEachFast(const BNTensor3 &in_data, + const BNTensor3 &out_data, const size_t channel, OnData onData) { - const size_t num = in_data.BatchSize(); - const size_t matrixSize = in_data.SpatialSize(); + const size_t num = in_data.OuterSize(); + const size_t matrixSize = in_data.InnerSize(); + const size_t skipLength = in_data.SkipLengthToNextSameChannelData(); + const size_t startOffset = in_data.StartOffset(channel); - size_t indices[2] = {0, channel}; + DType1 *data = in_data.dptr_ + startOffset; + DType2 *odata = out_data.dptr_ + startOffset; - for (size_t batchItem = 0; batchItem < num; ++batchItem) { - indices[0] = batchItem; - const size_t off = offset(in_data.shape_, &indices[0], sizeof(indices)/sizeof(indices[0])); - const DType1 *data = in_data.dptr_ + off; - DType2 *odata = out_data.dptr_ + off; + for (size_t outer = 0; outer < num; ++outer) { for (size_t i = 0; i < matrixSize; ++i) { onData(data++, odata++); } - } -} - -/*! \brief Fast-foreach when you don't care about the position other than channel */ -template -static inline void ForEachFast(const DeviceTensor3& tensor, - OnData onData) { - const size_t num = tensor.BatchSize(); - const size_t channels = tensor.ChannelCount(); - const size_t matrixSize = tensor.SpatialSize(); - - for (size_t batchItem = 0; batchItem < num; ++batchItem) { -#pragma openmp for - for (size_t channel = 0; channel < channels; ++channel) { - size_t indices[2] = { batchItem, channel }; - const size_t off = offset(tensor.shape_, &indices[0], sizeof(indices)/sizeof(indices[0])); - const DType *inData = tensor.dptr_ + off; - for (size_t i = 0; i < matrixSize; ++i) { - onData(channel, inData++); - } - } - } -} - -/*! \brief Fast-foreach when you don't care about the position other than channel */ -template -static inline void ForEachFast(const DeviceTensor3& in_data, - const DeviceTensor3& out_data, - OnData onData) { - const size_t num = in_data.BatchSize(); - const size_t channels = in_data.ChannelCount(); - const size_t matrixSize = in_data.SpatialSize(); - - for (size_t batchItem = 0; batchItem < num; ++batchItem) { -#pragma omp parallel for - for (int channel = 0; channel < channels; ++channel) { - size_t indices[2] = { batchItem, static_cast(channel) }; - const size_t off = offset(in_data.shape_, &indices[0], sizeof(indices)/sizeof(indices[0])); - const DType *inData = in_data.dptr_ + off; - DType *outData = out_data.dptr_ + off; - for (size_t i = 0; i < matrixSize; ++i) { - onData(channel, inData++, outData++); - } - } - } -} - -/*! \brief Compute the mean of each input channel */ -template -static inline void ComputeMean(const DeviceTensor3 &tensor, - AccReal *save_mean) { - const size_t channelCount = tensor.ChannelCount(); - - for (size_t i = 0; i < channelCount; ++i) { - save_mean[i] = 0; - } - - ForEachFast(tensor, - [&save_mean](const size_t channel, const DType *in_data){ - save_mean[channel] += *in_data; - }); - - const size_t itemCount = tensor.Size() / channelCount; - for (size_t i = 0, n = channelCount; i < n; ++i) { - save_mean[i] /= itemCount; - } -} - -/*! \brief Compute the variance of each input channel, as well as update moving mean/variants */ -template -static inline void ComputeVariance(const DeviceTensor3 &tensor, - const AccReal *mean_data, - const DType eps, - const TShape &oshape, - AccReal *save_std) { - const size_t channels = tensor.ChannelCount(); - for (size_t i = 0; i < channels; ++i) { - save_std[i] = 0; - } - ForEachFast(tensor, - [&save_std, &mean_data](const index_t channel, const DType *current_in_data) { - const AccReal mean = mean_data[channel]; - const AccReal current = *current_in_data; - save_std[channel] += (current - mean) * (current - mean); - }); - - const size_t itemCount = tensor.Size() / channels; -#pragma omp parallel for - for (int channel = 0; channel < channels; ++channel) { - const AccReal sum = save_std[channel]; - - AccReal invstd; - if (sum == 0 && eps == 0.0) { - // Nobody likes to divide by zero - invstd = 0; - } else { - const AccReal variance = sum/itemCount; - invstd = VARIANCE_TO_INVSTD(variance, eps); - } - save_std[channel] = invstd; + data += skipLength; + odata += skipLength; } } @@ -238,7 +77,7 @@ void BatchNormOp::DoForward(mshadow::Stream *, const std::vector &out_data, const std::vector &aux_states) { // Input - batchnorm::DeviceTensor3 inputData(in_data[batchnorm::kData], 1); + batchnorm::BNTensor3 inputData(in_data[batchnorm::kData], param_.axis); const TBlob &weights = in_data[batchnorm::kGamma]; const TBlob &bias = in_data[batchnorm::kBeta]; @@ -247,7 +86,7 @@ void BatchNormOp::DoForward(mshadow::Stream *, const TBlob &runningVariance = aux_states[batchnorm::kMovingVar]; // Output - batchnorm::DeviceTensor3 outputData(out_data[batchnorm::kOut], 1); + batchnorm::BNTensor3 outputData(out_data[batchnorm::kOut], param_.axis); const TBlob &meanVector = out_data[batchnorm::kMean]; const TBlob &varianceVector = out_data[batchnorm::kVar]; @@ -255,54 +94,79 @@ void BatchNormOp::DoForward(mshadow::Stream *, AccReal *var = varianceVector.dptr(); const bool is_train_and_not_global_stats = ctx.is_train && !param_.use_global_stats; + const size_t channelCount = inputData.ChannelCount(); + const size_t itemCountPerChannel = inputData.Size() / channelCount; + + #pragma omp parallel for + for (int channel = 0; channel < channelCount; ++channel) { + if (is_train_and_not_global_stats) { + // compute mean per input + mean[channel] = 0; + ForEachFast(inputData, channel, [mean, channel](const DType *in_data) { + mean[channel] += *in_data; }); + mean[channel] /= itemCountPerChannel; + + // compute variance per input + const AccReal thisMean = mean[channel]; + var[channel] = 0; + ForEachFast(inputData, channel, + [var, thisMean, channel](const DType *current_in_data) { + const AccReal current = *current_in_data; + var[channel] += (current - thisMean) * (current - thisMean); + }); + + const AccReal sum = var[channel]; + + AccReal invstd; + if (sum == 0 && param_.eps == 0.0) { + // Nobody likes to divide by zero + invstd = 0; + } else { + const AccReal variance = sum / itemCountPerChannel; + invstd = VARIANCE_TO_INVSTD(variance, param_.eps); + } + var[channel] = invstd; + } else { + const AccReal *rm = runningMean.dptr(); + const AccReal *rv = runningVariance.dptr(); - if (is_train_and_not_global_stats) { - // compute mean per input - ComputeMean(inputData, meanVector.dptr()); - - // compute variance per input - ComputeVariance(inputData, - meanVector.dptr(), - static_cast(param_.eps), - varianceVector.shape_, - var); // var is actually returned as invstd - } else { - const AccReal *rm = runningMean.dptr(); - const AccReal *rv = runningVariance.dptr(); - - for (size_t i = 0, n = inputData.shape_[1]; i < n; ++i) { - mean[i] = rm[i]; - var[i] = VARIANCE_TO_INVSTD(rv[i], param_.eps); + mean[channel] = rm[channel]; + var[channel] = VARIANCE_TO_INVSTD(rv[channel], param_.eps); } - } - // compute output - AccReal *w = weights.dptr(); - const AccReal *b = bias.dptr(); + // compute output + AccReal *w = weights.dptr(); + const AccReal *b = bias.dptr(); + + const AccReal thisMean = mean[channel]; + const AccReal thisInvstd = var[channel]; + const AccReal thisWeight = w[channel]; + const AccReal thisBias = b[channel]; // note that var is still invstd if (!param_.fix_gamma) { if (IsWriting(req[batchnorm::kData])) { - ForEachFast(inputData, outputData, - [w, b, mean, var](const size_t channel, const DType *in_data, DType *out_data) { + ForEachFast(inputData, outputData, channel, + [thisWeight, thisBias, thisMean, thisInvstd](const DType *in_data, + DType *out_data) { *out_data = static_cast( - ((*in_data - mean[channel]) * var[channel]) * w[channel] + b[channel]); + ((*in_data - thisMean) * thisInvstd) * thisWeight + thisBias); }); } } else { if (IsWriting(req[batchnorm::kGamma])) { - for (size_t i =0, n = weights.Size(); i < n; ++i) { - w[i] = AccReal(1); - } + w[channel] = AccReal(1); } if (IsWriting(req[batchnorm::kData])) { - ForEachFast(inputData, outputData, - [w, b, mean, var](const size_t channel, const DType *in_data, DType *out_data) { + ForEachFast(inputData, outputData, channel, + [thisWeight, thisBias, thisMean, thisInvstd](const DType *in_data, + DType *out_data) { *out_data = static_cast( - ((*in_data - mean[channel]) * var[channel]) + b[channel]); + ((*in_data - thisMean) * thisInvstd) + thisBias); }); } } + } } template @@ -315,11 +179,11 @@ void BatchNormOp::DoBackward(mshadow::Stream *, const std::vector &in_grad, const std::vector &aux_states) { // Input Data - batchnorm::DeviceTensor3 inputData(in_data[batchnorm::kData], 1); + batchnorm::BNTensor3 inputData(in_data[batchnorm::kData], param_.axis); const TBlob &weights = in_data[batchnorm::kGamma]; // Input Grad - batchnorm::DeviceTensor3 gradIn(in_grad[batchnorm::kData], 1); + batchnorm::BNTensor3 gradIn(in_grad[batchnorm::kData], param_.axis); const TBlob &gradWeight = in_grad[batchnorm::kGamma]; const TBlob &gradBias = in_grad[batchnorm::kBeta]; @@ -328,18 +192,18 @@ void BatchNormOp::DoBackward(mshadow::Stream *, const TBlob &runningVariance = aux_states[batchnorm::kMovingVar]; // Output - batchnorm::DeviceTensor3 gradOut(out_grad[batchnorm::kOut], 1); + batchnorm::BNTensor3 gradOut(out_grad[batchnorm::kOut], param_.axis); const TBlob &saveMean = out_data[batchnorm::kMean]; const TBlob &saveStd = out_data[batchnorm::kVar]; - const size_t channelCount = inputData.shape_[1]; + const size_t channelCount = inputData.ChannelCount(); const size_t itemCount = inputData.Size() / channelCount; // Avoid multiple dptr() call within the channel loop AccReal *runningMeanDataPtr = runningMean.dptr(); AccReal *runningVarDataPtr = runningVariance.dptr(); - AccReal *saveMeanDataPtr = saveMean.dptr(); - AccReal *saveInvStdDataPtr = saveStd.dptr(); + const AccReal *saveMeanDataPtr = saveMean.dptr(); + const AccReal *saveInvStdDataPtr = saveStd.dptr(); AccReal *gradWeightData = gradWeight.dptr(); AccReal *gradBiasData = gradBias.dptr(); @@ -347,7 +211,7 @@ void BatchNormOp::DoBackward(mshadow::Stream *, #pragma omp parallel for for (int channel = 0; channel < static_cast(channelCount); ++channel) { - AccReal *weight = weights.dptr(); + const AccReal *weight = weights.dptr(); const AccReal w = weight ? weight[channel] : AccReal(1); AccReal mean, invstd; if (is_train_and_not_global_stats) { @@ -381,7 +245,7 @@ void BatchNormOp::DoBackward(mshadow::Stream *, dotp += (*thisInputData - mean) * (*gradOut_data); }); - if (gradIn.shape_.ndim() && IsWriting(req[batchnorm::kData])) { // if there's a grad input + if (!gradIn.IsEmpty() && IsWriting(req[batchnorm::kData])) { // if there's a grad input if (is_train_and_not_global_stats) { // when in training mode // Q(X) = X - E[x] ; i.e. input centered to zero mean @@ -431,12 +295,14 @@ void BatchNormOp::DoBackward(mshadow::Stream *, } } - template<> -Operator *CreateOp(const BatchNormParam& param, const int dtype, const TShape& shape) { +Operator *CreateOp(BatchNormParam param, const int dtype, const TShape& shape) { + param.axis = mxnet::op::batchnorm::GetRealAxis(shape, param.axis); Operator *op = nullptr; #if MXNET_USE_MKL2017 == 1 - if (shape.ndim() == 4) { + if (shape.ndim() == 4 + && param.axis == mxnet::op::batchnorm::DEFAULT_AXIS + && !mxnet::op::batchnorm::disable_mkl) { switch (dtype) { case mshadow::kFloat32: op = new MKLBatchNormOp(param); @@ -449,9 +315,12 @@ Operator *CreateOp(const BatchNormParam& param, const int dtype, const TSha break; } } -#define BATCHNORM_LOG_MKL_INFO() do { \ - LOG(INFO) << MKLBatchNormOp::getName() \ - << " Skipping MKL optimization (unsupported dimension or type)"; \ +#define BATCHNORM_LOG_MKL_INFO() \ + do { \ + if (!mxnet::op::batchnorm::disable_mkl) { \ + LOG(INFO) << MKLBatchNormOp::getName() \ + << " Skipping MKL optimization (unsupported dimension, axis or type)"; \ + } \ } while (0) #else #define BATCHNORM_LOG_MKL_INFO() ((void)0) @@ -517,6 +386,10 @@ If ``use_global_stats`` is set to be true, then ``moving_mean`` and ``moving_var`` are used instead of ``data_mean`` and ``data_var`` to compute the output. It is often used during inference. +The parameter ``axis`` specifies which axis of the input shape denotes +the 'channel' (separately normalized groups). The default is 1. Specifying -1 sets the channel +axis to be the last item in the input shape. + Both ``gamma`` and ``beta`` are learnable parameters. But if ``fix_gamma`` is true, then set ``gamma`` to 1 and its gradient to 0. diff --git a/src/operator/batch_norm.cu b/src/operator/batch_norm.cu index f081383b8276..1d27427653b8 100755 --- a/src/operator/batch_norm.cu +++ b/src/operator/batch_norm.cu @@ -6,9 +6,7 @@ * Adapted from Torch */ #include -#include #include -#include #include "batch_norm-inl.h" #define WRITE_DATA_FLAG 1 @@ -22,9 +20,10 @@ #include "./cudnn_batch_norm-inl.h" #endif -#include #include "../common/cuda_utils.h" +using namespace mxnet; + /*! \brief inverse standard deviation <-> variance */ #define VARIANCE_TO_INVSTD(__var$, __eps$) (1.0/sqrt((__var$) + DType(__eps$))) #define INVSTD_TO_VARIANCE(__invstd$, __eps$) ((1.0 / ((__invstd$) * (__invstd$))) - (__eps$)) @@ -45,14 +44,15 @@ struct ScalarConvert { }; // Number of threads in a block given an input size up to MAX_BLOCK_SIZE -static unsigned getNumThreads(int nElem) { +static unsigned getNumThreads(int nElem, const bool smaller) { unsigned threadSizes[5] = {32, 64, 128, 256, MAX_BLOCK_SIZE}; - for (int i = 0; i != 5; ++i) { + const int maxi = smaller ? 4 : 5; + for (int i = 0; i != maxi; ++i) { if (static_cast(nElem) <= threadSizes[i]) { return threadSizes[i]; } } - return MAX_BLOCK_SIZE; + return smaller ? (MAX_BLOCK_SIZE >> 1) : MAX_BLOCK_SIZE; } // Returns the index of the most significant 1 bit in `val`. @@ -80,60 +80,60 @@ struct Float2 { } }; -template +template struct SumOp { - __device__ SumOp(const DeviceTensor3 t) : tensor(t) {} + __device__ SumOp(const DeviceTensor t) : tensor(t) {} __device__ __forceinline__ AccReal operator()(int batch, int plane, int n) { - return ScalarConvert::to(tensor(batch, plane, n)); + return ScalarConvert::to(tensor.get_ref(batch, plane, n)); } - const DeviceTensor3 tensor; + const DeviceTensor tensor; }; -template +template struct VarOp { - __device__ VarOp(AccReal m, const DeviceTensor3 t) + __device__ VarOp(AccReal m, const DeviceTensor t) : mean(m) , tensor(t) { } __device__ __forceinline__ AccReal operator()(int batch, int plane, int n) { - DType val = tensor(batch, plane, n); + DType val = tensor.get_ref(batch, plane, n); return (val - mean) * (val - mean); } const AccReal mean; - const DeviceTensor3 tensor; + const DeviceTensor tensor; }; -template +template struct GradOp { - __device__ GradOp(AccReal m, const DeviceTensor3 i, const DeviceTensor3 g) + __device__ GradOp(AccReal m, const DeviceTensor i, const DeviceTensor g) : mean(m), input(i), gradOutput(g) {} __device__ __forceinline__ Float2 operator()(int batch, int plane, int n) { - const DType g = gradOutput(batch, plane, n); - const DType c = ScalarConvert::to(input(batch, plane, n) - mean); + const DType g = gradOutput.get_ref(batch, plane, n); + const DType c = ScalarConvert::to(input.get_ref(batch, plane, n) - mean); return Float2(g, g * c); } const AccReal mean; - const DeviceTensor3 input; - const DeviceTensor3 gradOutput; + const DeviceTensor input; + const DeviceTensor gradOutput; }; // Sum across all threads within a warp template static __device__ __forceinline__ T warpSum(T val) { #if __CUDA_ARCH__ >= 300 - for (int i = 0; i < getMSB(WARP_SIZE); ++i) { +for (int i = 0; i < getMSB(WARP_SIZE); ++i) { val += __shfl_xor(val, 1 << i, WARP_SIZE); } #else - __shared__ T values[MAX_BLOCK_SIZE]; - values[threadIdx.x] = val; - __threadfence_block(); - const int base = (threadIdx.x / WARP_SIZE) * WARP_SIZE; - for (int i = 1; i < WARP_SIZE; i++) { - val += values[base + ((i + threadIdx.x) % WARP_SIZE)]; - } +__shared__ T values[MAX_BLOCK_SIZE]; +values[threadIdx.x] = val; +__threadfence_block(); +const int base = (threadIdx.x / WARP_SIZE) * WARP_SIZE; +for (int i = 1; i < WARP_SIZE; i++) { +val += values[base + ((i + threadIdx.x) % WARP_SIZE)]; +} #endif - return val; +return val; } template @@ -144,11 +144,11 @@ static __device__ __forceinline__ Float2 warpSum(Float2 -static __device__ T reduce(Op op, DeviceTensor3 tensor, int plane) { +template +static __device__ T reduce(Op op, DeviceTensor tensor, int plane) { T sum = (T) 0; - for (int batch = 0; batch < tensor.getSize(0); ++batch) { - for (int x = threadIdx.x; x < tensor.getSize(2); x += blockDim.x) { + for (int batch = 0; batch < tensor.OuterSize(); ++batch) { + for (int x = threadIdx.x; x < tensor.InnerSize(); x += blockDim.x) { sum += op(batch, plane, x); } } @@ -179,10 +179,10 @@ static __device__ T reduce(Op op, DeviceTensor3 tensor, int plane) { return shared[0]; } -template +template __global__ void BatchNormalizationUpdateOutputInferenceKernel( - DeviceTensor3 input, - DeviceTensor3 output, + DeviceTensor input, + DeviceTensor output, DeviceTensor1 runningMean, DeviceTensor1 runningVar, DeviceTensor1 saveMean, @@ -209,19 +209,19 @@ __global__ void BatchNormalizationUpdateOutputInferenceKernel( } } // Write normalized and update the output - for (int batch = 0, nbatch = input.getSize(0); batch < nbatch; ++batch) { - for (int x = threadIdx.x, nx = input.getSize(2); x < nx; x += blockDim.x) { - const DType inp = input(batch, plane, x); - output(batch, plane, x) = + for (int batch = 0, nbatch = input.OuterSize(); batch < nbatch; ++batch) { + for (int x = threadIdx.x, nx = input.InnerSize(); x < nx; x += blockDim.x) { + const DType inp = input.get_ref(batch, plane, x); + output.get_ref(batch, plane, x) = ScalarConvert::to(gamma * (inp - mean) * invstd + beta); } } } -template +template __global__ void BatchNormalizationUpdateOutputKernel( - DeviceTensor3 input, - DeviceTensor3 output, + DeviceTensor input, + DeviceTensor output, DeviceTensor1 weight, DeviceTensor1 bias, const AccReal epsilon, @@ -232,15 +232,15 @@ __global__ void BatchNormalizationUpdateOutputKernel( DeviceTensor1 saveInvStd, const uint32_t flags) { const int plane = blockIdx.x; - const int N = input.getSize(0) * input.getSize(2); + const int N = input.OuterSize() * input.InnerSize(); const AccReal norm = AccReal(1) / N; // Compute the mean and variance across (batch, x/y/z) const AccReal mean = reduce( - SumOp(input), input, plane) * norm; + SumOp(input), input, plane) * norm; __syncthreads(); - const AccReal varN = reduce(VarOp(mean, input), + const AccReal varN = reduce(VarOp(mean, input), input, plane); AccReal invStd = 0; if (varN != AccReal(0) || epsilon != AccReal(0)) { @@ -265,55 +265,60 @@ __global__ void BatchNormalizationUpdateOutputKernel( : ScalarConvert::to(1); const AccReal beta = bias.numElements() > 0 ? ScalarConvert::to(bias[plane]) : ScalarConvert::to(0); - for (int batch = 0, nbatch = input.getSize(0); batch < nbatch; ++batch) { - for (int x = threadIdx.x, nx = input.getSize(2); x < nx; x += blockDim.x) { - const DType inp = input(batch, plane, x); - output(batch, plane, x) = + for (int batch = 0, nbatch = input.OuterSize(); batch < nbatch; ++batch) { + for (int x = threadIdx.x, nx = input.InnerSize(); x < nx; x += blockDim.x) { + const DType inp = input.get_ref(batch, plane, x); + output.get_ref(batch, plane, x) = ScalarConvert::to(gamma * (inp - mean) * invStd + beta); } } } -template +template +struct CUDATensors { + DeviceTensor1 gradWeight; + DeviceTensor1 gradBias; + DeviceTensor1 weight; + DeviceTensor1 runningMean; + DeviceTensor1 runningVar; + DeviceTensor1 saveMean; + DeviceTensor1 saveInvStd; +}; + +template static __global__ void BatchNormalizationBackwardKernel( - const DeviceTensor3 input, - const DeviceTensor3 gradOutput, - DeviceTensor3 gradInput, - DeviceTensor1 gradWeight, - DeviceTensor1 gradBias, - const DeviceTensor1 weight, - const DeviceTensor1 runningMean, - const DeviceTensor1 runningVar, - const DeviceTensor1 saveMean, - const DeviceTensor1 saveInvstd, + const DeviceTensor input, + const DeviceTensor gradOutput, + DeviceTensor gradInput, + CUDATensors tensors, const uint32_t flags, const AccReal momentum, const double eps) { int plane = blockIdx.x; - int N = gradOutput.getSize(0) * gradOutput.getSize(2); + int N = gradOutput.OuterSize() * gradOutput.InnerSize(); const bool is_train_and_not_global_stats = (flags & IS_TRAINING_FLAG) != 0 && (flags & USE_GLOBAL_STATS_FLAG) == 0; AccReal mean, invstd; if (is_train_and_not_global_stats) { - mean = ScalarConvert::to(saveMean[plane]); - invstd = saveInvstd[plane]; + mean = ScalarConvert::to(tensors.saveMean[plane]); + invstd = tensors.saveInvStd[plane]; } else { - mean = ScalarConvert::to(runningMean[plane]); - invstd = VARIANCE_TO_INVSTD(runningVar[plane], eps); + mean = ScalarConvert::to(tensors.runningMean[plane]); + invstd = VARIANCE_TO_INVSTD(tensors.runningVar[plane], eps); } - const AccReal weightVal = weight.numElements() > 0 ? - ScalarConvert::to(weight[plane]) : AccReal(1); + const AccReal weightVal = tensors.weight.numElements() > 0 ? + ScalarConvert::to(tensors.weight[plane]) : AccReal(1); const AccReal norm = AccReal(1) / N; // Compute two values across (batch, x/y/z) in one pass: // 1. Sum(gradOutput) // 2. DotProduct(input - mean, gradOutput) - GradOp g(mean, input, gradOutput); + GradOp g(mean, input, gradOutput); Float2< DType, AccReal > res = reduce < Float2 < DType, AccReal >, - GradOp< DType, AccReal, DeviceTensor3 >, DeviceTensor3 > (g, gradOutput, plane); + GradOp< DType, AccReal, DeviceTensor >, DeviceTensor > (g, gradOutput, plane); const AccReal gradOutputSum = res.v1; const AccReal dotP = res.v2; @@ -322,46 +327,50 @@ static __global__ void BatchNormalizationBackwardKernel( const AccReal gradScale = invstd * weightVal; if (threadIdx.x == 0 && is_train_and_not_global_stats) { - const AccReal localVariance = INVSTD_TO_VARIANCE(saveInvstd[plane], eps); - const AccReal localMean = saveMean[plane]; + const AccReal localVariance = INVSTD_TO_VARIANCE(tensors.saveInvStd[plane], eps); + const AccReal localMean = tensors.saveMean[plane]; // update running averages - runningMean[plane] = runningMean[plane] * momentum + localMean * (AccReal(1) - momentum); - runningVar[plane] = runningVar[plane] * momentum + localVariance * (AccReal(1) - momentum); + tensors.runningMean[plane] = tensors.runningMean[plane] + * momentum + localMean * (AccReal(1) - momentum); + tensors.runningVar[plane] = tensors.runningVar[plane] + * momentum + localVariance * (AccReal(1) - momentum); } - if (gradInput.numElements() > 0 && (flags & WRITE_DATA_FLAG) != 0) { - for (int batch = 0, nbatch = gradOutput.getSize(0); batch < nbatch; ++batch) { - for (int x = threadIdx.x, nx = gradOutput.getSize(2); x < nx; x += blockDim.x) { - const DType gradOut = gradOutput(batch, plane, x); + if (gradInput.Size() > 0 && (flags & WRITE_DATA_FLAG) != 0) { + for (int batch = 0, nbatch = gradOutput.OuterSize(); batch < nbatch; ++batch) { + for (int x = threadIdx.x, nx = gradOutput.InnerSize(); x < nx; x += blockDim.x) { + const DType gradOut = gradOutput.get_ref(batch, plane, x); if (is_train_and_not_global_stats) { - const DType inp = input(batch, plane, x); + const DType inp = input.get_ref(batch, plane, x); const AccReal proj = (inp - mean) * projScale; - gradInput(batch, plane, x) = + gradInput.get_ref(batch, plane, x) = ScalarConvert::to((gradOut - proj - gradMean) * gradScale); } else { - gradInput(batch, plane, x) = ScalarConvert::to(gradOut * gradScale); + gradInput.get_ref(batch, plane, x) = ScalarConvert::to( + gradOut * gradScale); } } } } - if (gradWeight.numElements() > 0 && threadIdx.x == 0 && (flags & WRITE_GAMMA_FLAG) != 0) { + if (tensors.gradWeight.numElements() > 0 && threadIdx.x == 0 && (flags & WRITE_GAMMA_FLAG) != 0) { if ((flags & FIX_GAMMA_FLAG) == 0) { - gradWeight[plane] = ScalarConvert::to(dotP * invstd); + tensors.gradWeight[plane] = ScalarConvert::to(dotP * invstd); } else { - gradWeight[plane] = DType(0); + tensors.gradWeight[plane] = DType(0); } } - if (gradBias.numElements() > 0 && threadIdx.x == 0 && (flags & WRITE_BETA_FLAG) != 0) { - gradBias[plane] = ScalarConvert::to(gradOutputSum); + if (tensors.gradBias.numElements() > 0 && threadIdx.x == 0 && (flags & WRITE_BETA_FLAG) != 0) { + tensors.gradBias[plane] = ScalarConvert::to(gradOutputSum); } } template struct DeviceTensor { public: + inline DeviceTensor() {} inline DeviceTensor(DType *p, const int *size) : dptr_(p) { for (int i = 0; i < Dim; ++i) { @@ -369,13 +378,11 @@ struct DeviceTensor { } } - __host__ __device__ - __forceinline__ unsigned getSize(const int i) const { + MSHADOW_XINLINE unsigned getSize(const int i) const { return size_[i]; } - __host__ __device__ - __forceinline__ int numElements() const { + MSHADOW_XINLINE int numElements() const { int n = 1; for (int i = 0; i < Dim; ++i) { n *= size_[i]; @@ -383,8 +390,7 @@ struct DeviceTensor { return n; } - __host__ __device__ - __forceinline__ DType &operator()(const size_t batch, + MSHADOW_XINLINE DType &operator()(const size_t batch, const size_t plane, const size_t x) const { int offset = 0; @@ -401,12 +407,11 @@ struct DeviceTensor { return *(const_cast(dptr_ + offset)); } - __host__ __device__ - __forceinline__ DType &operator[](const size_t x) const { + MSHADOW_XINLINE DType &operator[](const size_t x) const { return *(dptr_ + x); } - __forceinline__ size_t SpatialSize() const { + MSHADOW_XINLINE size_t InnerSize() const { size_t sz = 1; for (size_t i = 2; i < Dim; ++i) { sz *= size_[i]; @@ -414,7 +419,7 @@ struct DeviceTensor { return sz; } - __forceinline__ size_t ChannelCount() const { + MSHADOW_XINLINE size_t ChannelCount() const { return size_[1]; } @@ -450,19 +455,23 @@ static DeviceTensor devicetensor(const TBlob &blob) { #define DeviceTensor1 DeviceTensor -#define DeviceTensor3 DeviceTensor + +using namespace mxnet::op; template static void BatchNormalizationUpdateOutput(mshadow::Stream *s, const OpContext &ctx, + const BatchNormParam& param, const std::vector &in_data, const std::vector &out_data, const std::vector &aux_states, const uint32_t flags, double momentum, double eps) { - DeviceTensor3 input = devicetensor(in_data[batchnorm::kData]); - DeviceTensor3 output = devicetensor(out_data[batchnorm::kOut]); + batchnorm::BNTensor3 input = batchnorm::BNTensor3( + in_data[batchnorm::kData], param.axis); + batchnorm::BNTensor3 output = batchnorm::BNTensor3( + out_data[batchnorm::kOut], param.axis); DeviceTensor1 weight = devicetensor(in_data[batchnorm::kGamma]); DeviceTensor1 bias = devicetensor(in_data[batchnorm::kBeta]); DeviceTensor1 runningMean = devicetensor(aux_states[batchnorm::kMovingMean]); @@ -474,15 +483,17 @@ static void BatchNormalizationUpdateOutput(mshadow::Stream *s, if ((flags & IS_TRAINING_FLAG) == 0 || (flags & USE_GLOBAL_STATS_FLAG) != 0) { dim3 blocks(input.ChannelCount()); - dim3 threads(getNumThreads(input.SpatialSize())); - BatchNormalizationUpdateOutputInferenceKernel + dim3 threads(batchnorm::cuda::getNumThreads(input.InnerSize(), false)); + BatchNormalizationUpdateOutputInferenceKernel> <<< blocks, threads, 0, mshadow::Stream::GetStream(s) >>> ( input, output, runningMean, runningVar, saveMean, saveInvStd, weight, bias, eps, flags); } else { dim3 blocks(input.ChannelCount()); - dim3 threads(getNumThreads(input.SpatialSize())); - BatchNormalizationUpdateOutputKernel + dim3 threads(batchnorm::cuda::getNumThreads(input.InnerSize(), false)); + BatchNormalizationUpdateOutputKernel> << < blocks, threads, 0, mshadow::Stream::GetStream(s) >> > ( input, output, weight, bias, eps, momentum, runningMean, runningVar, saveMean, saveInvStd, flags); @@ -493,6 +504,7 @@ static void BatchNormalizationUpdateOutput(mshadow::Stream *s, template static void BatchNormalizationBackward(mshadow::Stream *s, const OpContext &ctx, + const BatchNormParam& param, const std::vector &out_grad, const std::vector &in_data, const std::vector &out_data, @@ -501,25 +513,34 @@ static void BatchNormalizationBackward(mshadow::Stream *s, const uint32_t flags, double momentum, double eps) { - DeviceTensor3 input = devicetensor(in_data[batchnorm::kData]); - DeviceTensor3 gradOutput = devicetensor(out_grad[batchnorm::kOut]); - DeviceTensor3 gradInput = devicetensor(in_grad[batchnorm::kData]); - DeviceTensor1 gradWeight = devicetensor(in_grad[batchnorm::kGamma]); - DeviceTensor1 gradBias = devicetensor(in_grad[batchnorm::kBeta]); - DeviceTensor1 weight = devicetensor(in_data[batchnorm::kGamma]); - DeviceTensor1 runningMean = devicetensor(aux_states[batchnorm::kMovingMean]); - DeviceTensor1 runningVar = devicetensor(aux_states[batchnorm::kMovingVar]); - DeviceTensor1 saveMean = devicetensor(out_data[batchnorm::kMean]); - DeviceTensor1 saveInvStd = devicetensor(out_data[batchnorm::kVar]); - - DCHECK_GT(weight.numElements(), 0); - + batchnorm::BNTensor3 input = batchnorm::BNTensor3( + in_data[batchnorm::kData], param.axis); + batchnorm::BNTensor3gradOutput = batchnorm::BNTensor3( + out_grad[batchnorm::kOut], param.axis); + batchnorm::BNTensor3gradInput = batchnorm::BNTensor3( + in_grad[batchnorm::kData], param.axis); + + CUDATensors tensors; + + tensors.gradWeight = devicetensor(in_grad[batchnorm::kGamma]); + tensors.gradBias = devicetensor(in_grad[batchnorm::kBeta]); + tensors.weight = devicetensor(in_data[batchnorm::kGamma]); + tensors.runningMean = devicetensor(aux_states[batchnorm::kMovingMean]); + tensors.runningVar = devicetensor(aux_states[batchnorm::kMovingVar]); + tensors.saveMean = devicetensor(out_data[batchnorm::kMean]); + tensors.saveInvStd = devicetensor(out_data[batchnorm::kVar]); + + DCHECK_GT(tensors.weight.numElements(), 0); +#ifdef NDEBUG + constexpr bool SMALLER_THREADS = false; +#else + constexpr bool SMALLER_THREADS = true; +#endif dim3 blocks(gradOutput.ChannelCount()); - dim3 threads(getNumThreads(gradOutput.SpatialSize())); - BatchNormalizationBackwardKernel + dim3 threads(batchnorm::cuda::getNumThreads(gradOutput.InnerSize(), SMALLER_THREADS)); + BatchNormalizationBackwardKernel> <<< blocks, threads, 0, mshadow::Stream::GetStream(s) >>> ( - input, gradOutput, gradInput, gradWeight, gradBias, weight, runningMean, runningVar, - saveMean, saveInvStd, flags, momentum, eps); + input, gradOutput, gradInput, tensors, flags, momentum, eps); MSHADOW_CUDA_POST_KERNEL_CHECK(BatchNormalizationBackward); } @@ -557,6 +578,7 @@ void BatchNormOp::DoForward(mshadow::Stream *stream, batchnorm::cuda::BatchNormalizationUpdateOutput( stream, ctx, + param_, in_data, out_data, aux_states, @@ -579,6 +601,7 @@ void BatchNormOp::DoBackward(mshadow::Stream *stream, batchnorm::cuda::BatchNormalizationBackward( stream, ctx, + param_, out_grad, in_data, out_data, @@ -592,10 +615,12 @@ void BatchNormOp::DoBackward(mshadow::Stream *stream, /*! \brief Create GPU operator for batch normalization */ template<> -Operator *CreateOp(const BatchNormParam& param, const int dtype, const TShape& shape) { +Operator *CreateOp(BatchNormParam param, const int dtype, const TShape& shape) { + param.axis = mxnet::op::batchnorm::GetRealAxis(shape, param.axis); Operator *op = NULL; #if MXNET_USE_CUDNN == 1 && CUDNN_MAJOR >= 5 - if (!param.use_global_stats && !param.cudnn_off && shape.ndim() <= 4) { + if (!param.use_global_stats && !param.cudnn_off && shape.ndim() <= 4 + && param.axis == mxnet::op::batchnorm::DEFAULT_AXIS) { MSHADOW_REAL_TYPE_SWITCH(dtype, DType, { op = new CuDNNBatchNormOp(param); }) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index c0796f8e5e82..7b7f283b82d4 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -51,6 +51,7 @@ if(GTEST_FOUND) endif() add_test(AllTestsIn${PROJECT_NAME}UnitTests ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${PROJECT_NAME}_unit_tests) - +else() + message(WARNING "Google Test not found") endif() diff --git a/tests/cpp/operator/batchnorm_test.cc b/tests/cpp/operator/batchnorm_test.cc index cabddec7b83e..719980b5d4f5 100644 --- a/tests/cpp/operator/batchnorm_test.cc +++ b/tests/cpp/operator/batchnorm_test.cc @@ -243,7 +243,7 @@ class BatchNormValidator : public test::op::Validator { CHECK_EQ(info_2.prop_->getParam().use_global_stats, info_1.prop_->getParam().use_global_stats); -#if MXNET_USE_CUDNN != 1 /* CUDNN takes a slightly different approach here on first pass */ +#if MXNET_USE_CUDNN != 1 /* CUDNN takes a different approach here on first pass */ // Aux EXPECT_TRUE(compare(*info_1.data_, *info_2.data_, test::op::BasicOperatorData::kAux, @@ -533,6 +533,8 @@ static test::op::OpInfo runOperatorBackward( return *info; } +static constexpr size_t CYCLE_COUNT = 3; + template static test::op::OpInfoPair testForwardAndBackward( const bool isGPU1, @@ -541,7 +543,7 @@ static test::op::OpInfoPair testFo const test::op::kwargs_t& kwargs, const bool dumpC, const size_t count = 1, - const size_t cycleCount = 5) { + const size_t cycleCount = CYCLE_COUNT) { test::op::OpInfo info_1 = TestBatchNormOperatorForward(isGPU1, inputShape, kwargs, count); @@ -603,13 +605,18 @@ static test::op::OpInfoPair testForwardAndBackward(const bool isGPU, const TShape &inputShape, const test::op::kwargs_t kwargs, - const bool dumpC = false) { + const bool dumpC = false, + const size_t count = 1, + const size_t cycleCount = CYCLE_COUNT +) { return testForwardAndBackward( isGPU, isGPU, inputShape, kwargs, - dumpC); + dumpC, + count, + cycleCount); } template @@ -638,7 +645,6 @@ TEST(BATCH_NORM, Test2DForwardV1V2) { { auto infoA = testBNForwardAndBackward2D( false, {BATCH_SIZE, CHANNELS, DH, DW}, blank_kwargs); - dumpF(&std::cout, infoA); }); } @@ -794,15 +800,18 @@ TEST(BATCH_NORM, TestTiming_2D) { MSHADOW_REAL_TYPE_SWITCH_EX( mshadow::kFloat32, DType, AccReal, { - std::string prefix; -#if MXNET_USE_MKL2017 == 1 - prefix = "MKL "; -#endif timingTest("BatchNormV1Prop 2D", false, false, blank_kwargs, 2, THISCOUNT); - timingTest(prefix + "BatchNormProp 2D", +#if MXNET_USE_MKL2017 == 1 + timingTest("MKL BatchNormProp 2D", + false, false, + blank_kwargs_nocudnn, + 2, THISCOUNT); +#endif + test::ScopeSet disableMKL(&mxnet::op::batchnorm::disable_mkl, true); + timingTest("BatchNormProp 2D", false, false, blank_kwargs_nocudnn, 2, THISCOUNT); @@ -999,7 +1008,7 @@ struct Test2DBackward2DPlusLoadAndCompareLogicUtil { const TShape inputShape({1, 1, 2, 1}); test::op::OpInfoPair bi = testForwardAndBackward( - false, inputShape, blank_kwargs); + false, inputShape, blank_kwargs, false, 1, 5); #if MXNET_DUMP_C bi.info_1_.data_->dumpC(&std::cerr, "Test2DBackward2DPlusLoadAndCompareLogic"); @@ -1045,6 +1054,7 @@ struct Test2DBackward2DPlusLoadAndCompareLogicUtil { TEST(BATCH_NORM, Test2DBackward2DPlusLoadAndCompareLogic) { + test::ScopeSet disableMKL(&mxnet::op::batchnorm::disable_mkl, true); MSHADOW_REAL_TYPE_SWITCH_EX( mshadow::kFloat32, DType, AccReal, { @@ -1156,6 +1166,385 @@ TEST(BATCH_NORM, Test2DBackwardMixed_cpu_cpu_ugs) { }); } +template +class ChannelAxisTestData { + protected: + enum Mode { LOAD, SAVE }; + + void loadOrSave(const TBlob& blob, int channel_axis, const Mode mode) { + mxnet::op::batchnorm::BNTensor3 tensor3(blob, channel_axis); + const TShape &shape = blob.shape_; + CHECK_GT(shape.ndim(), 0); + if (channel_axis < 0) { + channel_axis = shape.ndim() + channel_axis; + } + CHECK_LT(channel_axis, shape.ndim()); + const size_t channel_count = shape[channel_axis]; + std::vector indexes(channel_count, 0); + for (size_t outer = 0, outerCount = tensor3.OuterSize(); outer < outerCount; ++outer) { + for (size_t channel = 0, channelCount = tensor3.ChannelCount(); + channel < channelCount; ++channel) { + CHECK_LT(channel, channel_data_.size()); + for (size_t inner = 0, innerCount = tensor3.InnerSize(); inner < innerCount; ++inner) { + CHECK_LT(indexes[channel], channel_data_[channel].size()); + if (mode == SAVE) { + tensor3.get_ref(outer, channel, inner) = channel_data_[channel][indexes[channel]++]; + } else { // mode == LOAD + channel_data_[channel][indexes[channel]++] = tensor3.get_ref(outer, channel, inner); + } + } + } + } + } + + public: + std::vector> channel_data_; + + static void print(const std::string& label, const std::vector>& m) { + if (test::debugOutput) { + if (!label.empty()) { + std::cout << label << ": "; + } + for (size_t i = 0, n = m.size(); i < n; ++i) { + const std::vector &vec = m[i]; + for (size_t j = 0, jn = vec.size(); j < jn; ++j) { + if (j) { + std::cout << ", "; + } + const DType val = vec[j]; + std::cout << std::fixed << std::setw(7) + << std::setprecision(mxnet::test::MPRINT_PRECISION) + << std::right << val; + } + std::cout << std::endl; + } + std::cout << "-----" << std::endl << std::flush; + } + } + + static void print(const std::string& label, const TBlob& blob) { + if (test::debugOutput) { + if (!label.empty()) { + std::cout << label << ": "; + } + const size_t totalSize = blob.Size(); + for (size_t i = 0; i < totalSize; ++i) { + const float val = blob.dptr()[i]; + if (i) { + std::cout << ", "; + } + std::cout << std::fixed << std::setw(7) << std::setprecision(mxnet::test::MPRINT_PRECISION) + << std::right << val; + } + std::cout << std::endl << std::flush; + } + } + + void save(const TBlob& blob, const int channel_axis) { + loadOrSave(blob, channel_axis, SAVE); + } + + void load(const TBlob& blob, const int channel_axis) { + loadOrSave(blob, channel_axis, LOAD); + } +}; + +template +static void compare(const TBlob& blob, const std::vector& vals) { + CHECK_EQ(blob.Size(), vals.size()); + const DType *v = blob.dptr(); + for (size_t i = 0, n = vals.size(); i < n; ++i) { + const DType vBlob = v[i]; + const DType vVect = vals[i]; + const bool near = test::op::Validator::isNear( + vBlob, vVect, test::op::Validator::ErrorBound(&blob)); + EXPECT_TRUE(near); + if (!near) { + LOG(WARNING) << vBlob << " is not near enough to " << vVect << std::endl; + } + } +} + +template +static void compare(const std::vector>& d1, + const std::vector>& d2) { + CHECK_EQ(d1.size(), d2.size()); + for (size_t x = 0, xn = d1.size(); x < xn; ++x) { + const std::vector &vec1 = d1[x]; + const std::vector &vec2 = d2[x]; + CHECK_EQ(vec1.size(), vec2.size()); + for (size_t i = 0, n = vec1.size(); i < n; ++i) { + const DType v1 = vec1[i]; + const DType v2 = vec2[i]; + const bool near = test::op::Validator::isNear( + v1, v2, test::op::Validator::ERROR_BOUND()); + EXPECT_TRUE(near); + if (!near) { + LOG(WARNING) << v1 << " is not near enough to " << v2 << std::endl; + } + } + } +} + +template +static void testSaveAndLoad(const std::vector& dims, + const int channelAxis, + const std::vector>& inputChannelData, + const std::vector& expectedBlobData) { + ChannelAxisTestData data; + data.channel_data_ = inputChannelData; + + TShape shape(dims.size()); + for (size_t i = 0, n = dims.size(); i < n; ++i) { + shape[i] = index_t(dims[i]); + } + + std::unique_ptr blob(new test::StandaloneBlob( + shape, false, mshadow::DataType::kFlag)); + + data.save(*blob, channelAxis); + ChannelAxisTestData::print("saved to blob", *blob); + compare(*blob, expectedBlobData); + data.load(*blob, channelAxis); + compare(data.channel_data_, inputChannelData); +} + +/*! \brief Check normalization/denormalization of various channel positions */ +TEST(BATCH_NORM, TestChannelAxisSaveAndLoad) { + std::cout << std::endl << std::flush; + + typedef float DType; + typedef float AccReal; + + const std::vector> myData = + { { 1.0f, 1.0f, 1.0, 1.0 }, + { 2.0f, 2.0f, 2.0f, 2.0f }, + { 3.0f, 3.0f, 3.0f, 3.0f } }; + + testSaveAndLoad({ 1, 3, 2, 2 }, 1, myData, + { 1.0f, 1.0f, 1.0f, 1.0f, + 2.0f, 2.0f, 2.0f, 2.0f, + 3.0f, 3.0f, 3.0f, 3.0f}); + + testSaveAndLoad({ 1, 2, 2, 3 }, 3, myData, + { 1.0f, 2.0f, 3.0f, + 1.0f, 2.0f, 3.0f, + 1.0f, 2.0f, 3.0f, + 1.0f, 2.0f, 3.0f}); + + testSaveAndLoad({ 1, 2, 3, 2 }, 2, myData, + { 1.0f, 1.0f, 2.0f, 2.0f, 3.0f, 3.0f, + 1.0f, 1.0f, 2.0f, 2.0f, 3.0f, 3.0f}); +} + +/*! \brief Insert the channel field `channelCount` into the shape at `channelAxis` position */ +static TShape MakeShape(const std::vector& shape, + signed int channelAxis, + const size_t channelCount) { + if (channelAxis < 0) { + channelAxis += shape.size() + 1; + } + CHECK_LT(channelAxis, shape.size() + 1); + const index_t dim = index_t(shape.size()) + 1; + TShape newShape(dim); + for (size_t x = 0; x < channelAxis; ++x) { + newShape[x] = index_t(shape[x]); + } + newShape[channelAxis] = index_t(channelCount); + for (int x = channelAxis + 1; x < dim; ++x) { + newShape[x] = shape[x - 1]; + } + return newShape; +} + +/*! \brief Create and arrange equivalent data with different channel axes, then compare + * normalized results */ +static void runChannelAxisTest( + const bool isGPU1, + const bool isGPU2, + const test::op::kwargs_t& base_kwargs, + const std::vector shape, + const signed int channelAxis1, + const signed int channelAxis2, + const size_t channelCount, + const bool simpleData, + const size_t numberOfPasses = 5 + +) { + typedef float DType; + typedef float AccReal; + + size_t spatialSize = 1; + for (size_t x = 1, n = shape.size(); x < n; ++x) { + spatialSize *= shape[x]; + } + + const size_t batchSize = shape[0]; + + // Create normalized input and output-grad data (inputs to forward and backward pass) + std::vector> myData, myGradOut; + DType ival = 1.0f, gval = 0.1f; + myData.resize(batchSize); + myData.resize(channelCount); + myGradOut.resize(channelCount); + for (size_t c = 0; c < channelCount; ++c) { + for (size_t i = 0; i < spatialSize; ++i) { + if (!simpleData) { + myData[c].push_back(ival += 1.0f); + myGradOut[c].push_back(gval += 0.1f); + } else { + myData[c].push_back(c + 1); + myGradOut[c].push_back(DType(c + 1) / 10.0f); + } + } + } + + ChannelAxisTestData::print("myData", myData); + ChannelAxisTestData::print("myGradOut", myGradOut); + ChannelAxisTestData data_c1, data_c2, grad_c1, grad_c2; + + // For forward pass + data_c1.channel_data_ = data_c2.channel_data_ = myData; + + // For backward pass + grad_c1.channel_data_ = grad_c2.channel_data_ = myGradOut; + + test::op::kwargs_t kwargs = base_kwargs; + + // Insert the channel field into the shape at channelAxis position + const TShape shape_c1 = MakeShape(shape, channelAxis1, channelCount); + const TShape shape_c2 = MakeShape(shape, channelAxis2, channelCount); + + // Create operator 1 with ChannelAxis2 (normally the experimental one) + kwargs.push_back({"axis", std::to_string(channelAxis1)}); + test::op::OpInfo info_c1 = test::op::createOpAndInfoF< + op::BatchNormProp, BNOperatorData, DType, AccReal>( + isGPU1, shape_c1, kwargs); + + // Create operator 2 with ChannelAxis2 (normally the control one) + kwargs.pop_back(); + kwargs.push_back({"axis", std::to_string(channelAxis2)}); + test::op::OpInfo info_c2 = test::op::createOpAndInfoF< + op::BatchNormProp, BNOperatorData, DType, AccReal>( + isGPU2, shape_c2, kwargs); + kwargs.pop_back(); + + // Init operators + info_c1.data_->initForward(*info_c1.prop_, &info_c1.in_type_); + info_c1.data_->initBackward(*info_c1.prop_, &info_c1.in_type_); + info_c2.data_->initForward(*info_c2.prop_, &info_c2.in_type_); + info_c2.data_->initBackward(*info_c2.prop_, &info_c2.in_type_); + + // Save input data to blob with new shape 1 + data_c1.save(info_c1.data_->c_.blob_input_vec_[0], channelAxis1); + ChannelAxisTestData::print("blob 1 input", info_c1.data_->c_.blob_input_vec_[0]); + + // Save input data to blob with new shape 2 + data_c2.save(info_c2.data_->c_.blob_input_vec_[0], channelAxis2); + ChannelAxisTestData::print("blob 2 input", info_c2.data_->c_.blob_input_vec_[0]); + + // Save output grad to blob with new shape 1 + grad_c1.save(info_c1.data_->c_.blob_out_grad_[0], channelAxis1); + ChannelAxisTestData::print("blob 1 output grad", info_c1.data_->c_.blob_out_grad_[0]); + + // Save output grad to blob with new shape 2 + grad_c2.save(info_c2.data_->c_.blob_out_grad_[0], channelAxis2); + ChannelAxisTestData::print("blob 2 output grad", info_c2.data_->c_.blob_out_grad_[0]); + + // Run both operators forward and backwards several times + for (int x = 0; x < numberOfPasses; ++x) { + info_c1.data_->forward(); + info_c2.data_->forward(); + + info_c1.data_->backward(); + info_c2.data_->backward(); + } + + // Transform operator 1's blob output to a normalized shape + data_c1.load(info_c1.data_->c_.blob_output_vec_[0], channelAxis1); + ChannelAxisTestData::print("channel data 1", data_c1.channel_data_); + + // Transform operator 2's blob output to a normalized shape + data_c2.load(info_c2.data_->c_.blob_output_vec_[0], channelAxis2); + ChannelAxisTestData::print("channel data 2", data_c2.channel_data_); + + // Compare the operators' output data while they're in a normalized shape + compare(data_c1.channel_data_, data_c2.channel_data_); + + // Transform operator 1's input-grad blob to a normalized shape + grad_c1.load(info_c1.data_->c_.blob_in_grad_[0], channelAxis1); + ChannelAxisTestData::print("input grad 1", grad_c1.channel_data_); + + // Transform operator 2's input-grad blob to a normalized shape + grad_c2.load(info_c2.data_->c_.blob_in_grad_[0], channelAxis2); + ChannelAxisTestData::print("input grad 2", grad_c2.channel_data_); + + // Compare the operators' input grad data while they're in a normalized shape + compare(grad_c1.channel_data_, grad_c2.channel_data_); +} + +TEST(BATCH_NORM, TestChannelAxisSimple) { + std::cout << std::endl << std::flush; + const size_t CHANNEL_COUNT = 4; + const int DEFAULT_AXIS = 1; + const int NEW_AXIS = -2; + const bool useSimpleData = true; // change to true sometimes for troubleshooting + const std::vector shape = {1, 2, 3}; + // Check against base-case of channel axis position 1 + runChannelAxisTest(false, false, + useglobalstats_kwargs_nocudnn, + shape, + DEFAULT_AXIS, + NEW_AXIS, + CHANNEL_COUNT, + useSimpleData); +} + +/*! \brief Test varying channel axis shapes + * For several channel counts (1-3), test that result data (after reshape) is + * equivalent for the default (channel position 1) and all other channel positions + * in the shape vector + * Channel position 1 (default) is checked everywhere else, so for and + * backward result equivalence here implies correctness for other channel positions + */ +TEST(BATCH_NORM, TestChannelAxis) { + test::ScopeSet noDebugOutput(&test::debugOutput, false); + + test::op::kwargs_t kwargs; + const std::vector> shapes = + { {1, 2}, {1, 2, 1}, {1, 2, 3}, {1, 2, 3, 4} }; + const char *tof[2] = { "False", "True" }; + + for (size_t x1 = 0; x1 < 2U; ++x1) { + kwargs.push_back({"fix_gamma", tof[x1]}); + for (size_t x2 = 0; x2 < 2U; ++x2) { + kwargs.push_back({"use_global_stats", tof[x2]}); + for (size_t x3 = 0; x3 < 2U; ++x3) { + kwargs.push_back({"cudnn_off", tof[x3]}); + for (int g1 = 0; g1 < 2U; ++g1) { + for (int g2 = 0; g2 < 2U; ++g2) { + for (const std::vector &simpleShape : shapes) { + const int dim = static_cast(simpleShape.size()); + for (signed int channelAxis = -dim, shapeDim = dim; + channelAxis <= shapeDim; + ++channelAxis) { + for (size_t channelCount = 1; channelCount <= 3; ++channelCount) { + // Check against base-case of channel axis position 1 + runChannelAxisTest(g1 != 0, g2 != 0, kwargs, simpleShape, + 1, channelAxis, channelCount, false); + } + } + } + } + } + kwargs.pop_back(); + } + kwargs.pop_back(); + } + kwargs.pop_back(); + } +} + #if MXNET_USE_CUDA TEST(BATCH_NORM, Test2DForwardV12D_gpu) { diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py index f0c4ea6bb376..924ef351dbe5 100644 --- a/tests/python/unittest/test_operator.py +++ b/tests/python/unittest/test_operator.py @@ -860,6 +860,39 @@ def test_batchnorm_training(): test = mx.symbol.BatchNorm(data, fix_gamma=False, use_global_stats=True) check_numeric_gradient(test, [data_tmp, gamma, beta], [rolling_mean, rolling_std], numeric_eps=1e-2, rtol=0.16) + # Test varying channel axis + dim = len(shape) + for chaxis in range(-dim, dim): + chaxis_true = chaxis + if chaxis < 0: + chaxis_true = dim + chaxis + + shapex = shape + + channel_count = shapex[chaxis_true] + data_tmp = np.random.normal(-0.1, 0.1, size=shapex) + + gamma = np.ones(channel_count) + beta = np.ones(channel_count) + if channel_count > 1: + gamma[1] = 3 + beta[0] = 3 + + xrolling_mean = np.random.uniform(size=channel_count) + xrolling_std = np.random.uniform(size=channel_count) + + test = mx.symbol.BatchNorm(data, fix_gamma=True, axis=chaxis) + check_numeric_gradient(test, [data_tmp, gamma, beta], [xrolling_mean, xrolling_std], numeric_eps=1e-2, rtol=0.2) + + test = mx.symbol.BatchNorm(data, fix_gamma=True, use_global_stats=True, axis=chaxis) + check_numeric_gradient(test, [data_tmp, gamma, beta], [xrolling_mean, xrolling_std], numeric_eps=1e-2, rtol=0.2) + + test = mx.symbol.BatchNorm(data, fix_gamma=False, axis=chaxis) + check_numeric_gradient(test, [data_tmp, gamma, beta], [xrolling_mean, xrolling_std], numeric_eps=1e-2, rtol=0.2) + + test = mx.symbol.BatchNorm(data, fix_gamma=False, use_global_stats=True, axis=chaxis) + check_numeric_gradient(test, [data_tmp, gamma, beta], [xrolling_mean, xrolling_std], numeric_eps=1e-2, rtol=0.2) + def test_convolution_grouping(): num_filter = 4 num_group = 2 From ba2d9f61aa7b744c4ec3243097fb7170ad06012c Mon Sep 17 00:00:00 2001 From: Haibin Lin Date: Mon, 5 Jun 2017 23:49:59 -0700 Subject: [PATCH 036/834] fix `out` option for mx.nd.zeros and mx.nd.ones" (#6589) --- python/mxnet/ndarray.py | 18 ++++++++++++------ tests/python/unittest/test_ndarray.py | 11 +++++++++++ 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/python/mxnet/ndarray.py b/python/mxnet/ndarray.py index c5d6754555d1..b167e8673ff4 100644 --- a/python/mxnet/ndarray.py +++ b/python/mxnet/ndarray.py @@ -1004,6 +1004,8 @@ def zeros(shape, ctx=None, dtype=mx_real_t, **kwargs): An optional device context (default is the current default context). dtype : str or numpy.dtype, optional An optional value type (default is `float32`). + out : NDArray, optional + The output NDArray (default is `None`). Returns ------- @@ -1023,7 +1025,7 @@ def zeros(shape, ctx=None, dtype=mx_real_t, **kwargs): if ctx is None: ctx = Context.default_ctx # pylint: disable= no-member, protected-access - return _internal._zeros(shape=shape, ctx=ctx, dtype=dtype) + return _internal._zeros(shape=shape, ctx=ctx, dtype=dtype, **kwargs) # pylint: enable= no-member, protected-access def ones(shape, ctx=None, dtype=mx_real_t, **kwargs): @@ -1038,6 +1040,8 @@ def ones(shape, ctx=None, dtype=mx_real_t, **kwargs): Defaults to the current default context (``mxnet.Context.default_ctx``). dtype : str or numpy.dtype, optional An optional value type (default is `float32`). + out : NDArray, optional + The output NDArray (default is `None`). Returns ------- @@ -1057,10 +1061,10 @@ def ones(shape, ctx=None, dtype=mx_real_t, **kwargs): if ctx is None: ctx = Context.default_ctx # pylint: disable= no-member, protected-access - return _internal._ones(shape=shape, ctx=ctx, dtype=dtype) + return _internal._ones(shape=shape, ctx=ctx, dtype=dtype, **kwargs) # pylint: enable= no-member, protected-access -def full(shape, val, ctx=None, dtype=mx_real_t): +def full(shape, val, ctx=None, dtype=mx_real_t, out=None): """Returns a new array of given shape and type, filled with the given value `val`. Parameters @@ -1073,6 +1077,8 @@ def full(shape, val, ctx=None, dtype=mx_real_t): Device context (default is the current default context). dtype : `str` or `numpy.dtype`, optional The data type of the returned `NDArray`. The default datatype is `float32`. + out : NDArray, optional + The output NDArray (default is `None`). Returns ------- @@ -1088,9 +1094,9 @@ def full(shape, val, ctx=None, dtype=mx_real_t): >>> mx.nd.full((1, 2), 2.0, dtype='float16').asnumpy() array([[ 2., 2.]], dtype=float16) """ - arr = empty(shape, ctx, dtype) - arr[:] = val - return arr + out = empty(shape, ctx, dtype) if out is None else out + out[:] = val + return out def array(source_array, ctx=None, dtype=None): diff --git a/tests/python/unittest/test_ndarray.py b/tests/python/unittest/test_ndarray.py index 2be95a9766af..dd38bdf98606 100644 --- a/tests/python/unittest/test_ndarray.py +++ b/tests/python/unittest/test_ndarray.py @@ -637,6 +637,17 @@ def test_cached(): o2 = mx.nd.invoke(op, [data, weight, bias]) assert_almost_equal(o2.asnumpy(), o1.asnumpy()+1) +def test_output(): + shape = (2,2) + ones = mx.nd.ones(shape) + zeros = mx.nd.zeros(shape) + out = mx.nd.zeros(shape) + mx.nd.ones(shape, out=out) + assert_almost_equal(out.asnumpy(), ones.asnumpy()) + mx.nd.zeros(shape, out=out) + assert_almost_equal(out.asnumpy(), zeros.asnumpy()) + mx.nd.full(shape, 2, out=out) + assert_almost_equal(out.asnumpy(), ones.asnumpy() * 2) if __name__ == '__main__': import nose From 59781868b8753aa407e07759f12ef8a04b4cd291 Mon Sep 17 00:00:00 2001 From: Arik Poznanski Date: Tue, 6 Jun 2017 19:41:40 +0300 Subject: [PATCH 037/834] Fixed caffe_converter and improved test_converter (#6489) * added support for running test_converter on cpu * Added a missing check before searching for the bias blob name. This prevents failure when converting the standard bvlc_googlenet model from the caffe model zoo. * accuracy drop allowed should be 3% not 30%! * fixed crash when converting resnet: rescale_factor = 1 / rescale_factor TypeError: unsupported operand type(s) for /: 'int' and 'google.protobuf.pyext._message.RepeatedScalarContainer' * fix lint issues: missing docstring and condition --- tools/caffe_converter/convert_model.py | 14 +++++++++----- tools/caffe_converter/test_converter.py | 19 ++++++++++++++----- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/tools/caffe_converter/convert_model.py b/tools/caffe_converter/convert_model.py index e4134e272283..03641aa80d51 100644 --- a/tools/caffe_converter/convert_model.py +++ b/tools/caffe_converter/convert_model.py @@ -82,6 +82,10 @@ def convert_model(prototxt_fname, caffemodel_fname, output_prefix=None): bias = bias.reshape((bias.shape[0], 1)) assert(bias.flags['C_CONTIGUOUS'] is True) bias_name = layer_name + "_bias" + + if bias_name not in arg_shape_dic: + print(bias_name + ' not found in arg_shape_dic.') + continue bias = bias.reshape(arg_shape_dic[bias_name]) arg_params[bias_name] = mx.nd.zeros(bias.shape) arg_params[bias_name][:] = bias @@ -105,8 +109,8 @@ def convert_model(prototxt_fname, caffemodel_fname, output_prefix=None): elif layer_type == 'Scale': bn_name = layer_name.replace('scale', 'bn') - gamma = layer_blobs[0].data - beta = layer_blobs[1].data + gamma = np.array(layer_blobs[0].data) + beta = np.array(layer_blobs[1].data) # beta = np.expand_dims(beta, 1) beta_name = '{}_beta'.format(bn_name) gamma_name = '{}_gamma'.format(bn_name) @@ -124,9 +128,9 @@ def convert_model(prototxt_fname, caffemodel_fname, output_prefix=None): beta.shape, gamma.shape)) elif layer_type == 'BatchNorm': bn_name = layer_name - mean = layer_blobs[0].data - var = layer_blobs[1].data - rescale_factor = layer_blobs[2].data + mean = np.array(layer_blobs[0].data) + var = np.array(layer_blobs[1].data) + rescale_factor = layer_blobs[2].data[0] if rescale_factor != 0: rescale_factor = 1 / rescale_factor mean_name = '{}_moving_mean'.format(bn_name) diff --git a/tools/caffe_converter/test_converter.py b/tools/caffe_converter/test_converter.py index 128e7c276c50..96947baf0c2b 100644 --- a/tools/caffe_converter/test_converter.py +++ b/tools/caffe_converter/test_converter.py @@ -1,6 +1,7 @@ """Test converted models """ import os +import argparse import sys import logging import mxnet as mx @@ -35,13 +36,21 @@ def test_imagenet_model(model_name, val_data, gpus, batch_size): logging.info('speed : %f image/sec', speed) for a in acc: logging.info(a.get()) - assert acc[0].get()[1] > meta_info['top-1-acc'] - 0.3 - assert acc[1].get()[1] > meta_info['top-5-acc'] - 0.3 + assert acc[0].get()[1] > meta_info['top-1-acc'] - 0.03 + assert acc[1].get()[1] > meta_info['top-5-acc'] - 0.03 def main(): - gpus = mx.test_utils.list_gpus() - assert len(gpus) > 0 - batch_size = 32 * len(gpus) + """Entrypoint for test_converter""" + parser = argparse.ArgumentParser(description='Test Caffe converter') + parser.add_argument('--cpu', action='store_true', help='use cpu?') + args = parser.parse_args() + if args.cpu: + gpus = '' + batch_size = 32 + else: + gpus = mx.test_utils.list_gpus() + assert gpus, 'At least one GPU is needed to run test_converter in GPU mode' + batch_size = 32 * len(gpus) models = ['bvlc_googlenet', 'vgg-16', 'resnet-50'] From 2428e00fcd507fb0d8cbc7f72304dd17b0865097 Mon Sep 17 00:00:00 2001 From: Indhu Bharathi Date: Tue, 6 Jun 2017 10:02:47 -0700 Subject: [PATCH 038/834] Add instructions to install OpenCV. (#6582) --- .../vision/large_scale_classification.md | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/docs/tutorials/vision/large_scale_classification.md b/docs/tutorials/vision/large_scale_classification.md index cf76c96b70a5..f1929980ad3c 100644 --- a/docs/tutorials/vision/large_scale_classification.md +++ b/docs/tutorials/vision/large_scale_classification.md @@ -83,7 +83,7 @@ done ``` ### Pack images into record files -While MXNet can read image files directly, it is recommended to pack the image files into a recordIO file for increased performance. MXNet provides a tool (tools/im2rec.py) to do this. To use this tool, MXNet and OpenCV’s python module needs to be installed in the system. OpenCV’s python module can be installed on Ubuntu using the command `sudo apt-get install python-opencv`. +While MXNet can read image files directly, it is recommended to pack the image files into a recordIO file for increased performance. MXNet provides a tool (tools/im2rec.py) to do this. To use this tool, MXNet and OpenCV’s python module needs to be installed in the system. [Here](#installing-opencv) are instructions to install OpenCV python module. Set the environment variable `MXNET` to point to the MXNet installation directory and `NAME` to the name of the dataset. Here, we assume MXNet is installed at `~/mxnet` @@ -244,3 +244,30 @@ It is often straightforward to achieve a reasonable validation accuracy, but ach If the batch size is too big, it can exhaust GPU memory. If this happens, you’ll see the error message “cudaMalloc failed: out of memory” or something similar. There are a couple of ways to fix this: - Reduce the batch size. - Set the environment variable `MXNET_BACKWARD_DO_MIRROR` to 1. It reduces the memory consumption by trading off speed. For example, with batch size 64, inception-v3 uses 10G memory and trains 30 image/sec on a single K80 GPU. When mirroring is enabled, with 10G GPU memory consumption, we can run inception-v3 using batch size of 128. The cost is that, the speed reduces to 27 images/sec. + +## Appendix +### Installing OpenCV +#### On Ubuntu +Install OpenCV: +``` +sudo apt-get install -y libopencv-dev +``` +Install OpenCV Python libraries: +``` +sudo apt-get install python-opencv +``` +#### On Amazon Linux +Install OpenCV: +``` +git clone https://github.com/opencv/opencv +cd opencv +mkdir -p build +cd build +cmake -D BUILD_opencv_gpu=OFF -D WITH_EIGEN=ON -D WITH_TBB=ON -D WITH_CUDA=OFF -D WITH_1394=OFF -D CMAKE_BUILD_TYPE=RELEASE -D CMAKE_INSTALL_PREFIX=/usr/local .. +make +sudo make PREFIX=/usr/local install +``` +Install OpenCV Python libraries: +``` +sudo yum install opencv-python +``` From fa709b288b6e46e9aa0b4343a0488df9d48cdcc9 Mon Sep 17 00:00:00 2001 From: Sandeep Krishnamurthy Date: Tue, 6 Jun 2017 10:27:57 -0700 Subject: [PATCH 039/834] Fixing obsolete installation guide (#6585) --- docs/get_started/amazonlinux_setup.md | 236 +------------------------- docs/get_started/centos_setup.md | 168 +----------------- docs/get_started/ubuntu_setup.md | 47 ++--- 3 files changed, 41 insertions(+), 410 deletions(-) diff --git a/docs/get_started/amazonlinux_setup.md b/docs/get_started/amazonlinux_setup.md index 6829acaa0465..054e0304e107 100644 --- a/docs/get_started/amazonlinux_setup.md +++ b/docs/get_started/amazonlinux_setup.md @@ -1,228 +1,8 @@ -# Installing MXNet on Amazon Linux - -**NOTE:** For MXNet with Python installation, please refer to the [new install guide](http://mxnet.io/get_started/install.html). - -Installing MXNet is a two-step process: - -1. Build the shared library from the MXNet C++ source code. -2. Install the supported language-specific packages for MXNet. - -**Note:** To change the compilation options for your build, edit the ```make/config.mk``` file and submit a build request with the ```make``` command. - -## Build the Shared Library -On Amazon Linux, you need the following dependencies: - -- Git (to pull code from GitHub) - -- libatlas-base-dev (for linear algebraic operations) - -- libopencv-dev (for computer vision operations) - -Install these dependencies using the following commands: - -```bash - # CMake is required for installing dependencies. - sudo yum install -y cmake - - # Set appropriate library path env variables - echo 'export PATH=/usr/local/bin:$PATH' >> ~/.profile - echo 'export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH' >> ~/.profile - echo 'export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH' >> ~/.profile - echo '. ~/.profile' >> ~/.bashrc - source ~/.profile - - # Install gcc-4.8/make and other development tools on Amazon Linux - # Reference: http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/compile-software.html - # Install Python, Numpy, Scipy and set up tools. - sudo yum groupinstall -y "Development Tools" - sudo yum install -y python27 python27-setuptools python27-tools python-pip - sudo yum install -y python27-numpy python27-scipy python27-nose python27-matplotlib graphviz - - # Install OpenBLAS at /usr/local/openblas - git clone https://github.com/xianyi/OpenBLAS - cd OpenBLAS - make FC=gfortran -j $(($(nproc) + 1)) - sudo make PREFIX=/usr/local install - cd .. - - # Install OpenCV at /usr/local/opencv - git clone https://github.com/opencv/opencv - cd opencv - mkdir -p build - cd build - cmake -D BUILD_opencv_gpu=OFF -D WITH_EIGEN=ON -D WITH_TBB=ON -D WITH_CUDA=OFF -D WITH_1394=OFF -D CMAKE_BUILD_TYPE=RELEASE -D CMAKE_INSTALL_PREFIX=/usr/local .. - sudo make PREFIX=/usr/local install - - # Install Graphviz for visualization and Jupyter notebook for running examples and tutorials - sudo pip install graphviz - sudo pip install jupyter - - # Export env variables for pkg config - export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH -``` -After installing the dependencies, use the following command to pull the MXNet source code from GitHub - -```bash - # Get MXNet source code - git clone https://github.com/dmlc/mxnet.git ~/mxnet --recursive - # Move to source code parent directory - cd ~/mxnet - cp make/config.mk . - echo "USE_BLAS=openblas" >>config.mk - echo "ADD_CFLAGS += -I/usr/include/openblas" >>config.mk - echo "ADD_LDFLAGS += -lopencv_core -lopencv_imgproc -lopencv_imgcodecs" >>config.mk -``` - -If building with ```GPU``` support, run below commands to add GPU dependency configurations to config.mk file: - -```bash - echo "USE_CUDA=1" >>config.mk - echo "USE_CUDA_PATH=/usr/local/cuda" >>config.mk - echo "USE_CUDNN=1" >>config.mk -``` - -Then build mxnet: - -```bash - make -j$(nproc) -``` - -Executing these commands creates a library called ```libmxnet.so``` - - -  - -We have installed MXNet core library. Next, we will install MXNet interface package for the programming language of your choice: -- [R](#install-the-mxnet-package-for-r) -- [Julia](#install-the-mxnet-package-for-julia) -- [Scala](#install-the-mxnet-package-for-scala) -- [Perl](#install-the-mxnet-package-for-perl) - -## Install the MXNet Package for R -Run the following commands to install the MXNet dependencies and build the MXNet R package. - -```r - Rscript -e "install.packages('devtools', repo = 'https://cran.rstudio.com')" -``` -```bash - cd R-package - Rscript -e "library(devtools); library(methods); options(repos=c(CRAN='https://cran.rstudio.com')); install_deps(dependencies = TRUE)" - cd .. - make rpkg -``` - -**Note:** R-package is a folder in the MXNet source. - -These commands create the MXNet R package as a tar.gz file that you can install as an R package. To install the R package, run the following command, use your MXNet version number: - -```bash - R CMD INSTALL mxnet_current_r.tar.gz -``` - -## Install the MXNet Package for Julia -The MXNet package for Julia is hosted in a separate repository, MXNet.jl, which is available on [GitHub](https://github.com/dmlc/MXNet.jl). To use Julia binding it with an existing libmxnet installation, set the ```MXNET_HOME``` environment variable by running the following command: - -```bash - export MXNET_HOME=//libmxnet -``` - -The path to the existing libmxnet installation should be the root directory of libmxnet. In other words, you should be able to find the ```libmxnet.so``` file at ```$MXNET_HOME/lib```. For example, if the root directory of libmxnet is ```~```, you would run the following command: - -```bash - export MXNET_HOME=/~/libmxnet -``` - -You might want to add this command to your ```~/.bashrc``` file. If you do, you can install the Julia package in the Julia console using the following command: - -```julia - Pkg.add("MXNet") -``` - -For more details about installing and using MXNet with Julia, see the [MXNet Julia documentation](http://dmlc.ml/MXNet.jl/latest/user-guide/install/). - -## Install the MXNet Package for Scala - -There are two ways to install the MXNet package for Scala: - -* Use the prebuilt binary package - -* Build the library from source code - -### Use the Prebuilt Binary Package -For Linux users, MXNet provides prebuilt binary packages that support computers with either GPU or CPU processors. To download and build these packages using ```Maven```, change the ```artifactId``` in the following Maven dependency to match your architecture: - -```HTML - - ml.dmlc.mxnet - mxnet-full_ - 0.1.1 - -``` - -For example, to download and build the 64-bit CPU-only version for Linux, use: - -```HTML - - ml.dmlc.mxnet - mxnet-full_2.10-linux-x86_64-cpu - 0.1.1 - -``` - -If your native environment differs slightly from the assembly package, for example, if you use the openblas package instead of the atlas package, it's better to use the mxnet-core package and put the compiled Java native library in your load path: - -```HTML - - ml.dmlc.mxnet - mxnet-core_2.10 - 0.1.1 - -``` - -### Build the Library from Source Code -Before you build MXNet for Scala from source code, you must complete [building the shared library](#build-the-shared-library). After you build the shared library, run the following command from the MXNet source root directory to build the MXNet Scala package: - -```bash - make scalapkg -``` - -This command creates the JAR files for the assembly, core, and example modules. It also creates the native library in the ```native/{your-architecture}/target directory```, which you can use to cooperate with the core module. - -To install the MXNet Scala package into your local Maven repository, run the following command from the MXNet source root directory: - -```bash - make scalainstall -``` - -## Install the MXNet Package for Perl - -Before you build MXNet for Perl from source code, you must complete [building the shared library](#build-the-shared-library). After you build the shared library, run the following command from the MXNet source root directory to build the MXNet Perl package: - -```bash - ## install PDL, Graphviz, Mouse, App::cpanminus, swig via yum before running these commands - cpanm -q -L "${HOME}/perl5" Function::Parameters - - MXNET_HOME=${PWD} - export LD_LIBRARY_PATH=${MXNET_HOME}/lib - export PERL5LIB=${HOME}/perl5/lib/perl5 - - cd ${MXNET_HOME}/perl-package/AI-MXNetCAPI/ - perl Makefile.PL INSTALL_BASE=${HOME}/perl5 - make install - - cd ${MXNET_HOME}/perl-package/AI-NNVMCAPI/ - perl Makefile.PL INSTALL_BASE=${HOME}/perl5 - make install - - cd ${MXNET_HOME}/perl-package/AI-MXNet/ - perl Makefile.PL INSTALL_BASE=${HOME}/perl5 - make install -``` - -**Note -** You are more than welcome to contribute easy installation scripts for other operating systems and programming languages, see [community page](http://mxnet.io/community/index.html) for contributors guidelines. - -## Next Steps - -* [Tutorials](http://mxnet.io/tutorials/index.html) -* [How To](http://mxnet.io/how_to/index.html) -* [Architecture](http://mxnet.io/architecture/index.html) + + + +

+ + This content is moved to a new MXNet install page. Redirecting... +

diff --git a/docs/get_started/centos_setup.md b/docs/get_started/centos_setup.md index 9cfa865b09d9..054e0304e107 100644 --- a/docs/get_started/centos_setup.md +++ b/docs/get_started/centos_setup.md @@ -1,160 +1,8 @@ -# Installing MXNet on CentOS - -**NOTE:** For MXNet with Python installation, please refer to the [new install guide](http://mxnet.io/get_started/install.html). - -MXNet currently supports Python, R, Julia, Scala, and Perl. For users on CentOS with Docker environment, MXNet provides [Docker installation guide](http://mxnet.io/get_started/docker_setup.html). If you do not have a Docker environment set up, follow below-provided step by step instructions. - - -## Minimum Requirements -Make sure you have the root permission, and `yum` is properly installed. Check it using the following command: - -```bash -sudo yum check-update -``` -If you don't get an error message, then `yum` is installed. - -**To install MXNet on CentOS, you must have the following:** - -1. gcc, g++ (4.8 or later) -2. python2, python-numpy, python-pip, clang -3. graphviz, jupyter (pip or yum install) -4. OpenBLAS -5. CUDA for GPU -6. cmake and opencv (do not use yum to install opencv, some shared libs may not be installed) - -## Install Dependencies -Make sure your machine is connected to Internet. A few installations need to download (`git clone` or `wget`) some packages from Internet. - -### Install Basic Environment -```bash - # Install gcc-4.8/make and other development tools - sudo yum install -y gcc - sudo yum install -y gcc-c++ - sudo yum install -y clang - - # Install Python, Numpy, pip and set up tools. - sudo yum groupinstall -y "Development Tools" - sudo yum install -y python27 python27-setuptools python27-tools python-pip - sudo yum install -y python27-numpy - - # install graphviz, jupyter - sudo pip install graphviz - sudo pip install jupyter -``` -### Install OpenBLAS -Note that OpenBLAS can be replaced by other BLAS libs, e.g, Intel MKL. - -```bash - # Install OpenBLAS at /usr/local/openblas - git clone https://github.com/xianyi/OpenBLAS - cd OpenBLAS - make -j $(($(nproc) + 1)) - sudo make PREFIX=/usr/local install - cd .. -``` -### Install CUDA for GPU -Note: Setting up CUDA is optional for MXNet. If you do not have a GPU machine (or if you want to train with CPU), you can skip this section and proceed with installation of OpenCV. - -If you plan to build with GPU, you need to set up the environment for CUDA and CUDNN. - -First, download and install [CUDA 8 toolkit](https://developer.nvidia.com/cuda-toolkit). - -Then download [cudnn 5](https://developer.nvidia.com/cudnn). - -Unzip the file and change to the cudnn root directory. Move the header and libraries to your local CUDA Toolkit folder: - -```bash - tar xvzf cudnn-8.0-linux-x64-v5.1-ga.tgz - sudo cp -P cuda/include/cudnn.h /usr/local/cuda/include - sudo cp -P cuda/lib64/libcudnn* /usr/local/cuda/lib64 - sudo chmod a+r /usr/local/cuda/include/cudnn.h /usr/local/cuda/lib64/libcudnn* - sudo ldconfig -``` -### Install opencv -Note: Setting up opencv is optional but strongly recommended for MXNet, unless you do not want to work on Computer Vision and Image Augmentation. If you are quite sure about that, skip this section and set `USE_OPENCV = 0` in `config.mk`. - -The Open Source Computer Vision (OpenCV) library contains programming functions for computer vision and image augmentation. For more information, see [OpenCV](https://en.wikipedia.org/wiki/OpenCV). - -```bash - # Install cmake for building opencv - sudo yum install -y cmake - # Install OpenCV at /usr/local/opencv - git clone https://github.com/opencv/opencv - cd opencv - mkdir -p build - cd build - cmake -D BUILD_opencv_gpu=OFF -D WITH_EIGEN=ON -D WITH_TBB=ON -D WITH_CUDA=OFF -D WITH_1394=OFF -D CMAKE_BUILD_TYPE=RELEASE -D CMAKE_INSTALL_PREFIX=/usr/local .. - sudo make PREFIX=/usr/local install -``` - -## Install MXNet - -### Build MXNet shared library -After installing the dependencies, use the following command to pull the MXNet source code from GitHub. - -```bash - # Download MXNet source code to ~/mxnet directory - git clone https://github.com/dmlc/mxnet.git ~/mxnet --recursive - # Move to source code parent directory - cd ~/mxnet - cp make/config.mk . - # Replace this line if you use other BLAS libs - echo "USE_BLAS=openblas" >>config.mk - echo "ADD_CFLAGS += -I/usr/include/openblas" >>config.mk - echo "ADD_LDFLAGS += -lopencv_core -lopencv_imgproc -lopencv_imgcodecs" >>config.mk -``` - -If building with ```GPU``` support, run below commands to add GPU dependency configurations to `config.mk` file: - -```bash - echo "USE_CUDA=1" >>config.mk - echo "USE_CUDA_PATH=/usr/local/cuda" >>config.mk - echo "USE_CUDNN=1" >>config.mk -``` - -Then build mxnet: - -```bash - make -j$(nproc) -``` - -Executing these commands creates a library called ```libmxnet.so``` in `~/mxnet/lib/`. - -### Install MXNet for R, Julia, Scala, and Perl. - -- [R](http://mxnet.io/get_started/amazonlinux_setup.html#install-the-mxnet-package-for-r) -- [Julia](http://mxnet.io/get_started/amazonlinux_setup.html#install-the-mxnet-package-for-julia) -- [Scala](http://mxnet.io/get_started/amazonlinux_setup.html#install-the-mxnet-package-for-scala) -- [Perl](http://mxnet.io/get_started/amazonlinux_setup.html#install-the-mxnet-package-for-perl) - -## Troubleshooting - -Here is some information to help you troubleshoot, in case you encounter error messages: - -**1. Cannot build opencv from source code** - -This may be caused by download failure during building, e.g., `ippicv`. - -Prepare some large packages by yourself, then copy them to the right place, e.g, `opencv/3rdparty/ippicv/downloads/linux-808XXXXXXXXX/`. - -**2. Link errors when building MXNet** - -```bash -/usr/bin/ld: /tmp/ccQ9qruP.o: undefined reference to symbol '_ZN2cv6String10deallocateEv' -/usr/local/lib/libopencv_core.so.3.2: error adding symbols: DSO missing from command line -``` -This error occurs when you already have old opencv (e.g, 2.4) installed using `yum` (in `/usr/lib64`). When g++ tries to link opencv libs, it will first find and link old opencv libs in `/usr/lib64`. - -Please modify `config.mk` in `mxnet` directory, and add `-L/usr/local/lib` to `ADD_CFLAGS`. - -```bash - ADD_CFLAGS += -I/usr/include/openblas -L/usr/local/lib -``` -This solution solves this link error, but there are still lots of warnings. - - -## Next Steps - -* [Tutorials](http://mxnet.io/tutorials/index.html) -* [How To](http://mxnet.io/how_to/index.html) -* [Architecture](http://mxnet.io/architecture/index.html) + + + +

+ + This content is moved to a new MXNet install page. Redirecting... +

diff --git a/docs/get_started/ubuntu_setup.md b/docs/get_started/ubuntu_setup.md index 95c59d3f2109..b7130bef4967 100644 --- a/docs/get_started/ubuntu_setup.md +++ b/docs/get_started/ubuntu_setup.md @@ -76,45 +76,48 @@ Installing MXNet is a two-step process: On Ubuntu versions 13.10 or later, you need the following dependencies: -- Git (to pull code from GitHub) - -- libatlas-base-dev (for linear algebraic operations) +**Step 1** Install build tools and git. +```bash + sudo apt-get update + sudo apt-get install -y build-essential git +``` -- libopencv-dev (for computer vision operations) +**Step 2** Install OpenBLAS. -Install these dependencies using the following commands: +*MXNet* uses [BLAS](https://en.wikipedia.org/wiki/Basic_Linear_Algebra_Subprograms) library for accelerated numerical computations on CPU machine. There are several flavors of BLAS libraries - [OpenBLAS](http://www.openblas.net/), [ATLAS](http://math-atlas.sourceforge.net/) and [MKL](https://software.intel.com/en-us/intel-mkl). In this step we install OpenBLAS. You can choose to install ATLAS or MKL. ```bash - sudo apt-get update - sudo apt-get install -y build-essential git libatlas-base-dev libopencv-dev + sudo apt-get install -y libopenblas-dev ``` -After installing the dependencies, use the following command to pull the MXNet source code from GitHub +**Step 3** Install OpenCV. + +*MXNet* uses [OpenCV](http://opencv.org/) for efficient image loading and augmentation operations. ```bash - # Get MXNet source code - git clone https://github.com/dmlc/mxnet.git ~/mxnet --recursive - # Move to source code parent directory - cd ~/mxnet - cp make/config.mk . - echo "USE_BLAS=openblas" >>config.mk - echo "ADD_CFLAGS += -I/usr/include/openblas" >>config.mk - echo "ADD_LDFLAGS += -lopencv_core -lopencv_imgproc -lopencv_imgcodecs" >>config.mk + sudo apt-get install -y libopencv-dev ``` -If building with ```GPU``` support, run below commands to add GPU dependency configurations to config.mk file: + +**Step 4** Download MXNet sources and build MXNet core shared library. + +If building on CPU: ```bash - echo "USE_CUDA=1" >>config.mk - echo "USE_CUDA_PATH=/usr/local/cuda" >>config.mk - echo "USE_CUDNN=1" >>config.mk + git clone --recursive https://github.com/dmlc/mxnet + cd mxnet + make -j $(nproc) USE_OPENCV=1 USE_BLAS=openblas ``` -Then build mxnet: +If building on GPU: ```bash - make -j$(nproc) + git clone --recursive https://github.com/dmlc/mxnet + cd mxnet + make -j $(nproc) USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 ``` +*Note* - USE_OPENCV and USE_BLAS are make file flags to set compilation options to use OpenCV and BLAS library. You can explore and use more compilation options in `make/config.mk`. + Executing these commands creates a library called ```libmxnet.so```. Next, we install ```graphviz``` library that we use for visualizing network graphs you build on MXNet. We will also install [Jupyter Notebook](http://jupyter.readthedocs.io/) used for running MXNet tutorials and examples. From 7d572e6cb22f1db916a0fb366c6062983141c319 Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Tue, 6 Jun 2017 16:03:12 -0700 Subject: [PATCH 040/834] fix rnn doc (#6595) --- python/mxnet/rnn/rnn_cell.py | 101 ++++++++++++++++++++++++----------- 1 file changed, 70 insertions(+), 31 deletions(-) diff --git a/python/mxnet/rnn/rnn_cell.py b/python/mxnet/rnn/rnn_cell.py index b7e07a59b4c1..180311a9ceed 100644 --- a/python/mxnet/rnn/rnn_cell.py +++ b/python/mxnet/rnn/rnn_cell.py @@ -96,7 +96,7 @@ class BaseRNNCell(object): Prefix for names of layers (this prefix is also used for names of weights if `params` is None i.e. if `params` are being created and not reused) - params : RNNParams or None, optional + params : RNNParams, default None. Container for weight sharing between cells. A new RNNParams container is created if `params` is None. """ @@ -277,7 +277,7 @@ def unroll(self, length, inputs, begin_state=None, layout='NTC', merge_outputs=N Parameters ---------- length : int - number of steps to unroll + Number of steps to unroll. inputs : Symbol, list of Symbol, or None If `inputs` is a single Symbol (usually the output of Embedding symbol), it should have shape @@ -287,7 +287,7 @@ def unroll(self, length, inputs, begin_state=None, layout='NTC', merge_outputs=N If `inputs` is a list of symbols (usually output of previous unroll), they should all have shape (batch_size, ...). - begin_state : nested list of Symbol, optional + begin_state : nested list of Symbol, default None Input states created by `begin_state()` or output state of another cell. Created from `begin_state()` if None. @@ -300,7 +300,7 @@ def unroll(self, length, inputs, begin_state=None, layout='NTC', merge_outputs=N and return a single symbol with shape (batch_size, length, ...) if layout == 'NTC', or (length, batch_size, ...) if layout == 'TNC'. - If None, output whatever is faster + If None, output whatever is faster. Returns ------- @@ -344,15 +344,13 @@ class RNNCell(BaseRNNCell): Parameters ---------- num_hidden : int - number of units in output symbol + Number of units in output symbol. activation : str or Symbol, default 'tanh' - type of activation function + Type of activation function. Options are 'relu' and 'tanh'. prefix : str, default 'rnn_' - prefix for name of layers - (and name of weight if params is None) - params : RNNParams or None - container for weight sharing between cells. - created if None. + Prefix for name of layers (and name of weight if params is None). + params : RNNParams, default None + Container for weight sharing between cells. Created if None. """ def __init__(self, num_hidden, activation='tanh', prefix='rnn_', params=None): super(RNNCell, self).__init__(prefix=prefix, params=params) @@ -392,13 +390,11 @@ class LSTMCell(BaseRNNCell): Parameters ---------- num_hidden : int - number of units in output symbol + Number of units in output symbol. prefix : str, default 'lstm_' - prefix for name of layers - (and name of weight if params is None) - params : RNNParams or None - container for weight sharing between cells. - created if None. + Prefix for name of layers (and name of weight if params is None). + params : RNNParams, default None + Container for weight sharing between cells. Created if None. forget_bias : bias added to forget gate, default 1.0. Jozefowicz et al. 2015 recommends setting this to 1.0 """ @@ -457,13 +453,11 @@ class GRUCell(BaseRNNCell): Parameters ---------- num_hidden : int - number of units in output symbol + Number of units in output symbol. prefix : str, default 'gru_' - prefix for name of layers - (and name of weight if params is None) - params : RNNParams or None - container for weight sharing between cells. - created if None. + Prefix for name of layers (and name of weight if params is None). + params : RNNParams, default None + Container for weight sharing between cells. Created if None. """ def __init__(self, num_hidden, prefix='gru_', params=None): super(GRUCell, self).__init__(prefix=prefix, params=params) @@ -525,6 +519,26 @@ class FusedRNNCell(BaseRNNCell): Parameters ---------- + num_hidden : int + Number of units in output symbol. + num_layers : int, default 1 + Number of layers in the cell. + mode : str, default 'lstm' + Type of RNN. options are 'rnn_relu', 'rnn_tanh', 'lstm', 'gru'. + bidirectional : bool, default False + Whether to use bidirectional unroll. The output dimension size is doubled if bidrectional. + dropout : float, default 0. + Fraction of the input that gets dropped out during training time. + get_next_state : bool, default False + Whether to return the states that can be used as starting states next time. + forget_bias : bias added to forget gate, default 1.0. + Jozefowicz et al. 2015 recommends setting this to 1.0 + prefix : str, default '$mode_' such as 'lstm_' + Prefix for names of layers + (this prefix is also used for names of weights if `params` is None + i.e. if `params` are being created and not reused) + params : RNNParams, default None + Container for weight sharing between cells. Created if None. """ def __init__(self, num_hidden, num_layers=1, mode='lstm', bidirectional=False, dropout=0., get_next_state=False, forget_bias=1.0, @@ -711,9 +725,8 @@ class SequentialRNNCell(BaseRNNCell): Parameters ---------- - params : RNNParams or None - container for weight sharing between cells. - created if None. + params : RNNParams, default None + Container for weight sharing between cells. Created if None. """ def __init__(self, params=None): super(SequentialRNNCell, self).__init__(prefix='', params=params) @@ -725,7 +738,9 @@ def add(self, cell): Parameters ---------- - cell : rnn cell + cell : BaseRNNCell + The cell to be appended. During unroll, previous cell's output (or raw inputs if + no previous cell) is used as the input to this cell. """ self._cells.append(cell) if self._override_cell_params: @@ -790,8 +805,14 @@ class DropoutCell(BaseRNNCell): Parameters ---------- dropout : float - percentage of elements to drop out, which + Percentage of elements to drop out, which is 1 - percentage to retain. + prefix : str, default 'dropout_' + Prefix for names of layers + (this prefix is also used for names of weights if `params` is None + i.e. if `params` are being created and not reused) + params : RNNParams, default None + Container for weight sharing between cells. Created if None. """ def __init__(self, dropout, prefix='dropout_', params=None): super(DropoutCell, self).__init__(prefix, params) @@ -861,7 +882,17 @@ def __call__(self, inputs, states): class ZoneoutCell(ModifierCell): - """Apply Zoneout on base cell.""" + """Apply Zoneout on base cell. + + Parameters + ---------- + base_cell : BaseRNNCell + Cell on whose states to perform zoneout. + zoneout_outputs : float, default 0. + Fraction of the output that gets dropped out during training time. + zoneout_states : float, default 0. + Fraction of the states that gets dropped out during training time. + """ def __init__(self, base_cell, zoneout_outputs=0., zoneout_states=0.): assert not isinstance(base_cell, FusedRNNCell), \ "FusedRNNCell doesn't support zoneout. " \ @@ -899,10 +930,15 @@ def __call__(self, inputs, states): class ResidualCell(ModifierCell): - """ - Adds residual connection as described in Wu et al, 2016 + """Adds residual connection as described in Wu et al, 2016 (https://arxiv.org/abs/1609.08144). + Output of the cell is output of the base cell plus input. + + Parameters + ---------- + base_cell : BaseRNNCell + Cell on whose outputs to add residual connection. """ def __init__(self, base_cell): @@ -943,6 +979,9 @@ class BidirectionalCell(BaseRNNCell): cell for forward unrolling r_cell : BaseRNNCell cell for backward unrolling + params : RNNParams, default None. + Container for weight sharing between cells. + A new RNNParams container is created if `params` is None. output_prefix : str, default 'bi_' prefix for name of output """ From 00151e901a1247a4a5a6177e9bb5878c236e69a0 Mon Sep 17 00:00:00 2001 From: moskewcz Date: Tue, 6 Jun 2017 22:43:58 -0700 Subject: [PATCH 041/834] tools/measure.py: add 'from functools import reduce' for python3 compatibility (note: needs python 2.6) (#6593) --- tools/bandwidth/measure.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/bandwidth/measure.py b/tools/bandwidth/measure.py index 749f258d98df..0cac3de26731 100644 --- a/tools/bandwidth/measure.py +++ b/tools/bandwidth/measure.py @@ -9,6 +9,7 @@ import numpy as np from importlib import import_module from collections import namedtuple +from functools import reduce logger = logging.getLogger() logger.setLevel(logging.INFO) From 0985c2e239fce5850ffc6588ab84c52f6b878972 Mon Sep 17 00:00:00 2001 From: reminisce Date: Wed, 7 Jun 2017 09:18:03 -0700 Subject: [PATCH 042/834] Clearer error message for simple_bind failure (#6597) * Clearer error message for simple_bind failure * Polish format * Remove words --- python/mxnet/symbol.py | 9 +++++---- src/executor/graph_executor.cc | 3 +-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/python/mxnet/symbol.py b/python/mxnet/symbol.py index d1f52b4b48f5..6d5e18766e73 100644 --- a/python/mxnet/symbol.py +++ b/python/mxnet/symbol.py @@ -1368,11 +1368,12 @@ def simple_bind(self, ctx, grad_req='write', type_dict=None, group2ctx=None, ctypes.byref(aux_state_handles), shared_exec_handle, ctypes.byref(exe_handle))) - except MXNetError: - print("simple_bind error. Arguments:") + except MXNetError as e: + error_msg = "simple_bind error. Arguments:\n" for k, v in kwargs.items(): - print(" %s: %s" % (k, v)) - raise RuntimeError('simple_bind failed') + error_msg += "%s: %s\n" % (k, v) + error_msg += "%s" % e + raise RuntimeError(error_msg) # update shared_buffer if shared_buffer is not None: diff --git a/src/executor/graph_executor.cc b/src/executor/graph_executor.cc index b41d1734d946..d60c5e46e52c 100644 --- a/src/executor/graph_executor.cc +++ b/src/executor/graph_executor.cc @@ -378,8 +378,7 @@ void HandleInferShapeError(const size_t num_forward_inputs, } } LOG(FATAL) << "InferShape pass cannot decide shapes for the following arguments " - "(0s in shapes mean unknown dimension size). Please consider " - "providing them as inputs:\n" + "(0s means unknown dimensions). Please consider providing them as inputs:\n" << oss.str(); } From 5e3f3e6a0cc62f47cfdb311356c19a9ad6b9308a Mon Sep 17 00:00:00 2001 From: Dick Carter Date: Wed, 7 Jun 2017 10:12:32 -0700 Subject: [PATCH 043/834] Changed make to support more gpu archs, multiple toolkits, reduce lib size. (#6588) * Updated make to support more gpu archs, tolerate multiple toolkit versions, reduce lib size. * Moved CUDA_ARCH setting to Makefile, removed from all make/*.mk files. --- Makefile | 33 +++++++++++++++++++++++++++++++-- make/config.mk | 7 ------- make/osx.mk | 7 ------- make/pip_linux_cpu.mk | 7 ------- 4 files changed, 31 insertions(+), 23 deletions(-) diff --git a/Makefile b/Makefile index 501a170abeda..d7053fb53524 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ ifndef NNVM_PATH endif ifndef DLPACK_PATH - DLPACK_PATH = $(ROOTDIR)/dlpack + DLPACK_PATH = $(ROOTDIR)/dlpack endif ifneq ($(USE_OPENMP), 1) @@ -58,7 +58,7 @@ LDFLAGS = -pthread $(MSHADOW_LDFLAGS) $(DMLC_LDFLAGS) ifeq ($(DEBUG), 1) NVCCFLAGS = -std=c++11 -Xcompiler -D_FORCE_INLINES -g -G -O0 -ccbin $(CXX) $(MSHADOW_NVCCFLAGS) else - NVCCFLAGS = -std=c++11 -Xcompiler -D_FORCE_INLINES -g -O3 -ccbin $(CXX) $(MSHADOW_NVCCFLAGS) + NVCCFLAGS = -std=c++11 -Xcompiler -D_FORCE_INLINES -O3 -ccbin $(CXX) $(MSHADOW_NVCCFLAGS) endif # CFLAGS for profiler @@ -129,6 +129,35 @@ ifneq ($(USE_CUDA_PATH), NONE) NVCC=$(USE_CUDA_PATH)/bin/nvcc endif +# Sets 'CUDA_ARCH', which determines the GPU architectures supported +# by the compiled kernels. Users can edit the KNOWN_CUDA_ARCHS list below +# to remove archs they don't wish to support to speed compilation, or they +# can pre-set the CUDA_ARCH args in config.mk for full control. +# +# For archs in this list, nvcc will create a fat-binary that will include +# the binaries (SASS) for all architectures supported by the installed version +# of the cuda toolkit, plus the assembly (PTX) for the most recent such architecture. +# If these kernels are then run on a newer-architecture GPU, the binary will +# be JIT-compiled by the updated driver from the included PTX. +ifeq ($(USE_CUDA), 1) +ifeq ($(origin CUDA_ARCH), undefined) + KNOWN_CUDA_ARCHS := 30 35 50 52 60 61 + # Run nvcc on a zero-length file to check architecture-level support. + # Create args to include SASS in the fat binary for supported levels. + CUDA_ARCH := $(foreach arch,$(KNOWN_CUDA_ARCHS), \ + $(shell $(NVCC) -arch=sm_$(arch) -E --x cu /dev/null >/dev/null 2>&1 && \ + echo -gencode arch=compute_$(arch),code=sm_$(arch))) + # Convert a trailing "code=sm_NN" to "code=[sm_NN,compute_NN]" to also + # include the PTX of the most recent arch in the fat-binaries for + # forward compatibility with newer GPUs. + CUDA_ARCH := $(shell echo $(CUDA_ARCH) | sed 's/sm_\([0-9]*\)$$/[sm_\1,compute_\1]/') + # Add fat binary compression if supported by nvcc. + COMPRESS := --fatbin-options -compress-all + CUDA_ARCH += $(shell $(NVCC) -cuda $(COMPRESS) --x cu /dev/null -o /dev/null >/dev/null 2>&1 && \ + echo $(COMPRESS)) +endif +endif + # ps-lite PS_PATH=$(ROOTDIR)/ps-lite DEPS_PATH=$(shell pwd)/deps diff --git a/make/config.mk b/make/config.mk index 7a98d94e7a86..ec52db770f7e 100644 --- a/make/config.mk +++ b/make/config.mk @@ -54,13 +54,6 @@ USE_CUDA_PATH = NONE # whether use CuDNN R3 library USE_CUDNN = 0 -# CUDA architecture setting: going with all of them. -# For CUDA < 6.0, comment the *_50 lines for compatibility. -CUDA_ARCH := -gencode arch=compute_30,code=sm_30 \ - -gencode arch=compute_35,code=sm_35 \ - -gencode arch=compute_50,code=sm_50 \ - -gencode arch=compute_50,code=compute_50 - # whether use cuda runtime compiling for writing kernels in native language (i.e. Python) USE_NVRTC = 0 diff --git a/make/osx.mk b/make/osx.mk index a14ba3f70ff0..115db34da730 100644 --- a/make/osx.mk +++ b/make/osx.mk @@ -48,13 +48,6 @@ USE_CUDA = 0 # USE_CUDA_PATH = /usr/local/cuda USE_CUDA_PATH = NONE -# CUDA architecture setting: going with all of them. -# For CUDA < 6.0, comment the *_50 lines for compatibility. -CUDA_ARCH := -gencode arch=compute_30,code=sm_30 \ - -gencode arch=compute_35,code=sm_35 \ - -gencode arch=compute_50,code=sm_50 \ - -gencode arch=compute_50,code=compute_50 - # whether use CUDNN R3 library USE_CUDNN = 0 diff --git a/make/pip_linux_cpu.mk b/make/pip_linux_cpu.mk index 82ba964169ac..f8b255e6deb8 100644 --- a/make/pip_linux_cpu.mk +++ b/make/pip_linux_cpu.mk @@ -50,13 +50,6 @@ USE_CUDA_PATH = NONE # whether use CuDNN R3 library USE_CUDNN = 0 -# CUDA architecture setting: going with all of them. -# For CUDA < 6.0, comment the *_50 lines for compatibility. -CUDA_ARCH := -gencode arch=compute_30,code=sm_30 \ - -gencode arch=compute_35,code=sm_35 \ - -gencode arch=compute_50,code=sm_50 \ - -gencode arch=compute_50,code=compute_50 - # whether use cuda runtime compiling for writing kernels in native language (i.e. Python) USE_NVRTC = 0 From 8147e7e1d5b79eda6ec95b8fe855ee2fb2495a87 Mon Sep 17 00:00:00 2001 From: Przemyslaw Tredak Date: Wed, 7 Jun 2017 22:53:01 -0700 Subject: [PATCH 044/834] Update to version 1.7.0 of CUB (#6607) * Update to version 1.7.0 of CUB * Removed "shallow=true" option for CUB --- .gitmodules | 1 - cub | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index bfe84d7f0615..465cc667eec2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -16,4 +16,3 @@ [submodule "cub"] path = cub url = https://github.com/NVlabs/cub - shallow=true diff --git a/cub b/cub index 89de7ab20167..01347a797c62 160000 --- a/cub +++ b/cub @@ -1 +1 @@ -Subproject commit 89de7ab20167909bc2c4f8acd397671c47cf3c0d +Subproject commit 01347a797c620618d09e7d2d90bce4be4c42513e From e885c1b92dbbc80d82bbbf8fa1d5a1a1b173bf4c Mon Sep 17 00:00:00 2001 From: Eric Junyuan Xie Date: Wed, 7 Jun 2017 23:16:46 -0700 Subject: [PATCH 045/834] Create .gitmodules --- .gitmodules | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitmodules b/.gitmodules index 465cc667eec2..03f4cf13e01f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -16,3 +16,4 @@ [submodule "cub"] path = cub url = https://github.com/NVlabs/cub + shallow = true From c078b7a64fe3443f507b00d36a9bbbe429f0f00a Mon Sep 17 00:00:00 2001 From: Chunyang Wen Date: Thu, 8 Jun 2017 14:21:13 +0800 Subject: [PATCH 046/834] data_shapes are extracted from dict, so its sequence is not determined. (#6610) data_names are provided by user, so its sequence is also not determined. When checking the equality, we should sort them first. --- python/mxnet/module/base_module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/mxnet/module/base_module.py b/python/mxnet/module/base_module.py index f998fbc27d6c..820841087a9c 100644 --- a/python/mxnet/module/base_module.py +++ b/python/mxnet/module/base_module.py @@ -55,7 +55,7 @@ def _check_input_names(symbol, names, typename, throw): def _check_names_match(data_names, data_shapes, name, throw): """Check that input names matches input data descriptors.""" actual = [x[0] for x in data_shapes] - if data_names != actual: + if sorted(data_names) != sorted(actual): msg = "Data provided by %s_shapes don't match names specified by %s_names (%s vs. %s)"%( name, name, str(data_shapes), str(data_names)) if throw: From 49b1513a43585ef4d5705d2a2023d0da821bf788 Mon Sep 17 00:00:00 2001 From: Madan Jampani Date: Thu, 8 Jun 2017 13:28:56 -0700 Subject: [PATCH 047/834] Remove references to deprecated API in how to docs (#6616) * Remove references to deprecated API in how to docs * ctx -> context --- docs/how_to/multi_devices.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/how_to/multi_devices.md b/docs/how_to/multi_devices.md index 9b6ba81a5cd4..327206224383 100644 --- a/docs/how_to/multi_devices.md +++ b/docs/how_to/multi_devices.md @@ -37,13 +37,13 @@ gradients are then summed over all GPUs before updating the model. If a machine has one or more GPU cards installed, then each card is labeled by a number starting from 0. To use a particular GPU, one can either -specify the context `ctx` in code +specify the context `context` in code or pass `--gpus` at the command line. For example, to use GPU 0 and 2 in python, -one can typically create a model with +one can typically create a module with ```python import mxnet as mx -model = mx.model.FeedForward(ctx=[mx.gpu(0), mx.gpu(2)], ...) +module = mx.module.Module(context=[mx.gpu(0), mx.gpu(2)], ...) ``` while if the program accepts a `--gpus` flag (as seen in [example/image-classification](https://github.com/dmlc/mxnet/tree/master/example/image-classification)), @@ -57,7 +57,7 @@ If the available GPUs are not all equally powerful, we can partition the workload accordingly. For example, if GPU 0 is 3 times faster than GPU 2, then we might use the workload option `work_load_list=[3, 1]`, -see [model.fit](../api/python/model.html#mxnet.model.FeedForward.fit) +see [Module](../api/python/module.html#mxnet.module.Module) for more details. Training with multiple GPUs should yield the same results From 4feb759fdcf401ca8b442887635a0f8425cae521 Mon Sep 17 00:00:00 2001 From: Mu Li Date: Thu, 8 Jun 2017 23:02:40 -0700 Subject: [PATCH 048/834] fix pylint (#6618) --- python/mxnet/_ctypes/common.py | 2 +- python/mxnet/_ctypes/symbol.py | 4 ++-- python/mxnet/initializer.py | 2 +- python/mxnet/io.py | 2 +- python/mxnet/metric.py | 2 +- python/mxnet/model.py | 2 +- python/mxnet/ndarray.py | 4 ++-- python/mxnet/rnn/rnn_cell.py | 6 +++--- python/mxnet/symbol.py | 6 +++--- tests/ci_build/pylintrc | 2 +- 10 files changed, 16 insertions(+), 16 deletions(-) diff --git a/python/mxnet/_ctypes/common.py b/python/mxnet/_ctypes/common.py index 5773a6a99c61..24e2048eee4c 100644 --- a/python/mxnet/_ctypes/common.py +++ b/python/mxnet/_ctypes/common.py @@ -22,7 +22,7 @@ def __init__(self, op, num_input, **kwargs): op_handle, ctypes.c_int(num_input), ctypes.c_int(len(kwargs)), - c_array(ctypes.c_char_p, [c_str(key) for key in kwargs.keys()]), + c_array(ctypes.c_char_p, [c_str(key) for key in kwargs]), c_array(ctypes.c_char_p, [c_str(str(val)) for val in kwargs.values()]), ctypes.byref(self.handle))) diff --git a/python/mxnet/_ctypes/symbol.py b/python/mxnet/_ctypes/symbol.py index 2ffa1a933540..9026b20cd7db 100644 --- a/python/mxnet/_ctypes/symbol.py +++ b/python/mxnet/_ctypes/symbol.py @@ -64,7 +64,7 @@ def _compose(self, *args, **kwargs): num_args = len(args) + len(kwargs) if len(kwargs) != 0: - keys = c_array(ctypes.c_char_p, [c_str(key) for key in kwargs.keys()]) + keys = c_array(ctypes.c_char_p, [c_str(key) for key in kwargs]) args = c_array(SymbolHandle, [s.handle for s in kwargs.values()]) else: keys = None @@ -81,7 +81,7 @@ def _set_attr(self, **kwargs): The attributes to set """ keys = c_array(ctypes.c_char_p, - [c_str(key) for key in kwargs.keys()]) + [c_str(key) for key in kwargs]) vals = c_array(ctypes.c_char_p, [c_str(str(val)) for val in kwargs.values()]) num_args = mx_uint(len(kwargs)) diff --git a/python/mxnet/initializer.py b/python/mxnet/initializer.py index 46f459830462..708419950950 100755 --- a/python/mxnet/initializer.py +++ b/python/mxnet/initializer.py @@ -640,7 +640,7 @@ def __init__(self, init, num_hidden, num_layers, mode, bidirectional=False, forg self._bidirectional = bidirectional self._forget_bias = forget_bias - def _init_weight(self, desc, arr): + def _init_weight(self, desc, arr): # pylint: disable=arguments-differ from .rnn import rnn_cell cell = rnn_cell.FusedRNNCell(self._num_hidden, self._num_layers, self._mode, self._bidirectional, diff --git a/python/mxnet/io.py b/python/mxnet/io.py index 50968dece35a..ec3c25f54d30 100644 --- a/python/mxnet/io.py +++ b/python/mxnet/io.py @@ -42,7 +42,7 @@ class DataDesc(namedtuple('DataDesc', ['name', 'shape'])): layout : str, optional Data layout. """ - def __new__(cls, name, shape, dtype=mx_real_t, layout='NCHW'): + def __new__(cls, name, shape, dtype=mx_real_t, layout='NCHW'): # pylint: disable=super-on-old-class ret = super(cls, DataDesc).__new__(cls, name, shape) ret.dtype = dtype ret.layout = layout diff --git a/python/mxnet/metric.py b/python/mxnet/metric.py index c57f12fc5d6f..736864324227 100644 --- a/python/mxnet/metric.py +++ b/python/mxnet/metric.py @@ -251,7 +251,7 @@ def get_metric(self, index): return ValueError("Metric index {} is out of range 0 and {}".format( index, len(self.metrics))) - def update_dict(self, labels, preds): + def update_dict(self, labels, preds): # pylint: disable=arguments-differ if self.label_names is not None: labels = OrderedDict([i for i in labels.items() if i[0] in self.label_names]) diff --git a/python/mxnet/model.py b/python/mxnet/model.py index 5eddfac47981..189f301e91f7 100644 --- a/python/mxnet/model.py +++ b/python/mxnet/model.py @@ -62,7 +62,7 @@ def _create_kvstore(kvstore, num_device, arg_params): kv = None else: kv = kvs.create(kvstore) - if kvstore is 'local': + if kvstore == 'local': # automatically select a proper local max_size = max(np.prod(param.shape) for param in arg_params.values()) diff --git a/python/mxnet/ndarray.py b/python/mxnet/ndarray.py index b167e8673ff4..8900843f5937 100644 --- a/python/mxnet/ndarray.py +++ b/python/mxnet/ndarray.py @@ -2375,14 +2375,14 @@ def %s(%s): keys = list(kwargs.keys()) vals = list(kwargs.values())"""%(func_name, ', '.join(signature))) # NDArray args - for name in ndarg_names: + for name in ndarg_names: # pylint: disable=redefined-argument-from-local code.append(""" if {name} is not None: assert isinstance({name}, NDArrayBase), \\ "Argument {name} must have NDArray type, but got %s"%str({name}) ndargs.append({name})""".format(name=name)) # kwargs - for name in kwarg_names: + for name in kwarg_names: # pylint: disable=redefined-argument-from-local code.append(""" if %s is not _Null: keys.append('%s') diff --git a/python/mxnet/rnn/rnn_cell.py b/python/mxnet/rnn/rnn_cell.py index 180311a9ceed..c00f8a39d8c3 100644 --- a/python/mxnet/rnn/rnn_cell.py +++ b/python/mxnet/rnn/rnn_cell.py @@ -754,7 +754,7 @@ def add(self, cell): def state_info(self): return _cells_state_info(self._cells) - def begin_state(self, **kwargs): + def begin_state(self, **kwargs): # pylint: disable=arguments-differ assert not self._modified, \ "After applying modifier cells (e.g. ZoneoutCell) the base " \ "cell cannot be called directly. Call the modifier cell instead." @@ -862,7 +862,7 @@ def params(self): def state_info(self): return self.base_cell.state_info - def begin_state(self, init_sym=symbol.zeros, **kwargs): + def begin_state(self, init_sym=symbol.zeros, **kwargs): # pylint: disable=arguments-differ assert not self._modified, \ "After applying modifier cells (e.g. DropoutCell) the base " \ "cell cannot be called directly. Call the modifier cell instead." @@ -1013,7 +1013,7 @@ def __call__(self, inputs, states): def state_info(self): return _cells_state_info(self._cells) - def begin_state(self, **kwargs): + def begin_state(self, **kwargs): # pylint: disable=arguments-differ assert not self._modified, \ "After applying modifier cells (e.g. DropoutCell) the base " \ "cell cannot be called directly. Call the modifier cell instead." diff --git a/python/mxnet/symbol.py b/python/mxnet/symbol.py index 6d5e18766e73..14203e59862d 100644 --- a/python/mxnet/symbol.py +++ b/python/mxnet/symbol.py @@ -410,7 +410,7 @@ def _compose(self, *args, **kwargs): num_args = len(args) + len(kwargs) if len(kwargs) != 0: - keys = c_array(ctypes.c_char_p, [c_str(key) for key in kwargs.keys()]) + keys = c_array(ctypes.c_char_p, [c_str(key) for key in kwargs]) args = c_array(SymbolHandle, [s.handle for s in kwargs.values()]) else: keys = None @@ -2165,14 +2165,14 @@ def %s(%s): keys.append(k) vals.append(v)""") # NDArray args - for name in ndarg_names: + for name in ndarg_names: # pylint: disable=redefined-argument-from-local code.append(""" if {name} is not None: assert isinstance({name}, SymbolBase), \\ "Argument {name} must be Symbol instances, but got %s"%str({name}) sym_kwargs['{name}'] = {name}""".format(name=name)) # kwargs - for name in kwarg_names: + for name in kwarg_names: # pylint: disable=redefined-argument-from-local code.append(""" if %s is not _Null: keys.append('%s') diff --git a/tests/ci_build/pylintrc b/tests/ci_build/pylintrc index e7dc2340d1bb..a33f4e76bf3d 100644 --- a/tests/ci_build/pylintrc +++ b/tests/ci_build/pylintrc @@ -65,7 +65,7 @@ enable=indexing-exception,old-raise-syntax # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use"--disable=all --enable=classes # --disable=W" -disable=design,similarities,no-self-use,attribute-defined-outside-init,locally-disabled,star-args,pointless-except,bad-option-value,global-statement,fixme,suppressed-message,useless-suppression,locally-enabled,no-member,no-name-in-module,import-error,unsubscriptable-object,unbalanced-tuple-unpacking,undefined-variable,protected-access,superfluous-parens,invalid-name +disable=design,similarities,no-self-use,attribute-defined-outside-init,locally-disabled,star-args,pointless-except,bad-option-value,global-statement,fixme,suppressed-message,useless-suppression,locally-enabled,no-member,no-name-in-module,import-error,unsubscriptable-object,unbalanced-tuple-unpacking,undefined-variable,protected-access,superfluous-parens,invalid-name,no-else-return,useless-super-delegation,len-as-condition,invalid-unary-operand-type # disable=unicode-builtin,delslice-method,using-cmp-argument,setslice-method,dict-view-method,parameter-unpacking,range-builtin-not-iterating,print-statement,file-builtin,old-raise-syntax,basestring-builtin,execfile-builtin,indexing-exception,import-star-module-level,coerce-method,long-builtin,old-ne-operator,old-division,no-absolute-import,raw_input-builtin,old-octal-literal,oct-method,xrange-builtin,hex-method,unpacking-in-except,nonzero-method,raising-string,intern-builtin,reload-builtin,metaclass-assignment,cmp-method,filter-builtin-not-iterating,apply-builtin,map-builtin-not-iterating,next-method-called,unichr-builtin,buffer-builtin,dict-iter-method,input-builtin,coerce-builtin,getslice-method,useless-suppression,standarderror-builtin,zip-builtin-not-iterating,suppressed-message,cmp-builtin,backtick,long-suffix,reduce-builtin,round-builtin From 05a04c8af2dc9609d1ebceb2a8a9cca18e1ad83a Mon Sep 17 00:00:00 2001 From: Mu Li Date: Fri, 9 Jun 2017 09:48:48 -0700 Subject: [PATCH 049/834] update cub url (#6625) * update cub url * update * update * update --- .gitmodules | 3 +-- Jenkinsfile | 26 ++++++++++++++++++-------- cub | 2 +- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/.gitmodules b/.gitmodules index 03f4cf13e01f..7a76cbaf78d9 100644 --- a/.gitmodules +++ b/.gitmodules @@ -15,5 +15,4 @@ url = https://github.com/dmlc/dlpack [submodule "cub"] path = cub - url = https://github.com/NVlabs/cub - shallow = true + url = https://github.com/dmlc/cub diff --git a/Jenkinsfile b/Jenkinsfile index 2f4406856288..e55ebcdc4d4f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -11,21 +11,31 @@ max_time = 60 // initialize source codes def init_git() { - checkout scm retry(5) { - timeout(time: 2, unit: 'MINUTES') { - sh 'git submodule update --init' + try { + timeout(time: 2, unit: 'MINUTES') { + checkout scm + sh 'git submodule update --init' + } + } catch (exc) { + deleteDir() + error "Failed to fetch source codes" } } } def init_git_win() { - checkout scm - retry(5) { - timeout(time: 2, unit: 'MINUTES') { - bat 'git submodule update --init' - } + retry(5) { + try { + timeout(time: 2, unit: 'MINUTES') { + checkout scm + bat 'git submodule update --init' + } + } catch (exc) { + deleteDir() + error "Failed to fetch source codes" } + } } stage("Sanity Check") { diff --git a/cub b/cub index 01347a797c62..80dbf02aa36d 160000 --- a/cub +++ b/cub @@ -1 +1 @@ -Subproject commit 01347a797c620618d09e7d2d90bce4be4c42513e +Subproject commit 80dbf02aa36d9ef881629e2ee2c15415ba07cef5 From 64b9c8829994c5eb7026cd6105fc4baff584fd38 Mon Sep 17 00:00:00 2001 From: Naveen Swamy Date: Fri, 9 Jun 2017 11:24:56 -0700 Subject: [PATCH 050/834] remove Python file from tutorials (#6623) --- docs/mxdoc.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/docs/mxdoc.py b/docs/mxdoc.py index 67f1e233bbd3..c442271d1a24 100644 --- a/docs/mxdoc.py +++ b/docs/mxdoc.py @@ -234,13 +234,9 @@ def _get_src_download_btn(out_prefix, langs, lines): ipynb = out_prefix + '_' + lang + '.ipynb' with open(ipynb, 'w') as f: json.dump(_get_jupyter_notebook(lang, lines), f) - src = out_prefix + '.' + _LANGS[lang][0] - with open(src, 'w') as f: - f.write('\n'.join(_get_source(lang, lines))) - for f in [ipynb, src]: - f = f.split('/')[-1] - btn += '\n' % (f, f) + f = ipynb.split('/')[-1] + btn += '\n' % (f, f) btn += '
\n' return btn From 25e719fcdb7b5eb30bcebfc9524997abc7b85fb6 Mon Sep 17 00:00:00 2001 From: Indhu Bharathi Date: Fri, 9 Jun 2017 11:28:56 -0700 Subject: [PATCH 051/834] Fix couple of print statements that weren't working on Python 3. (#6604) --- docs/tutorials/basic/symbol.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/tutorials/basic/symbol.md b/docs/tutorials/basic/symbol.md index 8de6653a664b..921f3ec1e11e 100644 --- a/docs/tutorials/basic/symbol.md +++ b/docs/tutorials/basic/symbol.md @@ -314,8 +314,8 @@ executor. The executor provides `forward` method for evaluation and an attribute ex = c.bind(ctx=mx.cpu(), args={'a' : mx.nd.ones([2,3]), 'b' : mx.nd.ones([2,3])}) ex.forward() -print 'number of outputs = %d\nthe first output = \n%s' % ( - len(ex.outputs), ex.outputs[0].asnumpy()) +print('number of outputs = %d\nthe first output = \n%s' % ( + len(ex.outputs), ex.outputs[0].asnumpy())) ``` We can evaluate the same symbol on GPU with different data. @@ -332,8 +332,8 @@ and `forward` methods. ```python ex = c.eval(ctx = mx.cpu(), a = mx.nd.ones([2,3]), b = mx.nd.ones([2,3])) -print 'number of outputs = %d\nthe first output = \n%s' % ( - len(ex), ex[0].asnumpy()) +print('number of outputs = %d\nthe first output = \n%s' % ( + len(ex), ex[0].asnumpy())) ``` For neural nets, a more commonly used pattern is ```simple_bind```, which From 0da55a767aa8c28a71caf186f73e5255d0da1521 Mon Sep 17 00:00:00 2001 From: Mu Li Date: Fri, 9 Jun 2017 12:27:26 -0700 Subject: [PATCH 052/834] [CI] remove COPY * in dockerfiles (#6515) --- tests/ci_build/Dockerfile.caffe_gpu | 4 ++-- tests/ci_build/Dockerfile.cpu | 5 +++-- tests/ci_build/Dockerfile.doc | 5 ++++- tests/ci_build/Dockerfile.gpu | 5 +++-- tests/ci_build/Dockerfile.mklml_gpu | 5 +++-- 5 files changed, 15 insertions(+), 9 deletions(-) diff --git a/tests/ci_build/Dockerfile.caffe_gpu b/tests/ci_build/Dockerfile.caffe_gpu index fff5a027e842..c971dfb90bf4 100644 --- a/tests/ci_build/Dockerfile.caffe_gpu +++ b/tests/ci_build/Dockerfile.caffe_gpu @@ -1,8 +1,8 @@ FROM nvidia/cuda:7.5-cudnn5-devel -COPY install/ubuntu_*.sh /install/ - +COPY install/ubuntu_install_core.sh /install/ RUN /install/ubuntu_install_core.sh +COPY install/ubuntu_install_python.sh /install/ RUN /install/ubuntu_install_python.sh RUN apt-get install -y libprotobuf-dev libleveldb-dev \ diff --git a/tests/ci_build/Dockerfile.cpu b/tests/ci_build/Dockerfile.cpu index 1be21b03b21d..aabda4e99ce6 100644 --- a/tests/ci_build/Dockerfile.cpu +++ b/tests/ci_build/Dockerfile.cpu @@ -1,7 +1,8 @@ FROM ubuntu:14.04 -COPY install/ubuntu_*.sh /install/ - +COPY install/ubuntu_install_core.sh /install/ RUN /install/ubuntu_install_core.sh +COPY install/ubuntu_install_python.sh /install/ RUN /install/ubuntu_install_python.sh +COPY install/ubuntu_install_scala.sh /install/ RUN /install/ubuntu_install_scala.sh diff --git a/tests/ci_build/Dockerfile.doc b/tests/ci_build/Dockerfile.doc index 73f138139922..622d946665cc 100644 --- a/tests/ci_build/Dockerfile.doc +++ b/tests/ci_build/Dockerfile.doc @@ -1,10 +1,13 @@ FROM ubuntu:14.04 -COPY install/ubuntu_*.sh /install/ +COPY install/ubuntu_install_core.sh /install/ RUN /install/ubuntu_install_core.sh +COPY install/ubuntu_install_python.sh /install/ RUN /install/ubuntu_install_python.sh +COPY install/ubuntu_install_scala.sh /install/ RUN /install/ubuntu_install_scala.sh + RUN wget http://downloads.lightbend.com/scala/2.11.8/scala-2.11.8.deb && \ dpkg -i scala-2.11.8.deb && rm scala-2.11.8.deb diff --git a/tests/ci_build/Dockerfile.gpu b/tests/ci_build/Dockerfile.gpu index be669dbd1635..46d36312de7c 100644 --- a/tests/ci_build/Dockerfile.gpu +++ b/tests/ci_build/Dockerfile.gpu @@ -1,7 +1,8 @@ FROM nvidia/cuda:7.5-cudnn5-devel -COPY install/ubuntu_*.sh /install/ - +COPY install/ubuntu_install_core.sh /install/ RUN /install/ubuntu_install_core.sh +COPY install/ubuntu_install_python.sh /install/ RUN /install/ubuntu_install_python.sh +COPY install/ubuntu_install_scala.sh /install/ RUN /install/ubuntu_install_scala.sh diff --git a/tests/ci_build/Dockerfile.mklml_gpu b/tests/ci_build/Dockerfile.mklml_gpu index 0bdda62ce9ca..eb488267c0db 100644 --- a/tests/ci_build/Dockerfile.mklml_gpu +++ b/tests/ci_build/Dockerfile.mklml_gpu @@ -2,10 +2,11 @@ FROM nvidia/cuda:7.5-cudnn5-devel # the reason we used a gpu base container because we are going to test MKLDNN # operator implementation against GPU implementation -COPY install/ubuntu_*.sh /install/ - +COPY install/ubuntu_install_core.sh /install/ RUN /install/ubuntu_install_core.sh +COPY install/ubuntu_install_python.sh /install/ RUN /install/ubuntu_install_python.sh +COPY install/ubuntu_install_scala.sh /install/ RUN /install/ubuntu_install_scala.sh RUN wget --no-check-certificate -O /tmp/mklml.tgz https://github.com/dmlc/web-data/raw/master/mxnet/mklml-release/mklml_lnx_2017.0.2.20170209.tgz From 8b3a56d28570466120c392ce75b52ce3cc5665b3 Mon Sep 17 00:00:00 2001 From: Mu Li Date: Fri, 9 Jun 2017 13:24:50 -0700 Subject: [PATCH 053/834] add doc build into ci (#6636) --- Jenkinsfile | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index e55ebcdc4d4f..08c1e000050d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -350,3 +350,15 @@ stage('Integration Test') { } } } + +stage('Deploy') { + parallel 'Doc': { + node('linux') { + ws('workspace/docs') { + if (env.BRANCH_NAME == "master") { + sh "make docs" + } + } + } + } +} From ffc90ee13467431d4606dbe6177086ca453ffb4d Mon Sep 17 00:00:00 2001 From: Yao Wang Date: Fri, 9 Jun 2017 14:59:06 -0700 Subject: [PATCH 054/834] Website improvement (#6609) * More Improvement for website * Small fix * Change background color --- docs/_static/js/navbar.js | 1 + docs/_static/js/sidebar.js | 2 ++ docs/_static/mxnet.css | 54 +++++++++++++++++++++++++++++++------- docs/mxdoc.py | 4 +-- 4 files changed, 50 insertions(+), 11 deletions(-) diff --git a/docs/_static/js/navbar.js b/docs/_static/js/navbar.js index 44764efe5ef0..9c3164ee18ea 100644 --- a/docs/_static/js/navbar.js +++ b/docs/_static/js/navbar.js @@ -52,6 +52,7 @@ function navbar() { /*Show bottom border of current tab*/ function showTab() { var url = window.location.href; + if(url.indexOf('/get_started/why_mxnet') != -1) return; for(var i = 0; i < TITLE.length; ++i) { if(url.indexOf(TITLE[i]) != -1) { var tab = $($('#main-nav').children().eq(i)); diff --git a/docs/_static/js/sidebar.js b/docs/_static/js/sidebar.js index 8b0327946357..1df628c4089f 100644 --- a/docs/_static/js/sidebar.js +++ b/docs/_static/js/sidebar.js @@ -222,6 +222,8 @@ $(document).ready(function () { render_righttoc(); if ($('.leftsidebar').length) render_lefttoc(); } + + if(url.indexOf('/api/') != -1) return; $(window).scroll(function () { scroll_righttoc(); }); diff --git a/docs/_static/mxnet.css b/docs/_static/mxnet.css index 7358f0ae5e75..06d976326086 100644 --- a/docs/_static/mxnet.css +++ b/docs/_static/mxnet.css @@ -823,24 +823,32 @@ div.informaltable { text-align: left; } /*----------------API class and function formatting---------------------*/ -dl > dt:before { - content: " "; - display: block; - height: 70px; /* fixed header height*/ - margin-top: -50px; /* negative fixed header height */ -} - p.rubric { margin-top: 10px; } +dl { + padding-top: 20px; +} + dt:target, .highlighted { - background-color: #fff; - background: transparent; + background-color: #e7f2fa; border-bottom: 3px solid #c7254e; margin-bottom: -3px; } +dt:target:before { + background-color: white; + content: ''; + display: block; + height: 60px; +} + +dt { + background: #e7f2fa; + border-bottom: solid #0079b2; +} + dt em { font-weight: normal; font-style: normal; @@ -852,6 +860,11 @@ code { background-color: #f5f5f5; } +dl.last.docutils dt{ + background-color: transparent; + border-bottom: none; +} + /*----------------Model zoo page style------------------*/ #mxnet-model-zoo table, #mxnet-model-zoo td, #mxnet-model-zoo th { border: 1px solid lightgray; @@ -1070,4 +1083,27 @@ table.docutils tr:nth-child(even) { button.download { color: #0079b2; +} + +/*----------------------Download button------------------------*/ +div.download_btn { + + border: solid 1px lightgray; + border-radius: 3px; + font-size: 90%; + height: 30px; + display: table; + float: left; +} + +div.download_btn a { + padding: 0 10px; + display: table-cell; + vertical-align: middle; +} + +div.download_btn a:hover { + background-color: #0079b2; + color: white; + text-decoration: none; } \ No newline at end of file diff --git a/docs/mxdoc.py b/docs/mxdoc.py index c442271d1a24..b1fdf3d05435 100644 --- a/docs/mxdoc.py +++ b/docs/mxdoc.py @@ -235,8 +235,8 @@ def _get_src_download_btn(out_prefix, langs, lines): with open(ipynb, 'w') as f: json.dump(_get_jupyter_notebook(lang, lines), f) f = ipynb.split('/')[-1] - btn += '\n' % (f, f) + btn += '
' \ + ' %s
' % (f, f, f) btn += '
\n' return btn From 5375a379f476fc14ed98359356fdf5fc987ae9f0 Mon Sep 17 00:00:00 2001 From: Naveen Swamy Date: Fri, 9 Jun 2017 23:15:46 -0700 Subject: [PATCH 055/834] Update Tutorial prerequisites (#6639) * Add GraphViz to Installation(https://github.com/dmlc/mxnet/pull/6627) * update prereq for data iterator tutorial * update prereq * update prereq for module tutorial(https://github.com/dmlc/mxnet/pull/6622) * tutorial prereq for ndarray/symbol/predict_image, fix print for python 3 * update to make Graphviz optional --- docs/get_started/install.md | 68 ++++++++++++++++--- docs/tutorials/basic/data.md | 39 +++++++---- docs/tutorials/basic/module.md | 10 ++- docs/tutorials/basic/ndarray.md | 27 ++++++-- docs/tutorials/basic/symbol.md | 19 +++++- docs/tutorials/python/linear-regression.md | 12 ++++ docs/tutorials/python/mnist.md | 11 +++ docs/tutorials/python/predict_image.md | 14 +++- .../vision/large_scale_classification.md | 46 ++++--------- 9 files changed, 178 insertions(+), 68 deletions(-) diff --git a/docs/get_started/install.md b/docs/get_started/install.md index f81307833029..28d95aa155ba 100644 --- a/docs/get_started/install.md +++ b/docs/get_started/install.md @@ -102,16 +102,22 @@ After activating the environment, you should see the prompt as below. Installing *MXNet* with pip requires a latest version of `pip`. Install the latest version of `pip` by issuing the following command. ```bash -(mxnet)$ pip install --upgrade pip +$ pip install --upgrade pip ``` Install *MXNet* with OpenBLAS acceleration. ```bash -(mxnet)$ pip install mxnet +$ pip install mxnet ``` -**Step 4** Validate the installation by running simple *MXNet* code described [here](#validate-mxnet-installation). +**Step 4** Install [Graphviz](http://www.graphviz.org/). (Optional, needed for graph visualization using `mxnet.viz` package). +```bash +sudo apt-get install graphviz +pip install graphviz +``` + +**Step 5** Validate the installation by running simple *MXNet* code described [here](#validate-mxnet-installation). **Note** You can read more about virtualenv [here](https://virtualenv.pypa.io/en/stable/userguide/). @@ -136,7 +142,13 @@ $ wget https://bootstrap.pypa.io/get-pip.py && sudo python get-pip.py $ pip install mxnet ``` -**Step 3** Validate the installation by running simple MXNet code described [here](#validate-mxnet-installation). +**Step 3** Install [Graphviz](http://www.graphviz.org/). (Optional, needed for graph visualization using `mxnet.viz` package). +```bash +sudo apt-get install graphviz +pip install graphviz +``` + +**Step 4** Validate the installation by running simple MXNet code described [here](#validate-mxnet-installation).
@@ -236,7 +248,13 @@ $ cd python $ sudo python setup.py install ``` -**Step 3** Validate the installation by running simple MXNet code described [here](#validate-mxnet-installation). +**Step 3** Install [Graphviz](http://www.graphviz.org/). (Optional, needed for graph visualization using `mxnet.viz` package). +```bash +sudo apt-get install graphviz +pip install graphviz +``` + +**Step 4** Validate the installation by running simple MXNet code described [here](#validate-mxnet-installation).
@@ -284,7 +302,13 @@ $ wget https://bootstrap.pypa.io/get-pip.py && sudo python get-pip.py $ pip install mxnet-cu80 ``` -**Step 3** Validate the installation by running simple MXNet code described [here](#validate-mxnet-installation). +**Step 3** Install [Graphviz](http://www.graphviz.org/). (Optional, needed for graph visualization using `mxnet.viz` package). +```bash +sudo apt-get install graphviz +pip install graphviz +``` + +**Step 4** Validate the installation by running simple MXNet code described [here](#validate-mxnet-installation).
@@ -333,7 +357,13 @@ Install *MXNet* with GPU support using CUDA 8.0. (mxnet)$ pip install mxnet-cu80 ``` -**Step 4** Validate the installation by running simple *MXNet* code described [here](#validate-mxnet-installation). +**Step 4** Install [Graphviz](http://www.graphviz.org/). (Optional, needed for graph visualization using `mxnet.viz` package). +```bash +sudo apt-get install graphviz +pip install graphviz +``` + +**Step 5** Validate the installation by running simple *MXNet* code described [here](#validate-mxnet-installation). **Note** You can read more about virtualenv [here](https://virtualenv.pypa.io/en/stable/userguide/). @@ -438,7 +468,13 @@ $ cd python $ sudo python setup.py install ``` -**Step 3** Validate the installation by running simple MXNet code described [here](#validate-mxnet-installation). +**Step 3** Install [Graphviz](http://www.graphviz.org/). (Optional, needed for graph visualization using `mxnet.viz` package). +```bash +sudo apt-get install graphviz +pip install graphviz +``` + +**Step 4** Validate the installation by running simple MXNet code described [here](#validate-mxnet-installation).
@@ -513,7 +549,13 @@ Install *MXNet* with OpenBLAS acceleration. (mxnet)$ pip install mxnet ``` -**Step 5** Validate the installation by running simple *MXNet* code described [here](#validate-mxnet-installation). +**Step 5** Install [Graphviz](http://www.graphviz.org/). (Optional, needed for graph visualization using `mxnet.viz` package). +```bash +sudo apt-get install graphviz +pip install graphviz +``` + +**Step 6** Validate the installation by running simple *MXNet* code described [here](#validate-mxnet-installation). **Note** You can read more about virtualenv [here](https://virtualenv.pypa.io/en/stable/userguide/). @@ -546,7 +588,13 @@ $ pip install --upgrade setuptools $ pip install mxnet ``` -**Step 3** Validate the installation by running simple MXNet code described [here](#validate-mxnet-installation). +**Step 3** Install [Graphviz](http://www.graphviz.org/). (Optional, needed for graph visualization using `mxnet.viz` package). +```bash +sudo apt-get install graphviz +pip install graphviz +``` + +**Step 4** Validate the installation by running simple MXNet code described [here](#validate-mxnet-installation). diff --git a/docs/tutorials/basic/data.md b/docs/tutorials/basic/data.md index 0b1582fb57a5..7cbd14eff3d8 100644 --- a/docs/tutorials/basic/data.md +++ b/docs/tutorials/basic/data.md @@ -4,6 +4,24 @@ Most training and inference modules in MXNet accept data iterators, which simplifies this procedure, especially when reading large datasets. Here we discuss the API conventions and several provided iterators. +## Prerequisites + +To complete this tutorial, we need: + +- MXNet. See the instructions for your operating system in [Setup and Installation](http://mxnet.io/get_started/install.html). + +- [OpenCV Python library](http://opencv.org/opencv-3-2.html), [Python Requests](http://docs.python-requests.org/en/master/), [Matplotlib](https://matplotlib.org/) and [Jupyter Notebook](http://jupyter.org/index.html). + +``` +$ pip install opencv-python requests matplotlib jupyter +``` +- Set the environment variable `MXNET_HOME` to the root of the MXNet source folder. + +``` +$ git clone https://github.com/dmlc/mxnet ~/mxnet +$ MXNET_HOME = '~/mxnet' +``` + ## MXNet Data Iterator Data Iterators in *MXNet* are similar to Python iterator objects. In Python the function `iter` allows fetching items sequentially by calling `next()` on @@ -283,13 +301,6 @@ There are 4 ways of loading image data in MXNet. 4. Creating a Custom iterator inheriting `mx.io.DataIter` -First, set the environment variable `MXNET_HOME` to the root of the MXNet source folder: - -```python -# change this to your mxnet location -MXNET_HOME = '/scratch/mxnet' -``` - ### Preprocessing Images Images can be preprocessed in different ways. We list some of them below: - Using `mx.io.ImageRecordIter` which is fast but not very flexible. It is great for simple tasks like image recognition but won't work for more complex tasks like detection and segmentation. @@ -302,20 +313,19 @@ Let's download sample images that we can work with. ```python -fname = mx.test_utils.download(url='http://data.mxnet.io/data/test_images.tar.gz') +fname = mx.test_utils.download(url='http://data.mxnet.io/data/test_images.tar.gz', dirname='data', overwrite=False) tar = tarfile.open(fname) -tar.extractall() +tar.extractall(path='./data') tar.close() ``` #### Loading raw images -`mx.image.imdecode` lets us load the images. `imdecode` provides a similar interface to ``OpenCV``. -**Note: ** You will still need ``OpenCV``(not the CV2 Python library) installed to use `mx.image.imdecode`. +`mx.image.imdecode` lets us load the images. `imdecode` provides a similar interface to ``OpenCV``. +**Note:** You will still need ``OpenCV``(not the CV2 Python library) installed to use `mx.image.imdecode`. ```python -import cv2 -img = mx.image.imdecode(open('test_images/ILSVRC2012_val_00000001.JPEG').read()) +img = mx.image.imdecode(open('data/test_images/ILSVRC2012_val_00000001.JPEG').read()) plt.imshow(img.asnumpy()); plt.show() ``` @@ -346,9 +356,8 @@ Download and unzip ```python fname = mx.test_utils.download(url='http://www.vision.caltech.edu/Image_Datasets/Caltech101/101_ObjectCategories.tar.gz', dirname='data', overwrite=False) tar = tarfile.open(fname) -tar.extractall() +tar.extractall(path='./data') tar.close() -os.chdir('../') ``` Let's take a look at the data. As you can see, under the root folder (./data/101_ObjectCategories) every category has a subfolder(./data/101_ObjectCategories/yin_yang). diff --git a/docs/tutorials/basic/module.md b/docs/tutorials/basic/module.md index 6f28bdeda182..15fdaeef68c4 100644 --- a/docs/tutorials/basic/module.md +++ b/docs/tutorials/basic/module.md @@ -17,8 +17,12 @@ this tutorial. To complete this tutorial, we need: -- MXNet. See the instructions for your operating system in [Setup and Installation](http://mxnet.io/get_started/install.html) -- [Python](https://www.python.org/downloads/) +- MXNet. See the instructions for your operating system in [Setup and Installation](http://mxnet.io/get_started/install.html). + +- [Jupyter Notebook](http://jupyter.org/index.html) and [Python Requests](http://docs.python-requests.org/en/master/) packages. +``` +pip install jupyter requests +``` ## Preliminary @@ -170,7 +174,7 @@ It can be used as follows: ```python score = mod.score(val_iter, ['mse', 'acc']) -print "Accuracy score is ", score +print("Accuracy score is %f" % (score)) ``` Some of the other metrics which can be used are `top_k_acc`(top-k-accuracy), diff --git a/docs/tutorials/basic/ndarray.md b/docs/tutorials/basic/ndarray.md index 34333d8d4f99..c7a3feb1d871 100644 --- a/docs/tutorials/basic/ndarray.md +++ b/docs/tutorials/basic/ndarray.md @@ -38,6 +38,19 @@ Each NDArray supports some important attributes that you'll often want to query: - **ndarray.context**: The device on which this array is stored, e.g. `cpu()` or `gpu(1)`. +## Prerequisites + +To complete this tutorial, we need: + +- MXNet. See the instructions for your operating system in [Setup and Installation](http://mxnet.io/get_started/install.html) +- [Jupyter](http://jupyter.org/) + ``` + pip install jupyter + ``` +- GPUs - A section of this tutorial uses GPUs. If you don't have GPUs on your +machine, simply set the variable gpu_device (set in the GPUs section of this +tutorial) to mx.cpu(). + ## Array Creation There are a few different ways to create an `NDArray`. @@ -279,7 +292,11 @@ can cause all computations to run on GPU 0 by using context `mx.gpu(0)`, or simply `mx.gpu()`. When we have access to two or more GPUs, the 2nd GPU is represented by `mx.gpu(1)`, etc. +**Note** In order to execute the following section on a cpu set gpu_device to mx.cpu(). ```python +gpu_device=mx.gpu() # Change this to mx.cpu() in absence of GPUs. + + def f(): a = mx.nd.ones((100,100)) b = mx.nd.ones((100,100)) @@ -288,14 +305,14 @@ def f(): # in default mx.cpu() is used f() # change the default context to the first GPU -with mx.Context(mx.gpu()): +with mx.Context(gpu_device): f() ``` We can also explicitly specify the context when creating an array: ```python -a = mx.nd.ones((100, 100), mx.gpu(0)) +a = mx.nd.ones((100, 100), gpu_device) a ``` @@ -304,8 +321,8 @@ computation. There are several methods for copying data between devices. ```python a = mx.nd.ones((100,100), mx.cpu()) -b = mx.nd.ones((100,100), mx.gpu()) -c = mx.nd.ones((100,100), mx.gpu()) +b = mx.nd.ones((100,100), gpu_device) +c = mx.nd.ones((100,100), gpu_device) a.copyto(c) # copy from CPU to GPU d = b + c e = b.as_in_context(c.context) + c # same to above @@ -432,7 +449,7 @@ first runs on CPU and then on GPU: ```python n = 10 a = mx.nd.ones((1000,1000)) -b = mx.nd.ones((6000,6000), mx.gpu()) +b = mx.nd.ones((6000,6000), gpu_device) tic = time.time() c = do(a, n) wait(c) diff --git a/docs/tutorials/basic/symbol.md b/docs/tutorials/basic/symbol.md index 921f3ec1e11e..dc7daaea857e 100644 --- a/docs/tutorials/basic/symbol.md +++ b/docs/tutorials/basic/symbol.md @@ -45,6 +45,18 @@ For a visual explanation of these concepts, see To make things concrete, let's take a hands-on look at the Symbol API. There are a few different ways to compose a `Symbol`. +## Prerequisites + +To complete this tutorial, we need: + +- MXNet. See the instructions for your operating system in [Setup and Installation](http://mxnet.io/get_started/install.html) +- [Jupyter](http://jupyter.org/) + ``` + pip install jupyter + ``` +- GPUs - A section of this tutorial uses GPUs. If you don't have GPUs on your machine, simply +set the variable gpu_device to mx.cpu(). + ## Basic Symbol Composition ### Basic Operators @@ -320,9 +332,12 @@ print('number of outputs = %d\nthe first output = \n%s' % ( We can evaluate the same symbol on GPU with different data. +**Note** In order to execute the following section on a cpu set gpu_device to mx.cpu(). ```python -ex_gpu = c.bind(ctx=mx.gpu(), args={'a' : mx.nd.ones([3,4], mx.gpu())*2, - 'b' : mx.nd.ones([3,4], mx.gpu())*3}) +gpu_device=mx.gpu() # Change this to mx.cpu() in absence of GPUs. + +ex_gpu = c.bind(ctx=gpu_device, args={'a' : mx.nd.ones([3,4], gpu_device)*2, + 'b' : mx.nd.ones([3,4], gpu_device)*3}) ex_gpu.forward() ex_gpu.outputs[0].asnumpy() ``` diff --git a/docs/tutorials/python/linear-regression.md b/docs/tutorials/python/linear-regression.md index eb99642179a6..b904792b8509 100644 --- a/docs/tutorials/python/linear-regression.md +++ b/docs/tutorials/python/linear-regression.md @@ -4,6 +4,18 @@ In this tutorial we'll walk through how one can implement *linear regression* us The function we are trying to learn is: *y = x1 + 2x2*, where *(x1,x2)* are input features and *y* is the corresponding label. +## Prerequisites + +To complete this tutorial, we need: + +- MXNet. See the instructions for your operating system in [Setup and Installation](http://mxnet.io/get_started/install.html). + +- [Jupyter Notebook](http://jupyter.org/index.html). + +``` +$ pip install jupyter +``` + To begin, the following code imports the necessary packages we'll need for this exercise. ```python diff --git a/docs/tutorials/python/mnist.md b/docs/tutorials/python/mnist.md index f3ce3f26dbf4..c8789e2bd6dc 100644 --- a/docs/tutorials/python/mnist.md +++ b/docs/tutorials/python/mnist.md @@ -8,6 +8,17 @@ MNIST is a widely used dataset for the hand-written digit classification task. I **Figure 1:** Sample images from the MNIST dataset. +## Prerequisites +To complete this tutorial, we need: + +- MXNet. See the instructions for your operating system in [Setup and Installation](http://mxnet.io/get_started/install.html). + +- [Python Requests](http://docs.python-requests.org/en/master/) and [Jupyter Notebook](http://jupyter.org/index.html). + +``` +$ pip install requests jupyter +``` + ## Loading Data Before we define the model, let's first fetch the [MNIST](http://yann.lecun.com/exdb/mnist/) dataset. diff --git a/docs/tutorials/python/predict_image.md b/docs/tutorials/python/predict_image.md index f473bdb1d6f5..c1d40a06930f 100644 --- a/docs/tutorials/python/predict_image.md +++ b/docs/tutorials/python/predict_image.md @@ -3,10 +3,22 @@ This tutorial explains how to recognize objects in an image with a pre-trained model, and how to perform feature extraction. +## Prerequisites + +To complete this tutorial, we need: + +- MXNet. See the instructions for your operating system in [Setup and Installation](http://mxnet.io/get_started/install.html) + +- [Python Requests](http://docs.python-requests.org/en/master/), [Matplotlib](https://matplotlib.org/) and [Jupyter Notebook](http://jupyter.org/index.html). + +``` +$ pip install requests matplotlib jupyter +``` + ## Loading We first download a pre-trained ResNet 152 layer that is trained on the full -Imagenet dataset with over 10 million images and 10 thousand classes. A +ImageNet dataset with over 10 million images and 10 thousand classes. A pre-trained model contains two parts, a json file containing the model definition and a binary file containing the parameters. In addition there may be a text file for the labels. diff --git a/docs/tutorials/vision/large_scale_classification.md b/docs/tutorials/vision/large_scale_classification.md index f1929980ad3c..09e85075bebb 100644 --- a/docs/tutorials/vision/large_scale_classification.md +++ b/docs/tutorials/vision/large_scale_classification.md @@ -2,10 +2,19 @@ Training a neural network with a large number of images presents several challenges. Even with the latest GPUs, it is not possible to train large networks using a large number of images in a reasonable amount of time using a single GPU. This problem can be somewhat mitigated by using multiple GPUs in a single machine. But there is a limit to the number of GPUs that can be attached to one machine (typically 8 or 16). This tutorial explains how to train large networks with terabytes of data using multiple machines each containing multiple GPUs. +## Prerequisites +- MXNet. See the instructions for your operating system in [Setup and Installation](http://mxnet.io/get_started/install.html). + +- [OpenCV Python library](http://opencv.org/opencv-3-2.html) + +``` +$ pip install opencv-python +``` + ## Preprocessing ### Disk space -The first step in training with large data is downloading the data and preprocessing it. For this tutorial, we will be using the full imagenet dataset. Note that, at least 2 TB of disk space is required to download and preprocess this data. It is strongly recommended to use SSD instead of HDD. SSD is much better at dealing with a large number of small image files. After the preprocessing completes and images are packed into recordIO files, HDD should be fine for training. +The first step in training with large data is downloading the data and preprocessing it. For this tutorial, we will be using the full ImageNet dataset. Note that, at least 2 TB of disk space is required to download and preprocess this data. It is strongly recommended to use SSD instead of HDD. SSD is much better at dealing with a large number of small image files. After the preprocessing completes and images are packed into recordIO files, HDD should be fine for training. In this tutorial, we will use an AWS storage instance for data preprocessing. The storage instance `i3.4xlarge` has 3.8 TB of disk space across two NVMe SSD disks. We will use software RAID to combine them into one disk and mount it at `~/data`. @@ -20,9 +29,9 @@ sudo chown ${whoami} ~/data We now have sufficient disk space to download and preprocess the data. -### Download imagenet +### Download ImageNet -In this tutorial, we will be using the full imagenet dataset which can be downloaded from http://www.image-net.org/download-images. `fall11_whole.tar` contains all the images. This file is 1.2 TB in size and could take a long time to download. +In this tutorial, we will be using the full ImageNet dataset which can be downloaded from http://www.image-net.org/download-images. `fall11_whole.tar` contains all the images. This file is 1.2 TB in size and could take a long time to download. After downloading, untar the file. ``` @@ -51,7 +60,7 @@ n00120010 ``` ### Remove uncommon classes for transfer learning (optional) -A common reason to train a network on Imagenet data is to use it for transfer learning (including feature extraction or fine-tuning other models). According to [this](https://arxiv.org/pdf/1608.08614v1.pdf) study, classes with too few images don’t help in transfer learning. So, we could remove classes with fewer than a certain number of images. The following code will remove classes with less than 500 images. +A common reason to train a network on ImageNet data is to use it for transfer learning (including feature extraction or fine-tuning other models). According to [this](https://arxiv.org/pdf/1608.08614v1.pdf) study, classes with too few images don’t help in transfer learning. So, we could remove classes with fewer than a certain number of images. The following code will remove classes with less than 500 images. ``` BAK=${ROOT}_filtered @@ -83,7 +92,7 @@ done ``` ### Pack images into record files -While MXNet can read image files directly, it is recommended to pack the image files into a recordIO file for increased performance. MXNet provides a tool (tools/im2rec.py) to do this. To use this tool, MXNet and OpenCV’s python module needs to be installed in the system. [Here](#installing-opencv) are instructions to install OpenCV python module. +While MXNet can read image files directly, it is recommended to pack the image files into a recordIO file for increased performance. MXNet provides a tool (tools/im2rec.py) to do this. To use this tool, MXNet and OpenCV’s python module needs to be installed in the system. Set the environment variable `MXNET` to point to the MXNet installation directory and `NAME` to the name of the dataset. Here, we assume MXNet is installed at `~/mxnet` @@ -244,30 +253,3 @@ It is often straightforward to achieve a reasonable validation accuracy, but ach If the batch size is too big, it can exhaust GPU memory. If this happens, you’ll see the error message “cudaMalloc failed: out of memory” or something similar. There are a couple of ways to fix this: - Reduce the batch size. - Set the environment variable `MXNET_BACKWARD_DO_MIRROR` to 1. It reduces the memory consumption by trading off speed. For example, with batch size 64, inception-v3 uses 10G memory and trains 30 image/sec on a single K80 GPU. When mirroring is enabled, with 10G GPU memory consumption, we can run inception-v3 using batch size of 128. The cost is that, the speed reduces to 27 images/sec. - -## Appendix -### Installing OpenCV -#### On Ubuntu -Install OpenCV: -``` -sudo apt-get install -y libopencv-dev -``` -Install OpenCV Python libraries: -``` -sudo apt-get install python-opencv -``` -#### On Amazon Linux -Install OpenCV: -``` -git clone https://github.com/opencv/opencv -cd opencv -mkdir -p build -cd build -cmake -D BUILD_opencv_gpu=OFF -D WITH_EIGEN=ON -D WITH_TBB=ON -D WITH_CUDA=OFF -D WITH_1394=OFF -D CMAKE_BUILD_TYPE=RELEASE -D CMAKE_INSTALL_PREFIX=/usr/local .. -make -sudo make PREFIX=/usr/local install -``` -Install OpenCV Python libraries: -``` -sudo yum install opencv-python -``` From 514d721e7a3df33af9b9c39c8efa049b68bd1b3e Mon Sep 17 00:00:00 2001 From: vsooda Date: Sat, 10 Jun 2017 14:16:22 +0800 Subject: [PATCH 056/834] fix python debug cpp markdown format (#6641) --- example/python-howto/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/example/python-howto/README.md b/example/python-howto/README.md index 5007e858be4c..2499c2ab078c 100644 --- a/example/python-howto/README.md +++ b/example/python-howto/README.md @@ -15,7 +15,7 @@ Python Howto Examples * run python under gdb: ```gdb --args python debug_conv.py``` * in gdb set break point on particular line of the code and run execution: - ``` +``` (gdb) break src/operator/convolution-inl.h:120 (gdb) run Breakpoint 1, mxnet::op::ConvolutionOp::Forward (this=0x12219d0, ctx=..., in_data=std::vector of length 3, capacity 4 = {...}, req=std::vector of length 1, capacity 1 = {...}, out_data=std::vector of length 1, capacity 1 = {...}, @@ -32,4 +32,4 @@ Breakpoint 1, mxnet::op::ConvolutionOp::Forward (this=0x122 123 in_data[conv::kWeight].get_with_shape(wmat_shape, s); 124 Tensor out = out_data[conv::kOut].get(s); 125 #if defined(__CUDACC__) - ``` \ No newline at end of file +``` From d75ef8eb65a56ecec6613458017a0c83f7a9ed34 Mon Sep 17 00:00:00 2001 From: Mu Li Date: Fri, 9 Jun 2017 23:17:50 -0700 Subject: [PATCH 057/834] fix doc build on ci (#6638) --- Jenkinsfile | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 08c1e000050d..df39672c5ed2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -352,12 +352,11 @@ stage('Integration Test') { } stage('Deploy') { - parallel 'Doc': { - node('linux') { - ws('workspace/docs') { - if (env.BRANCH_NAME == "master") { - sh "make docs" - } + node('linux') { + ws('workspace/docs') { + if (env.BRANCH_NAME == "master") { + init_git() + sh "make docs" } } } From cba42457cf98d21ca7daff8f2113cc974b38bf70 Mon Sep 17 00:00:00 2001 From: Przemyslaw Tredak Date: Sun, 11 Jun 2017 17:00:21 -0700 Subject: [PATCH 058/834] Added ResNet v1 fp32 and double buffering of input data to both resnet (#6661) scripts --- .../image-classification/symbols/resnet-v1.py | 173 ++++++++++++++++++ .../image-classification/symbols/resnet.py | 1 + 2 files changed, 174 insertions(+) create mode 100755 example/image-classification/symbols/resnet-v1.py diff --git a/example/image-classification/symbols/resnet-v1.py b/example/image-classification/symbols/resnet-v1.py new file mode 100755 index 000000000000..9c515ba82cb9 --- /dev/null +++ b/example/image-classification/symbols/resnet-v1.py @@ -0,0 +1,173 @@ +''' +Adapted from https://github.com/tornadomeet/ResNet/blob/master/symbol_resnet.py +(Original author Wei Wu) by Antti-Pekka Hynninen + +Implementing the original resnet ILSVRC 2015 winning network from: + +Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun. "Deep Residual Learning for Image Recognition" +''' +import mxnet as mx + +def residual_unit(data, num_filter, stride, dim_match, name, bottle_neck=True, bn_mom=0.9, workspace=256, memonger=False): + """Return ResNet Unit symbol for building ResNet + Parameters + ---------- + data : str + Input data + num_filter : int + Number of output channels + bnf : int + Bottle neck channels factor with regard to num_filter + stride : tupe + Stride used in convolution + dim_match : Boolen + True means channel number between input and output is the same, otherwise means differ + name : str + Base name of the operators + workspace : int + Workspace used in convolution operator + """ + if bottle_neck: + conv1 = mx.sym.Convolution(data=data, num_filter=int(num_filter*0.25), kernel=(1,1), stride=stride, pad=(0,0), + no_bias=True, workspace=workspace, name=name + '_conv1') + bn1 = mx.sym.BatchNorm(data=conv1, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn1') + act1 = mx.sym.Activation(data=bn1, act_type='relu', name=name + '_relu1') + conv2 = mx.sym.Convolution(data=act1, num_filter=int(num_filter*0.25), kernel=(3,3), stride=(1,1), pad=(1,1), + no_bias=True, workspace=workspace, name=name + '_conv2') + bn2 = mx.sym.BatchNorm(data=conv2, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn2') + act2 = mx.sym.Activation(data=bn2, act_type='relu', name=name + '_relu2') + conv3 = mx.sym.Convolution(data=act2, num_filter=num_filter, kernel=(1,1), stride=(1,1), pad=(0,0), no_bias=True, + workspace=workspace, name=name + '_conv3') + bn3 = mx.sym.BatchNorm(data=conv3, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn3') + + if dim_match: + shortcut = data + else: + conv1sc = mx.sym.Convolution(data=data, num_filter=num_filter, kernel=(1,1), stride=stride, no_bias=True, + workspace=workspace, name=name+'_conv1sc') + shortcut = mx.sym.BatchNorm(data=conv1sc, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_sc') + if memonger: + shortcut._set_attr(mirror_stage='True') + return mx.sym.Activation(data=bn3 + shortcut, act_type='relu', name=name + '_relu3') + else: + conv1 = mx.sym.Convolution(data=data, num_filter=num_filter, kernel=(3,3), stride=stride, pad=(1,1), + no_bias=True, workspace=workspace, name=name + '_conv1') + bn1 = mx.sym.BatchNorm(data=conv1, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_bn1') + act1 = mx.sym.Activation(data=bn1, act_type='relu', name=name + '_relu1') + conv2 = mx.sym.Convolution(data=act1, num_filter=num_filter, kernel=(3,3), stride=(1,1), pad=(1,1), + no_bias=True, workspace=workspace, name=name + '_conv2') + bn2 = mx.sym.BatchNorm(data=conv2, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_bn2') + + if dim_match: + shortcut = data + else: + conv1sc = mx.sym.Convolution(data=data, num_filter=num_filter, kernel=(1,1), stride=stride, no_bias=True, + workspace=workspace, name=name+'_conv1sc') + shortcut = mx.sym.BatchNorm(data=conv1sc, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_sc') + if memonger: + shortcut._set_attr(mirror_stage='True') + return mx.sym.Activation(data=bn2 + shortcut, act_type='relu', name=name + '_relu3') + +def resnet(units, num_stages, filter_list, num_classes, image_shape, bottle_neck=True, bn_mom=0.9, workspace=256, memonger=False): + """Return ResNet symbol of + Parameters + ---------- + units : list + Number of units in each stage + num_stages : int + Number of stage + filter_list : list + Channel size of each stage + num_classes : int + Ouput size of symbol + dataset : str + Dataset type, only cifar10 and imagenet supports + workspace : int + Workspace used in convolution operator + """ + num_unit = len(units) + assert(num_unit == num_stages) + data = mx.sym.Variable(name='data') + data = mx.sym.identity(data=data, name='id') + (nchannel, height, width) = image_shape + if height <= 32: # such as cifar10 + body = mx.sym.Convolution(data=data, num_filter=filter_list[0], kernel=(3, 3), stride=(1,1), pad=(1, 1), + no_bias=True, name="conv0", workspace=workspace) + # Is this BatchNorm supposed to be here? + body = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn0') + else: # often expected to be 224 such as imagenet + body = mx.sym.Convolution(data=data, num_filter=filter_list[0], kernel=(7, 7), stride=(2,2), pad=(3, 3), + no_bias=True, name="conv0", workspace=workspace) + body = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn0') + body = mx.sym.Activation(data=body, act_type='relu', name='relu0') + body = mx.symbol.Pooling(data=body, kernel=(3, 3), stride=(2,2), pad=(1,1), pool_type='max') + + for i in range(num_stages): + body = residual_unit(body, filter_list[i+1], (1 if i==0 else 2, 1 if i==0 else 2), False, + name='stage%d_unit%d' % (i + 1, 1), bottle_neck=bottle_neck, workspace=workspace, + memonger=memonger) + for j in range(units[i]-1): + body = residual_unit(body, filter_list[i+1], (1,1), True, name='stage%d_unit%d' % (i + 1, j + 2), + bottle_neck=bottle_neck, workspace=workspace, memonger=memonger) + # bn1 = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn1') + # relu1 = mx.sym.Activation(data=bn1, act_type='relu', name='relu1') + # Although kernel is not used here when global_pool=True, we should put one + pool1 = mx.symbol.Pooling(data=body, global_pool=True, kernel=(7, 7), pool_type='avg', name='pool1') + flat = mx.symbol.Flatten(data=pool1) + fc1 = mx.symbol.FullyConnected(data=flat, num_hidden=num_classes, name='fc1') + return mx.symbol.SoftmaxOutput(data=fc1, name='softmax') + +def get_symbol(num_classes, num_layers, image_shape, conv_workspace=256, **kwargs): + """ + Adapted from https://github.com/tornadomeet/ResNet/blob/master/symbol_resnet.py + (Original author Wei Wu) by Antti-Pekka Hynninen + Implementing the original resnet ILSVRC 2015 winning network from: + Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun. "Deep Residual Learning for Image Recognition" + """ + image_shape = [int(l) for l in image_shape.split(',')] + (nchannel, height, width) = image_shape + if height <= 28: + num_stages = 3 + if (num_layers-2) % 9 == 0 and num_layers >= 164: + per_unit = [(num_layers-2)//9] + filter_list = [16, 64, 128, 256] + bottle_neck = True + elif (num_layers-2) % 6 == 0 and num_layers < 164: + per_unit = [(num_layers-2)//6] + filter_list = [16, 16, 32, 64] + bottle_neck = False + else: + raise ValueError("no experiments done on num_layers {}, you can do it youself".format(num_layers)) + units = per_unit * num_stages + else: + if num_layers >= 50: + filter_list = [64, 256, 512, 1024, 2048] + bottle_neck = True + else: + filter_list = [64, 64, 128, 256, 512] + bottle_neck = False + num_stages = 4 + if num_layers == 18: + units = [2, 2, 2, 2] + elif num_layers == 34: + units = [3, 4, 6, 3] + elif num_layers == 50: + units = [3, 4, 6, 3] + elif num_layers == 101: + units = [3, 4, 23, 3] + elif num_layers == 152: + units = [3, 8, 36, 3] + elif num_layers == 200: + units = [3, 24, 36, 3] + elif num_layers == 269: + units = [3, 30, 48, 8] + else: + raise ValueError("no experiments done on num_layers {}, you can do it youself".format(num_layers)) + + return resnet(units = units, + num_stages = num_stages, + filter_list = filter_list, + num_classes = num_classes, + image_shape = image_shape, + bottle_neck = bottle_neck, + workspace = conv_workspace) diff --git a/example/image-classification/symbols/resnet.py b/example/image-classification/symbols/resnet.py index e56e634a8b6e..9c121e8f9da7 100644 --- a/example/image-classification/symbols/resnet.py +++ b/example/image-classification/symbols/resnet.py @@ -87,6 +87,7 @@ def resnet(units, num_stages, filter_list, num_classes, image_shape, bottle_neck num_unit = len(units) assert(num_unit == num_stages) data = mx.sym.Variable(name='data') + data = mx.sym.identity(data=data, name='id') data = mx.sym.BatchNorm(data=data, fix_gamma=True, eps=2e-5, momentum=bn_mom, name='bn_data') (nchannel, height, width) = image_shape if height <= 32: # such as cifar10 From f7d74385aa3b6aae729d82d8c0702d25f182ef3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=A2=81=E5=BE=B7=E6=BE=8E?= Date: Mon, 12 Jun 2017 23:45:35 +0800 Subject: [PATCH 059/834] [Scala] add eq, ne, gt, ge, lt, le to NDArray and Symbol (#6665) --- .../main/scala/ml/dmlc/mxnet/NDArray.scala | 176 ++++++++++ .../src/main/scala/ml/dmlc/mxnet/Symbol.scala | 91 +++++ .../scala/ml/dmlc/mxnet/NDArraySuite.scala | 95 ++++++ .../scala/ml/dmlc/mxnet/OperatorSuite.scala | 323 +++++++++++++++++- 4 files changed, 682 insertions(+), 3 deletions(-) diff --git a/scala-package/core/src/main/scala/ml/dmlc/mxnet/NDArray.scala b/scala-package/core/src/main/scala/ml/dmlc/mxnet/NDArray.scala index fa9488c00c79..49eea3dc9d05 100644 --- a/scala-package/core/src/main/scala/ml/dmlc/mxnet/NDArray.scala +++ b/scala-package/core/src/main/scala/ml/dmlc/mxnet/NDArray.scala @@ -291,6 +291,89 @@ object NDArray { NDArray.genericNDArrayFunctionInvoke("_minimum_scalar", Seq(lhs, rhs)) } + /** + * Returns the result of element-wise **equal to** (==) comparison operation with broadcasting. + * For each element in input arrays, return 1(true) if corresponding elements are same, + * otherwise return 0(false). + */ + def equal(lhs: NDArray, rhs: NDArray): NDArray = { + NDArray.genericNDArrayFunctionInvoke("broadcast_equal", Seq(lhs, rhs)) + } + + def equal(lhs: NDArray, rhs: Float): NDArray = { + NDArray.genericNDArrayFunctionInvoke("_equal_scalar", Seq(lhs, rhs)) + } + + /** + * Returns the result of element-wise **not equal to** (!=) comparison operation + * with broadcasting. + * For each element in input arrays, return 1(true) if corresponding elements are different, + * otherwise return 0(false). + */ + def notEqual(lhs: NDArray, rhs: NDArray): NDArray = { + NDArray.genericNDArrayFunctionInvoke("broadcast_not_equal", Seq(lhs, rhs)) + } + + def notEqual(lhs: NDArray, rhs: Float): NDArray = { + NDArray.genericNDArrayFunctionInvoke("_not_equal_scalar", Seq(lhs, rhs)) + } + + /** + * Returns the result of element-wise **greater than** (>) comparison operation + * with broadcasting. + * For each element in input arrays, return 1(true) if lhs elements are greater than rhs, + * otherwise return 0(false). + */ + def greater(lhs: NDArray, rhs: NDArray): NDArray = { + NDArray.genericNDArrayFunctionInvoke("broadcast_greater", Seq(lhs, rhs)) + } + + def greater(lhs: NDArray, rhs: Float): NDArray = { + NDArray.genericNDArrayFunctionInvoke("_greater_scalar", Seq(lhs, rhs)) + } + + /** + * Returns the result of element-wise **greater than or equal to** (>=) comparison + * operation with broadcasting. + * For each element in input arrays, return 1(true) if lhs elements are greater than equal to rhs, + * otherwise return 0(false). + */ + def greaterEqual(lhs: NDArray, rhs: NDArray): NDArray = { + NDArray.genericNDArrayFunctionInvoke("broadcast_greater_equal", Seq(lhs, rhs)) + } + + def greaterEqual(lhs: NDArray, rhs: Float): NDArray = { + NDArray.genericNDArrayFunctionInvoke("_greater_equal_scalar", Seq(lhs, rhs)) + } + + /** + * Returns the result of element-wise **lesser than** (<) comparison operation + * with broadcasting. + * For each element in input arrays, return 1(true) if lhs elements are less than rhs, + * otherwise return 0(false). + */ + def lesser(lhs: NDArray, rhs: NDArray): NDArray = { + NDArray.genericNDArrayFunctionInvoke("broadcast_lesser", Seq(lhs, rhs)) + } + + def lesser(lhs: NDArray, rhs: Float): NDArray = { + NDArray.genericNDArrayFunctionInvoke("_lesser_scalar", Seq(lhs, rhs)) + } + + /** + * Returns the result of element-wise **lesser than or equal to** (<=) comparison + * operation with broadcasting. + * For each element in input arrays, return 1(true) if lhs elements are + * lesser than equal to rhs, otherwise return 0(false). + */ + def lesserEqual(lhs: NDArray, rhs: NDArray): NDArray = { + NDArray.genericNDArrayFunctionInvoke("broadcast_lesser_equal", Seq(lhs, rhs)) + } + + def lesserEqual(lhs: NDArray, rhs: Float): NDArray = { + NDArray.genericNDArrayFunctionInvoke("_lesser_equal_scalar", Seq(lhs, rhs)) + } + /** * Create a new NDArray that copies content from source_array. * @param sourceArr Source data to create NDArray from. @@ -749,6 +832,54 @@ class NDArray private[mxnet](private[mxnet] val handle: NDArrayHandle, this } + def **(other: NDArray): NDArray = { + NDArray.power(this, other) + } + + def **(other: Float): NDArray = { + NDArray.power(this, other) + } + + def **=(other: NDArray): NDArray = { + NDArray.genericNDArrayFunctionInvoke("_power", Seq(this, other), Map("out" -> this)) + } + + def **=(other: Float): NDArray = { + NDArray.genericNDArrayFunctionInvoke("_power_scalar", Seq(this, other), Map("out" -> this)) + } + + def >(other: NDArray): NDArray = { + NDArray.greater(this, other) + } + + def >(other: Float): NDArray = { + NDArray.greater(this, other) + } + + def >=(other: NDArray): NDArray = { + NDArray.greaterEqual(this, other) + } + + def >=(other: Float): NDArray = { + NDArray.greaterEqual(this, other) + } + + def <(other: NDArray): NDArray = { + NDArray.lesser(this, other) + } + + def <(other: Float): NDArray = { + NDArray.lesser(this, other) + } + + def <=(other: NDArray): NDArray = { + NDArray.lesserEqual(this, other) + } + + def <=(other: Float): NDArray = { + NDArray.lesserEqual(this, other) + } + /** * Return a copied flat java array of current array (row-major). * @return A copy of array content. @@ -880,6 +1011,41 @@ private[mxnet] class NDArrayConversions(val value: Float) { def /(other: NDArrayFuncReturn): NDArray = { NDArray.genericNDArrayFunctionInvoke("_rdiv_scalar", Seq(other.head, value)) } + + def **(other: NDArray): NDArray = { + NDArray.power(value, other) + } + def **(other: NDArrayFuncReturn): NDArray = { + NDArray.power(value, other.head) + } + + def >(other: NDArray): NDArray = { + NDArray.lesser(other, value) + } + def >(other: NDArrayFuncReturn): NDArray = { + NDArray.lesser(other.head, value) + } + + def >=(other: NDArray): NDArray = { + NDArray.lesserEqual(other, value) + } + def >=(other: NDArrayFuncReturn): NDArray = { + NDArray.lesserEqual(other.head, value) + } + + def <(other: NDArray): NDArray = { + NDArray.greater(other, value) + } + def <(other: NDArrayFuncReturn): NDArray = { + NDArray.greater(other.head, value) + } + + def <=(other: NDArray): NDArray = { + NDArray.greaterEqual(other, value) + } + def <=(other: NDArrayFuncReturn): NDArray = { + NDArray.greaterEqual(other.head, value) + } } private case class NDArrayFunction(handle: NDArrayHandle, arguments: List[String]) @@ -927,6 +1093,16 @@ private[mxnet] class NDArrayFuncReturn(private[mxnet] val arr: Array[NDArray]) { def *=(other: NDArray): NDArray = head *= other def *=(other: Float): NDArray = head *= other def /(other: NDArray): NDArray = head / other + def **(other: NDArray): NDArray = head ** other + def **(other: Float): NDArray = head ** other + def >(other: NDArray): NDArray = head > other + def >(other: Float): NDArray = head > other + def >=(other: NDArray): NDArray = head >= other + def >=(other: Float): NDArray = head >= other + def <(other: NDArray): NDArray = head < other + def <(other: Float): NDArray = head < other + def <=(other: NDArray): NDArray = head <= other + def <=(other: Float): NDArray = head <= other def toArray: Array[Float] = head.toArray def toScalar: Float = head.toScalar def copyTo(other: NDArray): NDArray = head.copyTo(other) diff --git a/scala-package/core/src/main/scala/ml/dmlc/mxnet/Symbol.scala b/scala-package/core/src/main/scala/ml/dmlc/mxnet/Symbol.scala index 41ae59c907ff..de60e472e76c 100644 --- a/scala-package/core/src/main/scala/ml/dmlc/mxnet/Symbol.scala +++ b/scala-package/core/src/main/scala/ml/dmlc/mxnet/Symbol.scala @@ -70,6 +70,21 @@ class Symbol private(private[mxnet] val handle: SymbolHandle) { Symbol.createFromListedSymbols("_DivScalar")(Array(this), Map("scalar" -> other.toString)) } + def **(other: Symbol): Symbol = Symbol.pow(this, other) + def **[@specialized(Int, Float, Double) V](other: V): Symbol = Symbol.pow(this, other) + + def >(other: Symbol): Symbol = Symbol.greater(this, other) + def >[@specialized(Int, Float, Double) V](other: V): Symbol = Symbol.greater(this, other) + + def >=(other: Symbol): Symbol = Symbol.greaterEqual(this, other) + def >=[@specialized(Int, Float, Double) V](other: V): Symbol = Symbol.greaterEqual(this, other) + + def <(other: Symbol): Symbol = Symbol.lesser(this, other) + def <[@specialized(Int, Float, Double) V](other: V): Symbol = Symbol.lesser(this, other) + + def <=(other: Symbol): Symbol = Symbol.lesserEqual(this, other) + def <=[@specialized(Int, Float, Double) V](other: V): Symbol = Symbol.lesserEqual(this, other) + override def clone(): Symbol = { val clonedHandle = new SymbolHandleRef checkCall(_LIB.mxSymbolCopy(handle, clonedHandle)) @@ -852,6 +867,62 @@ object Symbol { createFromListedSymbols("_MinimumScalar")(Array(right), Map("scalar" -> left.toString)) } + def equal(left: Symbol, right: Symbol): Symbol = { + createFromListedSymbols("_equal")(Array(left, right)) + } + + def equal[@specialized(Int, Float, Double) V](left: Symbol, right: V): Symbol = { + createFromListedSymbols("_equal_scalar")(Array(left), Map("scalar" -> right.toString)) + } + + def equal[@specialized(Int, Float, Double) V](left: V, right: Symbol): Symbol = { + createFromListedSymbols("_equal_scalar")(Array(right), Map("scalar" -> left.toString)) + } + + def notEqual(left: Symbol, right: Symbol): Symbol = { + createFromListedSymbols("_not_equal")(Array(left, right)) + } + + def notEqual[@specialized(Int, Float, Double) V](left: Symbol, right: V): Symbol = { + createFromListedSymbols("_not_equal_scalar")(Array(left), Map("scalar" -> right.toString)) + } + + def notEqual[@specialized(Int, Float, Double) V](left: V, right: Symbol): Symbol = { + createFromListedSymbols("_not_equal_scalar")(Array(right), Map("scalar" -> left.toString)) + } + + def greater(left: Symbol, right: Symbol): Symbol = { + createFromListedSymbols("_greater")(Array(left, right)) + } + + def greater[@specialized(Int, Float, Double) V](left: Symbol, right: V): Symbol = { + createFromListedSymbols("_greater_scalar")(Array(left), Map("scalar" -> right.toString)) + } + + def greaterEqual(left: Symbol, right: Symbol): Symbol = { + createFromListedSymbols("_greater_equal")(Array(left, right)) + } + + def greaterEqual[@specialized(Int, Float, Double) V](left: Symbol, right: V): Symbol = { + createFromListedSymbols("_greater_equal_scalar")(Array(left), Map("scalar" -> right.toString)) + } + + def lesser(left: Symbol, right: Symbol): Symbol = { + createFromListedSymbols("_lesser")(Array(left, right)) + } + + def lesser[@specialized(Int, Float, Double) V](left: Symbol, right: V): Symbol = { + createFromListedSymbols("_lesser_scalar")(Array(left), Map("scalar" -> right.toString)) + } + + def lesserEqual(left: Symbol, right: Symbol): Symbol = { + createFromListedSymbols("_lesser_equal")(Array(left, right)) + } + + def lesserEqual[@specialized(Int, Float, Double) V](left: Symbol, right: V): Symbol = { + createFromListedSymbols("_lesser_equal_scalar")(Array(left), Map("scalar" -> right.toString)) + } + /** * Create a symbolic variable with specified name. * @param name Name of the variable. @@ -1145,6 +1216,26 @@ class SymbolConversions[@specialized(Int, Float, Double) V](val value: V) { Symbol.createFromListedSymbols("_RDivScalar")( Array(other), Map("scalar" -> value.toString)) } + + def **(other: Symbol): Symbol = { + Symbol.pow(value, other) + } + + def >(other: Symbol): Symbol = { + other < value + } + + def >=(other: Symbol): Symbol = { + other <= value + } + + def <(other: Symbol): Symbol = { + other > value + } + + def <=(other: Symbol): Symbol = { + other >= value + } } trait SymbolGenerator { diff --git a/scala-package/core/src/test/scala/ml/dmlc/mxnet/NDArraySuite.scala b/scala-package/core/src/test/scala/ml/dmlc/mxnet/NDArraySuite.scala index d3033ddffcf3..f30bab88aaa7 100644 --- a/scala-package/core/src/test/scala/ml/dmlc/mxnet/NDArraySuite.scala +++ b/scala-package/core/src/test/scala/ml/dmlc/mxnet/NDArraySuite.scala @@ -175,6 +175,101 @@ class NDArraySuite extends FunSuite with BeforeAndAfterAll with Matchers { val arrPower3 = NDArray.power(arr, arr) assert(arrPower3.shape === Shape(2, 1)) assert(arrPower3.toArray === Array(27f, 3125f)) + + val arrPower4 = arr ** 2f + assert(arrPower4.shape === Shape(2, 1)) + assert(arrPower4.toArray === Array(9f, 25f)) + + val arrPower5 = arr ** arr + assert(arrPower5.shape === Shape(2, 1)) + assert(arrPower5.toArray === Array(27f, 3125f)) + + arr **= 2f + assert(arr.shape === Shape(2, 1)) + assert(arr.toArray === Array(9f, 25f)) + + arr.set(Array(3f, 5f)) + arr **= arr + assert(arr.shape === Shape(2, 1)) + assert(arr.toArray === Array(27f, 3125f)) + } + + test("equal") { + val arr1 = NDArray.array(Array(1f, 2f, 3f, 5f), shape = Shape(2, 2)) + val arr2 = NDArray.array(Array(1f, 4f, 3f, 6f), shape = Shape(2, 2)) + + val arrEqual1 = NDArray.equal(arr1, arr2) + assert(arrEqual1.shape === Shape(2, 2)) + assert(arrEqual1.toArray === Array(1f, 0f, 1f, 0f)) + + val arrEqual2 = NDArray.equal(arr1, 3f) + assert(arrEqual2.shape === Shape(2, 2)) + assert(arrEqual2.toArray === Array(0f, 0f, 1f, 0f)) + } + + test("not_equal") { + val arr1 = NDArray.array(Array(1f, 2f, 3f, 5f), shape = Shape(2, 2)) + val arr2 = NDArray.array(Array(1f, 4f, 3f, 6f), shape = Shape(2, 2)) + + val arrEqual1 = NDArray.notEqual(arr1, arr2) + assert(arrEqual1.shape === Shape(2, 2)) + assert(arrEqual1.toArray === Array(0f, 1f, 0f, 1f)) + + val arrEqual2 = NDArray.notEqual(arr1, 3f) + assert(arrEqual2.shape === Shape(2, 2)) + assert(arrEqual2.toArray === Array(1f, 1f, 0f, 1f)) + } + + test("greater") { + val arr1 = NDArray.array(Array(1f, 2f, 4f, 5f), shape = Shape(2, 2)) + val arr2 = NDArray.array(Array(1f, 4f, 3f, 6f), shape = Shape(2, 2)) + + val arrEqual1 = arr1 > arr2 + assert(arrEqual1.shape === Shape(2, 2)) + assert(arrEqual1.toArray === Array(0f, 0f, 1f, 0f)) + + val arrEqual2 = arr1 > 2f + assert(arrEqual2.shape === Shape(2, 2)) + assert(arrEqual2.toArray === Array(0f, 0f, 1f, 1f)) + } + + test("greater_equal") { + val arr1 = NDArray.array(Array(1f, 2f, 4f, 5f), shape = Shape(2, 2)) + val arr2 = NDArray.array(Array(1f, 4f, 3f, 6f), shape = Shape(2, 2)) + + val arrEqual1 = arr1 >= arr2 + assert(arrEqual1.shape === Shape(2, 2)) + assert(arrEqual1.toArray === Array(1f, 0f, 1f, 0f)) + + val arrEqual2 = arr1 >= 2f + assert(arrEqual2.shape === Shape(2, 2)) + assert(arrEqual2.toArray === Array(0f, 1f, 1f, 1f)) + } + + test("lesser") { + val arr1 = NDArray.array(Array(1f, 2f, 4f, 5f), shape = Shape(2, 2)) + val arr2 = NDArray.array(Array(1f, 4f, 3f, 6f), shape = Shape(2, 2)) + + val arrEqual1 = arr1 < arr2 + assert(arrEqual1.shape === Shape(2, 2)) + assert(arrEqual1.toArray === Array(0f, 1f, 0f, 1f)) + + val arrEqual2 = arr1 < 2f + assert(arrEqual2.shape === Shape(2, 2)) + assert(arrEqual2.toArray === Array(1f, 0f, 0f, 0f)) + } + + test("lesser_equal") { + val arr1 = NDArray.array(Array(1f, 2f, 4f, 5f), shape = Shape(2, 2)) + val arr2 = NDArray.array(Array(1f, 4f, 3f, 6f), shape = Shape(2, 2)) + + val arrEqual1 = arr1 <= arr2 + assert(arrEqual1.shape === Shape(2, 2)) + assert(arrEqual1.toArray === Array(1f, 1f, 0f, 1f)) + + val arrEqual2 = arr1 <= 2f + assert(arrEqual2.shape === Shape(2, 2)) + assert(arrEqual2.toArray === Array(1f, 1f, 0f, 0f)) } test("choose_element_0index") { diff --git a/scala-package/core/src/test/scala/ml/dmlc/mxnet/OperatorSuite.scala b/scala-package/core/src/test/scala/ml/dmlc/mxnet/OperatorSuite.scala index 4a2ae75cc4b8..a870cabb568b 100644 --- a/scala-package/core/src/test/scala/ml/dmlc/mxnet/OperatorSuite.scala +++ b/scala-package/core/src/test/scala/ml/dmlc/mxnet/OperatorSuite.scala @@ -219,7 +219,7 @@ class OperatorSuite extends FunSuite with BeforeAndAfterAll val shape = Shape(1, 1) val dataTmp = NDArray.ones(shape) * 3 val dataTmpPowered = NDArray.ones(shape) * 9 - val test = Symbol.pow(data, 2) + val test = data ** 2 // TODO: check numeric gradient checkSymbolicForward(test, Array(dataTmp), Array(dataTmpPowered)) checkSymbolicBackward(test, Array(dataTmp), Array(NDArray.ones(shape)), Array(dataTmp * 2)) @@ -234,7 +234,7 @@ class OperatorSuite extends FunSuite with BeforeAndAfterAll val exp = Symbol.Variable("exp") val expTmp = NDArray.ones(shape) * 3 - val test = Symbol.pow(data, exp) + val test = data ** exp // TODO: check numeric gradient checkSymbolicForward(test, Seq(dataTmp, expTmp), Seq(NDArray.ones(shape) * 8)) @@ -249,7 +249,8 @@ class OperatorSuite extends FunSuite with BeforeAndAfterAll test("pow fn") { val shape = Shape(3, 4) val exp = Symbol.Variable("exp") - val y = Symbol.pow(2, exp) + import SymbolConversions._ + val y = 2 ** exp val x = NDArray.ones(shape) * 3 // TODO: check numeric gradient checkSymbolicForward(y, Seq(x), Seq(NDArray.ones(shape) * 8)) // 2**x @@ -258,6 +259,322 @@ class OperatorSuite extends FunSuite with BeforeAndAfterAll Seq(NDArray.ones(shape) * 8 * Math.log(2).toFloat)) } + test("scalar equal") { + val data = Symbol.Variable("datas") + val shape = Shape(2, 2) + val dataTmpExpected = NDArray.array(Array(0f, 1f, 0f, 0f), shape) + val test = Symbol.equal(data, 2f) + + val exec = test.simpleBind(Context.cpu(), gradReq = "write", shapeDict = Map("datas" -> shape)) + exec.argDict("datas").set(Array(1f, 2f, 3f, 4f)) + + exec.forward() + assert(reldiff(exec.outputs.head, dataTmpExpected) <= 1e-5f) + + exec.backward(NDArray.ones(shape)) + assert(exec.gradDict("datas").toArray === Array.fill[Float](shape.product)(0f)) + } + + test("symbol equal") { + val data = Symbol.Variable("datas") + val data2 = Symbol.Variable("datas2") + val shape = Shape(2, 2) + val dataTmpExpected = NDArray.array(Array(1f, 0f, 0f, 0f), shape) + val test = Symbol.equal(data, data2) + + val exec = test.simpleBind(Context.cpu(), gradReq = "write", + shapeDict = Map("datas" -> shape, "datas2" -> shape)) + exec.argDict("datas").set(Array(1f, 2f, 3f, 4f)) + exec.argDict("datas2").set(Array(1f, 3f, 2f, 6f)) + + exec.forward() + assert(reldiff(exec.outputs.head, dataTmpExpected) <= 1e-5f) + + exec.backward(NDArray.ones(shape)) + assert(exec.gradDict("datas").toArray === Array.fill[Float](shape.product)(0f)) + assert(exec.gradDict("datas2").toArray === Array.fill[Float](shape.product)(0f)) + } + + test("scalar equal 2") { + val data = Symbol.Variable("datas") + val shape = Shape(2, 2) + val dataTmpExpected = NDArray.array(Array(0f, 1f, 0f, 0f), shape) + val test = Symbol.equal(2f, data) + + val exec = test.simpleBind(Context.cpu(), gradReq = "write", shapeDict = Map("datas" -> shape)) + exec.argDict("datas").set(Array(1f, 2f, 3f, 4f)) + + exec.forward() + assert(reldiff(exec.outputs.head, dataTmpExpected) <= 1e-5f) + + exec.backward(NDArray.ones(shape)) + assert(exec.gradDict("datas").toArray === Array.fill[Float](shape.product)(0f)) + } + + test("scalar not_equal") { + val data = Symbol.Variable("datas") + val shape = Shape(2, 2) + val dataTmpExpected = NDArray.array(Array(1f, 0f, 1f, 1f), shape) + val test = Symbol.notEqual(data, 2f) + + val exec = test.simpleBind(Context.cpu(), gradReq = "write", shapeDict = Map("datas" -> shape)) + exec.argDict("datas").set(Array(1f, 2f, 3f, 4f)) + + exec.forward() + assert(reldiff(exec.outputs.head, dataTmpExpected) <= 1e-5f) + + exec.backward(NDArray.ones(shape)) + assert(exec.gradDict("datas").toArray === Array.fill[Float](shape.product)(0f)) + } + + test("symbol not_equal") { + val data = Symbol.Variable("datas") + val data2 = Symbol.Variable("datas2") + val shape = Shape(2, 2) + val dataTmpExpected = NDArray.array(Array(0f, 1f, 1f, 1f), shape) + val test = Symbol.notEqual(data, data2) + + val exec = test.simpleBind(Context.cpu(), gradReq = "write", + shapeDict = Map("datas" -> shape, "datas2" -> shape)) + exec.argDict("datas").set(Array(1f, 2f, 3f, 4f)) + exec.argDict("datas2").set(Array(1f, 3f, 2f, 6f)) + + exec.forward() + assert(reldiff(exec.outputs.head, dataTmpExpected) <= 1e-5f) + + exec.backward(NDArray.ones(shape)) + assert(exec.gradDict("datas").toArray === Array.fill[Float](shape.product)(0f)) + assert(exec.gradDict("datas2").toArray === Array.fill[Float](shape.product)(0f)) + } + + test("scalar not_equal 2") { + val data = Symbol.Variable("datas") + val shape = Shape(2, 2) + val dataTmpExpected = NDArray.array(Array(1f, 0f, 1f, 1f), shape) + val test = Symbol.notEqual(2f, data) + + val exec = test.simpleBind(Context.cpu(), gradReq = "write", shapeDict = Map("datas" -> shape)) + exec.argDict("datas").set(Array(1f, 2f, 3f, 4f)) + + exec.forward() + assert(reldiff(exec.outputs.head, dataTmpExpected) <= 1e-5f) + + exec.backward(NDArray.ones(shape)) + assert(exec.gradDict("datas").toArray === Array.fill[Float](shape.product)(0f)) + } + + test("scalar greater") { + val data = Symbol.Variable("datas") + val shape = Shape(2, 2) + val dataTmpExpected = NDArray.array(Array(0f, 0f, 1f, 1f), shape) + val test = data > 2f + + val exec = test.simpleBind(Context.cpu(), gradReq = "write", shapeDict = Map("datas" -> shape)) + exec.argDict("datas").set(Array(1f, 2f, 3f, 4f)) + + exec.forward() + assert(reldiff(exec.outputs.head, dataTmpExpected) <= 1e-5f) + + exec.backward(NDArray.ones(shape)) + assert(exec.gradDict("datas").toArray === Array.fill[Float](shape.product)(0f)) + } + + test("symbol greater") { + val data = Symbol.Variable("datas") + val data2 = Symbol.Variable("datas2") + val shape = Shape(2, 2) + val dataTmpExpected = NDArray.array(Array(0f, 0f, 1f, 0f), shape) + val test = data > data2 + + val exec = test.simpleBind(Context.cpu(), gradReq = "write", + shapeDict = Map("datas" -> shape, "datas2" -> shape)) + exec.argDict("datas").set(Array(1f, 2f, 3f, 4f)) + exec.argDict("datas2").set(Array(1f, 3f, 2f, 6f)) + + exec.forward() + assert(reldiff(exec.outputs.head, dataTmpExpected) <= 1e-5f) + + exec.backward(NDArray.ones(shape)) + assert(exec.gradDict("datas").toArray === Array.fill[Float](shape.product)(0f)) + assert(exec.gradDict("datas2").toArray === Array.fill[Float](shape.product)(0f)) + } + + test("scalar greater 2") { + val data = Symbol.Variable("datas") + val shape = Shape(2, 2) + val dataTmpExpected = NDArray.array(Array(1f, 0f, 0f, 0f), shape) + import SymbolConversions._ + val test = 2f > data + + val exec = test.simpleBind(Context.cpu(), gradReq = "write", shapeDict = Map("datas" -> shape)) + exec.argDict("datas").set(Array(1f, 2f, 3f, 4f)) + + exec.forward() + assert(reldiff(exec.outputs.head, dataTmpExpected) <= 1e-5f) + + exec.backward(NDArray.ones(shape)) + assert(exec.gradDict("datas").toArray === Array.fill[Float](shape.product)(0f)) + } + + test("scalar greater_equal") { + val data = Symbol.Variable("datas") + val shape = Shape(2, 2) + val dataTmpExpected = NDArray.array(Array(0f, 1f, 1f, 1f), shape) + val test = data >= 2f + + val exec = test.simpleBind(Context.cpu(), gradReq = "write", shapeDict = Map("datas" -> shape)) + exec.argDict("datas").set(Array(1f, 2f, 3f, 4f)) + + exec.forward() + assert(reldiff(exec.outputs.head, dataTmpExpected) <= 1e-5f) + + exec.backward(NDArray.ones(shape)) + assert(exec.gradDict("datas").toArray === Array.fill[Float](shape.product)(0f)) + } + + test("symbol greater_equal") { + val data = Symbol.Variable("datas") + val data2 = Symbol.Variable("datas2") + val shape = Shape(2, 2) + val dataTmpExpected = NDArray.array(Array(1f, 0f, 1f, 0f), shape) + val test = data >= data2 + + val exec = test.simpleBind(Context.cpu(), gradReq = "write", + shapeDict = Map("datas" -> shape, "datas2" -> shape)) + exec.argDict("datas").set(Array(1f, 2f, 3f, 4f)) + exec.argDict("datas2").set(Array(1f, 3f, 2f, 6f)) + + exec.forward() + assert(reldiff(exec.outputs.head, dataTmpExpected) <= 1e-5f) + + exec.backward(NDArray.ones(shape)) + assert(exec.gradDict("datas").toArray === Array.fill[Float](shape.product)(0f)) + assert(exec.gradDict("datas2").toArray === Array.fill[Float](shape.product)(0f)) + } + + test("scalar greater_equal 2") { + val data = Symbol.Variable("datas") + val shape = Shape(2, 2) + val dataTmpExpected = NDArray.array(Array(1f, 1f, 0f, 0f), shape) + import SymbolConversions._ + val test = 2f >= data + + val exec = test.simpleBind(Context.cpu(), gradReq = "write", shapeDict = Map("datas" -> shape)) + exec.argDict("datas").set(Array(1f, 2f, 3f, 4f)) + + exec.forward() + assert(reldiff(exec.outputs.head, dataTmpExpected) <= 1e-5f) + + exec.backward(NDArray.ones(shape)) + assert(exec.gradDict("datas").toArray === Array.fill[Float](shape.product)(0f)) + } + + test("scalar lesser") { + val data = Symbol.Variable("datas") + val shape = Shape(2, 2) + val dataTmpExpected = NDArray.array(Array(1f, 0f, 0f, 0f), shape) + val test = data < 2f + + val exec = test.simpleBind(Context.cpu(), gradReq = "write", shapeDict = Map("datas" -> shape)) + exec.argDict("datas").set(Array(1f, 2f, 3f, 4f)) + + exec.forward() + assert(reldiff(exec.outputs.head, dataTmpExpected) <= 1e-5f) + + exec.backward(NDArray.ones(shape)) + assert(exec.gradDict("datas").toArray === Array.fill[Float](shape.product)(0f)) + } + + test("symbol lesser") { + val data = Symbol.Variable("datas") + val data2 = Symbol.Variable("datas2") + val shape = Shape(2, 2) + val dataTmpExpected = NDArray.array(Array(0f, 1f, 0f, 1f), shape) + val test = data < data2 + + val exec = test.simpleBind(Context.cpu(), gradReq = "write", + shapeDict = Map("datas" -> shape, "datas2" -> shape)) + exec.argDict("datas").set(Array(1f, 2f, 3f, 4f)) + exec.argDict("datas2").set(Array(1f, 3f, 2f, 6f)) + + exec.forward() + assert(reldiff(exec.outputs.head, dataTmpExpected) <= 1e-5f) + + exec.backward(NDArray.ones(shape)) + assert(exec.gradDict("datas").toArray === Array.fill[Float](shape.product)(0f)) + assert(exec.gradDict("datas2").toArray === Array.fill[Float](shape.product)(0f)) + } + + test("scalar lesser 2") { + val data = Symbol.Variable("datas") + val shape = Shape(2, 2) + val dataTmpExpected = NDArray.array(Array(0f, 0f, 1f, 1f), shape) + import SymbolConversions._ + val test = 2f < data + + val exec = test.simpleBind(Context.cpu(), gradReq = "write", shapeDict = Map("datas" -> shape)) + exec.argDict("datas").set(Array(1f, 2f, 3f, 4f)) + + exec.forward() + assert(reldiff(exec.outputs.head, dataTmpExpected) <= 1e-5f) + + exec.backward(NDArray.ones(shape)) + assert(exec.gradDict("datas").toArray === Array.fill[Float](shape.product)(0f)) + } + + test("scalar lesser_equal") { + val data = Symbol.Variable("datas") + val shape = Shape(2, 2) + val dataTmpExpected = NDArray.array(Array(1f, 1f, 0f, 0f), shape) + val test = data <= 2f + + val exec = test.simpleBind(Context.cpu(), gradReq = "write", shapeDict = Map("datas" -> shape)) + exec.argDict("datas").set(Array(1f, 2f, 3f, 4f)) + + exec.forward() + assert(reldiff(exec.outputs.head, dataTmpExpected) <= 1e-5f) + + exec.backward(NDArray.ones(shape)) + assert(exec.gradDict("datas").toArray === Array.fill[Float](shape.product)(0f)) + } + + test("symbol lesser_equal") { + val data = Symbol.Variable("datas") + val data2 = Symbol.Variable("datas2") + val shape = Shape(2, 2) + val dataTmpExpected = NDArray.array(Array(1f, 1f, 0f, 1f), shape) + val test = data <= data2 + + val exec = test.simpleBind(Context.cpu(), gradReq = "write", + shapeDict = Map("datas" -> shape, "datas2" -> shape)) + exec.argDict("datas").set(Array(1f, 2f, 3f, 4f)) + exec.argDict("datas2").set(Array(1f, 3f, 2f, 6f)) + + exec.forward() + assert(reldiff(exec.outputs.head, dataTmpExpected) <= 1e-5f) + + exec.backward(NDArray.ones(shape)) + assert(exec.gradDict("datas").toArray === Array.fill[Float](shape.product)(0f)) + assert(exec.gradDict("datas2").toArray === Array.fill[Float](shape.product)(0f)) + } + + test("scalar lesser_equal 2") { + val data = Symbol.Variable("datas") + val shape = Shape(2, 2) + val dataTmpExpected = NDArray.array(Array(0f, 1f, 1f, 1f), shape) + import SymbolConversions._ + val test = 2f <= data + + val exec = test.simpleBind(Context.cpu(), gradReq = "write", shapeDict = Map("datas" -> shape)) + exec.argDict("datas").set(Array(1f, 2f, 3f, 4f)) + + exec.forward() + assert(reldiff(exec.outputs.head, dataTmpExpected) <= 1e-5f) + + exec.backward(NDArray.ones(shape)) + assert(exec.gradDict("datas").toArray === Array.fill[Float](shape.product)(0f)) + } + test("embedding") { val inDim = 10 val outDim = 4 From c09520d94743e3137d202f981bb96764ff825255 Mon Sep 17 00:00:00 2001 From: reminisce Date: Mon, 12 Jun 2017 09:58:54 -0700 Subject: [PATCH 060/834] Add backward infer shape in expand_dims (#6654) * Add backward infer shape in expand_dims * Change axis type from index_t to int * Fix corner case bug --- src/operator/tensor/matrix_op-inl.h | 46 +++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/src/operator/tensor/matrix_op-inl.h b/src/operator/tensor/matrix_op-inl.h index cdc8819da18e..9f48e0cf306c 100644 --- a/src/operator/tensor/matrix_op-inl.h +++ b/src/operator/tensor/matrix_op-inl.h @@ -294,10 +294,12 @@ inline bool TransposeShape(const nnvm::NodeAttrs& attrs, struct ExpandDimParam : public dmlc::Parameter { - index_t axis; + int axis; DMLC_DECLARE_PARAMETER(ExpandDimParam) { DMLC_DECLARE_FIELD(axis) - .describe("Position (amongst axes) where new axis is to be inserted."); + .describe("Position where new axis is to be inserted. Suppose that " + "the input `NDArray`'s dimension is `ndim`, the range of " + "the inserted axis is `[-ndim, ndim]`"); } }; @@ -308,14 +310,40 @@ inline bool ExpandDimShape(const nnvm::NodeAttrs& attrs, const ExpandDimParam& param = nnvm::get(attrs.parsed); CHECK_EQ(in_attrs->size(), 1U); CHECK_EQ(out_attrs->size(), 1U); - TShape& shp = (*in_attrs)[0]; - CHECK_LE(param.axis, shp.ndim()) - << "axis exceeds the dimension of the array"; - TShape ret(shp.ndim() + 1); - for (index_t i = 0; i < param.axis; ++i) ret[i] = shp[i]; - ret[param.axis] = 1; - for (index_t i = param.axis+1; i < ret.ndim(); ++i) ret[i] = shp[i-1]; + if (in_attrs->at(0).ndim() == 0U && out_attrs->at(0).ndim() == 0U) { + return false; + } + + TShape& ishape = (*in_attrs)[0]; + TShape& oshape = (*out_attrs)[0]; + int indim = ishape.ndim(); + bool unknown_ishape = false; + if (0 == indim) { + indim = oshape.ndim() - 1; + unknown_ishape = true; + } + + int axis = param.axis; + if (axis < 0) { + axis += indim; + } + CHECK(axis >= 0 && axis <= indim) + << "axis must be in the range [" << -indim << ", " << indim << "] (" + << param.axis << " provided)"; + TShape ret(indim + 1); + for (int i = 0; i < axis; ++i) { + ret[i] = (unknown_ishape? 0 : ishape[i]); + } + ret[axis] = 1; + for (int i = axis+1; i < indim+1; ++i) { + ret[i] = (unknown_ishape? 0 : ishape[i-1]); + } SHAPE_ASSIGN_CHECK(*out_attrs, 0, ret); + + ret = TShape(indim); + for (int i = 0; i < axis; ++i) ret[i] = oshape[i]; + for (int i = axis+1; i < indim+1; ++i) ret[i-1] = oshape[i]; + SHAPE_ASSIGN_CHECK(*in_attrs, 0, ret); return true; } From e39607f839269fa7e572e72136b906f8a1666311 Mon Sep 17 00:00:00 2001 From: Pracheer Gupta Date: Mon, 12 Jun 2017 14:57:09 -0700 Subject: [PATCH 061/834] Spellcheck for pre-trained model tutorial (#6669) * Minor spelling error. * Some more minor spelling errors. --- docs/tutorials/python/predict_image.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/tutorials/python/predict_image.md b/docs/tutorials/python/predict_image.md index c1d40a06930f..1c6cfa8e2e27 100644 --- a/docs/tutorials/python/predict_image.md +++ b/docs/tutorials/python/predict_image.md @@ -20,7 +20,7 @@ $ pip install requests matplotlib jupyter We first download a pre-trained ResNet 152 layer that is trained on the full ImageNet dataset with over 10 million images and 10 thousand classes. A pre-trained model contains two parts, a json file containing the model -definition and a binary file containing the parameters. In addition there may be +definition and a binary file containing the parameters. In addition, there may be a text file for the labels. ```python @@ -32,7 +32,7 @@ path='http://data.mxnet.io/models/imagenet-11k/' ``` Next, we load the downloaded model. *Note:* If GPU is available, we can replace all -occurances of `mx.cpu()` with `mx.gpu()` to accelerate the computation. +occurrences of `mx.cpu()` with `mx.gpu()` to accelerate the computation. ```python sym, arg_params, aux_params = mx.model.load_checkpoint('resnet-152', 0) @@ -98,7 +98,7 @@ predict('http://thenotoriouspug.com/wp-content/uploads/2015/01/Pug-Cookie-1920x1 ## Feature extraction -By feature extraction we mean presenting the input images by the output of an +By feature extraction, we mean presenting the input images by the output of an internal layer rather than the last softmax layer. These outputs, which can be viewed as the feature of the raw input image, can then be used by other applications such as object detection. @@ -113,10 +113,10 @@ all_layers.list_outputs()[-10:] ``` An often used layer for feature extraction is the one before the last fully -connected layer. For ResNet, and also Inception, it is the flatten layer with +connected layer. For ResNet, and also Inception, it is the flattened layer with name `flatten0` which reshapes the 4-D convolutional layer output into 2-D for the fully connected layer. The following source code extracts a new Symbol which -outputs the flatten layer and creates a model. +outputs the flattened layer and creates a model. ```python fe_sym = all_layers['flatten0_output'] From 6eb8bd16c9ef25bf039b60a32417e6fb2adb48e1 Mon Sep 17 00:00:00 2001 From: Andrei Paleyes Date: Mon, 12 Jun 2017 22:59:05 +0100 Subject: [PATCH 062/834] Truncate operator implementation (#6640) * Truncate operator implementation * Improved docs for floor and ceil operators --- docs/api/python/ndarray.md | 1 + docs/api/python/symbol.md | 1 + src/operator/mshadow_op.h | 8 ++++++++ src/operator/tensor/elemwise_unary_op.cc | 18 ++++++++++++++++++ src/operator/tensor/elemwise_unary_op.cu | 4 ++++ tests/python/unittest/test_operator.py | 13 +++++++++++++ 6 files changed, 45 insertions(+) diff --git a/docs/api/python/ndarray.md b/docs/api/python/ndarray.md index 1a2dfda650d5..2581c2c3354b 100644 --- a/docs/api/python/ndarray.md +++ b/docs/api/python/ndarray.md @@ -321,6 +321,7 @@ In the rest of this document, we first overview the methods provided by the fix floor ceil + trunc ``` diff --git a/docs/api/python/symbol.md b/docs/api/python/symbol.md index 4f45777da05d..00402dbc3af8 100644 --- a/docs/api/python/symbol.md +++ b/docs/api/python/symbol.md @@ -314,6 +314,7 @@ Composite multiple symbols into a new one by an operator. fix floor ceil + trunc ``` diff --git a/src/operator/mshadow_op.h b/src/operator/mshadow_op.h index 0a47db1a9b2b..7e950c980e53 100644 --- a/src/operator/mshadow_op.h +++ b/src/operator/mshadow_op.h @@ -593,6 +593,14 @@ struct floor { } }; +/*! \brief used to round towards zero */ +struct trunc { + template + MSHADOW_XINLINE static DType Map(DType a) { + return DType(truncf(a)); + } +}; + /*! \brief used to round number to nearest integer */ struct rint { template diff --git a/src/operator/tensor/elemwise_unary_op.cc b/src/operator/tensor/elemwise_unary_op.cc index 073bbe16d491..4af087749c2b 100644 --- a/src/operator/tensor/elemwise_unary_op.cc +++ b/src/operator/tensor/elemwise_unary_op.cc @@ -236,6 +236,8 @@ Example:: MXNET_OPERATOR_REGISTER_UNARY(ceil) .describe(R"code(Returns element-wise ceiling of the input. +The ceil of the scalar x is the smallest integer i, such that i >= x. + Example:: ceil([-2.1, -1.9, 1.5, 1.9, 2.1]) = [-2., -1., 2., 2., 3.] @@ -247,6 +249,8 @@ Example:: MXNET_OPERATOR_REGISTER_UNARY(floor) .describe(R"code(Returns element-wise floor of the input. +The floor of the scalar x is the largest integer i, such that i <= x. + Example:: floor([-2.1, -1.9, 1.5, 1.9, 2.1]) = [-3., -2., 1., 1., 2.] @@ -254,6 +258,20 @@ Example:: )code" ADD_FILELINE) .set_attr("FCompute", UnaryCompute); +// trunc +MXNET_OPERATOR_REGISTER_UNARY(trunc) +.describe(R"code(Return the element-wise truncated value of the input. + +The truncated value of the scalar x is the nearest integer i which is closer to +zero than x is. In short, the fractional part of the signed number x is discarded. + +Example:: + + trunc([-2.1, -1.9, 1.5, 1.9, 2.1]) = [-2., -1., 1., 1., 2.] + +)code" ADD_FILELINE) +.set_attr("FCompute", UnaryCompute); + // fix MXNET_OPERATOR_REGISTER_UNARY(fix) .describe(R"code(Returns element-wise rounded value to the nearest integer towards zero of the input. diff --git a/src/operator/tensor/elemwise_unary_op.cu b/src/operator/tensor/elemwise_unary_op.cu index 746b39fe4c8c..a3ecc8fab638 100644 --- a/src/operator/tensor/elemwise_unary_op.cu +++ b/src/operator/tensor/elemwise_unary_op.cu @@ -73,6 +73,10 @@ NNVM_REGISTER_OP(ceil) NNVM_REGISTER_OP(floor) .set_attr("FCompute", UnaryCompute); +// trunc +NNVM_REGISTER_OP(trunc) +.set_attr("FCompute", UnaryCompute); + // rint NNVM_REGISTER_OP(rint) .set_attr("FCompute", UnaryCompute); diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py index 924ef351dbe5..4a7860d0500c 100644 --- a/tests/python/unittest/test_operator.py +++ b/tests/python/unittest/test_operator.py @@ -517,6 +517,19 @@ def test_round_ceil_floor(): npout = np.round(data_tmp) + np.ceil(data_tmp) + np.floor(data_tmp) assert_almost_equal(out, npout) +def test_trunc(): + data_tmp = np.random.rand(3, 4) * 10 - 5 + arr_data = mx.nd.array(data_tmp) + data = mx.symbol.Variable('data') + test = mx.sym.trunc(data) + + exe_test = test.bind(default_context(), args=[arr_data]) + exe_test.forward(is_train=True) + out = exe_test.outputs[0].asnumpy() + npout = np.trunc(data_tmp) + + assert_almost_equal(out, npout) + def test_rsqrt_cos_sin(): data = mx.symbol.Variable('data') shape = (3, 4) From 3db253388f2796c1fc7e0d1b8ccd59788b5f8eaa Mon Sep 17 00:00:00 2001 From: Yao Wang Date: Mon, 12 Jun 2017 19:05:12 -0700 Subject: [PATCH 063/834] Change method and attribute (#6676) --- docs/_static/mxnet.css | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/_static/mxnet.css b/docs/_static/mxnet.css index 06d976326086..b0eff89003d6 100644 --- a/docs/_static/mxnet.css +++ b/docs/_static/mxnet.css @@ -849,6 +849,24 @@ dt { border-bottom: solid #0079b2; } +dl.method dt { + background: #f0f0f0; + border-bottom: solid #ccc; +} + +dl.method dt code.descname { + color:#555; +} + +dl.attribute dt { + background: #f0f0f0; + border-bottom: solid #ccc; +} + +dl.attribute dt code.descname { + color:#555; +} + dt em { font-weight: normal; font-style: normal; From 17332e5b54c37e08ad9637447025a329c457375d Mon Sep 17 00:00:00 2001 From: "Joshua Z. Zhang" Date: Mon, 12 Jun 2017 19:05:51 -0700 Subject: [PATCH 064/834] Ssd hotfix (#6675) * modify metric w.r.t. new metric api * update ssd example eval_metric and train/metric --- example/ssd/evaluate/eval_metric.py | 18 ++++++++++++------ example/ssd/train/metric.py | 16 +++++++++++++++- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/example/ssd/evaluate/eval_metric.py b/example/ssd/evaluate/eval_metric.py index f475bb336ddb..68d0fb2da443 100644 --- a/example/ssd/evaluate/eval_metric.py +++ b/example/ssd/evaluate/eval_metric.py @@ -17,16 +17,17 @@ class MApMetric(mx.metric.EvalMetric): prediction index in network output list """ def __init__(self, ovp_thresh=0.5, use_difficult=False, class_names=None, pred_idx=0): + super(MApMetric, self).__init__('mAP') if class_names is None: - super(MApMetric, self).__init__("mAP") + self.num = None else: - assert isinstance(class_names, list) + assert isinstance(class_names, (list, tuple)) for name in class_names: assert isinstance(name, str), "must provide names as str" num = len(class_names) - super(MApMetric, self).__init__(class_names + ["mAP"], num + 1) - self.records = dict() - self.counts = dict() + self.name = class_names + ['mAP'] + self.num = num + 1 + self.reset() self.ovp_thresh = ovp_thresh self.use_difficult = use_difficult self.class_names = class_names @@ -34,7 +35,12 @@ def __init__(self, ovp_thresh=0.5, use_difficult=False, class_names=None, pred_i def reset(self): """Clear the internal statistics to initial state.""" - super(MApMetric, self).reset() + if getattr(self, 'num', None) is None: + self.num_inst = 0 + self.sum_metric = 0.0 + else: + self.num_inst = [0] * self.num + self.sum_metric = [0.0] * self.num self.records = dict() self.counts = dict() diff --git a/example/ssd/train/metric.py b/example/ssd/train/metric.py index fa631a5263fc..c7d007832c4f 100644 --- a/example/ssd/train/metric.py +++ b/example/ssd/train/metric.py @@ -5,8 +5,22 @@ class MultiBoxMetric(mx.metric.EvalMetric): """Calculate metrics for Multibox training """ def __init__(self, eps=1e-8): - super(MultiBoxMetric, self).__init__(['CrossEntropy', 'SmoothL1'], 2) + super(MultiBoxMetric, self).__init__('MultiBox') self.eps = eps + self.num = 2 + self.name = ['CrossEntropy', 'SmoothL1'] + self.reset() + + def reset(self): + """ + override reset behavior + """ + if getattr(self, 'num', None) is None: + self.num_inst = 0 + self.sum_metric = 0.0 + else: + self.num_inst = [0] * self.num + self.sum_metric = [0.0] * self.num def update(self, labels, preds): """ From c43c90191068d04f42f50cb4938d7299b3d0c4f6 Mon Sep 17 00:00:00 2001 From: Arik Poznanski Date: Tue, 13 Jun 2017 20:52:07 +0300 Subject: [PATCH 065/834] updated version used of mshadow submodule (#6681) --- mshadow | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mshadow b/mshadow index c037b06ddd81..eda261eef135 160000 --- a/mshadow +++ b/mshadow @@ -1 +1 @@ -Subproject commit c037b06ddd810d39322cd056650f8b1f4763dd9d +Subproject commit eda261eef135a51e7388e680b295996d18d4e4d1 From e8520360c847120186bad37d74d43ab90f40a1f6 Mon Sep 17 00:00:00 2001 From: moin Date: Tue, 13 Jun 2017 14:13:20 -0700 Subject: [PATCH 066/834] Add BLAS3 and LAPACK routines (#6538) * Added linear algebra operators * more comments about style of wrapper interface * more appropriate fatal exit when lapack does not exist * more comments on row/col-major ordering * added config switch for lapack usage * switched lapack usage off by default --- CMakeLists.txt | 10 + Jenkinsfile | 4 +- Makefile | 10 + appveyor.yml | 2 +- docs/Dockerfile | 2 +- docs/api/python/symbol.md | 15 + docs/get_started/install.md | 16 +- include/mxnet/c_lapack_api.h | 91 ++++ include/mxnet/tensor_blob.h | 28 ++ make/config.mk | 3 + make/osx.mk | 4 + make/pip_linux_cpu.mk | 5 + make/readthedocs.mk | 2 + src/operator/elemwise_op_common.h | 17 + src/operator/tensor/la_op.cc | 399 +++++++++++++++++ src/operator/tensor/la_op.h | 420 ++++++++++++++++++ src/operator/tensor/la_op_inline.h | 335 ++++++++++++++ tests/ci_build/install/ubuntu_install_core.sh | 2 +- tests/python/unittest/test_operator.py | 250 +++++++++++ 19 files changed, 1602 insertions(+), 13 deletions(-) create mode 100644 include/mxnet/c_lapack_api.h create mode 100644 src/operator/tensor/la_op.cc create mode 100644 src/operator/tensor/la_op.h create mode 100644 src/operator/tensor/la_op_inline.h diff --git a/CMakeLists.txt b/CMakeLists.txt index f6878b80e7d5..73c55313acf9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -199,6 +199,16 @@ if(USE_OPENMP) endif() endif() +if(USE_LAPACK) + add_definitions(-DMXNET_USE_LAPACK=1) +else(USE_LAPACK) + # Workaround for Windows until using new Jenkinsfile. + if(USE_BLAS STREQUAL "open") + add_definitions(-DMXNET_USE_LAPACK=1) + endif() +endif() + + if(UNIX) find_library(RTLIB rt) if(RTLIB) diff --git a/Jenkinsfile b/Jenkinsfile index df39672c5ed2..f2beae0d4a31 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -157,7 +157,7 @@ USE_CPP_PACKAGE=1 \ init_git_win() bat """mkdir build_vc14_cpu cd build_vc14_cpu -cmake -G \"Visual Studio 14 2015 Win64\" -DUSE_CUDA=0 -DUSE_CUDNN=0 -DUSE_NVRTC=0 -DUSE_OPENCV=1 -DUSE_OPENMP=1 -DUSE_PROFILER=1 -DUSE_BLAS=open -DUSE_DIST_KVSTORE=0 ${env.WORKSPACE}""" +cmake -G \"Visual Studio 14 2015 Win64\" -DUSE_CUDA=0 -DUSE_CUDNN=0 -DUSE_NVRTC=0 -DUSE_OPENCV=1 -DUSE_OPENMP=1 -DUSE_PROFILER=1 -DUSE_BLAS=open -DUSE_LAPACK=1 -DUSE_DIST_KVSTORE=0 ${env.WORKSPACE}""" bat 'C:\\mxnet\\build_vc14_cpu.bat' bat '''rmdir /s/q pkg_vc14_gpu @@ -188,7 +188,7 @@ del /Q *.7z bat """mkdir build_vc14_gpu call "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\x86_amd64\\vcvarsx86_amd64.bat" cd build_vc14_gpu -cmake -G \"NMake Makefiles JOM\" -DUSE_CUDA=1 -DUSE_CUDNN=1 -DUSE_NVRTC=1 -DUSE_OPENCV=1 -DUSE_OPENMP=1 -DUSE_PROFILER=1 -DUSE_BLAS=open -DUSE_DIST_KVSTORE=0 -DCUDA_ARCH_NAME=All -DCMAKE_CXX_FLAGS_RELEASE="/FS /MD /O2 /Ob2 /DNDEBUG" -DCMAKE_BUILD_TYPE=Release ${env.WORKSPACE}""" +cmake -G \"NMake Makefiles JOM\" -DUSE_CUDA=1 -DUSE_CUDNN=1 -DUSE_NVRTC=1 -DUSE_OPENCV=1 -DUSE_OPENMP=1 -DUSE_PROFILER=1 -DUSE_BLAS=open -DUSE_LAPACK=1 -DUSE_DIST_KVSTORE=0 -DCUDA_ARCH_NAME=All -DCMAKE_CXX_FLAGS_RELEASE="/FS /MD /O2 /Ob2 /DNDEBUG" -DCMAKE_BUILD_TYPE=Release ${env.WORKSPACE}""" bat 'C:\\mxnet\\build_vc14_gpu.bat' bat '''rmdir /s/q pkg_vc14_gpu mkdir pkg_vc14_gpu\\lib diff --git a/Makefile b/Makefile index d7053fb53524..98eaba00231c 100644 --- a/Makefile +++ b/Makefile @@ -106,6 +106,16 @@ else endif endif +# lapack settings. +ifeq ($(USE_LAPACK), 1) +ifeq ($(USE_BLAS),$(filter $(USE_BLAS),openblas apple atlas mkl)) + CFLAGS += -DMXNET_USE_LAPACK +endif +ifeq ($(USE_BLAS),$(filter $(USE_BLAS),openblas atlas mkl)) + LDFLAGS += -llapack +endif +endif + ifeq ($(USE_CUDNN), 1) CFLAGS += -DMSHADOW_USE_CUDNN=1 LDFLAGS += -lcudnn diff --git a/appveyor.yml b/appveyor.yml index 54434bbf7b02..a5432b1483f5 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -52,7 +52,7 @@ before_build: set OpenCV_DIR=%APPVEYOR_BUILD_FOLDER%/%MXNET_OPENCV_DIR%/build - cmake .. -DOPENCV_DIR=%OpenCV_DIR% -DUSE_PROFILER=1 -DUSE_CUDA=0 -DUSE_CUDNN=0 -DUSE_NVRTC=0 -DUSE_OPENCV=1 -DUSE_OPENMP=1 -DUSE_BLAS=open -DUSE_DIST_KVSTORE=0 -G "Visual Studio 12 2013 Win64" + cmake .. -DOPENCV_DIR=%OpenCV_DIR% -DUSE_PROFILER=1 -DUSE_CUDA=0 -DUSE_CUDNN=0 -DUSE_NVRTC=0 -DUSE_OPENCV=1 -DUSE_OPENMP=1 -DUSE_BLAS=open -DUSE_LAPACK=1 -DUSE_DIST_KVSTORE=0 -G "Visual Studio 12 2013 Win64" build_script: - cmd: >- diff --git a/docs/Dockerfile b/docs/Dockerfile index bea556ed398a..99bb3d5be492 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -5,7 +5,7 @@ MAINTAINER Mu Li # First, build MXNet binaries (ref mxnet/docker/cpu/Dockerfile) # -RUN apt-get update && apt-get install -y build-essential git libopenblas-dev libopencv-dev +RUN apt-get update && apt-get install -y build-essential git libopenblas-dev liblapack-dev libopencv-dev RUN git clone --recursive https://github.com/dmlc/mxnet/ && cd mxnet && \ cp make/config.mk . && \ echo "USE_BLAS=openblas" >>config.mk && \ diff --git a/docs/api/python/symbol.md b/docs/api/python/symbol.md index 00402dbc3af8..14ed06b9db9b 100644 --- a/docs/api/python/symbol.md +++ b/docs/api/python/symbol.md @@ -393,6 +393,21 @@ Composite multiple symbols into a new one by an operator. argmin ``` +### Linear Algebra + +```eval_rst +.. autosummary:: + :nosignatures: + + linalg_gemm + linalg_gemm2 + linalg_potrf + linalg_potri + linalg_trmm + linalg_trsm + linalg_sumlogdiag +``` + ### Miscellaneous ```eval_rst diff --git a/docs/get_started/install.md b/docs/get_started/install.md index 28d95aa155ba..8f31d337ed22 100644 --- a/docs/get_started/install.md +++ b/docs/get_started/install.md @@ -209,9 +209,9 @@ $ sudo apt-get install -y build-essential git **Step 2** Install OpenBLAS. -*MXNet* uses [BLAS](https://en.wikipedia.org/wiki/Basic_Linear_Algebra_Subprograms) library for accelerated numerical computations on CPU machine. There are several flavors of BLAS libraries - [OpenBLAS](http://www.openblas.net/), [ATLAS](http://math-atlas.sourceforge.net/) and [MKL](https://software.intel.com/en-us/intel-mkl). In this step we install OpenBLAS. You can choose to install ATLAS or MKL. +*MXNet* uses [BLAS](https://en.wikipedia.org/wiki/Basic_Linear_Algebra_Subprograms) and [LAPACK](https://en.wikipedia.org/wiki/LAPACK) libraries for accelerated numerical computations on CPU machine. There are several flavors of BLAS/LAPACK libraries - [OpenBLAS](http://www.openblas.net/), [ATLAS](http://math-atlas.sourceforge.net/) and [MKL](https://software.intel.com/en-us/intel-mkl). In this step we install OpenBLAS. You can choose to install ATLAS or MKL. ```bash -$ sudo apt-get install -y libopenblas-dev +$ sudo apt-get install -y libopenblas-dev liblapack-dev ``` **Step 3** Install OpenCV. @@ -429,9 +429,9 @@ $ sudo apt-get install -y build-essential git ``` **Step 2** Install OpenBLAS. -*MXNet* uses [BLAS](https://en.wikipedia.org/wiki/Basic_Linear_Algebra_Subprograms) library for accelerated numerical computations. There are several flavors of BLAS libraries - [OpenBLAS](http://www.openblas.net/), [ATLAS](http://math-atlas.sourceforge.net/) and [MKL](https://software.intel.com/en-us/intel-mkl). In this step we install OpenBLAS. You can choose to install ATLAS or MKL. +*MXNet* uses [BLAS](https://en.wikipedia.org/wiki/Basic_Linear_Algebra_Subprograms) and [LAPACK](https://en.wikipedia.org/wiki/LAPACK) libraries for accelerated numerical computations on CPU machine. There are several flavors of BLAS/LAPACK libraries - [OpenBLAS](http://www.openblas.net/), [ATLAS](http://math-atlas.sourceforge.net/) and [MKL](https://software.intel.com/en-us/intel-mkl). In this step we install OpenBLAS. You can choose to install ATLAS or MKL. ```bash -$ sudo apt-get install -y libopenblas-dev +$ sudo apt-get install -y libopenblas-dev liblapack-dev ``` **Step 3** Install OpenCV. @@ -751,9 +751,9 @@ $ sudo apt-get install -y build-essential git **Step 2** Install OpenBLAS. -*MXNet* uses [BLAS](https://en.wikipedia.org/wiki/Basic_Linear_Algebra_Subprograms) library for accelerated numerical computations on CPU machine. There are several flavors of BLAS libraries - [OpenBLAS](http://www.openblas.net/), [ATLAS](http://math-atlas.sourceforge.net/) and [MKL](https://software.intel.com/en-us/intel-mkl). In this step we install OpenBLAS. You can choose to install ATLAS or MKL. +*MXNet* uses [BLAS](https://en.wikipedia.org/wiki/Basic_Linear_Algebra_Subprograms) and [LAPACK](https://en.wikipedia.org/wiki/LAPACK) libraries for accelerated numerical computations on CPU machine. There are several flavors of BLAS/LAPACK libraries - [OpenBLAS](http://www.openblas.net/), [ATLAS](http://math-atlas.sourceforge.net/) and [MKL](https://software.intel.com/en-us/intel-mkl). In this step we install OpenBLAS. You can choose to install ATLAS or MKL. ```bash -$ sudo apt-get install -y libopenblas-dev +$ sudo apt-get install -y libopenblas-dev liblapack-dev ``` **Step 3** Install OpenCV. @@ -823,9 +823,9 @@ $ sudo apt-get install -y build-essential git ``` **Step 2** Install OpenBLAS. -*MXNet* uses [BLAS](https://en.wikipedia.org/wiki/Basic_Linear_Algebra_Subprograms) library for accelerated numerical computations. There are several flavors of BLAS libraries - [OpenBLAS](http://www.openblas.net/), [ATLAS](http://math-atlas.sourceforge.net/) and [MKL](https://software.intel.com/en-us/intel-mkl). In this step we install OpenBLAS. You can choose to install ATLAS or MKL. +*MXNet* uses [BLAS](https://en.wikipedia.org/wiki/Basic_Linear_Algebra_Subprograms) and [LAPACK](https://en.wikipedia.org/wiki/LAPACK) libraries for accelerated numerical computations on CPU machine. There are several flavors of BLAS/LAPACK libraries - [OpenBLAS](http://www.openblas.net/), [ATLAS](http://math-atlas.sourceforge.net/) and [MKL](https://software.intel.com/en-us/intel-mkl). In this step we install OpenBLAS. You can choose to install ATLAS or MKL. ```bash -$ sudo apt-get install -y libopenblas-dev +$ sudo apt-get install -y libopenblas-dev liblapack-dev ``` **Step 3** Install OpenCV. diff --git a/include/mxnet/c_lapack_api.h b/include/mxnet/c_lapack_api.h new file mode 100644 index 000000000000..2e3ff49760b0 --- /dev/null +++ b/include/mxnet/c_lapack_api.h @@ -0,0 +1,91 @@ +/*! + * Copyright (c) 2017 by Contributors + * \file c_lapack_api.h + * \brief Unified interface for LAPACK calls from within mxnet. + * Purpose is to hide the platform specific differences. + */ +#ifndef MXNET_C_LAPACK_API_H_ +#define MXNET_C_LAPACK_API_H_ + +// Manually maintained list of LAPACK interfaces that can be used +// within MXNET. Conventions: +// - Interfaces must be compliant with lapacke.h in terms of signature and +// naming conventions so wrapping a function "foo" which has the +// signature +// lapack_int LAPACKE_foo(int, char, lapack_int, float* , lapack_int) +// within lapacke.h should result in a wrapper with the following signature +// int MXNET_LAPACK_foo(int, char, int, float* , int) +// Note that function signatures in lapacke.h will always have as first +// argument the storage order (row/col-major). All wrappers have to support +// that argument. The underlying fortran functions will always assume a +// column-major layout. It is the responsibility of the wrapper function +// to handle the (usual) case that it is called with data in row-major +// format, either by doing appropriate transpositions explicitly or using +// transposition options of the underlying fortran function. +// - It is ok to assume that matrices are stored in contiguous memory +// (which removes the need to do special handling for lda/ldb parameters +// and enables us to save additional matrix transpositions around +// the fortran calls). +// - It is desired to add some basic checking in the C++-wrappers in order +// to catch simple mistakes when calling these wrappers. +// - Must support compilation without lapack-package but issue runtime error in this case. + +#include + +extern "C" { + // Fortran signatures + #define MXNET_LAPACK_FSIGNATURE1(func, dtype) \ + void func##_(char* uplo, int* n, dtype* a, int* lda, int *info); + + MXNET_LAPACK_FSIGNATURE1(spotrf, float) + MXNET_LAPACK_FSIGNATURE1(dpotrf, double) + MXNET_LAPACK_FSIGNATURE1(spotri, float) + MXNET_LAPACK_FSIGNATURE1(dpotri, double) +} + +#define MXNET_LAPACK_ROW_MAJOR 101 +#define MXNET_LAPACK_COL_MAJOR 102 + +#define CHECK_LAPACK_CONTIGUOUS(a, b) \ + CHECK_EQ(a, b) << "non contiguous memory for array in lapack call"; + +#define CHECK_LAPACK_UPLO(a) \ + CHECK(a == 'U' || a == 'L') << "neither L nor U specified as triangle in lapack call"; + +inline char loup(char uplo, bool invert) { return invert ? (uplo == 'U' ? 'L' : 'U') : uplo; } + +#if MXNET_USE_LAPACK + + #define MXNET_LAPACK_CWRAPPER1(func, dtype) \ + inline int MXNET_LAPACK_##func(int matrix_layout, char uplo, int n, dtype* a, int lda ) { \ + CHECK_LAPACK_CONTIGUOUS(n, lda); \ + CHECK_LAPACK_UPLO(uplo); \ + char o(loup(uplo, (matrix_layout == MXNET_LAPACK_ROW_MAJOR))); \ + int ret(0); \ + func##_(&o, &n, a, &lda, &ret); \ + return ret; \ + } + MXNET_LAPACK_CWRAPPER1(spotrf, float) + MXNET_LAPACK_CWRAPPER1(dpotrf, double) + MXNET_LAPACK_CWRAPPER1(spotri, float) + MXNET_LAPACK_CWRAPPER1(dpotri, double) + +#else + // use pragma message instead of warning + #pragma message("Warning: lapack usage not enabled, linalg-operators will be not available." \ + " Build with USE_LAPACK=1 to get lapack functionalities.") + + // Define compilable stubs. + #define MXNET_LAPACK_CWRAPPER1(func, dtype) \ + inline int MXNET_LAPACK_##func(int matrix_layout, char uplo, int n, dtype* a, int lda ) { \ + LOG(FATAL) << "MXNet build without lapack. Function " << #func << " is not available."; \ + return 1; \ + } + MXNET_LAPACK_CWRAPPER1(spotrf, float) + MXNET_LAPACK_CWRAPPER1(dpotrf, double) + MXNET_LAPACK_CWRAPPER1(spotri, float) + MXNET_LAPACK_CWRAPPER1(dpotri, double) + +#endif + +#endif // MXNET_C_LAPACK_API_H_ diff --git a/include/mxnet/tensor_blob.h b/include/mxnet/tensor_blob.h index 9a9774acf14d..1928aa48c85f 100755 --- a/include/mxnet/tensor_blob.h +++ b/include/mxnet/tensor_blob.h @@ -291,6 +291,34 @@ class TBlob { return this->get_with_shape( this->shape_.FlatTo3D(axis_begin, axis_end), stream); } + /*! + * \brief flatten the tensor to specified number of dimensions, + * collapse the highest dimensions or pad with higher dimensions + * \param stream the possible stream target tensor should reside on + * \tparam Device which device the tensor is on + * \tparam dim desired number of dimensions of returned tensor + * \tparam DType the type of elements in the tensor + * \return tensor after flatten + */ + template + inline mshadow::Tensor FlatToKD( + mshadow::Stream *stream = NULL) const { + mshadow::Shape shape; + shape[0] = 1; + // Pad higher dimensions in case dim > ndim() + for (int i = 0; i < dim - ndim(); ++i) { + shape[i] = 1; + } + // Collapse higher dimensions in case dim < ndim() + for (int i = 0; i < ndim() - dim + 1; ++i) { + shape[0] *= shape_[i]; + } + // Preserve lower dimensions. + for (int i = std::max(0, ndim() - dim + 1); i < ndim(); ++i) { + shape[i - ndim() + dim] = shape_[i]; + } + return this->get_with_shape(shape, stream); + } private: static DLDataType DTypeTransform(int type_flag) { diff --git a/make/config.mk b/make/config.mk index ec52db770f7e..5a0c64c62598 100644 --- a/make/config.mk +++ b/make/config.mk @@ -65,6 +65,9 @@ USE_OPENCV = 1 # use openmp for parallelization USE_OPENMP = 1 +# whether use lapack during compilation +# only effective when compiled with blas versions openblas/apple/atlas/mkl +USE_LAPACK = 0 # MKL ML Library for Intel CPU/Xeon Phi # Please refer to MKL_README.md for details diff --git a/make/osx.mk b/make/osx.mk index 115db34da730..01e50c2e76bb 100644 --- a/make/osx.mk +++ b/make/osx.mk @@ -62,6 +62,10 @@ USE_OPENCV = 1 # use openmp for parallelization USE_OPENMP = 0 +# whether use lapack during compilation +# only effective when compiled with blas versions openblas/apple/atlas/mkl +USE_LAPACK = 0 + # choose the version of blas you want to use # can be: mkl, blas, atlas, openblas USE_BLAS = apple diff --git a/make/pip_linux_cpu.mk b/make/pip_linux_cpu.mk index f8b255e6deb8..d7e5fa862246 100644 --- a/make/pip_linux_cpu.mk +++ b/make/pip_linux_cpu.mk @@ -29,6 +29,11 @@ ADD_CFLAGS += -Ldeps/lib -Ideps/include # matrix computation libraries for CPU/GPU #--------------------------------------------- +# whether use lapack during compilation +# only effective when compiled with blas versions openblas/apple/atlas/mkl +# you can disable it, however, you will not be able to use linalg-operators +USE_LAPACK = 0 + # choose the version of blas you want to use # can be: mkl, blas, atlas, openblas # in default use atlas for linux while apple for osx diff --git a/make/readthedocs.mk b/make/readthedocs.mk index b14c4baf7482..5a33855a5e5c 100644 --- a/make/readthedocs.mk +++ b/make/readthedocs.mk @@ -32,6 +32,8 @@ USE_OPENMP = 0 # can be: mkl, blas, atlas, openblas USE_STATIC_MKL = NONE USE_BLAS = NONE +USE_LAPACK = 0 + # # add path to intel library, you may need it # for MKL, if you did not add the path to environment variable diff --git a/src/operator/elemwise_op_common.h b/src/operator/elemwise_op_common.h index def38126d08c..b7e87cf2bc70 100644 --- a/src/operator/elemwise_op_common.h +++ b/src/operator/elemwise_op_common.h @@ -96,6 +96,23 @@ struct ElemwiseGradUseOut { } }; +// Transfer gradient and input and output to FGradient function +struct ElemwiseGradUseInOut { + const char *op_name; + std::vector operator()(const nnvm::NodePtr& n, + const std::vector& ograds) { + std::vector heads(ograds.begin(), ograds.end()); + for (auto& h : n->inputs) { + heads.push_back(h); + } + index_t n_out = n->num_outputs(); + for (index_t i = 0; i < n_out; ++i) { + heads.emplace_back(nnvm::NodeEntry{n, i, 0}); + } + return MakeGradNode(op_name, n, heads, n->attrs.dict); + } +}; + // Transfer only gradient to FGradient function struct ElemwiseGradUseNone { const char *op_name; diff --git a/src/operator/tensor/la_op.cc b/src/operator/tensor/la_op.cc new file mode 100644 index 000000000000..47582d6a8bfe --- /dev/null +++ b/src/operator/tensor/la_op.cc @@ -0,0 +1,399 @@ +/*! + * Copyright (c) 2017 by Contributors + * \file la_op.cc + * \brief CPU-Operators for advanced linear algebra. + */ +#include "./la_op.h" +#include "./la_op_inline.h" + +namespace mxnet { +namespace op { + +DMLC_REGISTER_PARAMETER(LaMatrixMacParam); +DMLC_REGISTER_PARAMETER(LaMatrixMultParam); +DMLC_REGISTER_PARAMETER(LaTriangMatrixMultParam); + +NNVM_REGISTER_OP(linalg_gemm) +.describe(R"code(Performs general matrix multiplication and accumulation. +Input are three tensors *A*, *B*, *C* each of dimension *n >= 2* and each +having the same shape on the leading *n-2* dimensions. For every *n-2* dimensional index *i* let +*A*\ :sub:`i`\ , *B*\ :sub:`i`\ , *C*\ :sub:`i` be the matrices given by the last *2* dimensions. +The operator performs the BLAS3 function *gemm* + + *out*\ :sub:`i` = *alpha* \* *op*\ (*A*\ :sub:`i`\ ) \* *op*\ (*B*\ :sub:`i`\ ) + *beta* \* *C*\ :sub:`i` + +on all such triples of matrices. Here *alpha* and *beta* are scalar operator parameters and *op()* +is either the identity or the matrix transposition. + +In case of *n=2*, a single *gemm* function is performed on the matrices *A*, *B*, *C*. + +.. note:: The operator does only support float32 and float64 data types and provides + proper backward gradients. + +Examples:: + + // Single matrix multiply-add + A = [[1.0, 1.0], [1.0, 1.0]] + B = [[1.0, 1.0], [1.0, 1.0], [1.0, 1.0]] + C = [[1.0, 1.0, 1.0], [1.0, 1.0, 1.0]] + linalg_gemm(A, B, C, transpose_b = 1, alpha = 2.0 , beta = 10.0) + = [[14.0, 14.0, 14.0], [14.0, 14.0, 14.0]] + + // Batch matrix multiply-add + A = [[[1.0, 1.0]], [[0.1, 0.1]]] + B = [[[1.0, 1.0]], [[0.1, 0.1]]] + C = [[[10.0]], [[0.01]]] + linalg_gemm(A, B, C, transpose_b = 1, alpha = 2.0 , beta = 10.0) + = [[[104.0]], [[0.14]]] +)code" ADD_FILELINE) +.set_num_inputs(3) +.set_num_outputs(1) +.set_attr_parser(ParamParser) +.set_attr("FListInputNames", [](const NodeAttrs& attrs) + { return std::vector{"A", "B", "C"}; } ) +.set_attr("FInferShape", LaMatrixMultMacOpShape) +.set_attr("FInferType", ElemwiseType<3, 1>) +.set_attr("FInplaceOption", [](const NodeAttrs& attrs) + { return std::vector>{{2, 0}}; }) +.set_attr("FCompute", LaOpForward) +.set_attr("FGradient", ElemwiseGradUseIn{"_backward_linalg_gemm"}) +.add_argument("A", "NDArray-or-Symbol", "Tensor of input matrices") +.add_argument("B", "NDArray-or-Symbol", "Tensor of input matrices") +.add_argument("C", "NDArray-or-Symbol", "Tensor of input matrices") +.add_arguments(LaMatrixMacParam::__FIELDS__()); + +NNVM_REGISTER_OP(_backward_linalg_gemm) +.set_num_inputs(4) +.set_num_outputs(3) +.set_attr_parser(ParamParser) +.set_attr("FInplaceOption", [](const NodeAttrs& attrs) + { return std::vector >{{1, 0}, {2, 1}, {3, 2}}; }) +.set_attr("FResourceRequest", [](const NodeAttrs& attrs) + { return std::vector{ResourceRequest::kTempSpace}; }) +.set_attr("TIsBackward", true) +.set_attr("FCompute", LaOpBackward); + +NNVM_REGISTER_OP(linalg_gemm2) +.describe(R"code(Performs general matrix multiplication. +Input are two tensors *A*, *B* each of dimension *n >= 2* and each +having the same shape on the leading *n-2* dimensions. For every *n-2* dimensional index *i* let +*A*\ :sub:`i`\ , *B*\ :sub:`i`\ be the matrices given by the last *2* dimensions. +The operator performs the BLAS3 function *gemm* (restricted to two arguments) + + *out*\ :sub:`i` = *alpha* \* *op*\ (*A*\ :sub:`i`\ ) \* *op*\ (*B*\ :sub:`i`\ ) + +on all such pairs of matrices. Here *alpha* is a scalar operator parameter and *op()* is either +the identity or the matrix transposition. + +In case of *n=2*, a single *gemm* function is performed on the matrices *A*, *B*. + +.. note:: The operator does only support float32 and float64 data types and provides + proper backward gradients. + +Examples:: + + // Single matrix multiply + A = [[1.0, 1.0], [1.0, 1.0]] + B = [[1.0, 1.0], [1.0, 1.0], [1.0, 1.0]] + linalg_gemm2(A, B, transpose_b = 1, alpha = 2.0) + = [[4.0, 4.0, 4.0], [4.0, 4.0, 4.0]] + + // Batch matrix multiply + A = [[[1.0, 1.0]], [[0.1, 0.1]]] + B = [[[1.0, 1.0]], [[0.1, 0.1]]] + linalg_gemm2(A, B, transpose_b = 1, alpha = 2.0 ) + = [[[4.0]], [[0.04 ]]] +)code" ADD_FILELINE) +.set_num_inputs(2) +.set_num_outputs(1) +.set_attr_parser(ParamParser) +.set_attr("FListInputNames", [](const NodeAttrs& attrs) + { return std::vector{"A", "B"}; } ) +.set_attr("FInferShape", LaMatrixMultMacOpShape) +.set_attr("FInferType", ElemwiseType<2, 1>) +.set_attr("FCompute", LaOpForward) +.set_attr("FGradient", ElemwiseGradUseIn{"_backward_linalg_gemm2"}) +.add_argument("A", "NDArray-or-Symbol", "Tensor of input matrices") +.add_argument("B", "NDArray-or-Symbol", "Tensor of input matrices") +.add_arguments(LaMatrixMultParam::__FIELDS__()); + +NNVM_REGISTER_OP(_backward_linalg_gemm2) +.set_num_inputs(3) +.set_num_outputs(2) +.set_attr_parser(ParamParser) +.set_attr("FInplaceOption", [](const NodeAttrs& attrs) + { return std::vector >{{1, 0}, {2, 1}}; }) +.set_attr("FResourceRequest", [](const NodeAttrs& attrs) + { return std::vector{ResourceRequest::kTempSpace}; }) +.set_attr("TIsBackward", true) +.set_attr("FCompute", LaOpBackward); + +NNVM_REGISTER_OP(linalg_potrf) +.describe(R"code(Performs Cholesky factorization of a symmetric positive-definite matrix. +Input is a tensor *A* of dimension *n >= 2*. For every *n-2* dimensional index *i* let +*A*\ :sub:`i`\ be the matrix given by the last *2* dimensions. +The operator performs the Cholesky factorization (LAPACK function *potrf*) +on each *A*\ :sub:`i`\ , +i.e. it computes a lower triangular matrix *U*\ :sub:`i` such that + + *A*\ :sub:`i`\ = *U*\ :sub:`i`\ \* *U*\ :sub:`i`\ \ :sup:`T` + +for all such matrices. The matrices *A*\ :sub:`i` must be all symmetric and positive-definite. +The resulting matrices *U*\ :sub:`i` will contain zeros in the upper triangle +apart from the diagonal. + +In case of *n=2*, a single Cholesky factorization is performed on the matrix *A*. + +.. note:: The operator does only support float32 and float64 data types and provides + proper backward gradients. + +Examples:: + + // Single matrix factorization + A = [[4.0, 1.0], [1.0, 4.25]] + linalg_potrf(A) = [[2.0, 0], [0.5, 2.0]] + + // Batch matrix factorization + A = [[[4.0, 1.0], [1.0, 4.25]], [[16.0, 4.0], [4.0, 17.0]]] + linalg_potrf(A) = [[[2.0, 0], [0.5, 2.0]], [[4.0, 0], [1.0, 4.0]]] +)code" ADD_FILELINE) +.set_num_inputs(1) +.set_num_outputs(1) +.set_attr("FListInputNames", [](const NodeAttrs& attrs) + { return std::vector{"A"}; } ) +.set_attr("FInferShape", ElemwiseShape<1, 1>) +.set_attr("FInferType", ElemwiseType<1, 1>) +.set_attr("FInplaceOption", [](const NodeAttrs& attrs) + { return std::vector>{{0, 0}}; }) +.set_attr("FCompute", LaOpForward) +.set_attr("FGradient", ElemwiseGradUseOut{"_backward_linalg_potrf"}) +.add_argument("A", "NDArray-or-Symbol", "Tensor of input matrices to be decomposed"); + +NNVM_REGISTER_OP(_backward_linalg_potrf) +.set_num_inputs(2) +.set_num_outputs(1) +.set_attr("FInplaceOption", [](const NodeAttrs& attrs) + { return std::vector >{{0, 0}}; }) +.set_attr("FResourceRequest", [](const NodeAttrs& attrs) + { return std::vector{ResourceRequest::kTempSpace}; }) +.set_attr("TIsBackward", true) +.set_attr("FCompute", LaOpBackward); + + +NNVM_REGISTER_OP(linalg_potri) +.describe(R"code(Performs matrix inversion from a Cholesky factorization. +Input is a tensor *A* of dimension *n >= 2*. For every *n-2* dimensional index *i* let +*A*\ :sub:`i`\ be the matrix given by the last *2* dimensions. +The operator assumes that each *A*\ :sub:`i` is the Cholesky factorization of some symmetric +positive-definite matrix *B*\ :sub:`i` given as a lower triangular matrix +(so *A* is the output of a prior call to operator *linalg_potrf*). The operator computes the +inverse of each *B*\ :sub:`i` from this decomposition, i.e + + *out*\ :sub:`i` = *B*\ :sub:`i`\ \ :sup:`-1` + +for all such matrices. + +In case of *n=2*, the operation is performed on the matrix *A* itself. + +.. note:: The operator does only support float32 and float64 data types and provides + proper backward gradients. + +Examples:: + + // Single matrix inverse + A = [[2.0, 0], [0.5, 2.0]] + linalg_potri(A) = [[0.26563, -0.0625], [-0.0625, 0.25]] + + // Batch matrix inverse + A = [[[2.0, 0], [0.5, 2.0]], [[4.0, 0], [1.0, 4.0]]] + linalg_potri(A) = [[[0.26563, -0.0625], [-0.0625, 0.25]], + [[0.06641, -0.01562], [-0.01562, 0,0625]]] +)code" ADD_FILELINE) +.set_num_inputs(1) +.set_num_outputs(1) +.set_attr("FListInputNames", [](const NodeAttrs& attrs) + { return std::vector{"A"}; } ) +.set_attr("FInferShape", ElemwiseShape<1, 1>) +.set_attr("FInferType", ElemwiseType<1, 1>) +.set_attr("FInplaceOption", [](const NodeAttrs& attrs) + { return std::vector>{{0, 0}}; }) +.set_attr("FCompute", LaOpForward) +.set_attr("FGradient", ElemwiseGradUseInOut{"_backward_linalg_potri"}) +.add_argument("A", "NDArray-or-Symbol", "Tensor of lower triangular matrices"); + +NNVM_REGISTER_OP(_backward_linalg_potri) +.set_num_inputs(3) +.set_num_outputs(1) +.set_attr("FResourceRequest", [](const NodeAttrs& attrs) + { return std::vector{ResourceRequest::kTempSpace}; }) +.set_attr("TIsBackward", true) +.set_attr("FCompute", LaOpBackward); + +NNVM_REGISTER_OP(linalg_trmm) +.describe(R"code(Performs multiplication with a triangular matrix. +Input are two tensors *A*, *B* each of dimension *n >= 2* and each +having the same shape on the leading *n-2* dimensions. For every *n-2* dimensional index *i* let +*A*\ :sub:`i`\ , *B*\ :sub:`i`\ be the matrices given by the last *2* dimensions. +The operator performs the BLAS3 function *trmm* + + *out*\ :sub:`i` = *alpha* \* *op*\ (*A*\ :sub:`i`\ ) \* *B*\ :sub:`i` + +or + + *out*\ :sub:`i` = *alpha* \* *B*\ :sub:`i` \* *op*\ (*A*\ :sub:`i`\ ) + +on all such pairs of matrices. Here *alpha* is a scalar operator parameter, *op()* is either +the identity or the matrix transposition (depending on the parameter *transpose*) and the +order of matrix multiplication depends on the parameter *rightside*. +All matrices *A*\ :sub:`i` must be lower triangular. + +In case of *n=2*, a single *trmm* function is performed on the matrices *A*, *B*. + +.. note:: The operator does only support float32 and float64 data types and provides + proper backward gradients. + +Examples:: + + // Single matrix multiply + A = [[1.0, 0], [1.0, 1.0]] + B = [[1.0, 1.0, 1.0], [1.0, 1.0, 1.0]] + linalg_trmm(A, B, alpha = 2.0) = [[2.0, 2.0, 2.0], [4.0, 4.0, 4.0]] + + // Batch matrix multiply + A = [[[1.0, 0], [1.0, 1.0]], [[1.0, 0], [1.0, 1.0]]] + B = [[[1.0, 1.0, 1.0], [1.0, 1.0, 1.0]], [[0.5, 0.5, 0.5], [0.5, 0.5, 0.5]]] + linalg_trmm(A, B, alpha = 2.0 ) = [[[2.0, 2.0, 2.0], [4.0, 4.0, 4.0]], + [[1.0, 1.0, 1.0], [2.0, 2.0, 2.0]]] + +)code" ADD_FILELINE) +.set_num_inputs(2) +.set_num_outputs(1) +.set_attr_parser(ParamParser) +.set_attr("FListInputNames", [](const NodeAttrs& attrs) + { return std::vector{"A", "B"}; } ) +.set_attr("FInferShape", LaTriangMatrixMultOpShape) +.set_attr("FInferType", ElemwiseType<2, 1>) +.set_attr("FInplaceOption", [](const NodeAttrs& attrs) + { return std::vector>{{1, 0}}; }) +.set_attr("FCompute", LaOpForward) +.set_attr("FGradient", ElemwiseGradUseInOut{"_backward_linalg_trmm"}) +.add_argument("A", "NDArray-or-Symbol", "Tensor of lower triangular matrices") +.add_argument("B", "NDArray-or-Symbol", "Tensor of matrices") +.add_arguments(LaTriangMatrixMultParam::__FIELDS__()); + +NNVM_REGISTER_OP(_backward_linalg_trmm) +.set_num_inputs(4) +.set_num_outputs(2) +.set_attr_parser(ParamParser) +.set_attr("FInplaceOption", [](const NodeAttrs& attrs) + { return std::vector >{{0, 1}}; }) +.set_attr("FResourceRequest", [](const NodeAttrs& attrs) + { return std::vector{ResourceRequest::kTempSpace}; }) +.set_attr("TIsBackward", true) +.set_attr("FCompute", LaOpBackward); + +NNVM_REGISTER_OP(linalg_trsm) +.describe(R"code(Solves matrix equations involving a triangular matrix. +Input are two tensors *A*, *B* each of dimension *n >= 2* and each +having the same shape on the leading *n-2* dimensions. For every *n-2* dimensional index *i* let +*A*\ :sub:`i`\ , *B*\ :sub:`i`\ be the matrices given by the last *2* dimensions. +The operator performs the BLAS3 function *trsm*, i.e. it solves the equation + + *op*\ (*A*\ :sub:`i`\ ) \* *X*\ :sub:`i` = *alpha* \* *B*\ :sub:`i` + +or + + *X*\ :sub:`i` \* *op*\ (*A*\ :sub:`i`\ ) = *alpha* \* *B*\ :sub:`i` + +on all such pairs of matrices. Here *alpha* is a scalar operator parameter, *op()* is either +the identity or the matrix transposition (depending on the parameter *transpose*) and the +order of multiplication on the left depends on the parameter *rightside*. +All matrices *A*\ :sub:`i` must be lower triangular. + +In case of *n=2*, a single *trsm* function is performed on the matrices *A*, *B*. + +.. note:: The operator does only support float32 and float64 data types and provides + proper backward gradients. + +Examples:: + + // Single matrix solve + A = [[1.0, 0], [1.0, 1.0]] + B = [[2.0, 2.0, 2.0], [4.0, 4.0, 4.0]] + linalg_trsm(A, B, alpha = 0.5) = [[1.0, 1.0, 1.0], [1.0, 1.0, 1.0]] + + // Batch matrix solve + A = [[[1.0, 0], [1.0, 1.0]], [[1.0, 0], [1.0, 1.0]]] + B = [[[2.0, 2.0, 2.0], [4.0, 4.0, 4.0]], + [[4.0, 4.0, 4.0], [8.0, 8.0, 8.0]]] + linalg_trsm(A, B, alpha = 0.5 ) = [[[1.0, 1.0, 1.0], [1.0, 1.0, 1.0]], + [[2.0, 2.0, 2.0 ], [2.0, 2.0, 2.0]]] +)code" ADD_FILELINE) +.set_num_inputs(2) +.set_num_outputs(1) +.set_attr_parser(ParamParser) +.set_attr("FListInputNames", [](const NodeAttrs& attrs) + { return std::vector{"A", "B"}; } ) +.set_attr("FInferShape", LaTriangMatrixMultOpShape) +.set_attr("FInferType", ElemwiseType<2, 1>) +.set_attr("FInplaceOption", [](const NodeAttrs& attrs) + { return std::vector>{{1, 0}}; }) +.set_attr("FCompute", LaOpForward) +.set_attr("FGradient", ElemwiseGradUseInOut{"_backward_linalg_trsm"}) +.add_argument("A", "NDArray-or-Symbol", "Tensor of lower triangular matrices") +.add_argument("B", "NDArray-or-Symbol", "Tensor of matrices") +.add_arguments(LaTriangMatrixMultParam::__FIELDS__()); + +NNVM_REGISTER_OP(_backward_linalg_trsm) +.set_num_inputs(4) +.set_num_outputs(2) +.set_attr_parser(ParamParser) +.set_attr("FInplaceOption", [](const NodeAttrs& attrs) + { return std::vector >{{0, 1}}; }) +.set_attr("FResourceRequest", [](const NodeAttrs& attrs) + { return std::vector{ResourceRequest::kTempSpace}; }) +.set_attr("TIsBackward", true) +.set_attr("FCompute", LaOpBackward); + +NNVM_REGISTER_OP(linalg_sumlogdiag) +.describe(R"code(Computes the sum of the logarithms of all diagonal elements in a matrix. +Input is a tensor *A* of dimension *n >= 2*. For every *n-2* dimensional index *i* let +*A*\ :sub:`i`\ be the matrix given by the last *2* dimensions. +The operator performs a reduction of each such matrix to a scalar by summing up the logarithms +of all diagonal elements. All matrices must be square and all diagonal elements must be positive. + +In case of *n=2*, *A* represents a single matrix on which the reduction will be performed. + +.. note:: The operator does only support float32 and float64 data types and provides + proper backward gradients. + +Examples:: + + // Single matrix reduction + A = [[1.0, 1.0], [1.0, 7.0]] + linalg_sumlogdiag(A) = [1.9459] + + // Batch matrix reduction + A = [[[1.0, 1.0], [1.0, 7.0]], [[3.0, 0], [0, 17.0]]] + linalg_sumlogdiag(A) = [1.9459, 3.9318] +)code" ADD_FILELINE) +.set_num_inputs(1) +.set_num_outputs(1) +.set_attr("FListInputNames", [](const NodeAttrs& attrs) + { return std::vector{"A"}; } ) +.set_attr("FInferShape", LaReduceShape<2>) +.set_attr("FInferType", ElemwiseType<1, 1>) +.set_attr("FCompute", LaReduceForward) +.set_attr("FGradient", ElemwiseGradUseIn{"_backward_linalg_sumlogdiag"}) +.add_argument("A", "NDArray-or-Symbol", "Tensor of square matrices"); + +NNVM_REGISTER_OP(_backward_linalg_sumlogdiag) +.set_num_inputs(2) +.set_num_outputs(1) +.set_attr("FResourceRequest", [](const NodeAttrs& attrs) + { return std::vector{ResourceRequest::kTempSpace}; }) +.set_attr("TIsBackward", true) +.set_attr("FCompute", LaReduceBackward); + +} // namespace op +} // namespace mxnet diff --git a/src/operator/tensor/la_op.h b/src/operator/tensor/la_op.h new file mode 100644 index 000000000000..488a3edd6bd9 --- /dev/null +++ b/src/operator/tensor/la_op.h @@ -0,0 +1,420 @@ +/*! + * Copyright (c) 2017 by Contributors + * \file la_op.h + * \brief Operators for advanced linear algebra. + */ +#ifndef MXNET_OPERATOR_TENSOR_LA_OP_H_ +#define MXNET_OPERATOR_TENSOR_LA_OP_H_ + +#include +#include +#include +#include "../mshadow_op.h" +#include "../mxnet_op.h" +#include "../operator_common.h" +#include "../elemwise_op_common.h" + +namespace mxnet { +namespace op { + +// Parameters for general matrix-matrix multiply-accumulate (mac) +struct LaMatrixMacParam : public dmlc::Parameter { + bool transpose_a, transpose_b; + double alpha, beta; + DMLC_DECLARE_PARAMETER(LaMatrixMacParam) { + DMLC_DECLARE_FIELD(transpose_a) + .set_default(false) + .describe("Multiply with transposed of first input (A)."); + DMLC_DECLARE_FIELD(transpose_b) + .set_default(false) + .describe("Multiply with transposed of second input (B)."); + DMLC_DECLARE_FIELD(alpha) + .set_default(1.0) + .describe("Scalar factor multiplied with A*B."); + DMLC_DECLARE_FIELD(beta) + .set_default(1.0) + .describe("Scalar factor multiplied with C."); + } +}; + +// Parameters for general matrix-matrix multiply +struct LaMatrixMultParam : public dmlc::Parameter { + bool transpose_a, transpose_b; + double alpha; + DMLC_DECLARE_PARAMETER(LaMatrixMultParam) { + DMLC_DECLARE_FIELD(transpose_a) + .set_default(false) + .describe("Multiply with transposed of first input (A)."); + DMLC_DECLARE_FIELD(transpose_b) + .set_default(false) + .describe("Multiply with transposed of second input (B)."); + DMLC_DECLARE_FIELD(alpha) + .set_default(1.0) + .describe("Scalar factor multiplied with A*B."); + } +}; + +// Parameters for matrix-matrix multiplication where one is a triangular matrix. +struct LaTriangMatrixMultParam : public dmlc::Parameter { + bool transpose; + bool rightside; + double alpha; + DMLC_DECLARE_PARAMETER(LaTriangMatrixMultParam) { + DMLC_DECLARE_FIELD(transpose) + .set_default(false) + .describe("Use transposed of the triangular matrix"); + DMLC_DECLARE_FIELD(rightside) + .set_default(false) + .describe("Multiply triangular matrix from the right to non-triangular one."); + DMLC_DECLARE_FIELD(alpha) + .set_default(1.0) + .describe("Scalar factor to be applied to the result."); + } +}; + +// Common function for shape inference for matrix mult and matrix mac. +bool LaMatrixMultMacOpShape(const nnvm::NodeAttrs& attrs, + std::vector* in_attrs, + std::vector* out_attrs) { + CHECK_GE(in_attrs->size(), 2); + CHECK_EQ(out_attrs->size(), 1); + bool transpose_a(false), transpose_b(false); + if ( in_attrs->size() == 2 ) { + // Matrix-Matrix mult + transpose_a = nnvm::get(attrs.parsed).transpose_a; + transpose_b = nnvm::get(attrs.parsed).transpose_b; + } else { + // Matrix-Matrix mac + transpose_a = nnvm::get(attrs.parsed).transpose_a; + transpose_b = nnvm::get(attrs.parsed).transpose_b; + } + if ( (*in_attrs)[0].ndim() >= 2 && (*in_attrs)[0].ndim() == (*in_attrs)[1].ndim() ) { + // Forward shape inference. + const int ndim((*in_attrs)[0].ndim()); + std::vector oshape(ndim); + for ( int i = 0; i < ndim-2; ++i ) { + // Both inputs must have same shape except for last two dimensions. + if ( (*in_attrs)[0][i] != (*in_attrs)[1][i] ) return false; + oshape[i] = (*in_attrs)[0][i]; + } + CHECK_EQ((transpose_a ? (*in_attrs)[0][ndim-2] : (*in_attrs)[0][ndim-1]), + (transpose_b ? (*in_attrs)[1][ndim-1] : (*in_attrs)[1][ndim-2])) + << "Incompatible matrix dimensions for multiplication"; + oshape[ndim-2] = (transpose_a ? (*in_attrs)[0][ndim-1] : (*in_attrs)[0][ndim-2]); + oshape[ndim-1] = (transpose_b ? (*in_attrs)[1][ndim-2] : (*in_attrs)[1][ndim-1]); + TShape tshape(oshape.begin(), oshape.end()); + SHAPE_ASSIGN_CHECK(*out_attrs, 0, tshape); + if ( in_attrs->size() > 2 ) { + // Infer/check shape of third operand of a mac. + SHAPE_ASSIGN_CHECK(*in_attrs, 2, tshape); + } + return true; + } + // Can't do backward inference of shapes for this operator. + return false; +} + +bool LaTriangMatrixMultOpShape(const nnvm::NodeAttrs& attrs, + std::vector* in_attrs, + std::vector* out_attrs) { + const LaTriangMatrixMultParam& param = nnvm::get(attrs.parsed); + CHECK_EQ(in_attrs->size(), 2); + CHECK_EQ(out_attrs->size(), 1); + if ( (*in_attrs)[0].ndim() >= 2 && (*in_attrs)[0].ndim() == (*in_attrs)[1].ndim() ) { + // Forward shape inference. + const int ndim((*in_attrs)[0].ndim()); + CHECK_EQ((*in_attrs)[0][ndim-2], (*in_attrs)[0][ndim-1]) + << "First operand must be a tensor of square matrices"; + std::vector oshape(ndim); + for ( int i = 0; i < ndim-2; ++i ) { + // Must have same shape except for last two dimensions. + if ( (*in_attrs)[0][i] != (*in_attrs)[1][i] ) return false; + oshape[i] = (*in_attrs)[0][i]; + } + if ( param.rightside ) { + // We compute B * A where A is the first and B the second input. + CHECK_EQ((*in_attrs)[0][ndim-2], (*in_attrs)[1][ndim-1]) + << "Incompatible matrix dimensions for multiplication"; + oshape[ndim-2] = (*in_attrs)[1][ndim-2]; + oshape[ndim-1] = (param.transpose ? (*in_attrs)[0][ndim-2] : (*in_attrs)[0][ndim-1]); + } else { + // We compute A * B where A is the first and B the second input. + CHECK_EQ((*in_attrs)[1][ndim-2], (*in_attrs)[0][ndim-1]) + << "Incompatible matrix dimensions for multiplication"; + oshape[ndim-2] = (param.transpose ? (*in_attrs)[0][ndim-1] : (*in_attrs)[0][ndim-2]); + oshape[ndim-1] = (*in_attrs)[1][ndim-1]; + } + TShape tshape(oshape.begin(), oshape.end()); + SHAPE_ASSIGN_CHECK(*out_attrs, 0, tshape); + return true; + } + if ( (*out_attrs)[0].ndim() >= 2 ) { + // Backward shape inference. + const int odim((*out_attrs)[0].ndim()); + std::vector ishape1(odim), ishape2(odim); + for ( int i = 0; i < odim-2; ++i ) { + ishape1[i] = ishape2[i] = (*out_attrs)[0][i]; + } + if ( param.rightside ) { + // We compute B * A where A is the first and B the second input. + ishape2[odim-2] = (*out_attrs)[0][odim-2]; + ishape1[odim-2] = ishape1[odim-1] = ishape2[odim-1] = (*out_attrs)[0][odim-1]; + } else { + // We compute A * B where A is the first and B the second input. + ishape2[odim-1] = (*out_attrs)[0][odim-1]; + ishape1[odim-2] = ishape1[odim-1] = ishape2[odim-2] = (*out_attrs)[0][odim-2]; + } + TShape tshape1(ishape1.begin(), ishape1.end()); + SHAPE_ASSIGN_CHECK(*in_attrs, 0, tshape1); + TShape tshape2(ishape2.begin(), ishape2.end()); + SHAPE_ASSIGN_CHECK(*in_attrs, 1, tshape2); + return true; + } + return false; +} + +template +bool LaReduceShape(const nnvm::NodeAttrs& attrs, + std::vector* in_attrs, + std::vector* out_attrs) { + // Shape for reduction of the dim lowest dimensions to a scalar. + // Can only deduct in forward direction. + CHECK_EQ(in_attrs->size(), 1); + CHECK_EQ(out_attrs->size(), 1); + const int ndim((*in_attrs)[0].ndim()); + if ( ndim < dim ) { + return false; + } + std::vector oshape(std::max(1, ndim-dim), 1); + for ( int i = 0; i < ndim - dim; ++i ) { + oshape[i] = (*in_attrs)[0][i]; + } + // Will reduce all matrices/vectors to a scalar. + TShape tshape(oshape.begin(), oshape.end()); + SHAPE_ASSIGN_CHECK(*out_attrs, 0, tshape); + return true; +} + +// Adapters for calling the various operators with appropriate signatures. +template +struct LaOpCaller { + static void op(const std::vector& inputs, + const std::vector& outputs, + const int index, + const nnvm::NodeAttrs& attrs, + mshadow::Stream *s) { + CHECK(false) << "no specialized LaOpCaller defined for template parameters"; + } +}; +template +struct LaOpCaller { + static void op(const std::vector& inputs, + const std::vector& outputs, + const int index, + const nnvm::NodeAttrs& attrs, + mshadow::Stream *s) { + laop::op(inputs[0].FlatToKD(s)[index], + outputs[0].FlatToKD(s)[index], attrs); + } +}; +template +struct LaOpCaller { + static void op(const std::vector& inputs, + const std::vector& outputs, + const int index, + const nnvm::NodeAttrs& attrs, + mshadow::Stream *s) { + laop::op(inputs[0].FlatToKD(s)[index], + inputs[1].FlatToKD(s)[index], + outputs[0].FlatToKD(s)[index], + attrs); + } +}; +template +struct LaOpCaller { + static void op(const std::vector& inputs, + const std::vector& outputs, + const int index, + const nnvm::NodeAttrs& attrs, + mshadow::Stream *s) { + laop::op(inputs[0].FlatToKD(s)[index], + inputs[1].FlatToKD(s)[index], + inputs[2].FlatToKD(s)[index], + outputs[0].FlatToKD(s)[index], + attrs); + } +}; +template +struct LaOpCaller { + static void op(const std::vector& inputs, + const std::vector& outputs, + const int index, + const nnvm::NodeAttrs& attrs, + mshadow::Stream *s) { + laop::op(inputs[0].FlatToKD(s)[index], + inputs[1].FlatToKD(s)[index], + inputs[2].FlatToKD(s)[index], + outputs[0].FlatToKD(s)[index], + outputs[1].FlatToKD(s)[index], + attrs); + } +}; +template +struct LaOpCaller { + static void op(const std::vector& inputs, + const std::vector& outputs, + const int index, + const nnvm::NodeAttrs& attrs, + mshadow::Stream *s) { + laop::op(inputs[0].FlatToKD(s)[index], + inputs[1].FlatToKD(s)[index], + inputs[2].FlatToKD(s)[index], + inputs[3].FlatToKD(s)[index], + outputs[0].FlatToKD(s)[index], + outputs[1].FlatToKD(s)[index], + attrs); + } +}; +template +struct LaOpCaller { + static void op(const std::vector& inputs, + const std::vector& outputs, + const int index, + const nnvm::NodeAttrs& attrs, + mshadow::Stream *s) { + laop::op(inputs[0].FlatToKD(s)[index], + inputs[1].FlatToKD(s)[index], + inputs[2].FlatToKD(s)[index], + inputs[3].FlatToKD(s)[index], + outputs[0].FlatToKD(s)[index], + outputs[1].FlatToKD(s)[index], + outputs[2].FlatToKD(s)[index], + attrs); + } +}; + + +template +void LaOpForward(const nnvm::NodeAttrs& attrs, + const OpContext& ctx, + const std::vector& inputs, + const std::vector& req, + const std::vector& outputs) { + using namespace mshadow; + Stream *s = ctx.get_stream(); + CHECK_EQ(inputs.size(), inum); + CHECK_EQ(outputs.size(), onum); + MSHADOW_REAL_TYPE_SWITCH(outputs[0].type_flag_, OType, { + int N(-1); + for ( int i = 0; i < inum; ++i ) { + CHECK_EQ(inputs[i].CheckContiguous(), true); + const int M(inputs[i].FlatToKD(s).size(0)); + CHECK_EQ((N == -1 || N == M), true); + N = M; + } + for ( int i = 0; i < onum; ++i ) { + CHECK_EQ(outputs[i].CheckContiguous(), true); + CHECK_EQ((req[i] == kWriteTo || req[i] == kWriteInplace), true); + const int M(outputs[i].FlatToKD(s).size(0)); + CHECK_EQ((N == -1 || N == M), true); + N = M; + } + for ( int i = 0; i < N; ++i ) { + LaOpCaller::op(inputs, outputs, i, attrs, s); + } + }); +} + + +template +void LaOpBackward(const nnvm::NodeAttrs& attrs, + const OpContext& ctx, + const std::vector& inputs, + const std::vector& req, + const std::vector& outputs) { + using namespace mshadow; + Stream *s = ctx.get_stream(); + CHECK_EQ(inputs.size(), inum); + CHECK_EQ(outputs.size(), onum); + MSHADOW_REAL_TYPE_SWITCH(outputs[0].type_flag_, OType, { + int N(-1); + for ( int i = 0; i < inum; ++i ) { + CHECK_EQ(inputs[i].CheckContiguous(), true); + const int M(inputs[i].FlatToKD(s).size(0)); + CHECK_EQ((N == -1 || N == M), true); + N = M; + } + std::vector tspace(outputs); + for ( int i = 0; i < onum; ++i ) { + CHECK_EQ(outputs[i].CheckContiguous(), true); + const int M(outputs[i].FlatToKD(s).size(0)); + CHECK_EQ((N == -1 || N == M), true); + N = M; + if ( req[i] == kAddTo ) { + tspace[i].dptr_ = ctx.requested[ResourceRequest::kTempSpace] + .get_space_typed(Shape1(outputs[i].Size()), s).dptr_; + } + } + for ( int i = 0; i < N; ++i ) { + LaOpCaller::op(inputs, tspace, i, attrs, s); + } + for ( int i = 0; i < onum; ++i ) { + if ( req[i] == kAddTo ) { + Tensor out = outputs[i].FlatTo1D(s); + out += tspace[i].FlatTo1D(s); + } + } + }); +} + +template +void LaReduceForward(const nnvm::NodeAttrs& attrs, + const OpContext& ctx, + const std::vector& inputs, + const std::vector& req, + const std::vector& outputs) { + using namespace mshadow; + Stream *s = ctx.get_stream(); + CHECK_EQ(inputs.size(), 1); + CHECK_EQ(outputs.size(), 1); + CHECK_EQ(inputs[0].CheckContiguous(), true); + CHECK_EQ(outputs[0].CheckContiguous(), true); + MSHADOW_REAL_TYPE_SWITCH(outputs[0].type_flag_, OType, { + Tensor in(inputs[0].FlatToKD(s)); + Tensor out(outputs[0].FlatTo1D(s)); + const int N(outputs[0].Size()); + CHECK_EQ(in.size(0), N); + for ( int i = 0; i < N; ++i ) { + laop::op(in[i], out[i], attrs); + } + }); +} + +template +void LaReduceBackward(const nnvm::NodeAttrs& attrs, + const OpContext& ctx, + const std::vector& inputs, + const std::vector& req, + const std::vector& outputs) { + using namespace mshadow; + Stream *s = ctx.get_stream(); + CHECK_EQ(inputs.size(), 2); + CHECK_EQ(outputs.size(), 1); + CHECK_EQ(inputs[0].CheckContiguous(), true); + CHECK_EQ(inputs[1].CheckContiguous(), true); + CHECK_EQ(outputs[0].CheckContiguous(), true); + MSHADOW_REAL_TYPE_SWITCH(outputs[0].type_flag_, OType, { + const int N(inputs[0].Size()); + Tensor in0(inputs[0].FlatTo1D(s)); + Tensor in1(inputs[1].FlatToKD(s)); + Tensor out(outputs[0].FlatToKD(s)); + for ( int i = 0; i < N; ++i ) { + laop::op(in0[i], in1[i], out[i], attrs, (req[i] == kAddTo)); + } + }); +} + +} // namespace op +} // namespace mxnet + +#endif // MXNET_OPERATOR_TENSOR_LA_OP_H_ diff --git a/src/operator/tensor/la_op_inline.h b/src/operator/tensor/la_op_inline.h new file mode 100644 index 000000000000..efa6c538cfff --- /dev/null +++ b/src/operator/tensor/la_op_inline.h @@ -0,0 +1,335 @@ +/*! + * Copyright (c) 2017 by Contributors + * \file la_op_inline.h + * \brief Operators for advanced linear algebra. + */ +#ifndef MXNET_OPERATOR_TENSOR_LA_OP_INLINE_H_ +#define MXNET_OPERATOR_TENSOR_LA_OP_INLINE_H_ + +#include + +namespace mxnet { +namespace op { + +using namespace mshadow; + +#define LA_OP_NOT_AVAIL " operator can only be called with float/double data type." + +// Signature for single matrix operations (decomposition/inversion). +#define FUNC_SIGNATURE_1(fname, arg1) {CHECK_EQ(MXNET_LAPACK_##fname(MXNET_LAPACK_ROW_MAJOR, 'L', \ + arg1.size(0), arg1.dptr_, arg1.size(0)), 0) << "fname failed in lapack";} + +// Signature for matrix-matrix multiplications involving one diagonal matrix. +#define FUNC_SIGNATURE_2(fname, arg1, arg2) \ + { cblas_##fname(CblasRowMajor, (rightside ? CblasRight : CblasLeft), \ + CblasLower, (transpose ? CblasTrans : CblasNoTrans), \ + CblasNonUnit, arg2.size(0), arg2.size(1), alpha, arg1.dptr_, \ + (rightside ? arg2.size(1) : arg2.size(0)), arg2.dptr_, arg2.size(1)); } + + +// Helper functions. +template +void CopyLowerToUpper(DType *dptr, int N) + { for (int i = 1; i < N; ++i ) for ( int j = 0; j < i; ++j ) dptr[j*N+i] = dptr[i*N+j]; } +template +void ZeroUpper(DType *dptr, int N) + { for (int i = 0; i < N; ++i ) for ( int j = i+1; j < N; ++j ) dptr[i*N+j] = 0; } + +// Forward operators + +// D = gemm(A,B,C) +struct gemm { + template + static void op(const Tensor& A, const Tensor& B, + const Tensor& C, DType alpha, DType beta, bool tA, bool tB) + { CHECK(false) << "gemm" << LA_OP_NOT_AVAIL; } + template + static void op(const Tensor& A, const Tensor& B, + const Tensor& C, const Tensor& D, + const nnvm::NodeAttrs& attrs) { + if ( C.dptr_ != D.dptr_ ) Copy(D, C); + const LaMatrixMacParam& param = nnvm::get(attrs.parsed); + gemm::op(A, B, D, DType(param.alpha), DType(param.beta), param.transpose_a, param.transpose_b); + } +}; +template<> +void gemm::op(const Tensor& A, const Tensor& B, + const Tensor& C, + float alpha, float beta, bool tA, bool tB ) { + CHECK_EQ((tA ? A.size(1) : A.size(0)), C.size(0)) + << "Non compatible matrix dimensions between inputs A and C for gemm operator"; + CHECK_EQ((tB ? B.size(0) : B.size(1)), C.size(1)) + << "Non compatible matrix dimensions between inputs B and C for gemm operator"; + CHECK_EQ((tA ? A.size(0) : A.size(1)), (tB ? B.size(1) : B.size(0))) + << "Non compatible matrix dimensions between inputs A and B for gemm operator"; + cblas_sgemm(CblasRowMajor, (tA ? CblasTrans : CblasNoTrans), (tB ? CblasTrans : CblasNoTrans), + (tA ? A.size(1):A.size(0)), (tB ? B.size(0): B.size(1)), + (tA ? A.size(0):A.size(1)), alpha, A.dptr_, A.size(1), B.dptr_, B.size(1), + beta, C.dptr_, (tB ? B.size(0): B.size(1))); +} +template<> +void gemm::op(const Tensor& A, const Tensor& B, + const Tensor& C, + double alpha, double beta, bool tA, bool tB) { + CHECK_EQ((tA ? A.size(1) : A.size(0)), C.size(0)) + << "Non compatible matrix dimensions between inputs A and C for gemm operator"; + CHECK_EQ((tB ? B.size(0) : B.size(1)), C.size(1)) + << "Non compatible matrix dimensions between inputs B and C for gemm operator"; + CHECK_EQ((tA ? A.size(0) : A.size(1)), (tB ? B.size(1) : B.size(0))) + << "Non compatible matrix dimensions between inputs A and B for gemm operator"; + cblas_dgemm(CblasRowMajor, (tA ? CblasTrans : CblasNoTrans), (tB ? CblasTrans : CblasNoTrans), + (tA ? A.size(1):A.size(0)), (tB ? B.size(0): B.size(1)), + (tA ? A.size(0):A.size(1)), alpha, A.dptr_, A.size(1), B.dptr_, B.size(1), + beta, C.dptr_, (tB ? B.size(0): B.size(1))); +} + +// C = gemm2(A,B) +struct gemm2 { + template + static void op(const Tensor& A, const Tensor& B, + const Tensor& C, const nnvm::NodeAttrs& attrs) { + const LaMatrixMultParam& param = nnvm::get(attrs.parsed); + gemm::op(A, B, C, DType(param.alpha), DType(0), param.transpose_a, param.transpose_b); + } +}; + +// L = potrf(A). +struct potrf { + template + static void op(const Tensor& A, const Tensor& L, + const nnvm::NodeAttrs& attrs) + { CHECK(false) << "potrf" << LA_OP_NOT_AVAIL; } +}; +template<> +void potrf::op(const Tensor& A, const Tensor& L, + const nnvm::NodeAttrs& attrs) { + if ( A.dptr_ != L.dptr_ ) Copy(L, A); + FUNC_SIGNATURE_1(spotrf, L); + ZeroUpper(L.dptr_, L.size(0)); +} +template<> +void potrf::op(const Tensor& A, const Tensor& L, + const nnvm::NodeAttrs& attrs) { + if ( A.dptr_ != L.dptr_ ) Copy(L, A); + FUNC_SIGNATURE_1(dpotrf, L); + ZeroUpper(L.dptr_, L.size(0)); +} + +// A = potri(L). +struct potri { + template + static void op(const Tensor& L, const Tensor& A, + const nnvm::NodeAttrs& attrs) + { CHECK(false) << "potri" << LA_OP_NOT_AVAIL; } +}; +template<> +void potri::op(const Tensor& L, const Tensor& A, + const nnvm::NodeAttrs& attrs) { + if ( A.dptr_ != L.dptr_ ) Copy(A, L); + FUNC_SIGNATURE_1(spotri, A); + CopyLowerToUpper(A.dptr_, A.size(0)); +} +template<> +void potri::op(const Tensor& A, const Tensor& L, + const nnvm::NodeAttrs& attrs) { + if ( A.dptr_ != L.dptr_ ) Copy(L, A); + FUNC_SIGNATURE_1(dpotri, A); + CopyLowerToUpper(A.dptr_, A.size(0)); +} + +// B = trsm(L,A) +struct trsm { + template + static void op(const Tensor& L, const Tensor& B, + DType alpha, bool rightside, bool transpose) + { CHECK(false) << "trsm" << LA_OP_NOT_AVAIL; } + template + static void op(const Tensor& L, const Tensor& A, + const Tensor& B, const nnvm::NodeAttrs& attrs) { + if ( A.dptr_ != B.dptr_ ) Copy(B, A); + const LaTriangMatrixMultParam& param = nnvm::get(attrs.parsed); + op(L, B, DType(param.alpha), param.rightside, param.transpose); + } +}; +template<> +void trsm::op(const Tensor& L, const Tensor& B, + float alpha, bool rightside, bool transpose) { + FUNC_SIGNATURE_2(strsm, L, B); +} +template<> +void trsm::op(const Tensor& L, const Tensor& B, + double alpha, bool rightside, bool transpose) { + FUNC_SIGNATURE_2(dtrsm, L, B); +} + +// B = trmm(L,A) +struct trmm { + template + static void op(const Tensor& L, const Tensor& B, + DType alpha, bool rightside, bool transpose) + { CHECK(false) << "trmm" << LA_OP_NOT_AVAIL; } + template + static void op(const Tensor& L, const Tensor& A, + const Tensor& B, const nnvm::NodeAttrs& attrs) { + if ( A.dptr_ != B.dptr_ ) Copy(B, A); + const LaTriangMatrixMultParam& param = nnvm::get(attrs.parsed); + op(L, B, DType(param.alpha), param.rightside, param.transpose); + } +}; +template<> +void trmm::op(const Tensor& L, const Tensor& B, + float alpha, bool rightside, bool transpose) { + FUNC_SIGNATURE_2(strmm, L, B); +} +template<> +void trmm::op(const Tensor& L, const Tensor& B, + double alpha, bool rightside, bool transpose) { + FUNC_SIGNATURE_2(dtrmm, L, B); +} + +// Useful operator that is not part of BLAS/LAPACK. +struct sumlogdiag { + template::value, int>::type = 0> + static void op(const Tensor& A, DType& L, const nnvm::NodeAttrs& attrs) + { CHECK(false) << "sumlogdiag operator can only be called with float/double data type."; } + template::value, int>::type = 0> + static void op(const Tensor& A, DType& B, const nnvm::NodeAttrs& attrs) { + CHECK_EQ(A.size(0), A.size(1)) << "sumlogdiag operator requires a NxN matrix as input."; + const int N(A.size(0)); + DType sum(0); + DType *p(A.dptr_); + for ( int i = 0; i < N; ++i, p += N+1 ) { + sum += log(*p); + } + B = sum; + } +}; + +// Backward operators + +struct gemm_backward { + template + static void op(const Tensor& dD, const Tensor& A, + const Tensor& B, const Tensor& C, + const Tensor& dA, const Tensor& dB, + const Tensor& dC, const nnvm::NodeAttrs& attrs) { + const LaMatrixMacParam& param = nnvm::get(attrs.parsed); + (param.transpose_a ? gemm::op(B, dD, dA, DType(param.alpha), DType(0), param.transpose_b, true) + : gemm::op(dD, B, dA, DType(param.alpha), DType(0), false, !param.transpose_b)); + (param.transpose_b ? gemm::op(dD, A, dB, DType(param.alpha), DType(0), true, param.transpose_a) + : gemm::op(A, dD, dB, DType(param.alpha), DType(0), !param.transpose_a, false)); + const int N(dC.size(0)*dC.size(1)); + for ( int i = 0; i < N; ++i ) { + dC.dptr_[i] = param.beta * dD.dptr_[i]; + } + } +}; + +struct gemm2_backward { + template + static void op(const Tensor& dC, const Tensor& A, + const Tensor& B, const Tensor& dA, + const Tensor& dB, const nnvm::NodeAttrs& attrs) { + const LaMatrixMultParam& param = nnvm::get(attrs.parsed); + (param.transpose_a ? gemm::op(B, dC, dA, DType(param.alpha), DType(0), param.transpose_b, true) + : gemm::op(dC, B, dA, DType(param.alpha), DType(0), false, !param.transpose_b)); + (param.transpose_b ? gemm::op(dC, A, dB, DType(param.alpha), DType(0), true, param.transpose_a) + : gemm::op(A, dC, dB, DType(param.alpha), DType(0), !param.transpose_a, false)); + } +}; + +struct potrf_backward { + template + static void op(const Tensor& dL, const Tensor& L, + const Tensor& dA, const nnvm::NodeAttrs& attrs) { + // Backward of L = potrf(A). + // dA = 0.5 * L**T * symm(L**T * dL # E) * L**(-1) where + // '#' denotes Hadamard product + // E is the matrix having 1 on diagonal, 0 on upper and 2 on lower triagle + // symm(X) = 0.5 * (X + X**T) + // Hadamard product and symm can be realized by a single copy from lower to upper triangle. + if ( dL.dptr_ != dA.dptr_ ) { + Copy(dA, dL); + } + trmm::op(L, dA, DType(1.0), false, true); + CopyLowerToUpper(dA.dptr_, dA.size(0)); + trsm::op(L, dA, DType(1.0), false, true); + trsm::op(L, dA, DType(0.5), true, false); + } +}; + +struct potri_backward { + template + static void op(const Tensor& dA, const Tensor& L, + const Tensor& A, const Tensor& dL, + const nnvm::NodeAttrs& attrs) { + // Backward of A = potri(L). + // dL = -2 * tril(A * dA * L**(-T)), where tril() extracts lower triangle and diagonal. + gemm::op(A, dA, dL, DType(1.0), DType(0), false, false); + trsm::op(L, dL, DType(-2.0), true, true); + ZeroUpper(dL.dptr_, dL.size(0)); + } +}; + +struct trsm_backward { + template + static void op(const Tensor& dB, const Tensor& L, + const Tensor& A, const Tensor& B, + const Tensor& dL, const Tensor& dA, + const nnvm::NodeAttrs& attrs) { + // Backward of B = trsm(L,A). + const LaTriangMatrixMultParam& param = nnvm::get(attrs.parsed); + // Compute dA + if ( dA.dptr_ != dB.dptr_ ) Copy(dA, dB); + trsm::op(L, dA, DType(param.alpha), param.rightside, !param.transpose); + // Compute dL + const bool da_left(param.rightside == param.transpose); + (da_left ? + gemm::op(dA, B, dL, DType(-1.0/param.alpha), DType(0), param.transpose, !param.transpose) + : gemm::op(B, dA, dL, DType(-1.0/param.alpha), DType(0), !param.transpose, param.transpose)); + ZeroUpper(dL.dptr_, dL.size(0)); + } +}; + +struct trmm_backward { + template + static void op(const Tensor& dB, const Tensor& L, + const Tensor& A, const Tensor& B, + const Tensor& dL, const Tensor& dA, + const nnvm::NodeAttrs& attrs) { + // Backward of B = trmm(L,A). + const LaTriangMatrixMultParam& param = nnvm::get(attrs.parsed); + // Compute dL + const bool db_left(param.rightside == param.transpose); + (db_left ? gemm::op(dB, A, dL, DType(param.alpha), DType(0), param.transpose, !param.transpose) + : gemm::op(A, dB, dL, DType(param.alpha), DType(0), !param.transpose, param.transpose)); + ZeroUpper(dL.dptr_, dL.size(0)); + // Compute dA + if ( dA.dptr_ != dB.dptr_ ) Copy(dA, dB); + trmm::op(L, dA, DType(param.alpha), param.rightside, !param.transpose); + } +}; + +struct sumlogdiag_backward { + template + static void op(const DType& dB, const Tensor& A, const Tensor& dA, + const nnvm::NodeAttrs& attrs, bool add) { + // Backward of B = sumlogdiag(A). + const int N(A.size(0)); + if ( !add ) { + for ( int i = 0; i < N*N; ++i ) { + dA.dptr_[i] = 0; + } + } + for ( int i = 0; i < N; ++i ) { + dA.dptr_[i*(N+1)] += dB / A.dptr_[i*N+i]; + } + } +}; + +} // namespace op +} // namespace mxnet + +#endif // MXNET_OPERATOR_TENSOR_LA_OP_INLINE_H_ diff --git a/tests/ci_build/install/ubuntu_install_core.sh b/tests/ci_build/install/ubuntu_install_core.sh index dacd30b4af71..9ee76976e9e7 100755 --- a/tests/ci_build/install/ubuntu_install_core.sh +++ b/tests/ci_build/install/ubuntu_install_core.sh @@ -2,7 +2,7 @@ # install libraries for building mxnet c++ core on ubuntu apt-get update && apt-get install -y \ - build-essential git libopenblas-dev libopencv-dev \ + build-essential git libopenblas-dev liblapack-dev libopencv-dev \ libcurl4-openssl-dev libgtest-dev cmake wget unzip cd /usr/src/gtest && cmake CMakeLists.txt && make && cp *.a /usr/lib diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py index 4a7860d0500c..79795e9386b6 100644 --- a/tests/python/unittest/test_operator.py +++ b/tests/python/unittest/test_operator.py @@ -3160,6 +3160,256 @@ def create_operator(self, ctx, shapes, dtypes): check_numeric_gradient(op, [x]) +def test_laop(): + # Temporarily disabled until lapack is enabled by default + return + + # Currently no support for GPU. Will be added soon + # so keep these tests here in this file and activate + # gpu-testing when it is ready. + dev = default_context() + if dev.device_type == 'gpu': + return + + grad_check = 1 + + data1 = mx.symbol.Variable('data1') + data2 = mx.symbol.Variable('data2') + data3 = mx.symbol.Variable('data3') + data4 = mx.symbol.Variable('data4') + + # Test gemm separately from other la-operators. + shape1 = (2, 3) + shape2 = (3, 2) + shape3 = (3, 3) + shape4 = (2, 2) + #Ensure that ithis tests don't get changed by other calls to random. + np.random.seed(42) + data_in1 = np.random.uniform(1, 10, shape1) + data_in2 = np.random.uniform(1, 10, shape2) + data_in3 = np.random.uniform(1, 10, shape3) + data_in4 = np.random.uniform(1, 10, shape4) + # Check all transpositions of gemm operator. + data_in1_t = np.transpose(data_in1) + data_in2_t = np.transpose(data_in2) + res_gemm = 4*np.dot(data_in1,data_in2)+7*data_in4 + test_gemm = mx.sym.linalg_gemm(data1, data2, data3, alpha = 4, beta = 7) + check_symbolic_forward(test_gemm, [data_in1, data_in2, data_in4], [res_gemm]) + if grad_check == 1: + check_numeric_gradient(test_gemm, [data_in1, data_in2, data_in4], numeric_eps=1e-3, rtol=1e-1, atol=1e-1) + res_gemm = 4*np.dot(data_in1_t,data_in2_t)+7*data_in3 + test_gemm = mx.sym.linalg_gemm(data1, data2, data3, alpha = 4, beta = 7, transpose_a = 1, transpose_b = 1) + check_symbolic_forward(test_gemm, [data_in1, data_in2, data_in3], [res_gemm]) + if grad_check == 1: + check_numeric_gradient(test_gemm, [data_in1, data_in2, data_in3], numeric_eps=1e-3, rtol=1e-1, atol=1e-1) + res_gemm = 4*np.dot(data_in1_t,data_in1)+7*data_in3 + test_gemm = mx.sym.linalg_gemm(data1, data2, data3, alpha = 4, beta = 7, transpose_a = 1) + check_symbolic_forward(test_gemm, [data_in1, data_in1, data_in3], [res_gemm]) + if grad_check == 1: + check_numeric_gradient(test_gemm, [data_in1, data_in1, data_in3], numeric_eps=1e-3, rtol=1e-1, atol=1e-1) + res_gemm = 4*np.dot(data_in1,data_in1_t)+7*data_in4 + test_gemm = mx.sym.linalg_gemm(data1, data2, data3, alpha = 4, beta = 7, transpose_b = 1) + check_symbolic_forward(test_gemm, [data_in1, data_in1, data_in4], [res_gemm]) + if grad_check == 1: + check_numeric_gradient(test_gemm, [data_in1, data_in1, data_in4], numeric_eps=1e-3, rtol=1e-1, atol=1e-1) + + # Check batch of gemm. + a = np.tile(np.array(data_in1).flatten(),3) + a = np.reshape(a,(3,1,2,3)) + b = np.tile(np.array(data_in2).flatten(),3) + b = np.reshape(b,(3,1,3,2)) + c = np.tile(np.array(data_in4).flatten(),3) + c = np.reshape(c,(3,1,2,2)) + r = 4*np.dot(data_in1,data_in2)+7*data_in4 + r = np.tile(r.flatten(),3) + r = np.reshape(r,(3,1,2,2)) + test_gemm = mx.sym.linalg_gemm(data1, data2, data3, alpha = 4, beta = 7) + check_symbolic_forward(test_gemm, [a, b, c], [r]) + if grad_check == 1: + check_numeric_gradient(test_gemm, [a, b, c], numeric_eps=1e-3, rtol=1e-1, atol=1e-1) + + # Check gemm2 operator same way as gemm. + res_gemm = 4*np.dot(data_in1,data_in2) + test_gemm = mx.sym.linalg_gemm2(data1, data2, alpha = 4) + check_symbolic_forward(test_gemm, [data_in1, data_in2], [res_gemm]) + if grad_check == 1: + check_numeric_gradient(test_gemm, [data_in1, data_in2], numeric_eps=1e-3, rtol=1e-1, atol=1e-1) + res_gemm = 4*np.dot(data_in1_t, data_in2_t) + test_gemm = mx.sym.linalg_gemm2(data1, data2, alpha = 4, transpose_a = 1, transpose_b = 1) + check_symbolic_forward(test_gemm, [data_in1, data_in2], [res_gemm]) + if grad_check == 1: + check_numeric_gradient(test_gemm, [data_in1, data_in2], numeric_eps=1e-3, rtol=1e-1, atol=1e-1) + res_gemm = 4*np.dot(data_in1_t,data_in1) + test_gemm = mx.sym.linalg_gemm2(data1, data2, alpha = 4, transpose_a = 1) + check_symbolic_forward(test_gemm, [data_in1, data_in1], [res_gemm]) + if grad_check == 1: + check_numeric_gradient(test_gemm, [data_in1, data_in1], numeric_eps=1e-3, rtol=1e-1, atol=1e-1) + res_gemm = 4*np.dot(data_in1,data_in1_t) + test_gemm = mx.sym.linalg_gemm2(data1, data2, alpha = 4, transpose_b = 1) + check_symbolic_forward(test_gemm, [data_in1, data_in1], [res_gemm]) + if grad_check == 1: + check_numeric_gradient(test_gemm, [data_in1, data_in1], numeric_eps=1e-3, rtol=1e-1, atol=1e-1) + + # Check batch of gemm2. + a = np.tile(np.array(data_in1).flatten(),3) + a = np.reshape(a,(3,1,2,3)) + b = np.tile(np.array(data_in2).flatten(),3) + b = np.reshape(b,(3,1,3,2)) + r = 4*np.dot(data_in1,data_in2) + r = np.tile(r.flatten(),3) + r = np.reshape(r,(3,1,2,2)) + test_gemm = mx.sym.linalg_gemm2(data1, data2, alpha = 4) + check_symbolic_forward(test_gemm, [a, b], [r]) + if grad_check == 1: + check_numeric_gradient(test_gemm, [a, b], numeric_eps=1e-3, rtol=1e-1, atol=1e-1) + + # Now test all the other operators. + + # Tests with trivial 1x1 matrices. + shape = (4, 4, 1, 1 ) + data_in = np.random.uniform(1, 10, shape) + # test potrf + res_potrf = np.sqrt(data_in) + test_potrf = mx.sym.linalg_potrf(data1) + check_symbolic_forward(test_potrf, [data_in], [res_potrf]) + if grad_check == 1: + check_numeric_gradient(test_potrf, [data_in]) + # test potri + ones = mx.nd.ones(shape).asnumpy() + res_potri = np.divide(ones,data_in*data_in) + test_potri = mx.sym.linalg_potri(data1) + check_symbolic_forward(test_potri, [data_in], [res_potri]) + if grad_check == 1: + check_numeric_gradient(test_potri, [data_in], atol = 0.01, rtol = 1.5) + # test trsm + trian_in = data_in *7 + test_trsm = mx.sym.linalg_trsm(data1,data2,alpha = 7) + check_symbolic_forward(test_trsm, [trian_in,data_in], [ones]) + if grad_check == 1: + check_numeric_gradient(test_trsm, [trian_in,data_in], atol = 0.02, rtol = 2.0) + # test trmm + trian_in = np.divide(ones,trian_in) + test_trmm = mx.sym.linalg_trmm(data1,data2,alpha = 7, transpose = 1, rightside = 1) + check_symbolic_forward(test_trmm, [trian_in,data_in], [ones]) + if grad_check == 1: + check_numeric_gradient(test_trmm, [trian_in,data_in], atol = 0.02, rtol = 2.0) + # test sumlogdiag + res_sumlogdiag = np.reshape(np.log(data_in),(4,4)) + test_sumlogdiag = mx.sym.linalg_sumlogdiag(data1) + check_symbolic_forward(test_sumlogdiag, [data_in], [res_sumlogdiag]) + if grad_check == 1: + check_numeric_gradient(test_sumlogdiag, [data_in], atol = 0.01, rtol = 2.0) + + # more elaborate example of cholesky factorization + matrix = [ 9, 3, -6, 12, 3, 26, -7, -11, -6, -7, 9, 7, 12, -11, 7, 65 ] + trian = [ 3, 0, 0, 0, 1, 5, 0, 0, -2, -1, 2, 0, 4, -3, 6, 2 ] + pow = [ 2, 1, 1, 1, 1, 4, 1, 1, 1, 1, 8, 1, 1, 1, 1, 16 ] + inv = [ 2.98333, 0.01667, 2.65, -0.83333, 0.01667, 0.05, 0.05, 0, 2.65, 0.05, 2.5, -0.75, -0.83333, 0, -0.75, 0.25 ] + ident = [ 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1 ] + + # Tests for numeric gradients for potrf/potri/trmm/trsm are suppressed by default + # as they are very volatile and may often report false negatives which + # have to be excluded by manual inspection. + grad_check = 0 + + # test potrf + a = np.tile(np.array(matrix),3) + a = np.reshape(a,(3,1,4,4)) + r = np.tile(np.array(trian),3) + r = np.reshape(r,(3,1,4,4)) + check_symbolic_forward(test_potrf, [a], [r]) + if grad_check == 1: + check_numeric_gradient(test_potrf, [a], numeric_eps=1e-3, rtol=1e-2, atol=1e-1) + + #test potri + a = np.tile(np.array(trian),3) + a = np.reshape(a,(3,1,4,4)) + r = np.tile(np.array(inv),3) + r = np.reshape(r,(3,1,4,4)) + check_symbolic_forward(test_potri, [a], [r], atol=0.01) + if grad_check == 1: + check_numeric_gradient(test_potri, [a], numeric_eps=1e-3, rtol=1e-2, atol=1e-1) + + #test trsm + a = np.tile(np.array(trian),3) + a = np.reshape(a,(3,1,4,4)) + b = np.tile(np.array(matrix),3) + b = np.reshape(b,(3,1,4,4)) + r = 7*np.transpose(np.reshape(np.array(trian),(4,4))) + r = np.reshape(np.tile(np.reshape(r,(16)),3),(3,1,4,4)) + check_symbolic_forward(test_trsm, [a,b], [r]) + if grad_check == 1: + check_numeric_gradient(test_trsm, [a,b], numeric_eps=1e-3, rtol=1e-2, atol=1e-1) + + test_trsm2 = mx.sym.linalg_trsm(data1,data2,alpha = -2, rightside = 1, transpose = 1) + r = -2*np.reshape(np.array(trian),(4,4)) + r = np.reshape(np.tile(np.reshape(r,(16)),3),(3,1,4,4)) + check_symbolic_forward(test_trsm2, [a,b], [r]) + if grad_check == 1: + check_numeric_gradient(test_trsm2, [a,b], numeric_eps=1e-3, rtol=1e-2, atol=1e-1) + + test_trsm3 = mx.sym.linalg_trsm(data1,data2,alpha = 0.50, transpose = 1) + b = np.transpose(np.reshape(np.array(trian),(4,4))) + b = np.reshape(np.tile(np.reshape(b,(16)),3),(3,1,4,4)) + r = 0.5*np.reshape(np.array(ident),(4,4)) + r = np.reshape(np.tile(np.reshape(r,(16)),3),(3,1,4,4)) + check_symbolic_forward(test_trsm3, [a,b], [r]) + if grad_check == 1: + check_numeric_gradient(test_trsm3, [a,b], numeric_eps=1e-3, rtol=1e-2, atol=1e-1) + + test_trsm4 = mx.sym.linalg_trsm(data1,data2,alpha = -0.5, rightside = 1) + b = np.tile(np.array(trian),3) + b = np.reshape(b,(3,1,4,4)) + r = -0.5*np.reshape(np.array(ident),(4,4)) + r = np.reshape(np.tile(np.reshape(r,(16)),3),(3,1,4,4)) + check_symbolic_forward(test_trsm4, [a,b], [r]) + if grad_check == 1: + check_numeric_gradient(test_trsm4, [a,b], numeric_eps=1e-3, rtol=1e-2, atol=1e-1) + + #test trmm + a = np.tile(np.array(trian),3) + a = np.reshape(a,(3,1,4,4)) + b = np.tile(np.array(matrix),3) + b = np.reshape(b,(3,1,4,4)) + r = 7*np.dot(np.reshape(np.array(matrix),(4,4)),np.transpose(np.reshape(np.array(trian),(4,4)))) + r = np.reshape(np.tile(np.reshape(r,(16)),3),(3,1,4,4)) + check_symbolic_forward(test_trmm, [a,b], [r]) + if grad_check == 1: + check_numeric_gradient(test_trmm, [a,b], numeric_eps=1e-3, rtol=1e-2, atol=1e-1) + + test_trmm2 = mx.sym.linalg_trmm(data1,data2,alpha = -2) + r = -2*np.dot(np.reshape(np.array(trian),(4,4)),np.reshape(np.array(matrix),(4,4))) + r = np.reshape(np.tile(np.reshape(r,(16)),3),(3,1,4,4)) + check_symbolic_forward(test_trmm2, [a,b], [r]) + if grad_check == 1: + check_numeric_gradient(test_trmm2, [a,b], numeric_eps=1e-3, rtol=1e-2, atol=1e-1) + + test_trmm3 = mx.sym.linalg_trmm(data1,data2,rightside = 1) + r = np.dot(np.reshape(np.array(matrix),(4,4)),np.reshape(np.array(trian),(4,4))) + r = np.reshape(np.tile(np.reshape(r,(16)),3),(3,1,4,4)) + check_symbolic_forward(test_trmm3, [a,b], [r]) + if grad_check == 1: + check_numeric_gradient(test_trmm3, [a,b], numeric_eps=1e-3, rtol=1e-2, atol=1e-1) + + test_trmm4 = mx.sym.linalg_trmm(data1,data2,alpha = 1.2,transpose = 1) + r = 1.2*np.dot(np.transpose(np.reshape(np.array(trian),(4,4))),np.reshape(np.array(matrix),(4,4))) + r = np.reshape(np.tile(np.reshape(r,(16)),3),(3,1,4,4)) + check_symbolic_forward(test_trmm4, [a,b], [r]) + if grad_check == 1: + check_numeric_gradient(test_trmm4, [a,b], numeric_eps=1e-3, rtol=1e-2, atol=1e-1) + + # test sumlogdiag + a = np.array(pow) + a = np.tile(a,3) + a = np.reshape(a,(3,1,4,4)) + r = 10*np.log(np.array([2])) + r = np.tile(r,3) + r = np.reshape(r,(3)) + check_symbolic_forward(test_sumlogdiag, [a], [r]) + if grad_check == 1: + check_numeric_gradient(test_sumlogdiag, [a]) + + if __name__ == '__main__': import nose nose.runmodule() From 28f58438d4e93464310e1a5dc3162e5d87394845 Mon Sep 17 00:00:00 2001 From: Chunyang Wen Date: Wed, 14 Jun 2017 11:42:21 +0800 Subject: [PATCH 067/834] round batch should start with first element. (#6684) --- src/io/iter_csv.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/io/iter_csv.cc b/src/io/iter_csv.cc index c43f99911f69..9dcbcb8a681d 100644 --- a/src/io/iter_csv.cc +++ b/src/io/iter_csv.cc @@ -174,8 +174,8 @@ Examples:: [3. 4. 5.]] [[4. 5. 6.] - [2. 3. 4.] - [3. 4. 5.]] + [1. 2. 3.] + [2. 3. 4.]] // Now, `reset` method is called. CSVIter.reset() From 5a9c3c0e2fbd7405e8210f3f274e497f4871c52c Mon Sep 17 00:00:00 2001 From: Pracheer Gupta Date: Tue, 13 Jun 2017 22:15:22 -0700 Subject: [PATCH 068/834] Minor grammatical errors in 3 of the tutorials. (#6671) --- docs/tutorials/basic/data.md | 14 +++++++------- docs/tutorials/basic/ndarray.md | 4 ++-- .../tutorials/vision/large_scale_classification.md | 6 +++--- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/tutorials/basic/data.md b/docs/tutorials/basic/data.md index 7cbd14eff3d8..dba13918aa0e 100644 --- a/docs/tutorials/basic/data.md +++ b/docs/tutorials/basic/data.md @@ -1,5 +1,5 @@ # Iterators - Loading data -In this tutorial we focus on how to feed data into a training or inference program. +In this tutorial, we focus on how to feed data into a training or inference program. Most training and inference modules in MXNet accept data iterators, which simplifies this procedure, especially when reading large datasets. Here we discuss the API conventions and several provided iterators. @@ -24,7 +24,7 @@ $ MXNET_HOME = '~/mxnet' ## MXNet Data Iterator Data Iterators in *MXNet* are similar to Python iterator objects. -In Python the function `iter` allows fetching items sequentially by calling `next()` on +In Python, the function `iter` allows fetching items sequentially by calling `next()` on iterable objects such as a Python `list`. Iterators provide an abstract interface for traversing various types of iterable collections without needing to expose details about the underlying data source. @@ -162,7 +162,7 @@ The *data* variables are called free variables in MXNet's Symbol API. To execute a Symbol, they need to be bound with data. [Click here learn more about Symbol](http://mxnet.io/tutorials/basic/symbol.html). -We use the data iterator to feed examples to a neural networks via MXNet's `module` API. +We use the data iterator to feed examples to a neural network via MXNet's `module` API. [Click here to learn more about Module](http://mxnet.io/tutorials/basic/module.html). @@ -242,7 +242,7 @@ record.keys ### Packing and Unpacking data -Each record in a .rec file can contain arbitrary binary data. However most deep learning tasks require data to be input in label/data format. +Each record in a .rec file can contain arbitrary binary data. However, most deep learning tasks require data to be input in label/data format. The `mx.recordio` package provides a few utility functions for such operations, namely: `pack`, `unpack`, `pack_img`, and `unpack_img`. #### Packing/Unpacking Binary Data @@ -292,7 +292,7 @@ An example of how to use the script for converting to *RecordIO* format is shown ## Image IO -In this section we will learn how to preprocess and load image data in MXNet. +In this section, we will learn how to preprocess and load image data in MXNet. There are 4 ways of loading image data in MXNet. 1. Using [__mx.image.imdecode__](http://mxnet.io/api/python/io.html#mxnet.image.imdecode) to load raw image files. @@ -363,13 +363,13 @@ tar.close() Let's take a look at the data. As you can see, under the root folder (./data/101_ObjectCategories) every category has a subfolder(./data/101_ObjectCategories/yin_yang). Now let's convert them into record io format using the `im2rec.py` utility script. -First we need to make a list that contains all the image files and their categories: +First, we need to make a list that contains all the image files and their categories: ```python os.system('python %s/tools/im2rec.py --list=1 --recursive=1 --shuffle=1 --test-ratio=0.2 data/caltech data/101_ObjectCategories'%MXNET_HOME) ``` -The resulting list file (./data/caltech_train.lst) is in the format `index\t(one or more label)\tpath`. In this case there is only one label for each image but you can modify the list to add in more for multi label training. +The resulting list file (./data/caltech_train.lst) is in the format `index\t(one or more label)\tpath`. In this case, there is only one label for each image but you can modify the list to add in more for multi-label training. Then we can use this list to create our record io file: diff --git a/docs/tutorials/basic/ndarray.md b/docs/tutorials/basic/ndarray.md index c7a3feb1d871..bd76702aa376 100644 --- a/docs/tutorials/basic/ndarray.md +++ b/docs/tutorials/basic/ndarray.md @@ -66,7 +66,7 @@ b = mx.nd.array([[1,2,3], [2,3,4]]) {'a.shape':a.shape, 'b.shape':b.shape} ``` -* We can also create an MXNet NDArray from an `numpy.ndarray` object: +* We can also create an MXNet NDArray from a `numpy.ndarray` object: ```python import numpy as np @@ -370,7 +370,7 @@ c The `load` and `save` methods are preferable to pickle in two respects 1. When using these methods, you can save data from within the Python interface - and then use it later from another lanuage's binding. For example, if we save + and then use it later from another language's binding. For example, if we save the data in Python: ```python diff --git a/docs/tutorials/vision/large_scale_classification.md b/docs/tutorials/vision/large_scale_classification.md index 09e85075bebb..1cf22708efde 100644 --- a/docs/tutorials/vision/large_scale_classification.md +++ b/docs/tutorials/vision/large_scale_classification.md @@ -141,7 +141,7 @@ We now have all training and validation images in recordIO format in `train` and [ResNet](https://arxiv.org/abs/1512.03385) has shown its effectiveness on ImageNet competition. Our experiments also [reproduced](https://github.com/tornadomeet/ResNet) the results reported in the paper. As we increase the number of layers from 18 to 152, we see steady improvement in validation accuracy. Given this is a huge dataset, we will use Resnet with 152 layers. -Due to the huge computational complexity, even the fastest GPU needs more than one day for a single pass of the data. We often need tens of epochs before the training converges to good validation accuracy. While we can use multiple GPUs in a machine, number of GPUs in a machine is often limited to 8 or 16. For faster training, in this tutorial, we will use multiple machines each containing multiple GPUs to train the model. +Due to the huge computational complexity, even the fastest GPU needs more than one day for a single pass of the data. We often need tens of epochs before the training converges to good validation accuracy. While we can use multiple GPUs in a machine, the number of GPUs in a machine is often limited to 8 or 16. For faster training, in this tutorial, we will use multiple machines each containing multiple GPUs to train the model. ### Setup @@ -160,7 +160,7 @@ If you are setting up your cluster manually, without using AWS CloudFormation, r deeplearning-worker2 deeplearning-worker3 ``` - It should be possible to ssh into any of these machines from master by invoking `ssh` with just a hostname from the file. For example, + It should be possible to ssh into any of these machines from the master by invoking `ssh` with just a hostname from the file. For example, ``` $ ssh deeplearning-worker2 =================================== @@ -169,7 +169,7 @@ If you are setting up your cluster manually, without using AWS CloudFormation, r ... ubuntu@ip-10-0-1-199:~$ ``` - One way to do this is to use ssh agent forwarding. Please check [this](https://aws.amazon.com/blogs/security/securely-connect-to-linux-instances-running-in-a-private-amazon-vpc/) page to learn how to set this up. In short, you’ll configure all machines to login using a particular certificate (mycert.pem) which is present on your local machine. You then login to the master using the certificate and the `-A` switch to enable agent forwarding. Now, from master, you should be able to login to any other machine in the cluster by providing just the hostname (example: `ssh deeplearning-worker2`). + One way to do this is to use ssh agent forwarding. Please check [this](https://aws.amazon.com/blogs/security/securely-connect-to-linux-instances-running-in-a-private-amazon-vpc/) page to learn how to set this up. In short, you’ll configure all machines to login using a particular certificate (mycert.pem) which is present on your local machine. You then login to the master using the certificate and the `-A` switch to enable agent forwarding. Now, from the master, you should be able to login to any other machine in the cluster by providing just the hostname (example: `ssh deeplearning-worker2`). ### Run Training After the cluster is setup, login to master and run the following command from ${MXNET}/example/image-classification From b48ceae7ca8110b10a7a07d6eea3e7e7e29cc18b Mon Sep 17 00:00:00 2001 From: ziheng Date: Wed, 14 Jun 2017 09:29:58 -0700 Subject: [PATCH 069/834] Fix for broken link (#6690) --- example/image-classification/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/image-classification/README.md b/example/image-classification/README.md index 2c5f2d3a5409..25050f652c9f 100644 --- a/example/image-classification/README.md +++ b/example/image-classification/README.md @@ -108,7 +108,7 @@ to classify an image with jupyter notebook. ### ImageNet 1K It is first used by -[ImageNet challenge 2012](http://mxnet.io/tutorials/python/predict_imagenet.html), +[ImageNet challenge 2012](http://www.image-net.org/challenges/LSVRC/2012/), which contains about 1.2M images with 1000 classes. To test these models, one can use [data/imagenet1k-val.sh](https://github.com/dmlc/mxnet/blob/master/example/image-classification/data/imagenet1k-val.sh) From dc23888dbbf04e3cca3115ff36a944efb652ac71 Mon Sep 17 00:00:00 2001 From: Sergey Kolychev Date: Wed, 14 Jun 2017 09:32:06 -0700 Subject: [PATCH 070/834] sync with python. (#6655) --- perl-package/AI-MXNet/Changes | 3 + perl-package/AI-MXNet/META.json | 6 +- perl-package/AI-MXNet/META.yml | 6 +- perl-package/AI-MXNet/Makefile.PL | 10 +- perl-package/AI-MXNet/README | 2 +- perl-package/AI-MXNet/lib/AI/MXNet.pm | 7 +- .../AI-MXNet/lib/AI/MXNet/CachedOp.pm | 41 ++++ .../AI-MXNet/lib/AI/MXNet/Contrib/AutoGrad.pm | 74 +++++- .../AI-MXNet/lib/AI/MXNet/Executor/Group.pm | 210 +++++------------ perl-package/AI-MXNet/lib/AI/MXNet/NDArray.pm | 81 ++++++- .../AI-MXNet/lib/AI/MXNet/NDArray/Base.pm | 39 ++++ perl-package/AI-MXNet/lib/AI/MXNet/RNN.pm | 1 + .../AI-MXNet/lib/AI/MXNet/RNN/Cell.pm | 119 +++++++++- perl-package/AI-MXNet/lib/AI/MXNet/Symbol.pm | 221 +++++++++++------- .../AI-MXNet/lib/AI/MXNet/Symbol/Base.pm | 15 ++ .../lib/AI/MXNet/Symbol/NameManager.pm | 4 +- .../AI-MXNet/lib/AI/MXNet/TestUtils.pm | 35 ++- .../AI-MXNet/lib/AI/MXNet/Visualization.pm | 2 +- perl-package/AI-MXNet/t/test_executor.t | 4 +- perl-package/AI-MXNet/t/test_module.t | 124 +++++++++- perl-package/AI-MXNet/t/test_ndarray.t | 35 ++- perl-package/AI-MXNet/t/test_random.t | 13 +- perl-package/AI-MXNet/t/test_rnn.t | 84 ++++++- perl-package/AI-MXNet/t/test_symbol.t | 20 +- perl-package/AI-MXNetCAPI/Changes | 3 + perl-package/AI-MXNetCAPI/META.json | 2 +- perl-package/AI-MXNetCAPI/META.yml | 2 +- perl-package/AI-MXNetCAPI/README | 2 +- perl-package/AI-MXNetCAPI/lib/AI/MXNetCAPI.pm | 2 +- perl-package/AI-MXNetCAPI/mxnet.i | 147 ++++++++++-- perl-package/AI-MXNetCAPI/mxnet_typemaps.i | 214 +++++++++++++++-- perl-package/AI-NNVMCAPI/Changes | 3 + perl-package/AI-NNVMCAPI/META.json | 2 +- perl-package/AI-NNVMCAPI/META.yml | 2 +- perl-package/AI-NNVMCAPI/README | 2 +- perl-package/AI-NNVMCAPI/lib/AI/NNVMCAPI.pm | 2 +- 36 files changed, 1205 insertions(+), 334 deletions(-) create mode 100644 perl-package/AI-MXNet/lib/AI/MXNet/CachedOp.pm diff --git a/perl-package/AI-MXNet/Changes b/perl-package/AI-MXNet/Changes index 2664d2a1c8fc..f2663c01254d 100644 --- a/perl-package/AI-MXNet/Changes +++ b/perl-package/AI-MXNet/Changes @@ -1,5 +1,8 @@ Revision history for Perl extension AI::MXNet +1.01 Sat Jun 10 23:57:27 PDT 2017 + - sync with python. + 0.9507 Thu May 11 17:04:44 PDT 2017 - added AutoGrad, bugfixes. diff --git a/perl-package/AI-MXNet/META.json b/perl-package/AI-MXNet/META.json index 68afafd9e4e8..c2f75309c497 100644 --- a/perl-package/AI-MXNet/META.json +++ b/perl-package/AI-MXNet/META.json @@ -30,8 +30,8 @@ }, "runtime" : { "requires" : { - "AI::MXNetCAPI" : "0.95", - "AI::NNVMCAPI" : "0.95", + "AI::MXNetCAPI" : "1.01", + "AI::NNVMCAPI" : "1.01", "Function::Parameters" : "1.0705", "GraphViz" : "2.14", "Mouse" : "v2.1.0", @@ -43,5 +43,5 @@ } }, "release_status" : "stable", - "version" : "0.9506" + "version" : "1.01" } diff --git a/perl-package/AI-MXNet/META.yml b/perl-package/AI-MXNet/META.yml index 1abb0079a1ec..14d5dc3ec2d9 100644 --- a/perl-package/AI-MXNet/META.yml +++ b/perl-package/AI-MXNet/META.yml @@ -17,10 +17,10 @@ no_index: - t - inc requires: - AI::MXNetCAPI: '0.95' - AI::NNVMCAPI: '0.95' + AI::MXNetCAPI: '1.01' + AI::NNVMCAPI: '1.01' Function::Parameters: '1.0705' GraphViz: '2.14' Mouse: v2.1.0 PDL: '2.007' -version: '0.9507' +version: '1.01' diff --git a/perl-package/AI-MXNet/Makefile.PL b/perl-package/AI-MXNet/Makefile.PL index 0f6062ec5466..fc5abc0e5721 100644 --- a/perl-package/AI-MXNet/Makefile.PL +++ b/perl-package/AI-MXNet/Makefile.PL @@ -19,15 +19,15 @@ my %WriteMakefileArgs = ( "LICENSE" => "apache_2_0", "NAME" => "AI::MXNet", "PREREQ_PM" => { - "AI::MXNetCAPI" => "0.9507", - "AI::NNVMCAPI" => "0.95", + "AI::MXNetCAPI" => "1.01", + "AI::NNVMCAPI" => "1.01", "Function::Parameters" => "1.0705", "Mouse" => "2.1.0", "PDL" => "2.007", "GraphViz" => "2.14" }, "TEST_REQUIRES" => {}, - "VERSION" => "0.9507", + "VERSION" => "1.01", "test" => { "TESTS" => "t/*.t" } @@ -35,8 +35,8 @@ my %WriteMakefileArgs = ( my %FallbackPrereqs = ( - "AI::MXNetCAPI" => "0.9507", - "AI::NNVMCAPI" => "0.95", + "AI::MXNetCAPI" => "1.01", + "AI::NNVMCAPI" => "1.01", "Function::Parameters" => "1.0705", "Mouse" => "2.1.0", "PDL" => "2.007", diff --git a/perl-package/AI-MXNet/README b/perl-package/AI-MXNet/README index 9831038d54f9..85406f604808 100644 --- a/perl-package/AI-MXNet/README +++ b/perl-package/AI-MXNet/README @@ -1,5 +1,5 @@ This archive contains the distribution AI-MXNet, -version 0.9507: +version 1.01: Perl interface to MXNet machine learning library diff --git a/perl-package/AI-MXNet/lib/AI/MXNet.pm b/perl-package/AI-MXNet/lib/AI/MXNet.pm index f8866399d611..530b6eca23a4 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet.pm @@ -3,6 +3,7 @@ use v5.14.0; use strict; use warnings; use AI::MXNet::Base; +use AI::MXNet::CachedOp; use AI::MXNet::Callback; use AI::MXNet::NDArray; use AI::MXNet::Symbol; @@ -28,7 +29,7 @@ use AI::MXNet::RecordIO; use AI::MXNet::Image; use AI::MXNet::Contrib; use AI::MXNet::Contrib::AutoGrad; -our $VERSION = '0.9507'; +our $VERSION = '1.01'; sub import { @@ -64,9 +65,13 @@ sub import sub callback { 'AI::MXNet::Callback' } sub img { 'AI::MXNet::Image' } sub contrib { 'AI::MXNet::Contrib' } + sub name { '$short_name' } sub AttrScope { shift; AI::MXNet::Symbol::AttrScope->new(\@_) } *AI::MXNet::Symbol::AttrScope::current = sub { \$${short_name}::AttrScope; }; \$${short_name}::AttrScope = AI::MXNet::Symbol::AttrScope->new; + sub Prefix { AI::MXNet::Symbol::Prefix->new(prefix => \$_[1]) } + *AI::MXNet::Symbol::NameManager::current = sub { \$${short_name}::NameManager; }; + \$${short_name}::NameManager = AI::MXNet::Symbol::NameManager->new; *AI::MXNet::Context::current_ctx = sub { \$${short_name}::Context; }; \$${short_name}::Context = AI::MXNet::Context->new(device_type => 'cpu', device_id => 0); 1; diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/CachedOp.pm b/perl-package/AI-MXNet/lib/AI/MXNet/CachedOp.pm new file mode 100644 index 000000000000..bec3f5029c33 --- /dev/null +++ b/perl-package/AI-MXNet/lib/AI/MXNet/CachedOp.pm @@ -0,0 +1,41 @@ +package AI::MXNet::CachedOp; + +=head1 NAME + + AI::MXNet::CachedOp - A wrapper around CachedOpHandle +=cut + +use strict; +use warnings; +use AI::MXNet::Base; +use Mouse; + +has 'op' => (is => 'ro', isa => 'Str', required => 1); +has 'handle' => (is => 'ro', isa => 'CachedOpHandle', required => 1); +around BUILDARGS => sub { + my $orig = shift; + my $class = shift; + my ($op, $num_input, %kwargs) = @_; + for my $key (keys %kwargs) + { + $kwargs{ $key } = "(" .join(", ", @{ $kwargs{ $key } }) .")" + if ref $kwargs{ $key } eq 'ARRAY'; + } + my $AtomicSymbolCreator = check_call(AI::NNVMCAPI::GetOpHandle($op)); + my $handle = check_call( + AI::MXNetCAPI::CachedCreateOp( + $AtomicSymbolCreator, + $num_input, + scalar(keys %kwargs), + \%kwargs + ) + ); + return $class->$orig(op => $op, handle => $handle); +}; + +sub DEMOLISH +{ + check_call(AI::MXNetCAPI::CachedFree(shift->handle)); +} + +1; \ No newline at end of file diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Contrib/AutoGrad.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Contrib/AutoGrad.pm index 4fd910fb34c9..6d9c10340939 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Contrib/AutoGrad.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Contrib/AutoGrad.pm @@ -71,6 +71,58 @@ method mark_variables( ); } +=head2 backward + + Compute the gradients of outputs w.r.t variables. + + Parameters + ---------- + outputs: array ref of NDArray + out_grads: array ref of NDArray or undef + retain_graph: bool, defaults to false +=cut + + +method backward( + ArrayRef[AI::MXNet::NDArray] $outputs, + Maybe[ArrayRef[AI::MXNet::NDArray|Undef]] $out_grads=, + Bool $retain_graph=0 +) +{ + my @output_handles = map { $_->handle } @{ $outputs }; + if(not defined $out_grads) + { + check_call( + AI::MXNetCAPI::AutogradBackward( + scalar(@output_handles), + \@output_handles, + [], + $retain_graph + ) + ); + return; + } + + my @ograd_handles; + for my $arr (@$out_grads) + { + push @ograd_handles, (defined $arr ? $arr->handle : undef); + } + assert( + (@ograd_handles == @output_handles), + "outputs and out_grads must have the same length" + ); + + check_call( + AI::MXNetCAPI::AutogradBackward( + scalar(@output_handles), + \@output_handles, + \@ograd_handles, + $retain_graph + ) + ); +} + =head2 compute_gradient Compute the gradients of outputs w.r.t variables. @@ -87,13 +139,7 @@ method mark_variables( method compute_gradient(ArrayRef[AI::MXNet::NDArray] $outputs) { - my @output_handles = map { $_->handle } @{ $outputs }; - check_call( - AI::MXNetCAPI::AutogradComputeGradient( - scalar(@output_handles), - \@output_handles - ) - ); + __PACKAGE__->backward($outputs); } =head2 grad_and_loss @@ -164,4 +210,18 @@ method grad(CodeRef $func, Maybe[Int|ArrayRef[Int]] $argnum=) }; } +method train_section(CodeRef $sub) +{ + my $prev = __PACKAGE__->set_is_training(1); + $sub->(); + __PACKAGE__->set_is_training(0) unless $prev; +} + +method test_section(CodeRef $sub) +{ + my $prev = __PACKAGE__->set_is_training(0); + $sub->(); + __PACKAGE__->set_is_training(1) if $prev; +} + 1; \ No newline at end of file diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Executor/Group.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Executor/Group.pm index 33e54dc1e847..0ae2db0b1895 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Executor/Group.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Executor/Group.pm @@ -36,6 +36,7 @@ func _split_input_slice($batch_size, $work_load_list) } return \@slices; } + # Load a array ref of arrays into a array ref of arrays specified by slices func _load_general($data, $targets, $major_axis) { @@ -59,20 +60,45 @@ func _load_general($data, $targets, $major_axis) my ($slice_idx, $d_dst) = @{ $d }; if($axis >= 0) { - # copy slice - my $end = $d_src->shape; - my $begin = [(0) x @{ $end }]; - $begin->[$axis] = $slice_idx->[0]; - $end->[$axis] = $slice_idx->[1]; - if($d_src->context == $d_dst->context) + my $shape = $d_src->shape; + my $do_crop = ($slice_idx->[0] != 0 or $shape->[$axis] != $slice_idx->[1]); + if($do_crop) { - $d_src->crop({ begin => $begin, end => $end, out => $d_dst }); + if($axis == 0) + { + $d_src->slice([$slice_idx->[0], $slice_idx->[1] - 1])->copyto($d_dst); + } + else + { + if($d_src->context == $d_dst->context) + { + AI::MXNet::NDArray->slice_axis( + $d_src, + { + axis => $axis, + begin => $slice_idx->[0], + end => $slice_idx->[1], + out => $d_dst + } + ); + } + else + { + my $d_dst_copy = AI::MXNet::NDArray->slice_axis( + $d_src, + { + axis => $axis, + begin => $slice_idx->[0], + end => $slice_idx->[1] + } + ); + $d_dst_copy->copyto($d_dst); + } + } } else { - # on different device, crop and then do cross device copy - my $d_dst_copy = $d_src->crop({ begin => $begin, end => $end }); - $d_dst_copy->copyto($d_dst); + $d_src->copyto($d_dst); } } else @@ -787,8 +813,6 @@ method update_metric(AI::MXNet::EvalMetric $eval_metric, ArrayRef[AI::MXNet::NDA }, $self->_p->execs, $self->_p->slices); } -# Internal utility function to bind the i-th executor. - method _bind_ith_exec( Int $i, ArrayRef[AI::MXNet::DataDesc] $data_shapes, @@ -804,151 +828,15 @@ method _bind_ith_exec( { %input_shapes = (%input_shapes, map { $_->name => $_->shape } @{ $label_shapes }); } - my ($arg_shapes, undef, $aux_shapes) = $self->symbol->infer_shape(%input_shapes); - confess("shape inference failed") unless defined $arg_shapes; - my %input_types = map { $_->name => $_->dtype } @{ $data_shapes }; - my ($arg_types, undef, $aux_types) = $self->symbol->infer_type(%input_types); - confess("type inference failed") unless defined $arg_types; - my $arg_arrays = []; - my $grad_arrays = $self->for_training ? {} : undef; - - #Internal helper to get a memory block or re-use by re-shaping - my $_get_or_reshape = sub { - my ($name, $shared_data_arrays, $arg_shape, $arg_type, $context, $logger) = @_; - my $arg_arr; - if(exists $shared_data_arrays->{$name}) - { - $arg_arr = $shared_data_arrays->{$name}; - if(product(@{ $arg_arr->shape }) >= product(@{ $arg_shape })) - { - # nice, we can directly re-use this data blob - confess("dtypes do not match") - unless $arg_arr->dtype eq $arg_type; - $arg_arr = $arg_arr->reshape($arg_shape); - } - else - { - $logger->warning( - 'bucketing: data "%s" has a shape (%s)' - .', which is larger than already allocated ' - .'shape (%s)' - .'. Need to re-allocate. Consider putting ' - .'default_bucket_key to' - .' be the bucket taking the largest input for better ' - .'memory sharing.', - $name, join(',', $arg_shape), join(',', $arg_arr->shape) - ); - $arg_arr = AI::MXNet::NDArray->zeros( - $arg_shape, - ctx => $context, - dtype => $arg_type - ); - # replace existing shared array because the new one is bigger - $shared_data_arrays->{ $name } = $arg_arr; - } - } - else - { - $arg_arr = AI::MXNet::NDArray->zeros( - $arg_shape, - ctx => $context, - dtype => $arg_type - ); - $shared_data_arrays->{ $name } = $arg_arr; - } - return $arg_arr; - }; - my %param_names = map { $_ => 1 } @{ $self->param_names }; - # create or borrow arguments and gradients - for my $j (0..@{ $self->_p->arg_names }-1) - { - my $name = $self->_p->arg_names->[$j]; - my $arg_arr; - if(exists $param_names{ $name }) # model parameter - { - if(not defined $shared_exec) - { - $arg_arr = AI::MXNet::NDArray->zeros( - $arg_shapes->[$j], - ctx => $context, - dtype => $arg_types->[$j] - ); - - if($self->grad_req->{$name} ne 'null') - { - my $grad_arr = AI::MXNet::NDArray->zeros( - $arg_shapes->[$j], - ctx => $context, - dtype => $arg_types->[$j] - ); - $grad_arrays->{ $name } = $grad_arr; - } - } - else - { - $arg_arr = $shared_exec->arg_dict->{ $name }; - my $arg_arr_shape = $arg_arr->shape; - my $arg_shape = $arg_shapes->[$j]; - confess "shapes do not match (@$arg_arr_shape) != (@$arg_shape)" - unless "@$arg_arr_shape" eq "@$arg_shape"; - my $arg_arr_type = $arg_arr->dtype; - my $arg_type = $arg_types->[$j]; - confess "types do not match $arg_arr_type) != $arg_type" - unless $arg_arr_type eq $arg_type; - if($self->grad_req->{ $name } ne 'null') - { - $grad_arrays->{ $name } = $shared_exec->grad_dict->{ $name }; - } - } - } - else # data or label - { - $arg_arr = $_get_or_reshape->( - $name, $shared_data_arrays, $arg_shapes->[$j], - $arg_types->[$j], $context, $self->logger - ); - if($self->grad_req->{ $name } ne 'null') - { - $grad_arrays->{ $name } = $_get_or_reshape->( - "grad of $name", $shared_data_arrays, - $arg_shapes->[$j], $arg_types->[$j], - $context, $self->logger - ); - } - } - # data might also need grad if inputs_need_grad is True - push @{ $arg_arrays }, $arg_arr; - } - # create or borrow aux variables - my $aux_arrays = []; - if(not defined $shared_exec) - { - zip(sub{ - my ($s, $t) = @_; - push @{ $aux_arrays }, AI::MXNet::NDArray->zeros($s, ctx => $context, dtype => $t); - }, $aux_shapes, $aux_types); - } - else - { - for my $j (0..@{ $shared_exec->aux_arrays }-1) - { - my $arr = $shared_exec->aux_arrays->[$j]; - my $aux_shape = $aux_shapes->[$j]; - my $arr_shape = $arr->shape; - confess("aux shape (@$aux_shape) != array shape (@$arr_shape)") - unless "@$aux_shape" eq "@$arr_shape"; - my $aux_type = $aux_types->[$j]; - my $arr_type = $arr->dtype; - confess("aux_type $aux_type != array type $arr_type") - unless $aux_type ne $arr_type; - } - @{ $aux_arrays } = @{ $shared_exec->aux_arrays }; - } - my $executor = $self->symbol->bind( - ctx => $context, args => $arg_arrays, - args_grad => $grad_arrays, aux_states => $aux_arrays, - grad_req => $self->grad_req, shared_exec => $shared_exec + my $executor = $self->symbol->simple_bind( + ctx => $context, + grad_req => $self->grad_req, + type_dict => \%input_types, + shared_arg_names => $self->param_names, + shared_exec => $shared_exec, + shared_buffer => $shared_data_arrays, + shapes => \%input_shapes ); return $executor; } @@ -999,4 +887,14 @@ method install_monitor(AI::MXNet::Monitor $mon) $mon->install($_) for @{ $self->_p->execs }; } +method shared_data_arrays() +{ + $self->_p->shared_data_arrays; +} + +method execs() +{ + $self->_p->execs; +} + 1; diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/NDArray.pm b/perl-package/AI-MXNet/lib/AI/MXNet/NDArray.pm index 2871a62226e5..68c4e7061ec3 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/NDArray.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/NDArray.pm @@ -275,7 +275,7 @@ method asmpdl() Finishing index of slice. =cut -method _slice ( +method _slice ( Index $start, Index $stop ) @@ -918,9 +918,14 @@ method empty(Shape $shape, AI::MXNet::Context :$ctx=AI::MXNet::Context->current_ The created NDArray. =cut -method zeros(Shape $shape, AI::MXNet::Context :$ctx=AI::MXNet::Context->current_ctx, Dtype :$dtype='float32') +method zeros( + Shape $shape, + AI::MXNet::Context :$ctx=AI::MXNet::Context->current_ctx, + Dtype :$dtype='float32', + Maybe[AI::MXNet::NDArray] :$out= +) { - return __PACKAGE__->_zeros({ shape => $shape, ctx => "$ctx", dtype => $dtype }); + return __PACKAGE__->_zeros({ shape => $shape, ctx => "$ctx", dtype => $dtype, ($out ? (out => $out) : ()) }); } =head2 ones @@ -944,9 +949,14 @@ method zeros(Shape $shape, AI::MXNet::Context :$ctx=AI::MXNet::Context->current_ The created NDArray. =cut -method ones(Shape $shape, AI::MXNet::Context :$ctx=AI::MXNet::Context->current_ctx, Dtype :$dtype='float32') +method ones( + Shape $shape, + AI::MXNet::Context :$ctx=AI::MXNet::Context->current_ctx, + Dtype :$dtype='float32', + Maybe[AI::MXNet::NDArray] :$out= +) { - return __PACKAGE__->_ones({ shape => $shape, ctx => "$ctx", dtype => $dtype }); + return __PACKAGE__->_ones({ shape => $shape, ctx => "$ctx", dtype => $dtype, ($out ? (out => $out) : ()) }); } =head2 full @@ -973,9 +983,13 @@ method ones(Shape $shape, AI::MXNet::Context :$ctx=AI::MXNet::Context->current_c The created NDArray. =cut -method full(Shape $shape, Num $val, AI::MXNet::Context :$ctx=AI::MXNet::Context->current_ctx, Dtype :$dtype='float32') +method full( + Shape $shape, Num $val, + AI::MXNet::Context :$ctx=AI::MXNet::Context->current_ctx, + Dtype :$dtype='float32', Maybe[AI::MXNet::NDArray] :$out= +) { - return __PACKAGE__->_set_value({ src => $val, out => __PACKAGE__->empty($shape, ctx => $ctx, dtype => $dtype) }); + return __PACKAGE__->_set_value({ src => $val, out => $out ? $out : __PACKAGE__->empty($shape, ctx => $ctx, dtype => $dtype) }); } =head2 array @@ -1307,6 +1321,59 @@ method waitall() check_call(AI::MXNetCAPI::NDArrayWaitAll()); } +=head2 _fresh_grad + + Parameters: + ---------- + Maybe[Bool] $state= + + Whether this array's corresponding gradient array + (registered via `autograd->mark_variables`) has been + updated by `autograd->backward` since last reset. + + `_fresh_grad` need to be manually set to False + after consuming gradient (usually after updating this + array). +=cut + +method _fresh_grad(Maybe[Bool] $state=) +{ + if(defined $state) + { + check_call(AI::MXNetCAPI::NDArraySetGradState($self->handle, $state)); + return $state; + } + else + { + return scalar(check_call(AI::MXNetCAPI::NDArrayGetGradState($self->handle))); + } +} + +=head2 detach + + Returns a new NDArray, detached from the current graph. +=cut + +method detach() +{ + my $handle = check_call(AI::MXNetCAPI::NDArrayDetach($self->handle)); + return __PACKAGE__->new(handle => $handle); +} + +method backward(Maybe[AI::MXNet::NDArray] $out_grad=, Bool $retain_graph=0) +{ + check_call( + AI::MXNetCAPI::AutogradBackward( + 1, + [$self->handle], + [defined $out_grad ? $out_grad->handle : undef], + $retain_graph + ) + ) +} + +method CachedOp(@args) { AI::MXNet::CachedOp->new(@args) } + my $lvalue_methods = join "\n", map {"use attributes 'AI::MXNet::NDArray', \\&AI::MXNet::NDArray::$_, 'lvalue';"} qw/at slice aspdl asmpdl reshape copy sever T astype as_in_context copyto empty zero ones full array/; diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/NDArray/Base.pm b/perl-package/AI-MXNet/lib/AI/MXNet/NDArray/Base.pm index c962f0849733..0c48336c2aae 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/NDArray/Base.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/NDArray/Base.pm @@ -140,6 +140,45 @@ method _init_ndarray_module() } } +method invoke( + AI::MXNet::CachedOp $cached_op, + ArrayRef[AI::MXNet::NDArray] $args, + Maybe[AI::MXNet::NDArray|ArrayRef[AI::MXNet::NDArray]] $out=, + Maybe[Str] $name= +) +{ + my $original_output; + if(defined $out) + { + $original_output = $out; + if(not ref($out) eq 'ARRAY') + { + $out = [$out]; + } + } + else + { + $out = []; + } + my $output = check_call( + AI::MXNetCAPI::CachedInvoke( + $cached_op->handle, + scalar(@$args), + [map { $_->handle } @$args], + [map { $_->handle } @$out] + ) + ); + return $original_output if defined $original_output; + if(@$output == 1) + { + return $self->new(handle => $output->[0]); + } + else + { + return [map { $self->new(handle => $_) } @$output]; + } +} + __PACKAGE__->_init_ndarray_module; 1; diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/RNN.pm b/perl-package/AI-MXNet/lib/AI/MXNet/RNN.pm index cb3db9740868..13dc4f24de33 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/RNN.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/RNN.pm @@ -149,6 +149,7 @@ method SequentialRNNCell(@args) { AI::MXNet::RNN::SequentialCell->new(@args) } method BidirectionalCell(@args) { AI::MXNet::RNN::BidirectionalCell->new(@args) } method DropoutCell(@args) { AI::MXNet::RNN::DropoutCell->new(@args) } method ZoneoutCell(@args) { AI::MXNet::RNN::ZoneoutCell->new(@args) } +method ResidualCell(@args) { AI::MXNet::RNN::ResidualCell->new(@args) } method encode_sentences(@args) { AI::MXNet::RNN::IO->encode_sentences(@args) } method BucketSentenceIter(@args) { diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/RNN/Cell.pm b/perl-package/AI-MXNet/lib/AI/MXNet/RNN/Cell.pm index cea1af7cb9cc..89968491e153 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/RNN/Cell.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/RNN/Cell.pm @@ -1257,6 +1257,18 @@ sub BUILD { my ($self, $original_arguments) = @_; $self->_override_cell_params(defined $original_arguments->{params}); + if($self->_override_cell_params) + { + assert( + ($self->l_cell->_own_params and $self->r_cell->_own_params), + "Either specify params for BidirectionalCell ". + "or child cells, not both." + ); + %{ $self->l_cell->params->_params } = (%{ $self->l_cell->params->_params }, %{ $self->params->_params }); + %{ $self->r_cell->params->_params } = (%{ $self->r_cell->params->_params }, %{ $self->params->_params }); + } + %{ $self->params->_params } = (%{ $self->params->_params }, %{ $self->l_cell->params->_params }); + %{ $self->params->_params } = (%{ $self->params->_params }, %{ $self->r_cell->params->_params }); $self->_cells([$self->l_cell, $self->r_cell]); } @@ -1519,7 +1531,7 @@ has 'prev_output' => (is => 'rw', init_arg => undef); =head1 DESCRIPTION - Apply Zoneout on base cell + Apply Zoneout on base cell. =cut sub BUILD @@ -1555,8 +1567,7 @@ method call(AI::MXNet::Symbol $inputs, SymbolOrArrayOfSymbols $states) my $mask = sub { my ($p, $like) = @_; AI::MXNet::Symbol->Dropout( - AI::MXNet::Symbol->_identity_with_attr_like_rhs( - AI::MXNet::Symbol->ones(shape => [0, 0]), + AI::MXNet::Symbol->ones_like( $like ), p => $p @@ -1586,4 +1597,106 @@ method call(AI::MXNet::Symbol $inputs, SymbolOrArrayOfSymbols $states) return ($output, @states ? \@states : $next_states); } +package AI::MXNet::RNN::ResidualCell; +use Mouse; +use AI::MXNet::Base; +extends 'AI::MXNet::RNN::ModifierCell'; + +=head1 NAME + + AI::MXNet::RNN::ResidualCell +=cut + +=head1 DESCRIPTION + + Adds residual connection as described in Wu et al, 2016 + (https://arxiv.org/abs/1609.08144). + Output of the cell is output of the base cell plus input. +=cut + +method call(AI::MXNet::Symbol $inputs, SymbolOrArrayOfSymbols $states) +{ + my $output; + ($output, $states) = &{$self->base_cell}($inputs, $states); + $output = AI::MXNet::Symbol->elemwise_add($output, $inputs, name => $output->name.'_plus_residual'); + return ($output, $states) +} + +method unroll( + Int $length, + Maybe[AI::MXNet::Symbol|ArrayRef[AI::MXNet::Symbol]] :$inputs=, + Maybe[AI::MXNet::Symbol|ArrayRef[AI::MXNet::Symbol]] :$begin_state=, + Str :$input_prefix='', + Str :$layout='NTC', + Maybe[Bool] :$merge_outputs= +) +{ + $self->reset; + $self->base_cell->_modified(0); + my ($outputs, $states) = $self->base_cell->unroll($length, inputs=>$inputs, begin_state=>$begin_state, + layout=>$layout, merge_outputs=>$merge_outputs); + $self->base_cell->_modified(1); + $merge_outputs //= (blessed($outputs) and $outputs->isa('AI::MXNet::Symbol')); + ($inputs) = _normalize_sequence($length, $inputs, $layout, $merge_outputs); + if($merge_outputs) + { + $outputs = AI::MXNet::Symbol->elemwise_add($outputs, $inputs, name => $outputs->name . "_plus_residual"); + } + else + { + my @temp; + zip(sub { + my ($output_sym, $input_sym) = @_; + push @temp, AI::MXNet::Symbol->elemwise_add($output_sym, $input_sym, + name=>$output_sym->name."_plus_residual"); + }, [@{ $outputs }], [@{ $inputs }]); + $outputs = \@temp; + } + return ($outputs, $states); +} + +func _normalize_sequence($length, $inputs, $layout, $merge, $in_layout=) +{ + assert((defined $inputs), + "unroll(inputs=>undef) has been deprecated. ". + "Please create input variables outside unroll." + ); + + my $axis = index($layout, 'T'); + my $in_axis = defined $in_layout ? index($in_layout, 'T') : $axis; + if(blessed($inputs)) + { + if(not $merge) + { + assert( + (@{ $inputs->list_outputs() } == 1), + "unroll doesn't allow grouped symbol as input. Please " + ."convert to list first or let unroll handle splitting" + ); + $inputs = [ @{ AI::MXNet::Symbol->split( + $inputs, + axis => $in_axis, + num_outputs => $length, + squeeze_axis => 1 + ) }]; + } + } + else + { + assert(not defined $length or @$inputs == $length); + if($merge) + { + $inputs = [map { AI::MXNet::Symbol->expand_dims($_, axis=>$axis) } @{ $inputs }]; + $inputs = AI::MXNet::Symbol->Concat(@{ $inputs }, dim=>$axis); + $in_axis = $axis; + } + } + + if(blessed($inputs) and $axis != $in_axis) + { + $inputs = AI::MXNet::Symbol->swapaxes($inputs, dim0=>$axis, dim1=>$in_axis); + } + return ($inputs, $axis); +} + 1; diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Symbol.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Symbol.pm index f4f5f0de3efb..eec32640953c 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Symbol.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Symbol.pm @@ -732,6 +732,19 @@ method _get_ndarray_inputs( :$shapes : hash ref of str->Shape Input shape map, name->shape + :$shared_arg_names : Maybe[ArrayRef[Str]] + The argument names whose 'NDArray' of shared_exec can be reused for initializing + the current executor. + + :$shared_exec : Maybe[AI::MXNet::Executor] + The executor whose arg_arrays, arg_arrays, grad_arrays, and aux_arrays can be + reused for initializing the current executor. + + :$shared_buffer : Maybe[HashRef[AI::MXNet::NDArray]] + The dict mapping argument names to the `NDArray` that can be reused for initializing + the current executor. This buffer will be checked for reuse if one argument name + of the current executor is not found in `shared_arg_names`. + Returns ------- $executor : AI::MXNet::Executor @@ -739,115 +752,161 @@ method _get_ndarray_inputs( =cut method simple_bind( - AI::MXNet::Context :$ctx=AI::MXNet::Context->current_ctx, - Maybe[HashRef[Shape]] :$shapes=, - Str|HashRef[Str] :$grad_req='write', - Maybe[HashRef[Dtype]] :$type_dict=, - Maybe[HashRef[AI::MXNet::Context]] :$group2ctx= + AI::MXNet::Context :$ctx=AI::MXNet::Context->current_ctx, + GradReq|ArrayRef[GradReq]|HashRef[GradReq] :$grad_req='write', + Maybe[HashRef[Shape]] :$shapes=, + Maybe[HashRef[Dtype]] :$type_dict=, + Maybe[HashRef[AI::MXNet::Context]] :$group2ctx=, + Maybe[ArrayRef[Str]] :$shared_arg_names=, + Maybe[AI::MXNet::Executor] :$shared_exec=, + Maybe[HashRef[AI::MXNet::NDArray]] :$shared_buffer= ) { - $shapes //= {}; - if(not defined $type_dict) + my $num_provided_arg_types; + my @provided_arg_type_names; + my @provided_arg_type_data; + if(defined $type_dict) { - $type_dict = {}; - my $attrs = $self->attr_dict; - for my $k (@{ $self->list_arguments }) + while(my ($k, $v) = each %{ $type_dict }) { - if(not exists $attrs->{$k} or not exists $attrs->{$k}{__dtype__}) - { - $type_dict->{ $k } = 'float32'; - } + push @provided_arg_type_names, $k; + push @provided_arg_type_data, DTYPE_STR_TO_MX->{$v}; } + $num_provided_arg_types = @provided_arg_type_names; } - my @keys = keys %$shapes; - my @shape_input; - my @type_input; - for my $k (@keys) + my @provided_arg_shape_data; + # argument shape index in sdata, + # e.g. [sdata[indptr[0]], sdata[indptr[1]]) is the shape of the first arg + my @provided_arg_shape_idx = (0); + my @provided_arg_shape_names; + while(my ($k, $v) = each %{ $shapes//{} }) { - push @shape_input, ($k => $shapes->{$k}); - push @type_input, ($k => $type_dict->{$k}) + push @provided_arg_shape_names, $k; + push @provided_arg_shape_data, @{ $v }; + push @provided_arg_shape_idx, scalar(@provided_arg_shape_data); } - my ($arg_shapes, undef, $aux_shapes) = $self->infer_shape(@shape_input); - my ($arg_types, undef, $aux_types) = $self->infer_type(@type_input); - confess("Input node is not complete") - unless $arg_shapes and $arg_types; + $num_provided_arg_types = @provided_arg_type_names; - my ($arg_ctx, $aux_ctx) = ([], []); - if(defined $group2ctx) + my $provided_req_type_list_len = 0; + my @provided_grad_req_types; + my @provided_grad_req_names; + if(defined $grad_req) { - my $attr_dict = $self->attr_dict(); - for my $name (@{ $self->list_arguments() }) + if(not ref $grad_req) { - if( - exists $attr_dict->{ $name } - and - exists $attr_dict->{ $name }{ __ctx_group__ } - and - $group2ctx->{ $attr_dict->{ $name }{ __ctx_group__ } } - ) - { - push @{ $arg_ctx }, $group2ctx->{ $attr_dict->{ $name }{ __ctx_group__ } }; - } - else - { - push @{ $arg_ctx }, $ctx; - } + push @provided_grad_req_types, $grad_req; } - for my $name (@{ $self->list_auxiliary_states() }) + elsif(ref $grad_req eq 'ARRAY') { - if( - exists $attr_dict->{ $name } - and - exists $attr_dict->{ $name }{ __ctx_group__ } - and - $group2ctx->{ $attr_dict->{ $name }{ __ctx_group__ } } - ) - { - push @{ $aux_ctx }, $group2ctx->{ $attr_dict->{ $name }{ __ctx_group__ } }; - } - else + assert((@{ $grad_req } != 0), 'grad_req in simple_bind cannot be an empty list'); + @provided_grad_req_types = @{ $grad_req }; + $provided_req_type_list_len = @provided_grad_req_types; + } + elsif(ref $grad_req eq 'HASH') + { + assert((keys %{ $grad_req } != 0), 'grad_req in simple_bind cannot be an empty hash'); + while(my ($k, $v) = each %{ $grad_req }) { - push @{ $aux_ctx }, $ctx; + push @provided_grad_req_names, $k; + push @provided_grad_req_types, $v; } + $provided_req_type_list_len = @provided_grad_req_types; } } - else + my $num_ctx_map_keys = 0; + my @ctx_map_keys; + my @ctx_map_dev_types; + my @ctx_map_dev_ids; + if(defined $group2ctx) { - @{ $arg_ctx } = (($ctx) x @{ $arg_shapes }); - @{ $aux_ctx } = (($ctx) x @{ $aux_shapes }); + while(my ($k, $v) = each %{ $group2ctx }) + { + push @ctx_map_keys, $k; + push @ctx_map_dev_types, $v->device_type_id; + push @ctx_map_dev_ids, $v->device_id; + } + $num_ctx_map_keys = @ctx_map_keys; } - my @arg_ndarrays; - for (my $i = 0; $i < @{ $arg_types }; $i++) + + my @shared_arg_name_list; + if(defined $shared_arg_names) { - push @arg_ndarrays, AI::MXNet::NDArray->zeros( - $arg_shapes->[$i], ctx => $arg_ctx->[$i], dtype => $arg_types->[$i] - ); + @shared_arg_name_list = @{ $shared_arg_names }; } - my $grad_ndarrays; - if($grad_req ne 'null') + my %shared_data; + if(defined $shared_buffer) { - my $names = $self->list_arguments; - for (my $i = 0; $i < @{ $arg_types }; $i++) + while(my ($k, $v) = each %{ $shared_buffer }) { - if(not ref $grad_req eq 'HASH' or not ($grad_req->{ $names->[$i] }//'') eq 'null') - { - $grad_ndarrays->{ $names->[$i] } = AI::MXNet::NDArray->zeros( - $arg_shapes->[$i], ctx => $arg_ctx->[$i], dtype => $arg_types->[$i] - ); - } + $shared_data{$k} = $v->handle; } } - my @aux_ndarrays; - for (my $i = 0; $i < @{ $aux_types }; $i++) + my $shared_exec_handle = defined $shared_exec ? $shared_exec->handle : undef; + my ( + $updated_shared_data, + $in_arg_handles, + $arg_grad_handles, + $aux_state_handles, + $exe_handle + ); + eval { + ($updated_shared_data, $in_arg_handles, $arg_grad_handles, $aux_state_handles, $exe_handle) + = + check_call( + AI::MXNetCAPI::ExecutorSimpleBind( + $self->handle, + $ctx->device_type_id, + $ctx->device_id, + $num_ctx_map_keys, + \@ctx_map_keys, + \@ctx_map_dev_types, + \@ctx_map_dev_ids, + $provided_req_type_list_len, + \@provided_grad_req_names, + \@provided_grad_req_types, + scalar(@provided_arg_shape_names), + \@provided_arg_shape_names, + \@provided_arg_shape_data, + \@provided_arg_shape_idx, + $num_provided_arg_types, + \@provided_arg_type_names, + \@provided_arg_type_data, + scalar(@shared_arg_name_list), + \@shared_arg_name_list, + defined $shared_buffer ? \%shared_data : undef, + $shared_exec_handle + ) + ); + }; + if($@) { - push @aux_ndarrays, AI::MXNet::NDArray->zeros( - $aux_shapes->[$i], ctx => $aux_ctx->[$i], dtype => $aux_types->[$i] + confess( + "simple_bind failed: Error: $@; Arguments: ". + Data::Dumper->new( + [$shapes//{}] + )->Purity(1)->Deepcopy(1)->Terse(1)->Dump ); } - my $executor = $self->bind( - ctx => $ctx, args => \@arg_ndarrays, args_grad => $grad_ndarrays, - grad_req => $grad_req, aux_states => \@aux_ndarrays, group2ctx => $group2ctx + if(defined $shared_buffer) + { + while(my ($k, $v) = each %{ $updated_shared_data }) + { + $shared_buffer->{$k} = AI::MXNet::NDArray->new(handle => $v); + } + } + my @arg_arrays = map { AI::MXNet::NDArray->new(handle => $_) } @{ $in_arg_handles }; + my @grad_arrays = map { defined $_ ? AI::MXNet::NDArray->new(handle => $_) : undef } @{ $arg_grad_handles }; + my @aux_arrays = map { AI::MXNet::NDArray->new(handle => $_) } @{ $aux_state_handles }; + my $executor = AI::MXNet::Executor->new( + handle => $exe_handle, + symbol => $self, + ctx => $ctx, + grad_req => $grad_req, + group2ctx => $group2ctx ); + $executor->arg_arrays(\@arg_arrays); + $executor->grad_arrays(\@grad_arrays); + $executor->aux_arrays(\@aux_arrays); return $executor; } @@ -1288,6 +1347,8 @@ method arange(Index :$start=0, Index :$stop=, Num :$step=1.0, Index :$repeat=1, }); } +method CachedOp(@args) { AI::MXNet::CachedOp->new(@args) } + sub _parse_arguments { my $type = shift; diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Symbol/Base.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Symbol/Base.pm index 95b634024135..3eaee237bed0 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Symbol/Base.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Symbol/Base.pm @@ -167,6 +167,21 @@ method _init_symbol_module() } } +method invoke(AI::MXNet::CachedOp $cached_op, ArrayRef[AI::MXNet::Symbol] $args, Maybe[Str] $name=) +{ + my $hint = lc($cached_op->op); + $name = AI::MXNet::Symbol::NameManager->current->get($name, $hint); + my $handle = check_call( + AI::MXNetCAPI::CachedCreateSymbol( + $cached_op->handle, + $name, + scalar(@$args), + [map { $_->handle } @$args] + ) + ); + return $self->new(handle => $handle); +} + __PACKAGE__->_init_symbol_module; 1; diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Symbol/NameManager.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Symbol/NameManager.pm index 4791bc8b74a8..1e31730692a7 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Symbol/NameManager.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Symbol/NameManager.pm @@ -44,7 +44,7 @@ our $current; A canonical name for the symbol. =cut -method get(Str|Undef $name, Str $hint) +method get(Maybe[Str] $name, Str $hint) { return $name if $name; if(not exists $self->counter->{ $hint }) @@ -84,7 +84,7 @@ has prefix => ( required => 1 ); -method get(Str $name, Str $hint) +method get(Maybe[Str] $name, Str $hint) { $name = $self->SUPER::get($name, $hint); return $self->prefix . $name; diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/TestUtils.pm b/perl-package/AI-MXNet/lib/AI/MXNet/TestUtils.pm index d43d8eb09884..e6e3189646d8 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/TestUtils.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/TestUtils.pm @@ -9,7 +9,7 @@ use Exporter; use base qw(Exporter); @AI::MXNet::TestUtils::EXPORT_OK = qw(same reldiff almost_equal GetMNIST_ubyte GetCifar10 pdl_maximum pdl_minimum mlp2 conv - check_consistency zip assert enumerate); + check_consistency zip assert enumerate same_array); use constant default_numerical_threshold => 1e-6; =head1 NAME @@ -352,4 +352,37 @@ sub assert unless $input; } +=head2 same_array + + Check whether two NDArrays sharing the same memory block + + Parameters + ---------- + + array1 : NDArray + First NDArray to be checked + array2 : NDArray + Second NDArray to be checked + + Returns + ------- + bool + Whether two NDArrays share the same memory +=cut + +func same_array( + AI::MXNet::NDArray $array1, + AI::MXNet::NDArray $array2 +) +{ + $array1 += 1; + if(not same($array1->aspdl, $array2->aspdl)) + { + $array1 -= 1; + return 0 + } + $array1 -= 1; + return same($array1->aspdl, $array2->aspdl); +} + 1; \ No newline at end of file diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Visualization.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Visualization.pm index 9e90c69d9529..d6ea5aa97bf6 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Visualization.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Visualization.pm @@ -291,7 +291,7 @@ method plot_network( my $label = $name; if($op eq 'null') { - if($name =~ /(?:_weight|_bias)$/) + if($name =~ /(?:_weight|_bias|_beta|_gamma|_moving_var|_moving_mean)$/) { if($hide_weights) { diff --git a/perl-package/AI-MXNet/t/test_executor.t b/perl-package/AI-MXNet/t/test_executor.t index d6439b61aee6..026f1f13454a 100644 --- a/perl-package/AI-MXNet/t/test_executor.t +++ b/perl-package/AI-MXNet/t/test_executor.t @@ -151,12 +151,10 @@ sub test_reshape { my $x = mx->sym->Variable('x'); my $y = mx->sym->FullyConnected($x, num_hidden=>4); - - my $exe = $y->simple_bind(ctx => mx->cpu(), shapes => { x=>[5,4] }); + my $exe = $y->simple_bind(ctx => mx->cpu(), shapes => { x=>[5,4] }, grad_req=>'null'); $exe->arg_arrays->[0] .= 1; $exe->arg_arrays->[1] .= mx->nd->ones([4,4]); $exe->arg_arrays->[2] .= 0; - my $new_exe = $exe->reshape({ x=>[3,4] }); $new_exe->forward(0); # test sub exec forward diff --git a/perl-package/AI-MXNet/t/test_module.t b/perl-package/AI-MXNet/t/test_module.t index 89228c949aa8..c6e3c1a8ca0b 100644 --- a/perl-package/AI-MXNet/t/test_module.t +++ b/perl-package/AI-MXNet/t/test_module.t @@ -1,9 +1,10 @@ use strict; use warnings; -use Test::More tests => 23; +use Test::More tests => 247; use AI::MXNet qw(mx); use AI::MXNet::Base; -use AI::MXNet::TestUtils qw(almost_equal enumerate); +use AI::MXNet::TestUtils qw(almost_equal enumerate same_array); +use Data::Dumper; sub test_module_layout { @@ -332,6 +333,124 @@ sub test_module_input_grads ok(($c_grad == 3)->all); } +sub test_executor_group +{ + my $get_rnn_sym = sub { my ($num_layers, $num_words, $num_hidden, $num_embed, $seq_len) = @_; + my $stack = mx->rnn->SequentialRNNCell(); + for my $i (0..$num_layers-1) + { + $stack->add(mx->rnn->LSTMCell(num_hidden=>$num_hidden, prefix=>"lstm_l${i}_")); + } + my $data = mx->sym->Variable('data'); + my $label = mx->sym->Variable('softmax_label'); + my $embed = mx->sym->Embedding(data=>$data, input_dim=>$num_words, + output_dim=>$num_embed, name=>'embed'); + + $stack->reset(); + my ($outputs, $states) = $stack->unroll($seq_len, inputs=>$embed, merge_outputs=>1); + + my $pred = mx->sym->Reshape($outputs, shape=>[-1, $num_hidden]); + $pred = mx->sym->FullyConnected(data=>$pred, num_hidden=>$num_words, name=>'pred'); + + $label = mx->sym->Reshape($label, shape=>[-1]); + $pred = mx->sym->SoftmaxOutput(data=>$pred, label=>$label, name=>'softmax'); + return $pred; + }; + + my $test_shared_exec_group = sub { my ($exec_grp_shared, $exec_grp_created, $shared_arg_names, $extra_args) = @_; + # Test shared data arrays + for my $i (0..@{ $exec_grp_shared->execs }-1) + { + # test same shared_data_arrays for two exec groups + my $shared_data_array1 = $exec_grp_shared->shared_data_arrays->[$i]; + my $shared_data_array2 = $exec_grp_created->shared_data_arrays->[$i]; + if(defined $extra_args) + { + ok(keys(%$shared_data_array1) == @$extra_args); + } + ok(keys(%$shared_data_array1) == keys(%$shared_data_array2)); + while(my ($k, $v) = each %{ $shared_data_array1 }) + { + if(defined $extra_args) + { + ok(grep { $_ eq $k } @$extra_args); + } + ok(exists $shared_data_array2->{$k}); + ok(same_array($v, $shared_data_array2->{$k})); + } + # Test shared argument arrays and gradient arrays + my $exec_shared = $exec_grp_shared->execs->[$i]; + my $exec_created = $exec_grp_created->execs->[$i]; + if(defined $shared_arg_names) + { + # test shared arguments + for my $arg_name (@$shared_arg_names) + { + ok(exists $exec_created->arg_dict->{$arg_name}); + ok(same_array($exec_shared->arg_dict->{$arg_name}, $exec_created->arg_dict->{$arg_name})); + } + # test shared argument gradients + for my $arg_name (@$shared_arg_names) + { + ok(exists $exec_created->grad_dict->{$arg_name}); + ok(same_array($exec_shared->grad_dict->{$arg_name}, $exec_created->grad_dict->{$arg_name})); + } + } + my $grad_req = $exec_grp_shared->grad_req; + while(my ($arg_name, $grad) = each %{ $grad_req }) + { + ok($grad eq $exec_grp_created->grad_req->{$arg_name}); + } + } + }; + my $contexts = [mx->cpu(0), mx->cpu(1)]; + my $workload = [(1) x scalar(@$contexts)]; + my $batch_size = 32; + my $max_bucket_size = 80; + my $num_words = 1000; + my $num_hidden = 100; + my $num_embed = 200; + my $data_shapes = [['data', [$batch_size, $max_bucket_size]]]; + my $label_shapes = [['softmax_label', [$batch_size, $max_bucket_size]]]; + + # generate an rnn sym with #layers=5 + my $sym = $get_rnn_sym->(3, $num_words, $num_hidden, + $num_embed, $max_bucket_size); + my $arg_names1 = $sym->list_arguments(); + my $input_names = ['data', 'softmax_label']; + my $shared_arg_names = [grep { !/^(?:data|softmax_label)$/ } @$arg_names1]; + my $exec_group1 = AI::MXNet::DataParallelExecutorGroup->new( + symbol=>$sym, contexts=>$contexts, + workload=>$workload, data_shapes=>$data_shapes, + label_shapes=>$label_shapes, param_names=>$shared_arg_names, + for_training=>1, inputs_need_grad=>0 + ); + # shared_data_arrays should only have input "data" and "softmax_label" arrays + for my $i (0..@{$contexts}-1) + { + ok(keys(%{$exec_group1->shared_data_arrays->[$i]}) == @$input_names); + for my $name (@$input_names) + { + ok(exists $exec_group1->shared_data_arrays->[$i]->{$name}); + } + } + # generate an rnn sym with #layers=5 + $sym = $get_rnn_sym->(5, $num_words, $num_hidden, + $num_embed, $max_bucket_size); + my $arg_names2 = $sym->list_arguments(); + my $exec_group2 = AI::MXNet::DataParallelExecutorGroup->new(symbol=>$sym, contexts=>$contexts, + workload=>$workload, data_shapes=>$data_shapes, + label_shapes=>$label_shapes, param_names=>$shared_arg_names, + for_training=>1, inputs_need_grad=>0, + shared_group=>$exec_group1); + my %shared_arg_names = map { $_ => 1 } @$shared_arg_names; + my $extra_args = [grep { not exists $shared_arg_names{$_} } @$arg_names2]; + $test_shared_exec_group->( + $exec_group1, $exec_group2, + $shared_arg_names, $extra_args + ); +} + test_module_input_grads(); test_module_dtype(); test_monitor(); @@ -340,3 +459,4 @@ test_module_layout(); test_module_states(); test_module_reshape(); test_save_load(); +test_executor_group(); diff --git a/perl-package/AI-MXNet/t/test_ndarray.t b/perl-package/AI-MXNet/t/test_ndarray.t index 55350b70125a..53e5749d00f6 100644 --- a/perl-package/AI-MXNet/t/test_ndarray.t +++ b/perl-package/AI-MXNet/t/test_ndarray.t @@ -1,5 +1,8 @@ +use strict; +use warnings; use AI::MXNet qw(mx); -use Test::More tests => 5; +use AI::MXNet::TestUtils qw(almost_equal); +use Test::More tests => 9; sub test_ndarray_reshape { @@ -33,5 +36,33 @@ sub test_moveaxis is_deeply($X->moveaxis(2, 0)->shape, [3, 2, 2]); } +sub test_cached +{ + my $op = mx->nd->CachedOp('Convolution', 3, kernel=>[3, 3], num_filter=>10); + my $data = mx->nd->ones([3, 4, 10, 10]); + my $weight = mx->nd->ones([10, 4, 3, 3]); + my $bias = mx->nd->ones([10]); + my $o1 = mx->nd->invoke($op, [$data, $weight, $bias]); + $bias .= 2; + my $o2 = mx->nd->invoke($op, [$data, $weight, $bias]); + ok(almost_equal($o2->aspdl, $o1->aspdl + 1)); +} + +sub test_output +{ + my $shape = [2,2]; + my $ones = mx->nd->ones($shape); + my $zeros = mx->nd->zeros($shape); + my $out = mx->nd->zeros($shape); + mx->nd->ones($shape, out=>$out); + ok(almost_equal($out->aspdl, $ones->aspdl)); + mx->nd->zeros($shape, out=>$out); + ok(almost_equal($out->aspdl, $zeros->aspdl)); + mx->nd->full($shape, 2, out=>$out); + ok(almost_equal($out->aspdl, $ones->aspdl * 2)); +} + test_ndarray_reshape(); -test_moveaxis(); \ No newline at end of file +test_moveaxis(); +test_cached(); +test_output(); \ No newline at end of file diff --git a/perl-package/AI-MXNet/t/test_random.t b/perl-package/AI-MXNet/t/test_random.t index 7d7ef192fd0b..82175948efc4 100644 --- a/perl-package/AI-MXNet/t/test_random.t +++ b/perl-package/AI-MXNet/t/test_random.t @@ -1,6 +1,6 @@ use strict; use warnings; -use Test::More tests => 11; +use Test::More tests => 8; use AI::MXNet qw(mx); use AI::MXNet::TestUtils qw(same); @@ -44,17 +44,6 @@ sub check_symbolic_random my $un2 = ($yexec->outputs->[0] - $x)->copyto($dev); ok(same($un1->aspdl, $un2->aspdl)); ok(abs($un1->aspdl->avg - ($a+$b)/2) < 0.1); - - $Y = mx->sym->normal(loc=>$mu, scale=>$sigma, shape=>$shape); - $yexec = $Y->simple_bind(ctx => $dev); - mx->random->seed(128); - $yexec->forward; - my $ret1 = $yexec->outputs->[0]->copyto($dev); - mx->random->seed(128); - my $ret2 = mx->random->normal($mu, $sigma, $shape); - ok(same($ret1->aspdl, $ret2->aspdl)); - ok(abs($ret1->aspdl->avg - $mu) < 0.1); - ok(abs(($ret1->aspdl->stats)[6] - $sigma) < 0.1); } sub test_random diff --git a/perl-package/AI-MXNet/t/test_rnn.t b/perl-package/AI-MXNet/t/test_rnn.t index d314298c1eb9..77332b156441 100644 --- a/perl-package/AI-MXNet/t/test_rnn.t +++ b/perl-package/AI-MXNet/t/test_rnn.t @@ -1,8 +1,9 @@ use strict; use warnings; use AI::MXNet qw(mx); +use AI::MXNet::TestUtils qw(same); use PDL; -use Test::More tests => 37; +use Test::More tests => 45; sub test_rnn { @@ -60,12 +61,89 @@ sub test_gru is_deeply($outs, [[10, 100], [10, 100], [10, 100]]); } +sub test_residual +{ + my $cell = mx->rnn->ResidualCell(mx->rnn->GRUCell(50, prefix=>'rnn_')); + my $inputs = [map { mx->sym->Variable("rnn_t${_}_data") } 0..1]; + my ($outputs)= $cell->unroll(2, inputs => $inputs); + $outputs = mx->sym->Group($outputs); + is_deeply( + [sort keys %{ $cell->params->_params }], + ['rnn_h2h_bias', 'rnn_h2h_weight', 'rnn_i2h_bias', 'rnn_i2h_weight'] + ); + is_deeply( + $outputs->list_outputs, + ['rnn_t0_out_plus_residual_output', 'rnn_t1_out_plus_residual_output'] + ); + + my (undef, $outs) = $outputs->infer_shape(rnn_t0_data=>[10, 50], rnn_t1_data=>[10, 50]); + is_deeply($outs, [[10, 50], [10, 50]]); + $outputs = $outputs->eval(args => { + rnn_t0_data=>mx->nd->ones([10, 50]), + rnn_t1_data=>mx->nd->ones([10, 50]), + rnn_i2h_weight=>mx->nd->zeros([150, 50]), + rnn_i2h_bias=>mx->nd->zeros([150]), + rnn_h2h_weight=>mx->nd->zeros([150, 50]), + rnn_h2h_bias=>mx->nd->zeros([150]) + }); + my $expected_outputs = mx->nd->ones([10, 50])->aspdl; + same(@{$outputs}[0]->aspdl, $expected_outputs); + same(@{$outputs}[1]->aspdl, $expected_outputs); +} + +sub test_residual_bidirectional +{ + my $cell = mx->rnn->ResidualCell( + mx->rnn->BidirectionalCell( + mx->rnn->GRUCell(25, prefix=>'rnn_l_'), + mx->rnn->GRUCell(25, prefix=>'rnn_r_') + ) + ); + my $inputs = [map { mx->sym->Variable("rnn_t${_}_data") } 0..1]; + my ($outputs) = $cell->unroll(2, inputs => $inputs, merge_outputs=>0); + $outputs = mx->sym->Group($outputs); + is_deeply( + [sort keys %{ $cell->params->_params }], + ['rnn_l_h2h_bias', 'rnn_l_h2h_weight', 'rnn_l_i2h_bias', 'rnn_l_i2h_weight', + 'rnn_r_h2h_bias', 'rnn_r_h2h_weight', 'rnn_r_i2h_bias', 'rnn_r_i2h_weight'] + ); + is_deeply( + $outputs->list_outputs, + ['bi_t0_plus_residual_output', 'bi_t1_plus_residual_output'] + ); + + my (undef, $outs) = $outputs->infer_shape(rnn_t0_data=>[10, 50], rnn_t1_data=>[10, 50]); + is_deeply($outs, [[10, 50], [10, 50]]); + $outputs = $outputs->eval(args => { + rnn_t0_data=>mx->nd->ones([10, 50])+5, + rnn_t1_data=>mx->nd->ones([10, 50])+5, + rnn_l_i2h_weight=>mx->nd->zeros([75, 50]), + rnn_l_i2h_bias=>mx->nd->zeros([75]), + rnn_l_h2h_weight=>mx->nd->zeros([75, 25]), + rnn_l_h2h_bias=>mx->nd->zeros([75]), + rnn_r_i2h_weight=>mx->nd->zeros([75, 50]), + rnn_r_i2h_bias=>mx->nd->zeros([75]), + rnn_r_h2h_weight=>mx->nd->zeros([75, 25]), + rnn_r_h2h_bias=>mx->nd->zeros([75]) + }); + my $expected_outputs = (mx->nd->ones([10, 50])+5)->aspdl; + ok(same(@{$outputs}[0]->aspdl, $expected_outputs)); + ok(same(@{$outputs}[1]->aspdl, $expected_outputs)); +} + sub test_stack { my $cell = mx->rnn->SequentialRNNCell(); for my $i (0..4) { - $cell->add(mx->rnn->LSTMCell(100, prefix=>"rnn_stack${i}_")); + if($i == 1) + { + $cell->add(mx->rnn->ResidualCell(mx->rnn->LSTMCell(100, prefix=>"rnn_stack${i}_"))); + } + else + { + $cell->add(mx->rnn->LSTMCell(100, prefix=>"rnn_stack${i}_")); + } } my ($outputs) = $cell->unroll(3, input_prefix=>'rnn_'); $outputs = mx->sym->Group($outputs); @@ -127,6 +205,8 @@ test_rnn(); test_lstm(); test_lstm_forget_bias(); test_gru(); +test_residual(); +test_residual_bidirectional(); test_stack(); test_bidirectional(); test_unfuse(); diff --git a/perl-package/AI-MXNet/t/test_symbol.t b/perl-package/AI-MXNet/t/test_symbol.t index d6d79eaf30df..bf9e90598929 100644 --- a/perl-package/AI-MXNet/t/test_symbol.t +++ b/perl-package/AI-MXNet/t/test_symbol.t @@ -1,6 +1,6 @@ use strict; use warnings; -use Test::More tests => 98; +use Test::More tests => 102; use AI::MXNet qw(mx); use AI::MXNet::TestUtils qw(mlp2 conv check_consistency zip assert enumerate); use Storable qw(freeze thaw); @@ -221,6 +221,24 @@ sub test_load_000800 test_load_000800(); +sub test_cached +{ + my $op = mx->sym->CachedOp('Convolution', 3, kernel=>[3, 3], num_filter=>10); + my $data = mx->sym->var('data'); + my $weight = mx->sym->var('weight'); + my $bias = mx->sym->var('bias'); + my $out = mx->sym->invoke($op, [$data, $weight, $bias], 'conv'); + is_deeply($out->list_arguments, ['data', 'weight', 'bias']); + is_deeply($out->list_outputs, ['conv_output']); + { + local($mx::NameManager) = mx->name->Prefix('test_'); + is(mx->sym->invoke($op, [$data, $weight, $bias])->name,'test_convolution0'); + is(mx->sym->invoke($op, [$data, $weight, $bias])->name, 'test_convolution1'); + } +} + +test_cached(); + __DATA__ { "nodes": [ diff --git a/perl-package/AI-MXNetCAPI/Changes b/perl-package/AI-MXNetCAPI/Changes index 0cd407a2f8dc..df98bd9de411 100644 --- a/perl-package/AI-MXNetCAPI/Changes +++ b/perl-package/AI-MXNetCAPI/Changes @@ -1,5 +1,8 @@ Revision history for Perl extension AI::MXNetCAPI +1.01 Sat Jun 10 23:57:27 PDT 2017 + - sync with python. + 0.9507 Thu May 11 17:04:44 PDT 2017 - Added Autograd. diff --git a/perl-package/AI-MXNetCAPI/META.json b/perl-package/AI-MXNetCAPI/META.json index 137c542d47ac..579c81cd8995 100644 --- a/perl-package/AI-MXNetCAPI/META.json +++ b/perl-package/AI-MXNetCAPI/META.json @@ -37,5 +37,5 @@ } }, "release_status" : "stable", - "version" : "0.9507" + "version" : "1.01" } diff --git a/perl-package/AI-MXNetCAPI/META.yml b/perl-package/AI-MXNetCAPI/META.yml index 8191978fe68a..a36f94cfeecd 100644 --- a/perl-package/AI-MXNetCAPI/META.yml +++ b/perl-package/AI-MXNetCAPI/META.yml @@ -19,4 +19,4 @@ no_index: - inc requires: Test::More: '0' -version: '0.9507' +version: '1.01' diff --git a/perl-package/AI-MXNetCAPI/README b/perl-package/AI-MXNetCAPI/README index 2ddc00e7dfe5..3633756dae1c 100644 --- a/perl-package/AI-MXNetCAPI/README +++ b/perl-package/AI-MXNetCAPI/README @@ -1,4 +1,4 @@ -AI-MXNetCAPI version 0.9507 +AI-MXNetCAPI version 1.01 ===================== Swig interface to MXNet c api. diff --git a/perl-package/AI-MXNetCAPI/lib/AI/MXNetCAPI.pm b/perl-package/AI-MXNetCAPI/lib/AI/MXNetCAPI.pm index 5e3f44562b08..938146a30b6a 100644 --- a/perl-package/AI-MXNetCAPI/lib/AI/MXNetCAPI.pm +++ b/perl-package/AI-MXNetCAPI/lib/AI/MXNetCAPI.pm @@ -1,7 +1,7 @@ package AI::MXNetCAPI; use base qw(DynaLoader); bootstrap AI::MXNetCAPI; -our $VERSION = '0.9507'; +our $VERSION = '1.01'; 1; __END__ diff --git a/perl-package/AI-MXNetCAPI/mxnet.i b/perl-package/AI-MXNetCAPI/mxnet.i index 4b8afacfde1a..295832eb24dc 100644 --- a/perl-package/AI-MXNetCAPI/mxnet.i +++ b/perl-package/AI-MXNetCAPI/mxnet.i @@ -119,6 +119,7 @@ static void ExecutorMonitor_callback(const char* name, NDArrayHandle handle, voi SWIG_TypeClientData(SWIGTYPE_p_MXKVStore, (void *)"KVStoreHandle"); SWIG_TypeClientData(SWIGTYPE_p_MXRecordIO, (void *)"RecordIOHandle"); SWIG_TypeClientData(SWIGTYPE_p_MXRtc, (void *)"RtcHandle"); + SWIG_TypeClientData(SWIGTYPE_p_MXCachedOp, (void *)"CachedOpHandle"); %} /*! \brief manually define unsigned int */ @@ -130,7 +131,7 @@ typedef float mx_float; // these typedefs are mainly used for readablity reasons /*! \brief handle to NDArray */ typedef MXNDArray *NDArrayHandle; -/*! \brief handle to a mxnet narray function that changes NDArray */ +/*! \brief handle to a mxnet ndarray function that changes NDArray */ typedef MXFunction *FunctionHandle; /*! \brief handle to a function that takes param and creates symbol */ typedef MXAtomicSymbolCreator *AtomicSymbolCreator; @@ -150,6 +151,8 @@ typedef MXKVStore *KVStoreHandle; typedef MXRecordIO *RecordIOHandle; /*! \brief handle to MXRtc*/ typedef MXRtc *RtcHandle; +/*! \brief handle to cached operator */ +typedef MXCachedOp *CachedOpHandle; typedef void (*ExecutorMonitorCallback)(const char*, NDArrayHandle, @@ -234,6 +237,9 @@ int MXSetProfilerState(int state); /*! \brief Save profile and stop profiler */ int MXDumpProfile(); +/*! \brief Set the number of OMP threads to use */ +int MXSetNumOMPThreads(int thread_num); + //------------------------------------- // Part 1: NDArray creation and deletion //------------------------------------- @@ -252,7 +258,7 @@ int MXNDArrayCreateNone(NDArrayHandle *out); * \param dev_type device type, specify device we want to take * \param dev_id the device id of the specific device * \param delay_alloc whether to delay allocation until - * the narray is first mutated + * the ndarray is first mutated * \param out the returning handle * \return 0 when success, -1 when failure happens */ @@ -270,7 +276,7 @@ int MXNDArrayCreate(const mx_uint *in, * \param dev_type device type, specify device we want to take * \param dev_id the device id of the specific device * \param delay_alloc whether to delay allocation until - * the narray is first mutated + * the ndarray is first mutated * \param dtype data type of created array * \param out the returning handle * \return 0 when success, -1 when failure happens @@ -303,7 +309,7 @@ int MXNDArraySaveRawBytes(NDArrayHandle handle, size_t *out_size, const char **out_array); /*! - * \brief Save list of narray into the file. + * \brief Save list of ndarray into the file. * \param fname name of the file. * \param num_args number of arguments to save. * \param args the array of NDArrayHandles to be saved. @@ -315,10 +321,10 @@ int MXNDArraySave(const char* fname, NDArrayHandle* in, const char** in); /*! - * \brief Load list of narray from the file. + * \brief Load list of ndarray from the file. * \param fname name of the file. - * \param out_size number of narray loaded. - * \param out_arr head of the returning narray handles. + * \param out_size number of ndarray loaded. + * \param out_arr head of the returning ndarray handles. * \param out_name_size size of output name arrray. * \param out_names the names of returning NDArrays, can be NULL * \return 0 when success, -1 when failure happens @@ -377,7 +383,7 @@ int MXNDArrayWaitToWrite(NDArrayHandle handle); */ int MXNDArrayWaitAll(); /*! - * \brief free the narray handle + * \brief free the ndarray handle * \param handle the handle to be freed * \return 0 when success, -1 when failure happens */ @@ -406,7 +412,7 @@ int MXNDArrayAt(NDArrayHandle handle, NDArrayHandle *out); /*! * \brief Reshape the NDArray. - * \param handle the handle to the narray + * \param handle the handle to the ndarray * \param ndim number of dimensions of new shape * \param dims new shape * \param out the NDArrayHandle of reshaped NDArray @@ -418,7 +424,7 @@ int MXNDArrayReshape(NDArrayHandle handle, NDArrayHandle *out); /*! * \brief get the shape of the array - * \param handle the handle to the narray + * \param handle the handle to the ndarray * \param out_dim the output dimension * \param out_pdata pointer holder to get data pointer of the shape * \return 0 when success, -1 when failure happens @@ -428,7 +434,7 @@ int MXNDArrayGetShape(NDArrayHandle handle, const mx_uint **out_pdata); /*! * \brief get the content of the data in NDArray - * \param handle the handle to the narray + * \param handle the handle to the ndarray * \param out_pdata pointer holder to get pointer of data * \return 0 when success, -1 when failure happens */ @@ -436,7 +442,7 @@ int MXNDArrayGetData(NDArrayHandle handle, void **out_pdata); /*! * \brief get the type of the data in NDArray - * \param handle the handle to the narray + * \param handle the handle to the ndarray * \param out_dtype pointer holder to get type of data * \return 0 when success, -1 when failure happens */ @@ -444,7 +450,7 @@ int MXNDArrayGetDType(NDArrayHandle handle, int *out); /*! * \brief get the context of the NDArray - * \param handle the handle to the narray + * \param handle the handle to the ndarray * \param out_dev_type the output device type * \param out_dev_id the output device id * \return 0 when success, -1 when failure happens @@ -452,6 +458,28 @@ int MXNDArrayGetDType(NDArrayHandle handle, int MXNDArrayGetContext(NDArrayHandle handle, int *out, int *out); +/*! + * \brief detach and ndarray from computation graph by clearing entry_ + * \param handle NDArray handle + * \return 0 when success, -1 when failure happens + */ +int MXNDArrayDetach(NDArrayHandle handle, NDArrayHandle *out); + +/*! + * \brief set the flag for gradient array state. + * \param handle NDArray handle + * \param state the new state. + * \return 0 when success, -1 when failure happens + */ +int MXNDArraySetGradState(NDArrayHandle handle, int state); + +/*! + * \brief set the flag for gradient array state. + * \param handle NDArray handle + * \param state the new state. + * \return 0 when success, -1 when failure happens + */ +int MXNDArrayGetGradState(NDArrayHandle handle, int *out); //-------------------------------- // Part 2: functions on NDArray @@ -587,6 +615,42 @@ int MXAutogradMarkVariables(mx_uint num_var, */ int MXAutogradComputeGradient(mx_uint num_output, NDArrayHandle* in); +/*! + * \brief compute the gradient of outputs w.r.t variabels + * \param num_output number of output NDArray + * \param output_handles output NDArrays + * \param ograd_handles head gradient for NDArrays + * \param retain_graph whether to keep the graph after backward + * \return 0 when success, -1 when failure happens + */ +int MXAutogradBackward(mx_uint num_output, + NDArrayHandle* in, + NDArrayHandle* in, + int retain_graph); + +/*! + * \brief create cached operator + */ +int MXCachedCreateOp(AtomicSymbolCreator in, + int num_inputs, + int num_params, + const char **keys, + const char **vals, + CachedOpHandle *out); + +/*! + * \brief free cached operator + */ +int MXCachedFree(CachedOpHandle handle); + +/*! + * \brief invoke cached operator + */ +int MXCachedInvoke(CachedOpHandle handle, + int num_inputs, + NDArrayHandle *in, + int *out_size, + NDArrayHandle** out_array); //-------------------------------------------- // Part 3: symbolic configuration generation @@ -655,6 +719,20 @@ int MXSymbolCreateAtomicSymbol(AtomicSymbolCreator in, const char **keys, const char **vals, SymbolHandle *out); +/*! + * \brief Create an AtomicSymbol from cached op. + * \param handle cached node attribute. + * \param name name of new symbol. + * \param num_args the number of symbol arguments + * \param args symbol arguments + * \return 0 when success, -1 when failure happens + */ +int MXCachedCreateSymbol(CachedOpHandle handle, + const char* name, + mx_uint num_args, + SymbolHandle* in, + SymbolHandle* out); + /*! * \brief Create a Variable Symbol. * \param name name of the variable @@ -1016,8 +1094,8 @@ int MXExecutorBackward(ExecutorHandle handle, * \brief Get executor's head NDArray * * \param handle executor handle - * \param out_size output narray vector size - * \param out out put narray handles + * \param out_size output ndarray vector size + * \param out out put ndarray handles * \return 0 when success, -1 when failure happens */ int MXExecutorOutputs(ExecutorHandle handle, @@ -1121,6 +1199,45 @@ int MXExecutorBindEX(SymbolHandle symbol_handle, NDArrayHandle *in, ExecutorHandle shared_exec, ExecutorHandle *out); + +int MXExecutorSimpleBind(SymbolHandle symbol_handle, + int dev_type, + int dev_id, + const mx_uint num_g2c_keys, + const char** in, // g2c_keys, + const int* in, // g2c_dev_types, + const int* in, // g2c_dev_ids, + const mx_uint provided_grad_req_list_len, + const char** in, // provided_grad_req_names, + const char** in, // provided_grad_req_types, + const mx_uint num_provided_arg_shapes, + const char** in, // provided_arg_shape_names, + const mx_uint* in, // provided_arg_shape_data, + const mx_uint* in, // provided_arg_shape_idx, + const mx_uint num_provided_arg_dtypes, + const char** in, // provided_arg_dtype_names, + const int* in, // provided_arg_dtypes, + const mx_uint num_shared_arg_names, + const char** in, // shared_arg_name_list, +//------------ + int* shared_buffer_len, + const char** shared_buffer_name_list, + NDArrayHandle* shared_buffer_handle_list, + const char*** updated_shared_buffer_name_list, + NDArrayHandle** updated_shared_buffer_handle_list, +//------------------ + + mx_uint* num_in_args, + NDArrayHandle** in_args, + NDArrayHandle** arg_grads, +//----------------- + mx_uint* num_aux_states, + NDArrayHandle** aux_states, +//---------- + ExecutorHandle shared_exec_handle, + ExecutorHandle* out +); + /*! * \brief set a call back to notify the completion of operation */ diff --git a/perl-package/AI-MXNetCAPI/mxnet_typemaps.i b/perl-package/AI-MXNetCAPI/mxnet_typemaps.i index 8e035f1b4973..8574647512f5 100644 --- a/perl-package/AI-MXNetCAPI/mxnet_typemaps.i +++ b/perl-package/AI-MXNetCAPI/mxnet_typemaps.i @@ -15,13 +15,13 @@ { $1 = (char **) safemalloc((len)*sizeof(char *)); for (i = 0; i < len; i++) { - tv = av_fetch(tempav, i, 0); + tv = av_fetch(tempav, i, 0); $1[i] = (char *) SvPV(*tv,len2); } } else { - $1 = NULL; + $1 = NULL; } } %typemap(freearg) (const char** in), (char** in) { @@ -47,13 +47,18 @@ { $1 = (char **)safemalloc(hash_len*sizeof(char *)); $2 = (char **)safemalloc(hash_len*sizeof(char *)); - while ((val = hv_iternextsv(temphv, &key, &len))) + while ((val = hv_iternextsv(temphv, &key, &len))) { $1[i] = key; $2[i] = SvPV(val, len2); ++i; } } + else + { + $1 = NULL; + $2 = NULL; + } } %typemap(freearg) (const char **keys, const char **vals), (char **keys, char **vals) { @@ -197,6 +202,10 @@ $1[i] = (mx_uint)SvIV(*tv); } } + else + { + $1 = NULL; + } } %typemap(freearg) (const mx_uint *in), (mx_uint *in) { @@ -215,11 +224,19 @@ croak("Argument $argnum is not an array."); tempav = (AV*)SvRV($input); av_len = av_top_index(tempav) + 1; - $1 = (int *)safemalloc(av_len*sizeof(int)); - for (i = 0; i < av_len; i++) { - tv = av_fetch(tempav, i, 0); - $1[i] = (int)SvIV(*tv); + if(av_len) + { + $1 = (int *)safemalloc(av_len*sizeof(int)); + for (i = 0; i < av_len; i++) { + tv = av_fetch(tempav, i, 0); + $1[i] = (int)SvIV(*tv); + } } + else + { + $1 = NULL; + } + } %typemap(freearg) (const int *in), (int *in) { @@ -250,6 +267,10 @@ } } } + else + { + $1 = NULL; + } } %typemap(freearg) (NDArrayHandle* in), (SymbolHandle* in) { Safefree($1); @@ -270,10 +291,14 @@ { $1 = (mx_float *)safemalloc(len*sizeof(mx_float)); for (i = 0; i < len; i++) { - tv = av_fetch(tempav, i, 0); + tv = av_fetch(tempav, i, 0); $1[i] = (mx_float)SvNV(*tv); } } + else + { + $1 = NULL; + } } %typemap(freearg) (mx_float *in) { @@ -286,16 +311,16 @@ (DataIterHandle *out) (ExecutorHandle temp), (KVStoreHandle *out) (KVStoreHandle temp), (RecordIOHandle *out) (RecordIOHandle temp), - (RtcHandle *out) (RtcHandle temp) - + (RtcHandle *out) (RtcHandle temp), + (CachedOpHandle *out) (CachedOpHandle temp) { $1 = &temp; } %typemap(argout) (NDArrayHandle *out), (FunctionHandle* out), (SymbolHandle *out), (ExecutorHandle *out), (DataIterHandle *out), - (KVStoreHandle *out), (RecordIOHandle *out), (RtcHandle *out) (RtcHandle temp) + (KVStoreHandle *out), (RecordIOHandle *out), (RtcHandle *out) (RtcHandle temp), (CachedOpHandle *out) (CachedOpHandle temp) { if(!result) - { + { $result = SWIG_NewPointerObj(SWIG_as_voidptr(*$1), $*1_descriptor, 0); argvi++; } } @@ -520,7 +545,7 @@ SWIG_exception_fail(SWIG_ArgError(res), "in method '" "$symname" "', argument " "$argnum"" of type '" "NDArray""'"); } } - } + } temp = av_len; $1 = &temp; $2 = &temp_array; @@ -660,7 +685,7 @@ for (i = 0; i < *$3 ; i++) { av_push(names, newSVpv((*$4)[i],0)); av_push(types, newSVpv((*$5)[i],0)); - av_push(descs, newSVpv((*$6)[i],0)); + av_push(descs, newSVpv((*$6)[i],0)); } } av_push(container, newRV_noinc((SV*)names)); @@ -695,9 +720,10 @@ (mx_uint *aux_shape_size, const mx_uint **aux_shape_ndim, const mx_uint ***aux_shape_data) (mx_uint temp1, mx_uint *temp2, mx_uint **temp3) { - $1 = &temp1; + $1 = &temp1; $2 = &temp2; - $3 = &temp3; + $3 = &temp3; + *$1 = 0; } %typemap(argout) (mx_uint *in_shape_size, const mx_uint **in_shape_ndim, const mx_uint ***in_shape_data), @@ -733,13 +759,14 @@ (mx_uint *aux_type_size, const int **aux_type_data) (mx_uint temp1, int *temp2) { - $1 = &temp1; + $1 = &temp1; $2 = &temp2; + *$1 = 0; } %typemap(argout) (mx_uint *in_type_size, const int **in_type_data), - (mx_uint *out_type_size, const int **out_type_data), - (mx_uint *aux_type_size, const int **aux_type_data) + (mx_uint *out_type_size, const int **out_type_data), + (mx_uint *aux_type_size, const int **aux_type_data) { if(!result && *arg11) @@ -757,6 +784,155 @@ } } +%typemap(in,numinputs=0) (mx_uint* num_in_args, + NDArrayHandle** in_args, + NDArrayHandle** arg_grads) + (mx_uint temp1, + NDArrayHandle* temp2, + NDArrayHandle* temp3) +{ + $1 = &temp1; + $2 = &temp2; + $3 = &temp3; + *$1 = 0; +} + +%typemap(argout) (mx_uint* num_in_args, + NDArrayHandle** in_args, + NDArrayHandle** arg_grads) +{ + if(!result) + { + AV *container1 = newAV(); + AV *container2 = newAV(); + for (int i = 0; i < *$1 ; i++) + { + av_push(container1, SvREFCNT_inc(SWIG_NewPointerObj(SWIG_as_voidptr((*$2)[i]), SWIGTYPE_p_MXNDArray, 0))); + av_push(container2, (*$3)[i] ? SvREFCNT_inc(SWIG_NewPointerObj(SWIG_as_voidptr((*$3)[i]), SWIGTYPE_p_MXNDArray, 0)) : newSV(0)); + } + $result = newRV_noinc((SV*)container1); + sv_2mortal($result); + argvi++; + $result = newRV_noinc((SV*)container2); + sv_2mortal($result); + argvi++; + } +} + +%typemap(in,numinputs=0) (mx_uint* num_aux_states, + NDArrayHandle** aux_states) + (mx_uint temp1, + NDArrayHandle* temp2) +{ + $1 = &temp1; + $2 = &temp2; + *$1 = 0; +} + +%typemap(argout) (mx_uint* num_aux_states, + NDArrayHandle** aux_states) +{ + if(!result) + { + AV *container = newAV(); + for (int i = 0; i < *$1 ; i++) + { + av_push(container, SvREFCNT_inc(SWIG_NewPointerObj(SWIG_as_voidptr((*$2)[i]), SWIGTYPE_p_MXNDArray, 0))); + } + $result = newRV_noinc((SV*)container); + sv_2mortal($result); + argvi++; + } +} + +%typemap(in) (int* shared_buffer_len, + const char** shared_buffer_name_list, + NDArrayHandle* shared_buffer_handle_list, + const char*** updated_shared_buffer_name_list, + NDArrayHandle** updated_shared_buffer_handle_list) + (int temp1, + char* temp2, + NDArrayHandle temp3, + char** temp4, + NDArrayHandle* temp5) +{ + HV *temphv; + char *key; + SV *val; + I32 len; + int res; + int i = 0; + int hash_len; + $1 = &temp1; + $2 = &temp2; + $3 = &temp3; + $4 = &temp4; + $5 = &temp5; + if (!SvROK($input)) + { + *$1 = -1; + $2 = NULL; + $3 = NULL; + } + else + { + if (SvTYPE(SvRV($input)) != SVt_PVHV) + croak("Argument $argnum is not a hash."); + temphv = (HV*)SvRV($input); + *$1 = hv_iterinit(temphv); + if(*$1) + { + $2 = (char**)safemalloc((*$1)*sizeof(char*)); + $3 = (void**)safemalloc((*$1)*sizeof(void*)); + while ((val = hv_iternextsv(temphv, &key, &len))) + { + $2[i] = key; + res = SWIG_ConvertPtr(val,SWIG_as_voidptrptr(&($3[i])), 0, 0); + if (!SWIG_IsOK(res)) { + SWIG_exception_fail(SWIG_ArgError(res), "in method '" "$symname" "', argument " "$argnum"" of type '" "NDArray""'"); + } + i++; + } + } + else + { + $2 = NULL; + $3 = NULL; + } + } +} + +%typemap(freearg) (int* shared_buffer_len, + const char** shared_buffer_name_list, + NDArrayHandle* shared_buffer_handle_list, + const char*** updated_shared_buffer_name_list, + NDArrayHandle** updated_shared_buffer_handle_list) +{ + Safefree($2); + Safefree($3); +} + +%typemap(argout) (int* shared_buffer_len, + const char** shared_buffer_name_list, + NDArrayHandle* shared_buffer_handle_list, + const char*** updated_shared_buffer_name_list, + NDArrayHandle** updated_shared_buffer_handle_list) + +{ + if(!result) + { + HV* hash = newHV(); + for(int j = 0; j < *$1; j++) + { + hv_store(hash, (*$4)[j], strlen((*$4)[j]), SvREFCNT_inc(SWIG_NewPointerObj(SWIG_as_voidptr((*$5)[j]), SWIGTYPE_p_MXNDArray, 0)), 0); + } + $result = newRV_noinc((SV*)hash); + sv_2mortal($result); + argvi++; + } +} + + %typemap(in) (uint32_t x) { union fbits u; diff --git a/perl-package/AI-NNVMCAPI/Changes b/perl-package/AI-NNVMCAPI/Changes index 6539ee0e4ef9..09395184e3c6 100644 --- a/perl-package/AI-NNVMCAPI/Changes +++ b/perl-package/AI-NNVMCAPI/Changes @@ -1,5 +1,8 @@ Revision history for Perl extension AI::NNVMCAPI. +1.01 Sat Jun 10 23:57:27 PDT 2017 + - sync with python. + 0.95 Sun Mar 26 17:42:02 PDT 2017 - visible on http://mxnet.io diff --git a/perl-package/AI-NNVMCAPI/META.json b/perl-package/AI-NNVMCAPI/META.json index 7c0329d4c359..42247c6b98ff 100644 --- a/perl-package/AI-NNVMCAPI/META.json +++ b/perl-package/AI-NNVMCAPI/META.json @@ -37,5 +37,5 @@ } }, "release_status" : "stable", - "version" : "0.95" + "version" : "1.01" } diff --git a/perl-package/AI-NNVMCAPI/META.yml b/perl-package/AI-NNVMCAPI/META.yml index d43e8ee22389..6d48cc7b8578 100644 --- a/perl-package/AI-NNVMCAPI/META.yml +++ b/perl-package/AI-NNVMCAPI/META.yml @@ -19,4 +19,4 @@ no_index: - inc requires: Test::More: '0' -version: '0.95' +version: '1.01' diff --git a/perl-package/AI-NNVMCAPI/README b/perl-package/AI-NNVMCAPI/README index fa7870d301ee..50579140de82 100644 --- a/perl-package/AI-NNVMCAPI/README +++ b/perl-package/AI-NNVMCAPI/README @@ -1,4 +1,4 @@ -AI-NNVMCAPI version 0.95 +AI-NNVMCAPI version 1.01 ===================== Swig interface to MXNet c api. diff --git a/perl-package/AI-NNVMCAPI/lib/AI/NNVMCAPI.pm b/perl-package/AI-NNVMCAPI/lib/AI/NNVMCAPI.pm index 577f7dad2faf..62d4dd2b09ab 100644 --- a/perl-package/AI-NNVMCAPI/lib/AI/NNVMCAPI.pm +++ b/perl-package/AI-NNVMCAPI/lib/AI/NNVMCAPI.pm @@ -1,7 +1,7 @@ package AI::NNVMCAPI; use base qw(DynaLoader); bootstrap AI::NNVMCAPI; -our $VERSION = '0.95'; +our $VERSION = '1.01'; 1; __END__ From 5aeffd31cd1046268cd56d57dc1b5d7a5d5195f9 Mon Sep 17 00:00:00 2001 From: Przemyslaw Tredak Date: Wed, 14 Jun 2017 09:53:45 -0700 Subject: [PATCH 071/834] Use pinned memory in IO iterator to avoid unnecessary memory copies (#6660) * Use pinned memory in IO iterator to avoid unnecessary memory copies * Letting CPUPinned to fallback to normal CPU if CUDA is not enabled or no GPUs are present --- src/io/iter_image_recordio_2.cc | 2 +- src/kvstore/comm.h | 9 +-------- src/storage/storage.cc | 17 +++++++++++------ 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/src/io/iter_image_recordio_2.cc b/src/io/iter_image_recordio_2.cc index ace42855b6a7..18d12ff7e25a 100644 --- a/src/io/iter_image_recordio_2.cc +++ b/src/io/iter_image_recordio_2.cc @@ -266,7 +266,7 @@ inline bool ImageRecordIOParser2::ParseNext(DataBatch *out) { auto dtype = prefetch_param_.dtype ? prefetch_param_.dtype.value() : first_batch.data[i].type_flag_; - out->data.at(i) = NDArray(dst_shape, Context::CPU(), false, src_type_flag); + out->data.at(i) = NDArray(dst_shape, Context::CPUPinned(0), false, src_type_flag); unit_size_[i] = src_shape.Size(); } } diff --git a/src/kvstore/comm.h b/src/kvstore/comm.h index 1197d4ef3edb..07f2d24bd223 100644 --- a/src/kvstore/comm.h +++ b/src/kvstore/comm.h @@ -18,14 +18,7 @@ namespace kvstore { class Comm { public: Comm() { -#if MXNET_USE_CUDA - int gpu_num; - int ret = cudaGetDeviceCount(&gpu_num); - pinned_ctx_ = (ret == 0 && gpu_num > 0) ? - Context::CPUPinned(0) : Context::CPU(); -#else - pinned_ctx_ = Context::CPU(); -#endif + pinned_ctx_ = Context::CPUPinned(0); } virtual ~Comm() { } /** diff --git a/src/storage/storage.cc b/src/storage/storage.cc index 64731cf92456..1418216c7a76 100644 --- a/src/storage/storage.cc +++ b/src/storage/storage.cc @@ -32,13 +32,18 @@ class StorageImpl : public Storage { switch (ctx.dev_type) { case Context::kCPU: break; case Context::kGPU: - case Context::kCPUPinned: + case Context::kCPUPinned: { + int gpu_num = 0; #if MXNET_USE_CUDA - CUDA_CALL(cudaSetDevice(ctx.dev_id)); -#else // MXNET_USE_CUDA - LOG(FATAL) << "Please compile with CUDA enabled"; + CUDA_CALL(cudaGetDeviceCount(&gpu_num)); #endif // MXNET_USE_CUDA - break; + if (gpu_num > 0) { +#if MXNET_USE_CUDA + CUDA_CALL(cudaSetDevice(ctx.dev_id)); +#endif // MXNET_USE_CUDA + } + break; + } default: LOG(FATAL) << "Unimplemented device"; } @@ -66,7 +71,7 @@ Storage::Handle StorageImpl::Alloc(size_t size, Context ctx) { #if MXNET_USE_CUDA ptr = new storage::NaiveStorageManager(); #else - LOG(FATAL) << "Compile with USE_CUDA=1 to enable GPU usage"; + ptr = new storage::NaiveStorageManager(); #endif // MXNET_USE_CUDA break; } From 17ea9e3d99d4a923c68c39f29930036ed66d922e Mon Sep 17 00:00:00 2001 From: vsooda Date: Thu, 15 Jun 2017 00:56:52 +0800 Subject: [PATCH 072/834] fix cpp-package charRNN shape (#6626) --- cpp-package/example/charRNN.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cpp-package/example/charRNN.cpp b/cpp-package/example/charRNN.cpp index 87393cc544ed..5cb6382137c7 100644 --- a/cpp-package/example/charRNN.cpp +++ b/cpp-package/example/charRNN.cpp @@ -115,7 +115,7 @@ Symbol LSTMUnroll(int num_lstm_layer, int sequence_length, int input_dim, auto label = Symbol::Variable("softmax_label"); label = transpose(label); - label = Reshape(label, Shape(), false, Shape(-1), false); // -1: infer from graph + label = Reshape(label, Shape(), false, Shape(0), false); // -1: infer from graph auto sm = SoftmaxOutput("softmax", pred, label); if (isTrain) return sm; @@ -141,7 +141,7 @@ Symbol LSTMWithBuiltInRNNOp(int num_lstm_layer, int sequence_length, int input_d auto label = Symbol::Variable("softmax_label"); label = transpose(label); label = Reshape(label, Shape(), false, - Shape(-1), false); // FullyConnected requires one dimension + Shape(0), false); // FullyConnected requires one dimension if (!TIME_MAJOR && isTrain) embed = SwapAxis(embed, 0, 1); // Change to time-major as cuDNN requires @@ -151,7 +151,7 @@ Symbol LSTMWithBuiltInRNNOp(int num_lstm_layer, int sequence_length, int input_d auto rnn_params = Symbol::Variable("LSTM_parameters"); // See explanations near RNNXavier class auto rnn = RNN(embed, rnn_params, rnn_h_init, rnn_c_init, num_hidden, num_lstm_layer, RNNMode::kLstm, false, dropout, !isTrain); - auto hidden = Reshape(rnn[0], Shape(), false, Shape(-1, num_hidden), false); + auto hidden = Reshape(rnn[0], Shape(), false, Shape(0, num_hidden), false); auto cls_weight = Symbol::Variable("cls_weight"); auto cls_bias = Symbol::Variable("cls_bias"); From 7fcaf15a3a597cc72a342d1bdb00273dec00e78c Mon Sep 17 00:00:00 2001 From: Leonard Date: Thu, 15 Jun 2017 02:05:24 +0900 Subject: [PATCH 073/834] Update rnn.md (#6571) * Update rnn.md Via https://github.com/dmlc/mxnet/blob/master/example/rnn/cudnn_lstm_bucketing.py#L40-L45 * Restrict comment to multi-GPU setting --- docs/api/python/rnn.md | 7 +++++++ example/rnn/cudnn_lstm_bucketing.py | 11 ++++++----- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/docs/api/python/rnn.md b/docs/api/python/rnn.md index 2047d4d9f174..4021b26af154 100644 --- a/docs/api/python/rnn.md +++ b/docs/api/python/rnn.md @@ -250,6 +250,13 @@ outputs, _ = fused_lstm_cell.unroll(length=sequence_length, \ of `FusedRNNCell` is twice the size specified by `num_hidden`. ``` +When training a deep, complex model *on multiple GPUs* it's recommended to stack +fused RNN cells (one layer per cell) together instead of one with all layers. +The reason is that fused RNN cells don't set gradients to be ready until the +computation for the entire layer is completed. Breaking a multi-layer fused RNN +cell into several one-layer ones allows gradients to be processed ealier. This +reduces communication overhead, especially with multiple GPUs. + The `unfuse()` method can be used to convert the `FusedRNNCell` into an equivalent and CPU-compatible `SequentialRNNCell` that mirrors the settings of the `FusedRNNCell`. ```python diff --git a/example/rnn/cudnn_lstm_bucketing.py b/example/rnn/cudnn_lstm_bucketing.py index 35914dea8fac..140f2e697015 100644 --- a/example/rnn/cudnn_lstm_bucketing.py +++ b/example/rnn/cudnn_lstm_bucketing.py @@ -37,11 +37,12 @@ help='the batch size.') parser.add_argument('--disp-batches', type=int, default=50, help='show progress for every n batches') -# When training a deep, complex model, it's recommended to stack fused RNN cells (one -# layer per cell) together instead of one with all layers. The reason is that fused RNN -# cells doesn't set gradients to be ready until the computation for the entire layer is -# completed. Breaking a multi-layer fused RNN cell into several one-layer ones allows -# gradients to be processed ealier. This reduces communication overhead, especially with +# When training a deep, complex model *on multiple GPUs* it's recommended to +# stack fused RNN cells (one layer per cell) together instead of one with all +# layers. The reason is that fused RNN cells don't set gradients to be ready +# until the computation for the entire layer is completed. Breaking a +# multi-layer fused RNN cell into several one-layer ones allows gradients to be +# processed ealier. This reduces communication overhead, especially with # multiple GPUs. parser.add_argument('--stack-rnn', default=False, help='stack fused RNN cells to reduce communication overhead') From a98e502b35b841a9e4a83b008dfd381a8b62b0fb Mon Sep 17 00:00:00 2001 From: "Qiang Kou (KK)" Date: Thu, 15 Jun 2017 13:08:02 -0700 Subject: [PATCH 074/834] [R] fix accessing variables in environment. close #4282 (#6688) --- R-package/R/context.R | 4 ++-- R-package/R/optimizer.R | 48 ++++++++++++++++++++--------------------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/R-package/R/context.R b/R-package/R/context.R index fdcb48a857d6..604224e74fa7 100644 --- a/R-package/R/context.R +++ b/R-package/R/context.R @@ -1,6 +1,6 @@ # Initialize the global context init.context.default <- function() { - assign("mx.ctx.internal.default.value", mx.cpu(), envir = .MXNetEnv) + .MXNetEnv[["mx.ctx.internal.default.value"]] <- mx.cpu() } #' Set/Get default context for array creation. @@ -11,7 +11,7 @@ init.context.default <- function() { #' @export mx.ctx.default <- function(new = NULL) { if (!is.null(new)) { - assign("mx.ctx.internal.default.value", new, envir = .MXNetEnv) + .MXNetEnv[["mx.ctx.internal.default.value"]] <- new } return (.MXNetEnv$mx.ctx.internal.default.value) } diff --git a/R-package/R/optimizer.R b/R-package/R/optimizer.R index 820e382cb9ed..883dca647356 100644 --- a/R-package/R/optimizer.R +++ b/R-package/R/optimizer.R @@ -32,12 +32,12 @@ mx.opt.sgd <- function(learning.rate, lr <- sgd$lr ## update count indexKey <- paste0('ik', index) - if (!exists(envir = sgd, x = indexKey)){ - assign(x = indexKey, value = 0, envir = sgd) + if (!exists(envir = sgd, x = indexKey, inherits = FALSE)){ + sgd[[indexKey]] <- 0 } else { - indexValue <- get(envir = sgd, x = indexKey) - assign(x = indexKey, value = indexValue + 1, envir = sgd) - sgd$num_update <- max(sgd$num_update, get(envir = sgd, x = indexKey)) + indexValue <- sgd[[indexKey]] + sgd[[indexKey]] <- indexValue + sgd$num_update <- max(sgd$num_update, sgd[[indexKey]]) } } grad <- grad * rescale.grad @@ -114,12 +114,12 @@ mx.opt.rmsprop <- function(learning.rate=0.002, lr <- rmsprop$lr ## update count indexKey <- paste0('ik', index) - if (!exists(envir = rmsprop, x = indexKey)){ - assign(x = indexKey, value = 0, envir = rmsprop) + if (!exists(envir = rmsprop, x = indexKey, inherits = FALSE)){ + rmsprop[[indexKey]] <- 0 } else { - indexValue <- get(envir = rmsprop, x = indexKey) - assign(x = indexKey, value = indexValue + 1, envir = rmsprop) - rmsprop$num_update <- max(rmsprop$num_update, get(envir = rmsprop, x = indexKey)) + indexValue <- rmsprop[[indexKey]] + rmsprop[[indexKey]] <- indexValue + 1 + rmsprop$num_update <- max(rmsprop$num_update, rmsprop[[indexKey]]) } } grad <- grad * rescale.grad @@ -201,23 +201,23 @@ mx.opt.adam <- function(learning.rate=0.001, lr <- adam$lr ## update count indexKey <- paste0('ik', index) - if (!exists(envir = adam, x = indexKey)){ - assign(x = indexKey, value = 0, envir = adam) + if (!exists(envir = adam, x = indexKey, inherits = FALSE)){ + adam[[indexKey]] <- 0 } else { - indexValue <- get(envir = adam, x = indexKey) - assign(x = indexKey, value = indexValue + 1, envir = adam) - adam$num_update <- max(adam$num_update, get(envir = adam, x = indexKey)) + indexValue <- adam[[indexKey]] + adam[[indexKey]] <- indexValue + 1 + adam$num_update <- max(adam$num_update, adam[[indexKey]]) } } # increment time time.key <- paste0('t', index) - if (!exists(envir = adam, x = time.key)){ - assign(x = time.key, value = 0, envir = adam) + if (!exists(envir = adam, x = time.key, inherits = FALSE)){ + adam[[time.key]] <- 0 } - t <- get(envir = adam, x = time.key) + t <- adam[[time.key]] t <- t + 1 - assign(x = time.key, value = t, envir = adam) + adam[[time.key]] <- t mean <- state$mean variance <- state$variance @@ -297,12 +297,12 @@ mx.opt.adagrad <- function(learning.rate=0.05, lr <- adagrad$lr ## update count indexKey <- paste0('ik', index) - if (!exists(envir = adagrad, x = indexKey)){ - assign(x = indexKey, value = 0, envir = adagrad) + if (!exists(envir = adagrad, x = indexKey, inherits = FALSE)){ + adagrad[[indexKey]] <- 0 } else { - indexValue <- get(envir = adagrad, x = indexKey) - assign(x = indexKey, value = indexValue + 1, envir = adagrad) - adagrad$num_update <- max(adagrad$num_update, get(envir = adagrad, x = indexKey)) + indexValue <- adagrad[[indexKey]] + adagrad[[indexKey]] <- indexValue + 1 + adagrad$num_update <- max(adagrad$num_update, adagrad[[indexKey]]) } } From fc32a5ad00bd56181aa2e2e1d36b1e5e6b1f488b Mon Sep 17 00:00:00 2001 From: Hexiang Hu Date: Thu, 15 Jun 2017 15:14:27 -0700 Subject: [PATCH 075/834] Fix a typo (#6709) --- tools/caffe_converter/convert_caffe_modelzoo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/caffe_converter/convert_caffe_modelzoo.py b/tools/caffe_converter/convert_caffe_modelzoo.py index f900a6cc7d06..2cb3da9149fe 100644 --- a/tools/caffe_converter/convert_caffe_modelzoo.py +++ b/tools/caffe_converter/convert_caffe_modelzoo.py @@ -60,7 +60,7 @@ 'top-1-acc' : 0.753, 'top-5-acc' : 0.922 }, - 'resnt-101' : { + 'resnet-101' : { 'prototxt' : _mx_caffe_model+'ResNet-101-deploy.prototxt', 'caffemodel' : _mx_caffe_model+'ResNet-101-model.caffemodel', 'mean' : _mx_caffe_model+'ResNet_mean.binaryproto', From 54346122653d307952a55ac9fa731782b6b4485d Mon Sep 17 00:00:00 2001 From: "Qiang Kou (KK)" Date: Thu, 15 Jun 2017 23:25:50 -0700 Subject: [PATCH 076/834] [R] fix the predict function and optimizer (#6711) --- R-package/R/model.R | 7 ------- R-package/R/optimizer.R | 2 +- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/R-package/R/model.R b/R-package/R/model.R index 80edbc804b06..ccdd93635dc2 100644 --- a/R-package/R/model.R +++ b/R-package/R/model.R @@ -503,14 +503,7 @@ predict.MXFeedForwardModel <- function(model, X, ctx=NULL, array.batch.size=128, X$reset() if (!X$iter.next()) stop("Cannot predict on empty iterator") dlist = X$value() - arg_names <- arguments(model$symbol) - tmp <- unlist(lapply(arg_names, function(a) { - mxnet:::mx.util.str.endswith(a, "label") - })) - label_name <- arg_names[tmp] arg_lst <- list(symbol = model$symbol, ctx = ctx, data = dim(dlist$data), grad.req="null") - arg_lst[[label_name]] <- dim(dlist$label) - pexec <- do.call(mx.simple.bind, arg_lst) mx.exec.update.arg.arrays(pexec, model$arg.params, match.name=TRUE) diff --git a/R-package/R/optimizer.R b/R-package/R/optimizer.R index 883dca647356..52fc1f24e5c1 100644 --- a/R-package/R/optimizer.R +++ b/R-package/R/optimizer.R @@ -36,7 +36,7 @@ mx.opt.sgd <- function(learning.rate, sgd[[indexKey]] <- 0 } else { indexValue <- sgd[[indexKey]] - sgd[[indexKey]] <- indexValue + sgd[[indexKey]] <- indexValue + 1 sgd$num_update <- max(sgd$num_update, sgd[[indexKey]]) } } From eeec2c84022b31f07f3f9dab9594e799922a6f5c Mon Sep 17 00:00:00 2001 From: Leonard Date: Fri, 16 Jun 2017 15:26:35 +0900 Subject: [PATCH 077/834] Fix data layout handling in BucketSentenceIter (Fixes #5509) (#6707) * Fix data layout handling in BucketSentenceIter (Fixes #5509) BucketSentenceIter did not correctly specify the used data layout in the provide_data and provide_label attributes. This lead to the executor_group always assuming an NCHW layout, trying to split up the time dimension instead of the batch_size dimension over all contexts in case that TNC layout was used. For some reason this nevertheless worked for the provided FusedRNNCell, resulting in model parallelism. Interestingly this patch reduces performance by 25% as it replaces the unmeant model parallelism with the "correct" data parallelism. (Reference architecture is a simple single-layer CharRNN trained on 4 GTX 1080 cards. Without the patch ~2000 samples/sec, with the patch ~1500 samples/sec) * BucketSentenceIter: Correct default layout * BucketSentenceIter: Fix lint --- python/mxnet/rnn/io.py | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/python/mxnet/rnn/io.py b/python/mxnet/rnn/io.py index 73e8b5eb10be..8cfce9647374 100644 --- a/python/mxnet/rnn/io.py +++ b/python/mxnet/rnn/io.py @@ -7,7 +7,7 @@ import random import numpy as np -from ..io import DataIter, DataBatch +from ..io import DataIter, DataBatch, DataDesc from .. import ndarray def encode_sentences(sentences, vocab=None, invalid_label=-1, invalid_key='\n', start_label=0): @@ -85,7 +85,7 @@ class BucketSentenceIter(DataIter): """ def __init__(self, sentences, batch_size, buckets=None, invalid_label=-1, data_name='data', label_name='softmax_label', dtype='float32', - layout='NTC'): + layout='NT'): super(BucketSentenceIter, self).__init__() if not buckets: buckets = [i for i, j in enumerate(np.bincount([len(s) for s in sentences])) @@ -116,14 +116,23 @@ def __init__(self, sentences, batch_size, buckets=None, invalid_label=-1, self.nddata = [] self.ndlabel = [] self.major_axis = layout.find('N') + self.layout = layout self.default_bucket_key = max(buckets) if self.major_axis == 0: - self.provide_data = [(data_name, (batch_size, self.default_bucket_key))] - self.provide_label = [(label_name, (batch_size, self.default_bucket_key))] + self.provide_data = [DataDesc( + name=self.data_name, shape=(batch_size, self.default_bucket_key), + layout=self.layout)] + self.provide_label = [DataDesc( + name=self.label_name, shape=(batch_size, self.default_bucket_key), + layout=self.layout)] elif self.major_axis == 1: - self.provide_data = [(data_name, (self.default_bucket_key, batch_size))] - self.provide_label = [(label_name, (self.default_bucket_key, batch_size))] + self.provide_data = [DataDesc( + name=self.data_name, shape=(self.default_bucket_key, batch_size), + layout=self.layout)] + self.provide_label = [DataDesc( + name=self.label_name, shape=(self.default_bucket_key, batch_size), + layout=self.layout)] else: raise ValueError("Invalid layout %s: Must by NT (batch major) or TN (time major)") @@ -166,5 +175,9 @@ def next(self): return DataBatch([data], [label], pad=0, bucket_key=self.buckets[i], - provide_data=[(self.data_name, data.shape)], - provide_label=[(self.label_name, label.shape)]) + provide_data=[DataDesc( + name=self.data_name, shape=data.shape, + layout=self.layout)], + provide_label=[DataDesc( + name=self.label_name, shape=label.shape, + layout=self.layout)]) From 9073bc8436ab35bf18f466b9ee42989090df2101 Mon Sep 17 00:00:00 2001 From: bhavinthaker Date: Fri, 16 Jun 2017 10:39:04 -0700 Subject: [PATCH 078/834] macOS install script: Support $MXNET_HOME; Print MXNet version; Added few package dependencies (#6670) * Support $MXNET_HOME; Print MXNet version; Added few package dependencies * incorporate review comments: rm commented lines * Use [ -z ${var} ] to test for null/empty variable setting --- setup-utils/install-mxnet-osx-python.sh | 149 ++++++++++++++++++------ 1 file changed, 111 insertions(+), 38 deletions(-) mode change 100644 => 100755 setup-utils/install-mxnet-osx-python.sh diff --git a/setup-utils/install-mxnet-osx-python.sh b/setup-utils/install-mxnet-osx-python.sh old mode 100644 new mode 100755 index 2c772cf0f85a..b9b1ddc9a00f --- a/setup-utils/install-mxnet-osx-python.sh +++ b/setup-utils/install-mxnet-osx-python.sh @@ -8,32 +8,54 @@ #set -ex +export MXNET_GITPATH="https://github.com/dmlc/mxnet.git" +if [ -z ${MXNET_TAG} ]; +then + # + # TODO: Change this to latest tag + # to avoid updating this value for every release + # + export MXNET_TAG="v0.10.0" +fi + export TARIKH=`/bin/date +%Y-%m-%d-%H:%M:%S` -export MXNET_HOME="$HOME/mxnet" +if [ -z ${MXNET_HOME} ]; +then + export MXNET_HOME="$HOME/mxnet" +fi export MXNET_HOME_OLD="$HOME/mxnet_${TARIKH}" export MXNET_LOG=${MXNET_HOME}/buildMXNet_mac.log # Insert the Homebrew directory at the top of your PATH environment variable export PATH=/usr/local/bin:/usr/local/sbin:$PATH +export SLEEP_TIME=2 LINE="########################################################################" echo $LINE echo " " -echo "This script installs MXNet on MacOS in ${MXNET_HOME}" -echo "If this directory is already present, it is renamed to ${MXNET_HOME_OLD}" -echo "It has been tested to work successfully on MacOS El Capitan and Sierra" -echo "and is expected to work fine on other versions as well." +echo "This script installs MXNet on MacOS in \${MXNET_HOME}" +echo "If not set, the default value of \${MXNET_HOME} = ~/mxnet" +echo "The current value of \${MXNET_HOME} = ${MXNET_HOME}" +echo " " +echo "If this directory is already present, it is renamed to retain earlier contents." +echo "You may want to check and delete this directory if not required." +echo " " +echo "This script has been tested on: MacOS El Capitan and Sierra" +echo " " +echo "If you face any problems with this script, please let us know at:" +echo " https://stackoverflow.com/questions/tagged/mxnet" +echo " " +echo "Typical run-time for this script is around 7 minutes." +echo "If your environment has never been setup for development (e.g. gcc), " +echo "it could take up to 30 minutes or longer." echo " " -echo "Approximate run-time is around 5 minutes." +MACOS_VERSION=`/usr/bin/uname -r` +echo "Your macOS version is: $MACOS_VERSION" echo " " echo $LINE -sleep 2 +sleep ${SLEEP_TIME} -# -# Install dependencies for MXNet -# - -# Install Homebrew -yes '' | /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" +echo "You may have to enter your password for sudo access to install python for MXNet." +sudo ls > /dev/null brew_pkg_install () { pkg=$1 @@ -62,34 +84,80 @@ runme() { download_mxnet() { if [ -d ${MXNET_HOME} ]; then - echo "Renaming directory ${MXNET_HOME} to ${MXNET_HOME_OLD}" mv ${MXNET_HOME} ${MXNET_HOME_OLD} + echo " " + echo "Renamed directory ${MXNET_HOME} to ${MXNET_HOME_OLD}" + echo "You may want to check and delete this directory if not required." + echo " " + sleep ${SLEEP_TIME} fi - echo "Downloading MXNET source repositories from github" - git clone https://github.com/dmlc/mxnet.git ${MXNET_HOME} --recursive + + echo " " + echo "MXNET GIT Path = ${MXNET_GITPATH}" + #echo "MXNET Tag = ${MXNET_TAG}" + #echo "You can set \$MXNET_TAG to the appropriate github repo tag" + #echo "If not set, the default value used is the latest release" + echo " " + sleep ${SLEEP_TIME} + + runme git clone ${MXNET_GITPATH} ${MXNET_HOME} --recursive + sleep ${SLEEP_TIME} + cd ${MXNET_HOME} + echo " " + #echo "Checkout tag = ${MXNET_TAG}" + #runme git checkout ${MXNET_TAG} + #echo " " + sleep ${SLEEP_TIME} } -download_mxnet -runme brew update +echo " " +echo "BEGIN: Check/Install/Update Homebrew" +BREW_PATH=`/usr/bin/which brew` +if [[ (-z ${BREW_PATH}) || (! -f ${BREW_PATH}) ]]; +then + yes '' | /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" +else + runme brew update +fi +echo "END: Check/Install/Update Homebrew" +echo $LINE +echo " " + +echo " " +echo "BEGIN: Install dependent brew packages for MXNet" + +runme brew tap homebrew/science + runme brew_pkg_install pkg-config runme brew_pkg_install python -runme pip install --upgrade pip -brew install homebrew/science/openblas runme brew_pkg_install opencv -# Needed for /usr/local/lib/graphviz to be created -runme brew_pkg_install graphviz runme brew_pkg_install numpy +runme brew_pkg_install homebrew/science/openblas -runme brew tap homebrew/science +echo "END: Install dependent brew packages for MXNet" +echo $LINE +echo " " +echo "BEGIN: Install dependent pip packages for MXNet" +runme pip install --upgrade pip +runme pip install --user requests runme pip install graphviz runme pip install jupyter runme pip install cython +runme pip install --user opencv-python +echo "END: Install dependent pip packages for MXNet" +echo $LINE +echo " " -# -# Compile MXNet. It assumes you have checked out MXNet source to ~/mxnet -# +echo "BEGIN: Download MXNet" +download_mxnet +echo "END: Download MXNet" +sleep ${SLEEP_TIME} +echo $LINE +echo " " +# Compile MXNet: It assumes MXNet source is in ${MXNET_HOME} +echo "BEGIN: Compile MXNet" cd ${MXNET_HOME} runme cp make/osx.mk ./config.mk runme echo "USE_BLAS = openblas" >> ./config.mk @@ -100,18 +168,21 @@ echo " " echo "Running Make" echo " " runme make -j$(sysctl -n hw.ncpu) +echo "END: Compile MXNet" +sleep ${SLEEP_TIME} +echo $LINE +echo " " -# -# Install MXNet package for Python -# -echo "Installing MXNet package for Python..." +echo "BEGIN: Install MXNet package for Python" runme cd ${MXNET_HOME}/python runme sudo python setup.py install +echo "END: Install MXNet package for Python" +sleep ${SLEEP_TIME} +echo $LINE +echo " " -# -# Test MXNet -# -echo "Testing MXNet now..." + +echo "BEGIN: Test MXNet" python << END > mxnet_test.log import mxnet as mx a = mx.nd.ones((2, 3)); @@ -127,16 +198,18 @@ if [[ $? = 0 ]]; then echo " " echo "SUCCESS: MXNet test passed" echo "SUCCESS: MXNet is successfully installed and works fine!" - echo ":-)" | banner -w 40 + export MXNET_VERSION=`echo "import mxnet as mx; print(mx.__version__)" | python` + echo "SUCCESS: MXNet Version is: $MXNET_VERSION" + echo "END: Test MXNet" echo " " - echo $LINE + echo ":-)" exit 0 else echo $LINE echo " " echo "ERROR: MXNet test failed" - echo ":-(" | banner -w 40 + echo "END: Test MXNet" echo " " - echo $LINE + echo ":-(" exit 1 fi From 790328f37e043c87f712d3a842ffaa3306da265a Mon Sep 17 00:00:00 2001 From: formath Date: Sat, 17 Jun 2017 02:02:50 +0800 Subject: [PATCH 079/834] train text cnn using mx.module (#6717) --- .../cnn_text_classification/old/text_cnn.py | 261 ++++++++++++++ example/cnn_text_classification/text_cnn.py | 326 ++++++------------ 2 files changed, 372 insertions(+), 215 deletions(-) create mode 100644 example/cnn_text_classification/old/text_cnn.py diff --git a/example/cnn_text_classification/old/text_cnn.py b/example/cnn_text_classification/old/text_cnn.py new file mode 100644 index 000000000000..e41af36cf2ff --- /dev/null +++ b/example/cnn_text_classification/old/text_cnn.py @@ -0,0 +1,261 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +from __future__ import print_function +import sys,os +import mxnet as mx +import numpy as np +import time +import math +import data_helpers +from collections import namedtuple + +import logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) # get a logger to accuracies are printed + +logs = sys.stderr + +CNNModel = namedtuple("CNNModel", ['cnn_exec', 'symbol', 'data', 'label', 'param_blocks']) + +def make_text_cnn(sentence_size, num_embed, batch_size, vocab_size, + num_label=2, filter_list=[3, 4, 5], num_filter=100, + dropout=0., with_embedding=True): + + input_x = mx.sym.Variable('data') # placeholder for input + input_y = mx.sym.Variable('softmax_label') # placeholder for output + + # embedding layer + if not with_embedding: + embed_layer = mx.sym.Embedding(data=input_x, input_dim=vocab_size, output_dim=num_embed, name='vocab_embed') + conv_input = mx.sym.Reshape(data=embed_layer, target_shape=(batch_size, 1, sentence_size, num_embed)) + else: + conv_input = input_x + + # create convolution + (max) pooling layer for each filter operation + pooled_outputs = [] + for i, filter_size in enumerate(filter_list): + convi = mx.sym.Convolution(data=conv_input, kernel=(filter_size, num_embed), num_filter=num_filter) + relui = mx.sym.Activation(data=convi, act_type='relu') + pooli = mx.sym.Pooling(data=relui, pool_type='max', kernel=(sentence_size - filter_size + 1, 1), stride=(1,1)) + pooled_outputs.append(pooli) + + # combine all pooled outputs + total_filters = num_filter * len(filter_list) + concat = mx.sym.Concat(*pooled_outputs, dim=1) + h_pool = mx.sym.Reshape(data=concat, target_shape=(batch_size, total_filters)) + + # dropout layer + if dropout > 0.0: + h_drop = mx.sym.Dropout(data=h_pool, p=dropout) + else: + h_drop = h_pool + + # fully connected + cls_weight = mx.sym.Variable('cls_weight') + cls_bias = mx.sym.Variable('cls_bias') + + fc = mx.sym.FullyConnected(data=h_drop, weight=cls_weight, bias=cls_bias, num_hidden=num_label) + + # softmax output + sm = mx.sym.SoftmaxOutput(data=fc, label=input_y, name='softmax') + + return sm + + +def setup_cnn_model(ctx, batch_size, sentence_size, num_embed, vocab_size, + dropout=0.5, initializer=mx.initializer.Uniform(0.1), with_embedding=True): + + cnn = make_text_cnn(sentence_size, num_embed, batch_size=batch_size, + vocab_size=vocab_size, dropout=dropout, with_embedding=with_embedding) + arg_names = cnn.list_arguments() + + input_shapes = {} + if with_embedding: + input_shapes['data'] = (batch_size, 1, sentence_size, num_embed) + else: + input_shapes['data'] = (batch_size, sentence_size) + + arg_shape, out_shape, aux_shape = cnn.infer_shape(**input_shapes) + arg_arrays = [mx.nd.zeros(s, ctx) for s in arg_shape] + args_grad = {} + for shape, name in zip(arg_shape, arg_names): + if name in ['softmax_label', 'data']: # input, output + continue + args_grad[name] = mx.nd.zeros(shape, ctx) + + cnn_exec = cnn.bind(ctx=ctx, args=arg_arrays, args_grad=args_grad, grad_req='add') + + param_blocks = [] + arg_dict = dict(zip(arg_names, cnn_exec.arg_arrays)) + for i, name in enumerate(arg_names): + if name in ['softmax_label', 'data']: # input, output + continue + initializer(name, arg_dict[name]) + + param_blocks.append( (i, arg_dict[name], args_grad[name], name) ) + + out_dict = dict(zip(cnn.list_outputs(), cnn_exec.outputs)) + + data = cnn_exec.arg_dict['data'] + label = cnn_exec.arg_dict['softmax_label'] + + return CNNModel(cnn_exec=cnn_exec, symbol=cnn, data=data, label=label, param_blocks=param_blocks) + + +def train_cnn(model, X_train_batch, y_train_batch, X_dev_batch, y_dev_batch, batch_size, + optimizer='rmsprop', max_grad_norm=5.0, learning_rate=0.0005, epoch=200): + m = model + # create optimizer + opt = mx.optimizer.create(optimizer) + opt.lr = learning_rate + + updater = mx.optimizer.get_updater(opt) + + for iteration in range(epoch): + tic = time.time() + num_correct = 0 + num_total = 0 + for begin in range(0, X_train_batch.shape[0], batch_size): + batchX = X_train_batch[begin:begin+batch_size] + batchY = y_train_batch[begin:begin+batch_size] + if batchX.shape[0] != batch_size: + continue + + m.data[:] = batchX + m.label[:] = batchY + + # forward + m.cnn_exec.forward(is_train=True) + + # backward + m.cnn_exec.backward() + + # eval on training data + num_correct += sum(batchY == np.argmax(m.cnn_exec.outputs[0].asnumpy(), axis=1)) + num_total += len(batchY) + + # update weights + norm = 0 + for idx, weight, grad, name in m.param_blocks: + grad /= batch_size + l2_norm = mx.nd.norm(grad).asscalar() + norm += l2_norm * l2_norm + + norm = math.sqrt(norm) + for idx, weight, grad, name in m.param_blocks: + if norm > max_grad_norm: + grad *= (max_grad_norm / norm) + + updater(idx, grad, weight) + + # reset gradient to zero + grad[:] = 0.0 + + # decay learning rate + if iteration % 50 == 0 and iteration > 0: + opt.lr *= 0.5 + print('reset learning rate to %g' % opt.lr,file=logs) + + # end of training loop + toc = time.time() + train_time = toc - tic + train_acc = num_correct * 100 / float(num_total) + + # saving checkpoint + if (iteration + 1) % 10 == 0: + prefix = 'cnn' + m.symbol.save('checkpoint/%s-symbol.json' % prefix) + save_dict = {('arg:%s' % k) :v for k, v in m.cnn_exec.arg_dict.items()} + save_dict.update({('aux:%s' % k) : v for k, v in m.cnn_exec.aux_dict.items()}) + param_name = 'checkpoint/%s-%04d.params' % (prefix, iteration) + mx.nd.save(param_name, save_dict) + print('Saved checkpoint to %s' % param_name,file=logs) + + + # evaluate on dev set + num_correct = 0 + num_total = 0 + for begin in range(0, X_dev_batch.shape[0], batch_size): + batchX = X_dev_batch[begin:begin+batch_size] + batchY = y_dev_batch[begin:begin+batch_size] + + if batchX.shape[0] != batch_size: + continue + + m.data[:] = batchX + m.cnn_exec.forward(is_train=False) + + num_correct += sum(batchY == np.argmax(m.cnn_exec.outputs[0].asnumpy(), axis=1)) + num_total += len(batchY) + + dev_acc = num_correct * 100 / float(num_total) + print('Iter [%d] Train: Time: %.3fs, Training Accuracy: %.3f \ + --- Dev Accuracy thus far: %.3f' % (iteration, train_time, train_acc, dev_acc), file=logs) + + +def main(): + print('Loading data...') + # word2vec = data_helpers.load_google_word2vec('data/GoogleNews-vectors-negative300.bin') + word2vec = data_helpers.load_pretrained_word2vec('data/rt.vec') + x, y = data_helpers.load_data_with_word2vec(word2vec) + + # randomly shuffle data + np.random.seed(10) + shuffle_indices = np.random.permutation(np.arange(len(y))) + x_shuffled = x[shuffle_indices] + y_shuffled = y[shuffle_indices] + + # split train/dev set + x_train, x_dev = x_shuffled[:-1000], x_shuffled[-1000:] + y_train, y_dev = y_shuffled[:-1000], y_shuffled[-1000:] + print('Train/Dev split: %d/%d' % (len(y_train), len(y_dev))) + print('train shape:', x_train.shape) + print('dev shape:', x_dev.shape) + + # reshpae for convolution input + x_train = np.reshape(x_train, (x_train.shape[0], 1, x_train.shape[1], x_train.shape[2])) + x_dev = np.reshape(x_dev, (x_dev.shape[0], 1, x_dev.shape[1], x_dev.shape[2])) + + num_embed = x_train.shape[-1] + sentence_size = x_train.shape[2] + print('sentence max words', sentence_size) + print('embedding size', num_embed) + batch_size = 50 + + cnn_model = setup_cnn_model(mx.gpu(1), batch_size, sentence_size, num_embed, dropout=0.5) + train_cnn(cnn_model, x_train, y_train, x_dev, y_dev, batch_size) + +def train_without_pretrained_embedding(): + x, y, vocab, vocab_inv = data_helpers.load_data() + vocab_size = len(vocab) + + # randomly shuffle data + np.random.seed(10) + shuffle_indices = np.random.permutation(np.arange(len(y))) + x_shuffled = x[shuffle_indices] + y_shuffled = y[shuffle_indices] + + # split train/dev set + x_train, x_dev = x_shuffled[:-1000], x_shuffled[-1000:] + y_train, y_dev = y_shuffled[:-1000], y_shuffled[-1000:] + print('Train/Dev split: %d/%d' % (len(y_train), len(y_dev))) + print('train shape:', x_train.shape) + print('dev shape:', x_dev.shape) + print('vocab_size', vocab_size) + + batch_size = 50 + num_embed = 300 + sentence_size = x_train.shape[1] + + print('batch size', batch_size) + print('sentence max words', sentence_size) + print('embedding size', num_embed) + + cnn_model = setup_cnn_model(mx.gpu(0), batch_size, sentence_size, num_embed, vocab_size, dropout=0.5, with_embedding=False) + train_cnn(cnn_model, x_train, y_train, x_dev, y_dev, batch_size) + + +if __name__ == '__main__': + if not os.path.exists("checkpoint"): + os.mkdir("checkpoint") + train_without_pretrained_embedding() diff --git a/example/cnn_text_classification/text_cnn.py b/example/cnn_text_classification/text_cnn.py index e41af36cf2ff..16d3dca260fd 100644 --- a/example/cnn_text_classification/text_cnn.py +++ b/example/cnn_text_classification/text_cnn.py @@ -1,31 +1,93 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from __future__ import print_function -import sys,os + +import sys +import os import mxnet as mx import numpy as np -import time -import math -import data_helpers -from collections import namedtuple - +import argparse import logging -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) # get a logger to accuracies are printed +import data_helpers -logs = sys.stderr +logging.basicConfig(level=logging.DEBUG) + +parser = argparse.ArgumentParser(description="CNN for text classification", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument('--pretrained-embedding', type=bool, default=False, + help='use pre-trained word2vec') +parser.add_argument('--num-embed', type=int, default=300, + help='embedding layer size') +parser.add_argument('--gpus', type=str, default='', + help='list of gpus to run, e.g. 0 or 0,2,5. empty means using cpu. ') +parser.add_argument('--kv-store', type=str, default='local', + help='key-value store type') +parser.add_argument('--num-epochs', type=int, default=200, + help='max num of epochs') +parser.add_argument('--batch-size', type=int, default=50, + help='the batch size.') +parser.add_argument('--optimizer', type=str, default='rmsprop', + help='the optimizer type') +parser.add_argument('--lr', type=float, default=0.0005, + help='initial learning rate') +parser.add_argument('--dropout', type=float, default=0.0, + help='dropout rate') +parser.add_argument('--disp-batches', type=int, default=50, + help='show progress for every n batches') +parser.add_argument('--save-period', type=int, default=10, + help='save checkpoint for every n epochs') + +def save_model(): + if not os.path.exists("checkpoint"): + os.mkdir("checkpoint") + return mx.callback.do_checkpoint("checkpoint/checkpoint", args.save_period) -CNNModel = namedtuple("CNNModel", ['cnn_exec', 'symbol', 'data', 'label', 'param_blocks']) +def data_iter(batch_size, num_embed, pre_trained_word2vec=False): + print('Loading data...') + if pre_trained_word2vec: + word2vec = data_helpers.load_pretrained_word2vec('data/rt.vec') + x, y = data_helpers.load_data_with_word2vec(word2vec) + # reshpae for convolution input + x = np.reshape(x, (x.shape[0], 1, x.shape[1], x.shape[2])) + embed_size = x.shape[-1] + sentence_size = x.shape[2] + vocab_size = -1 + else: + x, y, vocab, vocab_inv = data_helpers.load_data() + embed_size = num_embed + sentence_size = x.shape[1] + vocab_size = len(vocab) -def make_text_cnn(sentence_size, num_embed, batch_size, vocab_size, - num_label=2, filter_list=[3, 4, 5], num_filter=100, - dropout=0., with_embedding=True): + # randomly shuffle data + np.random.seed(10) + shuffle_indices = np.random.permutation(np.arange(len(y))) + x_shuffled = x[shuffle_indices] + y_shuffled = y[shuffle_indices] - input_x = mx.sym.Variable('data') # placeholder for input - input_y = mx.sym.Variable('softmax_label') # placeholder for output + # split train/valid set + x_train, x_dev = x_shuffled[:-1000], x_shuffled[-1000:] + y_train, y_dev = y_shuffled[:-1000], y_shuffled[-1000:] + print('Train/Valid split: %d/%d' % (len(y_train), len(y_dev))) + print('train shape:', x_train.shape) + print('valid shape:', x_dev.shape) + print('sentence max words', sentence_size) + print('embedding size', embed_size) + print('vocab size', vocab_size) + + train = mx.io.NDArrayIter( + x_train, y_train, batch_size, shuffle=True) + valid = mx.io.NDArrayIter( + x_dev, y_dev, batch_size) + + return (train, valid, sentence_size, embed_size, vocab_size) + +def sym_gen(batch_size, sentence_size, num_embed, vocab_size, + num_label=2, filter_list=[3, 4, 5], num_filter=100, + dropout=0.0, pre_trained_word2vec=False): + input_x = mx.sym.Variable('data') + input_y = mx.sym.Variable('softmax_label') # embedding layer - if not with_embedding: + if not pre_trained_word2vec: embed_layer = mx.sym.Embedding(data=input_x, input_dim=vocab_size, output_dim=num_embed, name='vocab_embed') conv_input = mx.sym.Reshape(data=embed_layer, target_shape=(batch_size, 1, sentence_size, num_embed)) else: @@ -59,203 +121,37 @@ def make_text_cnn(sentence_size, num_embed, batch_size, vocab_size, # softmax output sm = mx.sym.SoftmaxOutput(data=fc, label=input_y, name='softmax') - return sm - - -def setup_cnn_model(ctx, batch_size, sentence_size, num_embed, vocab_size, - dropout=0.5, initializer=mx.initializer.Uniform(0.1), with_embedding=True): - - cnn = make_text_cnn(sentence_size, num_embed, batch_size=batch_size, - vocab_size=vocab_size, dropout=dropout, with_embedding=with_embedding) - arg_names = cnn.list_arguments() - - input_shapes = {} - if with_embedding: - input_shapes['data'] = (batch_size, 1, sentence_size, num_embed) - else: - input_shapes['data'] = (batch_size, sentence_size) - - arg_shape, out_shape, aux_shape = cnn.infer_shape(**input_shapes) - arg_arrays = [mx.nd.zeros(s, ctx) for s in arg_shape] - args_grad = {} - for shape, name in zip(arg_shape, arg_names): - if name in ['softmax_label', 'data']: # input, output - continue - args_grad[name] = mx.nd.zeros(shape, ctx) - - cnn_exec = cnn.bind(ctx=ctx, args=arg_arrays, args_grad=args_grad, grad_req='add') - - param_blocks = [] - arg_dict = dict(zip(arg_names, cnn_exec.arg_arrays)) - for i, name in enumerate(arg_names): - if name in ['softmax_label', 'data']: # input, output - continue - initializer(name, arg_dict[name]) - - param_blocks.append( (i, arg_dict[name], args_grad[name], name) ) - - out_dict = dict(zip(cnn.list_outputs(), cnn_exec.outputs)) - - data = cnn_exec.arg_dict['data'] - label = cnn_exec.arg_dict['softmax_label'] - - return CNNModel(cnn_exec=cnn_exec, symbol=cnn, data=data, label=label, param_blocks=param_blocks) - - -def train_cnn(model, X_train_batch, y_train_batch, X_dev_batch, y_dev_batch, batch_size, - optimizer='rmsprop', max_grad_norm=5.0, learning_rate=0.0005, epoch=200): - m = model - # create optimizer - opt = mx.optimizer.create(optimizer) - opt.lr = learning_rate - - updater = mx.optimizer.get_updater(opt) - - for iteration in range(epoch): - tic = time.time() - num_correct = 0 - num_total = 0 - for begin in range(0, X_train_batch.shape[0], batch_size): - batchX = X_train_batch[begin:begin+batch_size] - batchY = y_train_batch[begin:begin+batch_size] - if batchX.shape[0] != batch_size: - continue - - m.data[:] = batchX - m.label[:] = batchY - - # forward - m.cnn_exec.forward(is_train=True) - - # backward - m.cnn_exec.backward() - - # eval on training data - num_correct += sum(batchY == np.argmax(m.cnn_exec.outputs[0].asnumpy(), axis=1)) - num_total += len(batchY) - - # update weights - norm = 0 - for idx, weight, grad, name in m.param_blocks: - grad /= batch_size - l2_norm = mx.nd.norm(grad).asscalar() - norm += l2_norm * l2_norm - - norm = math.sqrt(norm) - for idx, weight, grad, name in m.param_blocks: - if norm > max_grad_norm: - grad *= (max_grad_norm / norm) - - updater(idx, grad, weight) - - # reset gradient to zero - grad[:] = 0.0 - - # decay learning rate - if iteration % 50 == 0 and iteration > 0: - opt.lr *= 0.5 - print('reset learning rate to %g' % opt.lr,file=logs) - - # end of training loop - toc = time.time() - train_time = toc - tic - train_acc = num_correct * 100 / float(num_total) - - # saving checkpoint - if (iteration + 1) % 10 == 0: - prefix = 'cnn' - m.symbol.save('checkpoint/%s-symbol.json' % prefix) - save_dict = {('arg:%s' % k) :v for k, v in m.cnn_exec.arg_dict.items()} - save_dict.update({('aux:%s' % k) : v for k, v in m.cnn_exec.aux_dict.items()}) - param_name = 'checkpoint/%s-%04d.params' % (prefix, iteration) - mx.nd.save(param_name, save_dict) - print('Saved checkpoint to %s' % param_name,file=logs) - - - # evaluate on dev set - num_correct = 0 - num_total = 0 - for begin in range(0, X_dev_batch.shape[0], batch_size): - batchX = X_dev_batch[begin:begin+batch_size] - batchY = y_dev_batch[begin:begin+batch_size] - - if batchX.shape[0] != batch_size: - continue - - m.data[:] = batchX - m.cnn_exec.forward(is_train=False) - - num_correct += sum(batchY == np.argmax(m.cnn_exec.outputs[0].asnumpy(), axis=1)) - num_total += len(batchY) - - dev_acc = num_correct * 100 / float(num_total) - print('Iter [%d] Train: Time: %.3fs, Training Accuracy: %.3f \ - --- Dev Accuracy thus far: %.3f' % (iteration, train_time, train_acc, dev_acc), file=logs) - - -def main(): - print('Loading data...') - # word2vec = data_helpers.load_google_word2vec('data/GoogleNews-vectors-negative300.bin') - word2vec = data_helpers.load_pretrained_word2vec('data/rt.vec') - x, y = data_helpers.load_data_with_word2vec(word2vec) - - # randomly shuffle data - np.random.seed(10) - shuffle_indices = np.random.permutation(np.arange(len(y))) - x_shuffled = x[shuffle_indices] - y_shuffled = y[shuffle_indices] - - # split train/dev set - x_train, x_dev = x_shuffled[:-1000], x_shuffled[-1000:] - y_train, y_dev = y_shuffled[:-1000], y_shuffled[-1000:] - print('Train/Dev split: %d/%d' % (len(y_train), len(y_dev))) - print('train shape:', x_train.shape) - print('dev shape:', x_dev.shape) - - # reshpae for convolution input - x_train = np.reshape(x_train, (x_train.shape[0], 1, x_train.shape[1], x_train.shape[2])) - x_dev = np.reshape(x_dev, (x_dev.shape[0], 1, x_dev.shape[1], x_dev.shape[2])) - - num_embed = x_train.shape[-1] - sentence_size = x_train.shape[2] - print('sentence max words', sentence_size) - print('embedding size', num_embed) - batch_size = 50 - - cnn_model = setup_cnn_model(mx.gpu(1), batch_size, sentence_size, num_embed, dropout=0.5) - train_cnn(cnn_model, x_train, y_train, x_dev, y_dev, batch_size) - -def train_without_pretrained_embedding(): - x, y, vocab, vocab_inv = data_helpers.load_data() - vocab_size = len(vocab) - - # randomly shuffle data - np.random.seed(10) - shuffle_indices = np.random.permutation(np.arange(len(y))) - x_shuffled = x[shuffle_indices] - y_shuffled = y[shuffle_indices] - - # split train/dev set - x_train, x_dev = x_shuffled[:-1000], x_shuffled[-1000:] - y_train, y_dev = y_shuffled[:-1000], y_shuffled[-1000:] - print('Train/Dev split: %d/%d' % (len(y_train), len(y_dev))) - print('train shape:', x_train.shape) - print('dev shape:', x_dev.shape) - print('vocab_size', vocab_size) - - batch_size = 50 - num_embed = 300 - sentence_size = x_train.shape[1] - - print('batch size', batch_size) - print('sentence max words', sentence_size) - print('embedding size', num_embed) - - cnn_model = setup_cnn_model(mx.gpu(0), batch_size, sentence_size, num_embed, vocab_size, dropout=0.5, with_embedding=False) - train_cnn(cnn_model, x_train, y_train, x_dev, y_dev, batch_size) - + return sm, ('data',), ('softmax_label',) + +def train(symbol, train_iter, valid_iter, data_names, label_names): + devs = mx.cpu() if args.gpus is None or args.gpus is '' else [ + mx.gpu(int(i)) for i in args.gpus.split(',')] + module = mx.mod.Module(symbol, data_names=data_names, label_names=label_names, context=devs) + module.fit(train_data = train_iter, + eval_data = valid_iter, + eval_metric = 'acc', + kvstore = args.kv_store, + optimizer = args.optimizer, + optimizer_params = { 'learning_rate': args.lr }, + initializer = mx.initializer.Uniform(0.1), + num_epoch = args.num_epochs, + batch_end_callback = mx.callback.Speedometer(args.batch_size, args.disp_batches), + epoch_end_callback = save_model()) if __name__ == '__main__': - if not os.path.exists("checkpoint"): - os.mkdir("checkpoint") - train_without_pretrained_embedding() + # parse args + args = parser.parse_args() + + # data iter + train_iter, valid_iter, sentence_size, embed_size, vocab_size = data_iter(args.batch_size, + args.num_embed, + args.pretrained_embedding) + # network symbol + symbol, data_names, label_names = sym_gen(args.batch_size, + sentence_size, + embed_size, + vocab_size, + num_label=2, filter_list=[3, 4, 5], num_filter=100, + dropout=args.dropout, pre_trained_word2vec=args.pretrained_embedding) + # train cnn model + train(symbol, train_iter, valid_iter, data_names, label_names) From 02634c9a01bfd3ff9a3b705449f00827aeaf806d Mon Sep 17 00:00:00 2001 From: lxn2 Date: Fri, 16 Jun 2017 14:54:23 -0700 Subject: [PATCH 080/834] Add python-opencv installation to Docker image (#6699) * Empty commit * Add python-opencv installation --- docker/install/cpp.sh | 2 +- emptycommit | 0 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 emptycommit diff --git a/docker/install/cpp.sh b/docker/install/cpp.sh index 91b8b8db0607..f30ab52f9a52 100755 --- a/docker/install/cpp.sh +++ b/docker/install/cpp.sh @@ -2,7 +2,7 @@ # libraries for building mxnet c++ core on ubuntu apt-get update && apt-get install -y \ - build-essential git libatlas-base-dev libopencv-dev \ + build-essential git libatlas-base-dev libopencv-dev python-opencv \ libcurl4-openssl-dev libgtest-dev cmake wget unzip cd /usr/src/gtest && cmake CMakeLists.txt && make && cp *.a /usr/lib diff --git a/emptycommit b/emptycommit new file mode 100644 index 000000000000..e69de29bb2d1 From aacfef45b00feeb31c37b0ba397ade78098d842b Mon Sep 17 00:00:00 2001 From: lxn2 Date: Fri, 16 Jun 2017 16:51:50 -0700 Subject: [PATCH 081/834] Fix docs typo from gpu0 -> cpu (#6727) * Empty commit * Fix typo from gpu0 -> cpu --- docs/architecture/note_engine.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/architecture/note_engine.md b/docs/architecture/note_engine.md index 8fa8b52b3a8c..dc0b84aa53d1 100644 --- a/docs/architecture/note_engine.md +++ b/docs/architecture/note_engine.md @@ -136,8 +136,8 @@ training one batch on a two-layer neural network. # aggregate gradient and update fc1_wgrad[cpu] = fc1_wgrad[gpu0] + fc1_wgrad[gpu1] fc2_wgrad[cpu] = fc2_wgrad[gpu0] + fc2_wgrad[gpu1] - fc1_weight[cpu] -= lr * fc1_wgrad[gpu0] - fc2_weight[cpu] -= lr * fc2_wgrad[gpu0] + fc1_weight[cpu] -= lr * fc1_wgrad[cpu] + fc2_weight[cpu] -= lr * fc2_wgrad[cpu] fc1_weight[cpu].copyto(fc1_weight[gpu0] , fc1_weight[gpu1]) fc2_weight[cpu].copyto(fc2_weight[gpu0] , fc2_weight[gpu1]) ``` From ce2bca6e6847f54fc37a9c295e96f14f681ae2b9 Mon Sep 17 00:00:00 2001 From: Yuwen Xiong Date: Sun, 18 Jun 2017 01:53:37 +0800 Subject: [PATCH 082/834] Add operators for Deformable ConvNets/DFF (#6298) * Add operators for Deformable ConvNets/FCIS/DFF * fix programming rule to meet pr rule * fix programming rule to meet pr rule * fix programming rule to meet pr rule * minor fix channel operator * minor fix deformable conv * fix a stupid error * add test code * add test code * remove channel operator and add gradient check * remove redundant print * fix unittest code * dummy commit to trigger building check * dummy commit to trigger building check * Update deformable_convolution-inl.h * Update deformable_im2col.h * Update deformable_convolution.cc * Update deformable_psroi_pooling.cc --- CONTRIBUTORS.md | 5 + .../contrib/deformable_convolution-inl.h | 488 ++++++++++++++ .../contrib/deformable_convolution.cc | 89 +++ .../contrib/deformable_convolution.cu | 29 + .../contrib/deformable_psroi_pooling-inl.h | 286 +++++++++ .../contrib/deformable_psroi_pooling.cc | 96 +++ .../contrib/deformable_psroi_pooling.cu | 415 ++++++++++++ src/operator/contrib/multi_proposal-inl.h | 301 +++++++++ src/operator/contrib/multi_proposal.cc | 63 ++ src/operator/contrib/multi_proposal.cu | 593 ++++++++++++++++++ src/operator/contrib/nn/deformable_im2col.cuh | 525 ++++++++++++++++ src/operator/contrib/nn/deformable_im2col.h | 157 +++++ src/operator/contrib/psroi_pooling-inl.h | 222 +++++++ src/operator/contrib/psroi_pooling.cc | 80 +++ src/operator/contrib/psroi_pooling.cu | 260 ++++++++ tests/python/gpu/test_operator_gpu.py | 165 +++++ tests/python/unittest/test_operator.py | 98 ++- 17 files changed, 3871 insertions(+), 1 deletion(-) create mode 100644 src/operator/contrib/deformable_convolution-inl.h create mode 100644 src/operator/contrib/deformable_convolution.cc create mode 100644 src/operator/contrib/deformable_convolution.cu create mode 100644 src/operator/contrib/deformable_psroi_pooling-inl.h create mode 100644 src/operator/contrib/deformable_psroi_pooling.cc create mode 100644 src/operator/contrib/deformable_psroi_pooling.cu create mode 100644 src/operator/contrib/multi_proposal-inl.h create mode 100644 src/operator/contrib/multi_proposal.cc create mode 100644 src/operator/contrib/multi_proposal.cu create mode 100644 src/operator/contrib/nn/deformable_im2col.cuh create mode 100644 src/operator/contrib/nn/deformable_im2col.h create mode 100644 src/operator/contrib/psroi_pooling-inl.h create mode 100644 src/operator/contrib/psroi_pooling.cc create mode 100644 src/operator/contrib/psroi_pooling.cu diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index dbbb73b1361a..843ad8a056fc 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -137,3 +137,8 @@ List of Contributors * [Roshani Nagmote](https://github.com/Roshrini) * [Chetan Khatri](https://github.com/chetkhatri/) * [James Liu](https://github.com/jamesliu/) +* [Yuwen Xiong](https://github.com/Orpine/) +* [Haozhi Qi](https://github.com/Oh233/) +* [Yi Li](https://github.com/liyi14/) +* [Guodong Zhang](https://github.com/gd-zhang/) +* [Xizhou Zhu](https://github.com/einsiedler0408/) diff --git a/src/operator/contrib/deformable_convolution-inl.h b/src/operator/contrib/deformable_convolution-inl.h new file mode 100644 index 000000000000..da979e707aee --- /dev/null +++ b/src/operator/contrib/deformable_convolution-inl.h @@ -0,0 +1,488 @@ +/*! + * Copyright (c) 2017 Microsoft + * Licensed under The Apache-2.0 License [see LICENSE for details] + * \file deformable_convolution-inl.h + * \brief + * \ref: https://github.com/Yangqing/caffe/wiki/Convolution-in-Caffe:-a-memo + * \ref: https://arxiv.org/abs/1703.06211 + * \author Yuwen Xiong, Haozhi Qi, Jifeng Dai +*/ +#ifndef MXNET_OPERATOR_CONTRIB_DEFORMABLE_CONVOLUTION_INL_H_ +#define MXNET_OPERATOR_CONTRIB_DEFORMABLE_CONVOLUTION_INL_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../operator_common.h" +#include "../nn/im2col.h" +#include "./nn/deformable_im2col.h" + + +namespace mxnet { +namespace op { + +namespace conv { + enum DeformableConvolutionOpInputs { kData, kOffset, kWeight, kBias }; + enum DeformableConvolutionOpOutputs { kOut }; + enum DeformableConvolutionOpResource { kTempSpace }; +} + +struct DeformableConvolutionParam : public dmlc::Parameter { + TShape kernel; + TShape stride; + TShape dilate; + TShape pad; + uint32_t num_filter; + uint32_t num_group; + uint32_t num_deformable_group; + uint64_t workspace; + bool no_bias; + dmlc::optional layout; + DMLC_DECLARE_PARAMETER(DeformableConvolutionParam) { + DMLC_DECLARE_FIELD(kernel).describe("convolution kernel size: (h, w) or (d, h, w)"); + DMLC_DECLARE_FIELD(stride).set_default(TShape()) + .describe("convolution stride: (h, w) or (d, h, w)"); + DMLC_DECLARE_FIELD(dilate).set_default(TShape()) + .describe("convolution dilate: (h, w) or (d, h, w)"); + DMLC_DECLARE_FIELD(pad).set_default(TShape()) + .describe("pad for convolution: (h, w) or (d, h, w)"); + DMLC_DECLARE_FIELD(num_filter).set_range(1, 100000) + .describe("convolution filter(channel) number"); + DMLC_DECLARE_FIELD(num_group).set_default(1) + .describe("Number of group partitions."); + DMLC_DECLARE_FIELD(num_deformable_group).set_default(1) + .describe("Number of deformable group partitions."); + DMLC_DECLARE_FIELD(workspace).set_default(1024).set_range(0, 8192) + .describe("Maximum temperal workspace allowed for convolution (MB)."); + DMLC_DECLARE_FIELD(no_bias).set_default(false) + .describe("Whether to disable bias parameter."); + DMLC_DECLARE_FIELD(layout) + .add_enum("NCW", mshadow::kNCW) + .add_enum("NCHW", mshadow::kNCHW) + .add_enum("NCDHW", mshadow::kNCDHW) + .set_default(dmlc::optional()) + .describe("Set layout for input, output and weight. Empty for\n " + "default layout: NCW for 1d, NCHW for 2d and NCDHW for 3d."); + } +}; + +template +class DeformableConvolutionOp : public Operator { + public: + explicit DeformableConvolutionOp(DeformableConvolutionParam p) { + this->param_ = p; + // convert MBytes first to Bytes and then to elements. + param_.workspace = (param_.workspace << 20) / sizeof(DType); + CHECK(param_.layout.value() == mshadow::kNCW || + param_.layout.value() == mshadow::kNCHW || + param_.layout.value() == mshadow::kNCDHW) + << "Only support NCW, NCHW and NCDHW layout"; + } + + virtual void Forward(const OpContext &ctx, + const std::vector &in_data, + const std::vector &req, + const std::vector &out_data, + const std::vector &aux_args) { + using namespace mshadow; + using namespace mshadow::expr; + CHECK_EQ(req[conv::kOut], kWriteTo); + size_t expected = param_.no_bias ? 3 : 4; + CHECK_EQ(in_data.size(), expected); + CHECK_EQ(out_data.size(), 1U); + LayerSetUp(in_data[conv::kData].shape_, + in_data[conv::kOffset].shape_, + out_data[conv::kOut].shape_); + Stream* s = ctx.get_stream(); + // allocate workspace for col_buffer + Tensor workspace = ctx.requested[conv::kTempSpace] + .get_space_typed(Shape1(col_buffer_size_), s); + // calculate the shape of col_buffer + TShape col_buffer_shape(num_spatial_axes_ + 1); + col_buffer_shape[0] = conv_in_channels_ * param_.kernel.Size(); + for (index_t i = 1; i < col_buffer_shape.ndim(); ++i) { + col_buffer_shape[i] = out_data[0].shape_[i + 1]; + } + // create a column buffer using workspace and col_buffer_shape + TBlob col_buffer(workspace.dptr_, col_buffer_shape, xpu::kDevMask, DataType::kFlag); + + // initialize weight and col_buffer 3D tensors for using gemm + index_t M = conv_out_channels_ / group_; + index_t N = conv_out_spatial_dim_; + index_t K = kernel_dim_; + Tensor weight_3d = in_data[conv::kWeight].get_with_shape( + Shape3(group_, M, K), s); + Tensor col_buffer_3d = col_buffer.get_with_shape( + Shape3(group_, K, N), s); + Tensor output_4d = out_data[conv::kOut].get_with_shape( + Shape4(num_, group_, M, N), s); + for (index_t n = 0; n < num_; ++n) { + // transform image to col_buffer in order to use gemm + deformable_im2col(s, in_data[conv::kData].dptr() + n*input_dim_, + in_data[conv::kOffset].dptr() + n*input_offset_dim_, in_data[conv::kData].shape_, + col_buffer.shape_, param_.kernel, param_.pad, param_.stride, param_.dilate, + param_.num_deformable_group, col_buffer.dptr()); + Tensor output_3d = output_4d[n]; + for (index_t g = 0; g < group_; ++g) { + ASSIGN_DISPATCH(output_3d[g], req[conv::kOut], dot(weight_3d[g], col_buffer_3d[g])); + } + } + if (bias_term_) { + Tensor bias = in_data[conv::kBias].get(s); + Tensor output_3d = out_data[conv::kOut].get_with_shape( + Shape3(num_, conv_out_channels_, conv_out_spatial_dim_), s); + // has bias term, broadcast it to the same shape of output_3d in channel dim + output_3d += mshadow::expr::broadcast<1>(bias, output_3d.shape_); + } + } + + virtual void Backward(const OpContext &ctx, + const std::vector& out_grad, + const std::vector& in_data, + const std::vector& out_data, + const std::vector& req, + const std::vector& in_grad, + const std::vector& aux_args) { + using namespace mshadow; + using namespace mshadow::expr; + CHECK_EQ(out_grad.size(), 1U); + size_t expected = param_.no_bias == 0 ? 4 : 3; + CHECK(in_data.size() == expected && in_grad.size() == expected); + CHECK_EQ(req.size(), expected); + CHECK_EQ(in_data[conv::kWeight].CheckContiguous(), true); + LayerSetUp(in_grad[conv::kData].shape_, + in_grad[conv::kOffset].shape_, + out_grad[conv::kOut].shape_); + Stream *s = ctx.get_stream(); + // allocate workspace for col_buffer + Tensor workspace = ctx.requested[conv::kTempSpace] + .get_space_typed(Shape1(col_buffer_size_), s); + // calculate the shape of col_buffer + TShape col_buffer_shape(num_spatial_axes_ + 1); + col_buffer_shape[0] = conv_in_channels_ * param_.kernel.Size(); + for (index_t i = 1; i < col_buffer_shape.ndim(); ++i) { + col_buffer_shape[i] = out_grad[conv::kData].shape_[i + 1]; + } + // create a column buffer using workspace and col_buffer_shape + TBlob col_buffer(workspace.dptr_, col_buffer_shape, xpu::kDevMask, DataType::kFlag); + + // initialize weight and col_buffer 3D tensors for using gemm + // For computing dLoss/d(in_data[kData]) + index_t M = kernel_dim_; + index_t N = conv_out_spatial_dim_; + index_t K = conv_out_channels_ / group_; + Tensor weight_3d = in_data[conv::kWeight].get_with_shape( + Shape3(group_, K, M), s); + Tensor out_grad_4d = out_grad[conv::kOut].get_with_shape( + Shape4(num_, group_, K, N), s); + Tensor col_buffer_3d = col_buffer.get_with_shape( + Shape3(group_, M, N), s); + // For computing dLoss/dWeight + Tensor dweight_3d = in_grad[conv::kWeight].get_with_shape( + Shape3(group_, K, M), s); + + Tensor data_grad = in_grad[conv::kData].FlatTo1D(s); + data_grad = 0; + + + for (index_t n = 0; n < num_; ++n) { + Tensor out_grad_3d = out_grad_4d[n]; + for (index_t g = 0; g < group_; ++g) { + col_buffer_3d[g] = dot(weight_3d[g].T(), out_grad_3d[g]); + } + + // gradient w.r.t. input coordinate data + deformable_col2im_coord(s, col_buffer.dptr(), + in_data[conv::kData].dptr() + n*input_dim_, + in_data[conv::kOffset].dptr() + n*input_offset_dim_, + in_grad[conv::kData].shape_, col_buffer.shape_, + param_.kernel, param_.pad, param_.stride, param_.dilate, param_.num_deformable_group, + in_grad[conv::kOffset].dptr() + n*input_offset_dim_, + req[conv::kData]); + + // gradient w.r.t. input data + deformable_col2im(s, col_buffer.dptr(), + in_data[conv::kOffset].dptr() + n*input_offset_dim_, + in_grad[conv::kData].shape_, col_buffer.shape_, + param_.kernel, param_.pad, param_.stride, param_.dilate, param_.num_deformable_group, + in_grad[conv::kData].dptr() + n*input_dim_, + req[conv::kData]); + + // gradient w.r.t. weight, dWeight should accumulate across the batch and group + im2col(s, in_data[conv::kData].dptr() + n*input_dim_, in_data[conv::kData].shape_, + col_buffer.shape_, param_.kernel, param_.pad, param_.stride, param_.dilate, + col_buffer.dptr()); + for (index_t g = 0; g < group_; ++g) { + if (0 == n) { + ASSIGN_DISPATCH(dweight_3d[g], req[conv::kWeight], + dot(out_grad_3d[g], col_buffer_3d[g].T())); + } else { + dweight_3d[g] += dot(out_grad_3d[g], col_buffer_3d[g].T()); + } + } + } + + // gradient w.r.t bias + if (bias_term_) { + Tensor dbias = in_grad[conv::kBias].get(s); + Tensor dout = out_grad[conv::kOut].get_with_shape( + Shape3(num_, conv_out_channels_, conv_out_spatial_dim_), s); + ASSIGN_DISPATCH(dbias, req[conv::kBias], sumall_except_dim<1>(dout)); + } + } + + private: + void LayerSetUp(const TShape& ishape, const TShape& offset_shape, const TShape& oshape) { + channel_axis_ = 1; // hard code channel axis + const index_t first_spatial_axis = channel_axis_ + 1; + const index_t num_axes = param_.kernel.ndim() + 2; + num_spatial_axes_ = num_axes - first_spatial_axis; + is_1x1_ = true; + for (index_t i = 0; i < param_.kernel.ndim(); ++i) { + is_1x1_ &= param_.kernel[i] == 1 && param_.stride[i] == 1 && param_.pad[i] == 0; + if (!is_1x1_) break; + } + + // batch size + num_ = ishape[0]; + // number of input channels + channels_ = ishape[1]; + group_ = param_.num_group; + conv_out_channels_ = param_.num_filter; + conv_in_channels_ = channels_; + bias_term_ = !param_.no_bias; + kernel_dim_ = conv_in_channels_ / group_ * param_.kernel.Size(); + weight_offset_ = conv_out_channels_ * kernel_dim_ / group_; + conv_out_spatial_dim_ = oshape.ProdShape(2, oshape.ndim()); + col_offset_ = kernel_dim_ * conv_out_spatial_dim_; + output_offset_ = conv_out_channels_ * conv_out_spatial_dim_ / group_; + // size of the column buffer used for storing im2col-ed pixels + col_buffer_size_ = kernel_dim_ * group_ * conv_out_spatial_dim_; + // input/output image size (#channels * height * width) + input_dim_ = ishape.ProdShape(1, ishape.ndim()); + input_offset_dim_ = offset_shape.ProdShape(1, offset_shape.ndim()); + output_dim_ = oshape.ProdShape(1, oshape.ndim()); + num_kernels_im2col_ = conv_in_channels_ * conv_out_spatial_dim_; + num_kernels_col2im_ = input_dim_; + } + + private: + DeformableConvolutionParam param_; + index_t channel_axis_; // channel axis of the input + index_t channels_; // number of channels of input image + index_t num_spatial_axes_; // number of spatial axes + index_t num_; // batch size + index_t group_; // number of groups + index_t conv_out_channels_; // number of output channels (num_filter) + index_t conv_out_spatial_dim_; // number of pixels of output images per channel + index_t conv_in_channels_; // number of input channels + index_t kernel_dim_; // number of input channels per group * kernel size + index_t weight_offset_; // number of output channels per group * kernel_dim_ + index_t col_offset_; + index_t output_offset_; + index_t col_buffer_size_; + index_t input_dim_; + index_t input_offset_dim_; + index_t output_dim_; + index_t num_kernels_im2col_; + index_t num_kernels_col2im_; + bool bias_term_; // has bias term? + bool is_1x1_; +}; // class ConvolutionOp + +template +Operator* CreateOp(DeformableConvolutionParam param, int dtype, + std::vector *in_shape, + std::vector *out_shape, + Context ctx); + +#if DMLC_USE_CXX11 +class DeformableConvolutionProp : public OperatorProperty { + public: + std::vector ListArguments() const override { + if (!param_.no_bias) { + return{ "data", "offset", "weight", "bias" }; + } else { + return{ "data", "offset", "weight" }; + } + } + + void Init(const std::vector >& kwargs) override { + using namespace mshadow; + param_.Init(kwargs); + if (param_.kernel.ndim() == 2) { + param_.layout = param_.layout ? param_.layout.value() : mshadow::kNCHW; + if (param_.stride.ndim() == 0) param_.stride = Shape2(1, 1); + if (param_.dilate.ndim() == 0) param_.dilate = Shape2(1, 1); + if (param_.pad.ndim() == 0) param_.pad = Shape2(0, 0); + } else { + LOG(FATAL) << "not implemented"; + } + } + + std::map GetParams() const override { + return param_.__DICT__(); + } + + bool InferShape(std::vector *in_shape, + std::vector *out_shape, + std::vector *aux_shape) const override { + using namespace mshadow; + if (!param_.no_bias) { + CHECK_EQ(in_shape->size(), 4U) << "Input:[data, offset, weight, bias]"; + } else { + CHECK_EQ(in_shape->size(), 3U) << "Input:[data, offset, weight]"; + } + out_shape->resize(1, TShape()); + const TShape &dshp = (*in_shape)[conv::kData]; + const TShape &oshp = (*in_shape)[conv::kOffset]; + if (dshp.ndim() == 0) return false; + if (param_.kernel.ndim() == 2) { + // 2d conv + CHECK_EQ(dshp.ndim(), 4U) \ + << "Input data should be 4D in batch-num_filter-y-x"; + CHECK_EQ(oshp.ndim(), 4U) \ + << "Input offset should be 4D in batch-num_filter-y-x"; + Shape<4> dshape = ConvertLayout(dshp.get<4>(), param_.layout.value(), kNCHW); + Shape<4> offsetshape = ConvertLayout(oshp.get<4>(), param_.layout.value(), kNCHW); + Shape<4> wshape = Shape4(param_.num_filter / param_.num_group, dshape[1] / param_.num_group, + param_.kernel[0], param_.kernel[1]); + wshape = ConvertLayout(wshape, kNCHW, param_.layout.value()); + wshape[0] *= param_.num_group; + SHAPE_ASSIGN_CHECK(*in_shape, conv::kWeight, wshape); + if (!param_.no_bias) { + SHAPE_ASSIGN_CHECK(*in_shape, conv::kBias, Shape1(param_.num_filter)); + } + + const index_t ksize_y = static_cast(param_.kernel[0]); + const index_t ksize_x = static_cast(param_.kernel[1]); + CHECK_EQ(dshape[1] % param_.num_group, 0U) \ + << "input num_filter must divide group size"; + CHECK_EQ(dshape[1] % param_.num_deformable_group, 0U) \ + << "input num_filter must divide deformable group size"; + CHECK_EQ(param_.num_filter % param_.num_group, 0U) \ + << "output num_filter must divide group size"; + CHECK_GT(param_.kernel.Size(), 0U) \ + << "incorrect kernel size: " << param_.kernel; + CHECK_GT(param_.stride.Size(), 0U) \ + << "incorrect stride size: " << param_.stride; + CHECK_GT(param_.dilate.Size(), 0U) \ + << "incorrect dilate size: " << param_.dilate; + Shape<4> oshape; + oshape[0] = dshape[0]; + oshape[1] = param_.num_filter; + oshape[2] = (dshape[2] + 2 * param_.pad[0] - + (param_.dilate[0] * (ksize_y - 1) + 1)) / param_.stride[0] + 1; + oshape[3] = (dshape[3] + 2 * param_.pad[1] - + (param_.dilate[1] * (ksize_x - 1) + 1)) / param_.stride[1] + 1; + SHAPE_ASSIGN_CHECK(*out_shape, 0, ConvertLayout(oshape, kNCHW, param_.layout.value())); + CHECK_EQ(oshape[1] % param_.num_deformable_group, 0U) \ + << "output num_filter must divide deformable group size"; + CHECK_EQ(oshape[2], offsetshape[2]) \ + << "output height must equal to offset map height"; + CHECK_EQ(oshape[3], offsetshape[3]) \ + << "output width must equal to offset map width"; + CHECK_EQ(offsetshape[1] % (param_.kernel[0] * param_.kernel[1]), 0U) \ + << "offset filter must divide deformable group size"; + CHECK_EQ(offsetshape[1] / (2 * param_.kernel[0] * param_.kernel[1]), \ + param_.num_deformable_group) \ + << "offset filter must divide deformable group size"; + // Perform incomplete shape inference. Fill in the missing values in data shape. + // 1) We can always fill in the batch_size. + // 2) We can back-calculate the input height/width if the corresponding stride is 1. + oshape = ConvertLayout((*out_shape)[0].get<4>(), param_.layout.value(), kNCHW); + dshape[0] = oshape[0]; + if (param_.stride[0] == 1) { + dshape[2] = oshape[2] + param_.dilate[0] * (ksize_y - 1) - 2 * param_.pad[0]; + } + if (param_.stride[1] == 1) { + dshape[3] = oshape[3] + param_.dilate[1] * (ksize_x - 1) - 2 * param_.pad[1]; + } + SHAPE_ASSIGN_CHECK(*in_shape, conv::kData, + ConvertLayout(dshape, kNCHW, param_.layout.value())); + // Check whether the kernel sizes are valid + if (dshape[2] != 0) { + CHECK_LE(ksize_y, dshape[2] + 2 * param_.pad[0]) << "kernel size exceed input"; + } + if (dshape[3] != 0) { + CHECK_LE(ksize_x, dshape[3] + 2 * param_.pad[1]) << "kernel size exceed input"; + } + return true; + } else { + LOG(FATAL) << "not implemented"; + return false; + } + } + + bool InferType(std::vector *in_type, + std::vector *out_type, + std::vector *aux_type) const override { + CHECK_GE(in_type->size(), 1U); + int dtype = (*in_type)[0]; + CHECK_NE(dtype, -1) << "First input must have specified type"; + for (index_t i = 0; i < in_type->size(); ++i) { + if ((*in_type)[i] == -1) { + (*in_type)[i] = dtype; + } else { + CHECK_EQ((*in_type)[i], dtype) << "This layer requires uniform type. " + << "Expected " << dtype << " v.s. given " + << (*in_type)[i] << " at " << ListArguments()[i]; + } + } + out_type->clear(); + out_type->push_back(dtype); + return true; + } + + OperatorProperty* Copy() const override { + auto ptr = new DeformableConvolutionProp(); + ptr->param_ = param_; + return ptr; + } + + std::string TypeString() const override { + return "_contrib_DeformableConvolution"; + } + + std::vector DeclareBackwardDependency( + const std::vector &out_grad, + const std::vector &in_data, + const std::vector &out_data) const override { + return{ out_grad[conv::kOut], in_data[conv::kData], + in_data[conv::kOffset], in_data[conv::kWeight] }; + } + + std::vector ForwardResource( + const std::vector &in_shape) const override { + return{ ResourceRequest::kTempSpace }; + } + + std::vector BackwardResource( + const std::vector &in_shape) const override { + return{ ResourceRequest::kTempSpace }; + } + + Operator* CreateOperator(Context ctx) const override { + LOG(FATAL) << "Not Implemented."; + return NULL; + } + + Operator* CreateOperatorEx(Context ctx, std::vector *in_shape, + std::vector *in_type) const override; + + private: + DeformableConvolutionParam param_; +}; // class ConvolutionProp +#endif // DMLC_USE_CXX11 +} // namespace op +} // namespace mxnet +#endif // MXNET_OPERATOR_CONTRIB_DEFORMABLE_CONVOLUTION_INL_H_ diff --git a/src/operator/contrib/deformable_convolution.cc b/src/operator/contrib/deformable_convolution.cc new file mode 100644 index 000000000000..5af91a0aa407 --- /dev/null +++ b/src/operator/contrib/deformable_convolution.cc @@ -0,0 +1,89 @@ +/*! + * Copyright (c) 2017 Microsoft + * Licensed under The Apache-2.0 License [see LICENSE for details] + * \file deformable_convolution.cc + * \brief + * \author Yuwen Xiong, Haozhi Qi, Jifeng Dai +*/ + +#include "./deformable_convolution-inl.h" + +namespace mxnet { +namespace op { +DMLC_REGISTER_PARAMETER(DeformableConvolutionParam); + +template<> +Operator* CreateOp(DeformableConvolutionParam param, int dtype, + std::vector *in_shape, + std::vector *out_shape, + Context ctx) { + Operator *op = NULL; + MSHADOW_REAL_TYPE_SWITCH(dtype, DType, { + op = new DeformableConvolutionOp(param); + }) + return op; +} + +// DO_BIND_DISPATCH comes from operator_common.h +Operator *DeformableConvolutionProp::CreateOperatorEx(Context ctx, + std::vector *in_shape, + std::vector *in_type) const { + std::vector out_shape, aux_shape; + std::vector out_type, aux_type; + CHECK(InferType(in_type, &out_type, &aux_type)); + CHECK(InferShape(in_shape, &out_shape, &aux_shape)); + DO_BIND_DISPATCH(CreateOp, param_, (*in_type)[0], in_shape, &out_shape, ctx); +} + +MXNET_REGISTER_OP_PROPERTY(_contrib_DeformableConvolution, DeformableConvolutionProp) +.describe(R"code(Compute 2-D deformable convolution on 4-D input. + +The deformable convolution operation is described in https://arxiv.org/abs/1703.06211 + +For 2-D deformable convolution, the shapes are + +- **data**: *(batch_size, channel, height, width)* +- **offset**: *(batch_size, num_deformable_group * kernel[0] * kernel[1], height, width)* +- **weight**: *(num_filter, channel, kernel[0], kernel[1])* +- **bias**: *(num_filter,)* +- **out**: *(batch_size, num_filter, out_height, out_width)*. + +Define:: + + f(x,k,p,s,d) = floor((x+2*p-d*(k-1)-1)/s)+1 + +then we have:: + + out_height=f(height, kernel[0], pad[0], stride[0], dilate[0]) + out_width=f(width, kernel[1], pad[1], stride[1], dilate[1]) + +If ``no_bias`` is set to be true, then the ``bias`` term is ignored. + +The default data ``layout`` is *NCHW*, namely *(batch_size, channle, height, +width)*. + +If ``num_group`` is larger than 1, denoted by *g*, then split the input ``data`` +evenly into *g* parts along the channel axis, and also evenly split ``weight`` +along the first dimension. Next compute the convolution on the *i*-th part of +the data with the *i*-th weight part. The output is obtained by concating all +the *g* results. + +If ``num_deformable_group`` is larger than 1, denoted by *dg*, then split the +input ``offset`` evenly into *dg* parts along the channel axis, and also evenly +split ``out`` evenly into *dg* parts along the channel axis. Next compute the +deformable convolution, apply the *i*-th part of the offset part on the *i*-th +out. + + +Both ``weight`` and ``bias`` are learnable parameters. + + +)code" ADD_FILELINE) +.add_argument("data", "NDArray-or-Symbol", "Input data to the DeformableConvolutionOp.") +.add_argument("offset", "NDArray-or-Symbol", "Input offset to the DeformableConvolutionOp.") +.add_argument("weight", "NDArray-or-Symbol", "Weight matrix.") +.add_argument("bias", "NDArray-or-Symbol", "Bias parameter.") +.add_arguments(DeformableConvolutionParam::__FIELDS__()); + +} // namespace op +} // namespace mxnet diff --git a/src/operator/contrib/deformable_convolution.cu b/src/operator/contrib/deformable_convolution.cu new file mode 100644 index 000000000000..f690cc1ce24c --- /dev/null +++ b/src/operator/contrib/deformable_convolution.cu @@ -0,0 +1,29 @@ +/*! + * Copyright (c) 2017 Microsoft + * Licensed under The Apache-2.0 License [see LICENSE for details] + * \file deformable_convolution.cu + * \brief + * \author Yuwen Xiong, Haozhi Qi, Jifeng Dai +*/ + +#include "./deformable_convolution-inl.h" +#include + +namespace mxnet { +namespace op { + + template<> + Operator* CreateOp(DeformableConvolutionParam param, int dtype, + std::vector *in_shape, + std::vector *out_shape, + Context ctx) { + Operator *op = NULL; + MSHADOW_REAL_TYPE_SWITCH(dtype, DType, { + op = new DeformableConvolutionOp(param); + }) + return op; + } + +} // namespace op +} // namespace mxnet + diff --git a/src/operator/contrib/deformable_psroi_pooling-inl.h b/src/operator/contrib/deformable_psroi_pooling-inl.h new file mode 100644 index 000000000000..16a98f76bcff --- /dev/null +++ b/src/operator/contrib/deformable_psroi_pooling-inl.h @@ -0,0 +1,286 @@ +/*! +* Copyright (c) 2017 Microsoft +* Licensed under The Apache-2.0 License [see LICENSE for details] +* \file deformable_psroi_pooling-inl.h +* \brief deformable psroi pooling operator and symbol +* \author Yi Li, Guodong Zhang, Jifeng Dai +*/ +#ifndef MXNET_OPERATOR_CONTRIB_DEFORMABLE_PSROI_POOLING_INL_H_ +#define MXNET_OPERATOR_CONTRIB_DEFORMABLE_PSROI_POOLING_INL_H_ + +#include +#include +#include +#include +#include +#include +#include +#include "../mshadow_op.h" +#include "../operator_common.h" + + +namespace mxnet { +namespace op { + + // Declare enumeration of input order to make code more intuitive. + // These enums are only visible within this header +namespace deformablepsroipool { + enum DeformablePSROIPoolingOpInputs { kData, kBox, kTrans }; + enum DeformablePSROIPoolingOpOutputs { kOut, kTopCount }; +} // deformablepsroipool + +struct DeformablePSROIPoolingParam : public dmlc::Parameter { + // TShape pooled_size; + float spatial_scale; + int output_dim; + int group_size; + int pooled_size; + int part_size; + int sample_per_part; + float trans_std; + bool no_trans; + DMLC_DECLARE_PARAMETER(DeformablePSROIPoolingParam) { + DMLC_DECLARE_FIELD(spatial_scale).set_range(0.0, 1.0) + .describe("Ratio of input feature map height (or w) to raw image height (or w). " + "Equals the reciprocal of total stride in convolutional layers"); + DMLC_DECLARE_FIELD(output_dim).describe("fix output dim"); + DMLC_DECLARE_FIELD(group_size).describe("fix group size"); + DMLC_DECLARE_FIELD(pooled_size).describe("fix pooled size"); + DMLC_DECLARE_FIELD(part_size).set_default(0).describe("fix part size"); + DMLC_DECLARE_FIELD(sample_per_part).set_default(1).describe("fix samples per part"); + DMLC_DECLARE_FIELD(trans_std).set_default(0.0).set_range(0.0, 1.0) + .describe("fix transition std"); + DMLC_DECLARE_FIELD(no_trans).set_default(false) + .describe("Whether to disable trans parameter."); + } +}; + +template +class DeformablePSROIPoolingOp : public Operator { + public: + explicit DeformablePSROIPoolingOp(DeformablePSROIPoolingParam p) { + this->param_ = p; + } + + virtual void Forward(const OpContext &ctx, + const std::vector &in_data, + const std::vector &req, + const std::vector &out_data, + const std::vector &aux_args) { + using namespace mshadow; + size_t in_expected = param_.no_trans? 2 : 3; + size_t out_expected = 2; + CHECK_EQ(in_data.size(), in_expected); + CHECK_EQ(out_data.size(), out_expected); + CHECK_EQ(out_data[deformablepsroipool::kOut].shape_[0], + in_data[deformablepsroipool::kBox].shape_[0]); + CHECK_EQ(out_data[deformablepsroipool::kTopCount].shape_[0], + in_data[deformablepsroipool::kBox].shape_[0]); + Stream *s = ctx.get_stream(); + + Tensor data = in_data[deformablepsroipool::kData].get(s); + Tensor bbox = in_data[deformablepsroipool::kBox].get(s); + Tensor out = out_data[deformablepsroipool::kOut].get(s); + Tensor top_count = out_data[deformablepsroipool::kTopCount] + .get(s); + CHECK_EQ(data.CheckContiguous(), true); + CHECK_EQ(bbox.CheckContiguous(), true); + CHECK_EQ(out.CheckContiguous(), true); + CHECK_EQ(top_count.CheckContiguous(), true); + out = -FLT_MAX; + top_count = 0.0f; + + Tensor trans; + if (!param_.no_trans) { + trans = in_data[deformablepsroipool::kTrans].get(s); + } + DeformablePSROIPoolForward(out, data, bbox, trans, top_count, param_.no_trans, + param_.spatial_scale, param_.output_dim, param_.group_size, param_.pooled_size, + param_.part_size, param_.sample_per_part, param_.trans_std); + } + + virtual void Backward(const OpContext &ctx, + const std::vector &out_grad, + const std::vector &in_data, + const std::vector &out_data, + const std::vector &req, + const std::vector &in_grad, + const std::vector &aux_args) { + using namespace mshadow; + size_t in_expected = param_.no_trans ? 2 : 3; + size_t out_expected = 2; + CHECK_EQ(in_data.size(), in_expected); + CHECK_EQ(out_data.size(), out_expected); + CHECK_EQ(out_grad[deformablepsroipool::kOut].shape_[0], + in_data[deformablepsroipool::kBox].shape_[0]); + CHECK_EQ(out_data[deformablepsroipool::kTopCount].shape_[0], + in_data[deformablepsroipool::kBox].shape_[0]); + CHECK_NE(req[deformablepsroipool::kData], kWriteInplace) << + "DeformablePSROIPooling: Backward doesn't support kWriteInplace."; + CHECK_NE(req[deformablepsroipool::kBox], kWriteInplace) << + "DeformablePSROIPooling: Backward doesn't support kWriteInplace."; + // CHECK_NE(req[deformablepsroipool::kTrans], kWriteInplace) << + // "DeformablePSROIPooling: Backward doesn't support kWriteInplace."; + Stream *s = ctx.get_stream(); + + Tensor grad_out = out_grad[deformablepsroipool::kOut].get(s); + Tensor data = in_data[deformablepsroipool::kData].get(s); + Tensor bbox = in_data[deformablepsroipool::kBox].get(s); + Tensor top_count = out_data[deformablepsroipool::kTopCount] + .get(s); + Tensor grad_in = in_grad[deformablepsroipool::kData].get(s); + Tensor grad_roi = in_grad[deformablepsroipool::kBox].get(s); + Tensor grad_trans; + Tensor trans; + if (!param_.no_trans) { + CHECK_EQ(in_grad.size(), 3); + trans = in_data[deformablepsroipool::kTrans].get(s); + grad_trans = in_grad[deformablepsroipool::kTrans].get(s); + } + + CHECK_EQ(grad_out.CheckContiguous(), true); + CHECK_EQ(data.CheckContiguous(), true); + CHECK_EQ(bbox.CheckContiguous(), true); + CHECK_EQ(top_count.CheckContiguous(), true); + CHECK_EQ(grad_in.CheckContiguous(), true); + + Assign(grad_in, req[deformablepsroipool::kData], 0); + if (!param_.no_trans) { + Assign(grad_trans, req[deformablepsroipool::kTrans], 0); + } + DeformablePSROIPoolBackwardAcc(grad_in, grad_trans, grad_out, data, bbox, trans, + top_count, param_.no_trans, param_.spatial_scale, param_.output_dim, param_.group_size, + param_.pooled_size, param_.part_size, param_.sample_per_part, param_.trans_std); + Assign(grad_roi, req[deformablepsroipool::kBox], 0); + } + + private: + DeformablePSROIPoolingParam param_; +}; // class DeformablePSROIPoolingOp + +// Decalre Factory function, used for dispatch specialization +template +Operator* CreateOp(DeformablePSROIPoolingParam param, int dtype); + +#if DMLC_USE_CXX11 +class DeformablePSROIPoolingProp : public OperatorProperty { + public: + std::vector ListArguments() const override { + if (param_.no_trans) { + return{ "data", "rois" }; + } else { + return{ "data", "rois", "trans" }; + } + } + + std::vector ListOutputs() const override { + return{ "output", "top_count" }; + } + + int NumOutputs() const override { + return 2; + } + + int NumVisibleOutputs() const override { + return 1; + } + + void Init(const std::vector >& kwargs) override { + param_.Init(kwargs); + if (param_.part_size == 0) { + param_.part_size = param_.pooled_size; + } + } + + std::map GetParams() const override { + return param_.__DICT__(); + } + + bool InferShape(std::vector *in_shape, + std::vector *out_shape, + std::vector *aux_shape) const override { + using namespace mshadow; + if (param_.no_trans) { + CHECK_EQ(in_shape->size(), 2) << "Input:[data, rois]"; + } else { + CHECK_EQ(in_shape->size(), 3) << "Input:[data, rois, trans]"; + // trans: [num_rois, 2, pooled_h, pooled_w] + TShape tshape = in_shape->at(deformablepsroipool::kTrans); + CHECK_EQ(tshape.ndim(), 4) << "trans should be a 4D tensor of shape"; + } + + // data: [batch_size, c, h, w] + TShape dshape = in_shape->at(deformablepsroipool::kData); + CHECK_EQ(dshape.ndim(), 4) << "data should be a 4D tensor"; + + // bbox: [num_rois, 5] + TShape bshape = in_shape->at(deformablepsroipool::kBox); + CHECK_EQ(bshape.ndim(), 2) << "bbox should be a 2D tensor of shape [batch, 5]"; + CHECK_EQ(bshape[1], 5) << "bbox should be a 2D tensor of shape [batch, 5]"; + + // out: [num_rois, c, pooled_h, pooled_w] + // top_count: [num_rois, c, pooled_h, pooled_w] + out_shape->clear(); + out_shape->push_back( + Shape4(bshape[0], param_.output_dim, param_.pooled_size, param_.pooled_size)); + out_shape->push_back( + Shape4(bshape[0], param_.output_dim, param_.pooled_size, param_.pooled_size)); + return true; + } + + bool InferType(std::vector *in_type, + std::vector *out_type, + std::vector *aux_type) const override { + CHECK_GE(in_type->size(), 2); + int dtype = (*in_type)[0]; + CHECK_EQ(dtype, (*in_type)[1]); + CHECK_NE(dtype, -1) << "Input must have specified type"; + + out_type->clear(); + out_type->push_back(dtype); + out_type->push_back(dtype); + return true; + } + + OperatorProperty* Copy() const override { + DeformablePSROIPoolingProp* deformable_psroi_pooling_sym = new DeformablePSROIPoolingProp(); + deformable_psroi_pooling_sym->param_ = this->param_; + return deformable_psroi_pooling_sym; + } + + std::string TypeString() const override { + return "_contrib_DeformablePSROIPooling"; + } + + // decalre dependency and inplace optimization options + std::vector DeclareBackwardDependency( + const std::vector &out_grad, + const std::vector &in_data, + const std::vector &out_data) const override { + if (param_.no_trans) { + return{ out_grad[deformablepsroipool::kOut], in_data[deformablepsroipool::kData], + in_data[deformablepsroipool::kBox], out_data[deformablepsroipool::kTopCount] }; + } else { + return{ out_grad[deformablepsroipool::kOut], in_data[deformablepsroipool::kData], + in_data[deformablepsroipool::kBox], in_data[deformablepsroipool::kTrans], + out_data[deformablepsroipool::kTopCount] }; + } + } + + + Operator* CreateOperator(Context ctx) const override { + LOG(FATAL) << "Not Implemented."; + return NULL; + } + + Operator* CreateOperatorEx(Context ctx, std::vector *in_shape, + std::vector *in_type) const override; + + + private: + DeformablePSROIPoolingParam param_; +}; // class DeformablePSROIPoolingProp +#endif +} // namespace op +} // namespace mxnet +#endif // MXNET_OPERATOR_CONTRIB_DEFORMABLE_PSROI_POOLING_INL_H_ diff --git a/src/operator/contrib/deformable_psroi_pooling.cc b/src/operator/contrib/deformable_psroi_pooling.cc new file mode 100644 index 000000000000..290bad2a76cd --- /dev/null +++ b/src/operator/contrib/deformable_psroi_pooling.cc @@ -0,0 +1,96 @@ +/*! + * Copyright (c) 2017 Microsoft + * Licensed under The Apache-2.0 License [see LICENSE for details] + * \file deformable_psroi_pooling.cc + * \brief + * \author Yi Li, Guodong Zhang, Jifeng Dai +*/ +#include "./deformable_psroi_pooling-inl.h" +#include +#include +#include +#include +#include + +using std::max; +using std::min; +using std::floor; +using std::ceil; + +namespace mshadow { + template + inline void DeformablePSROIPoolForward(const Tensor &out, + const Tensor &data, + const Tensor &bbox, + const Tensor &trans, + const Tensor &top_count, + const bool no_trans, + const float spatial_scale, + const int output_dim, + const int group_size, + const int pooled_size, + const int part_size, + const int sample_per_part, + const float trans_std) { + // NOT_IMPLEMENTED; + return; + } + + template + inline void DeformablePSROIPoolBackwardAcc(const Tensor &in_grad, + const Tensor &trans_grad, + const Tensor &out_grad, + const Tensor &data, + const Tensor &bbox, + const Tensor &trans, + const Tensor &top_count, + const bool no_trans, + const float spatial_scale, + const int output_dim, + const int group_size, + const int pooled_size, + const int part_size, + const int sample_per_part, + const float trans_std) { + // NOT_IMPLEMENTED; + return; + } +} // namespace mshadow + +namespace mxnet { +namespace op { + + template<> + Operator *CreateOp(DeformablePSROIPoolingParam param, int dtype) { + Operator* op = NULL; + MSHADOW_REAL_TYPE_SWITCH(dtype, DType, { + op = new DeformablePSROIPoolingOp(param); + }); + return op; + } + + Operator *DeformablePSROIPoolingProp::CreateOperatorEx( + Context ctx, std::vector *in_shape, + std::vector *in_type) const { + std::vector out_shape, aux_shape; + std::vector out_type, aux_type; + CHECK(InferType(in_type, &out_type, &aux_type)); + CHECK(InferShape(in_shape, &out_shape, &aux_shape)); + DO_BIND_DISPATCH(CreateOp, param_, in_type->at(0)); + } + + DMLC_REGISTER_PARAMETER(DeformablePSROIPoolingParam); + + MXNET_REGISTER_OP_PROPERTY(_contrib_DeformablePSROIPooling, DeformablePSROIPoolingProp) + .describe("Performs deformable position-sensitive region-of-interest pooling on inputs." + "The DeformablePSROIPooling operation is described in https://arxiv.org/abs/1703.06211." + "batch_size will change to the number of region bounding boxes after DeformablePSROIPooling") + .add_argument("data", "Symbol", "Input data to the pooling operator, a 4D Feature maps") + .add_argument("rois", "Symbol", "Bounding box coordinates, a 2D array of " + "[[batch_index, x1, y1, x2, y2]]. (x1, y1) and (x2, y2) are top left and down right corners " + "of designated region of interest. batch_index indicates the index of corresponding image " + "in the input data") + .add_argument("trans", "Symbol", "transition parameter") + .add_arguments(DeformablePSROIPoolingParam::__FIELDS__()); +} // namespace op +} // namespace mxnet diff --git a/src/operator/contrib/deformable_psroi_pooling.cu b/src/operator/contrib/deformable_psroi_pooling.cu new file mode 100644 index 000000000000..f9eb01a26e38 --- /dev/null +++ b/src/operator/contrib/deformable_psroi_pooling.cu @@ -0,0 +1,415 @@ +/*! + * Copyright (c) 2017 Microsoft + * Licensed under The Apache-2.0 License [see LICENSE for details] + * \file deformable_psroi_pooling.cu + * \brief + * \author Yi Li, Guodong Zhang, Jifeng Dai +*/ +#include "./deformable_psroi_pooling-inl.h" +#include +#include +#include +#include +#include "../../common/cuda_utils.h" +#include "../mxnet_op.h" + +#define DeformablePSROIPOOLING_CUDA_CHECK(condition) \ + /* Code block avoids redefinition of cudaError_t error */ \ + do { \ + cudaError_t error = condition; \ + CHECK_EQ(error, cudaSuccess) << " " << cudaGetErrorString(error); \ + } while (0) +#define CUDA_KERNEL_LOOP(i, n) \ +for (int i = blockIdx.x * blockDim.x + threadIdx.x; \ + i < (n); \ + i += blockDim.x * gridDim.x) + +namespace mshadow { +namespace cuda { + template + __device__ DType bilinear_interp( + const DType* data, + const DType x, + const DType y, + const int width, + const int height) { + int x1 = floor(x); + int x2 = ceil(x); + int y1 = floor(y); + int y2 = ceil(y); + DType dist_x = static_cast(x - x1); + DType dist_y = static_cast(y - y1); + DType value11 = data[y1*width + x1]; + DType value12 = data[y2*width + x1]; + DType value21 = data[y1*width + x2]; + DType value22 = data[y2*width + x2]; + DType value = (1 - dist_x)*(1 - dist_y)*value11 + (1 - dist_x)*dist_y*value12 + + dist_x*(1 - dist_y)*value21 + dist_x*dist_y*value22; + return value; + } + + template + __global__ void DeformablePSROIPoolForwardKernel( + const int count, + const DType* bottom_data, + const DType spatial_scale, + const int channels, + const int height, const int width, + const int pooled_height, const int pooled_width, + const DType* bottom_rois, const DType* bottom_trans, + const bool no_trans, + const DType trans_std, + const int sample_per_part, + const int output_dim, + const int group_size, + const int part_size, + const int num_classes, + const int channels_each_class, + DType* top_data, + DType* top_count) { + CUDA_KERNEL_LOOP(index, count) { + // The output is in order (n, ctop, ph, pw) + int pw = index % pooled_width; + int ph = (index / pooled_width) % pooled_height; + int ctop = (index / pooled_width / pooled_height) % output_dim; + int n = index / pooled_width / pooled_height / output_dim; + + // [start, end) interval for spatial sampling + const DType* offset_bottom_rois = bottom_rois + n * 5; + int roi_batch_ind = offset_bottom_rois[0]; + DType roi_start_w = static_cast(round(offset_bottom_rois[1])) * spatial_scale - 0.5; + DType roi_start_h = static_cast(round(offset_bottom_rois[2])) * spatial_scale - 0.5; + DType roi_end_w = static_cast(round(offset_bottom_rois[3]) + 1.) * spatial_scale - 0.5; + DType roi_end_h = static_cast(round(offset_bottom_rois[4]) + 1.) * spatial_scale - 0.5; + + // Force too small ROIs to be 1x1 + DType roi_width = max(roi_end_w - roi_start_w, 0.1); // avoid 0 + DType roi_height = max(roi_end_h - roi_start_h, 0.1); + + // Compute w and h at bottom + DType bin_size_h = roi_height / static_cast(pooled_height); + DType bin_size_w = roi_width / static_cast(pooled_width); + + DType sub_bin_size_h = bin_size_h / static_cast(sample_per_part); + DType sub_bin_size_w = bin_size_w / static_cast(sample_per_part); + + int part_h = floor(static_cast(ph) / pooled_height*part_size); + int part_w = floor(static_cast(pw) / pooled_width*part_size); + int class_id = ctop / channels_each_class; + DType trans_x = no_trans ? static_cast(0) : + bottom_trans[(((n * num_classes + class_id) * 2) + * part_size + part_h) + * part_size + part_w] * trans_std; + DType trans_y = no_trans ? static_cast(0) : + bottom_trans[(((n * num_classes + class_id) * 2 + 1) + * part_size + part_h) + * part_size + part_w] * trans_std; + + DType wstart = static_cast(pw)* bin_size_w + + roi_start_w; + wstart += trans_x * roi_width; + DType hstart = static_cast(ph) * bin_size_h + + roi_start_h; + hstart += trans_y * roi_height; + + DType sum = 0; + int count = 0; + int gw = floor(static_cast(pw) * group_size / pooled_width); + int gh = floor(static_cast(ph)* group_size / pooled_height); + gw = min(max(gw, 0), group_size - 1); + gh = min(max(gh, 0), group_size - 1); + + const DType* offset_bottom_data = bottom_data + (roi_batch_ind * channels) * height * width; + for (int ih = 0; ih < sample_per_part; ih++) { + for (int iw = 0; iw < sample_per_part; iw++) { + DType w = wstart + iw*sub_bin_size_w; + DType h = hstart + ih*sub_bin_size_h; + // bilinear interpolation + if (w<-0.5 || w>width - 0.5 || h<-0.5 || h>height - 0.5) { + continue; + } + w = min(max(w, 0.), width - 1.); + h = min(max(h, 0.), height - 1.); + int c = (ctop*group_size + gh)*group_size + gw; + DType val = bilinear_interp(offset_bottom_data + c*height*width, w, h, width, height); + sum += val; + count++; + } + } + top_data[index] = count == 0 ? static_cast(0) : sum / count; + top_count[index] = count; + } + } + + template + inline void DeformablePSROIPoolForward(const Tensor &out, + const Tensor &data, + const Tensor &bbox, + const Tensor &trans, + const Tensor &top_count, + const bool no_trans, + const float spatial_scale, + const int output_dim, + const int group_size, + const int pooled_size, + const int part_size, + const int sample_per_part, + const float trans_std) { + // LOG(INFO) << "DeformablePSROIPoolForward"; + const DType *bottom_data = data.dptr_; + const DType *bottom_rois = bbox.dptr_; + const DType *bottom_trans = no_trans ? NULL : trans.dptr_; + DType *top_data = out.dptr_; + DType *top_count_data = top_count.dptr_; + const int count = out.shape_.Size(); + const int channels = data.size(1); + const int height = data.size(2); + const int width = data.size(3); + const int pooled_height = pooled_size; + const int pooled_width = pooled_size; + const int num_classes = no_trans ? 1 : trans.size(1) / 2; + const int channels_each_class = no_trans ? output_dim : output_dim / num_classes; + + cudaStream_t stream = Stream::GetStream(out.stream_); + DeformablePSROIPoolForwardKernel << > >( + count, bottom_data, spatial_scale, channels, height, width, pooled_height, pooled_width, + bottom_rois, bottom_trans, no_trans, trans_std, sample_per_part, output_dim, + group_size, part_size, num_classes, channels_each_class, top_data, top_count_data); + DeformablePSROIPOOLING_CUDA_CHECK(cudaPeekAtLastError()); + } + + + template + __global__ void DeformablePSROIPoolBackwardAccKernel( + const int count, + const DType* top_diff, + const DType* top_count, + const int num_rois, + const DType spatial_scale, + const int channels, + const int height, const int width, + const int pooled_height, const int pooled_width, + const int output_dim, + DType* bottom_data_diff, DType* bottom_trans_diff, + const DType* bottom_data, + const DType* bottom_rois, + const DType* bottom_trans, + const bool no_trans, + const DType trans_std, + const int sample_per_part, + const int group_size, + const int part_size, + const int num_classes, + const int channels_each_class) { + CUDA_KERNEL_LOOP(index, count) { + // The output is in order (n, ctop, ph, pw) + int pw = index % pooled_width; + int ph = (index / pooled_width) % pooled_height; + int ctop = (index / pooled_width / pooled_height) % output_dim; + int n = index / pooled_width / pooled_height / output_dim; + + // [start, end) interval for spatial sampling + const DType* offset_bottom_rois = bottom_rois + n * 5; + int roi_batch_ind = offset_bottom_rois[0]; + DType roi_start_w = static_cast(round(offset_bottom_rois[1])) * spatial_scale - 0.5; + DType roi_start_h = static_cast(round(offset_bottom_rois[2])) * spatial_scale - 0.5; + DType roi_end_w = static_cast(round(offset_bottom_rois[3]) + 1.) * spatial_scale - 0.5; + DType roi_end_h = static_cast(round(offset_bottom_rois[4]) + 1.) * spatial_scale - 0.5; + + // Force too small ROIs to be 1x1 + DType roi_width = max(roi_end_w - roi_start_w, 0.1); // avoid 0 + DType roi_height = max(roi_end_h - roi_start_h, 0.1); + + // Compute w and h at bottom + DType bin_size_h = roi_height / static_cast(pooled_height); + DType bin_size_w = roi_width / static_cast(pooled_width); + + DType sub_bin_size_h = bin_size_h / static_cast(sample_per_part); + DType sub_bin_size_w = bin_size_w / static_cast(sample_per_part); + + int part_h = floor(static_cast(ph) / pooled_height*part_size); + int part_w = floor(static_cast(pw) / pooled_width*part_size); + int class_id = ctop / channels_each_class; + DType trans_x = no_trans ? static_cast(0) : + bottom_trans[(((n * num_classes + class_id) * 2) + * part_size + part_h) + * part_size + part_w] * trans_std; + DType trans_y = no_trans ? static_cast(0) : + bottom_trans[(((n * num_classes + class_id) * 2 + 1) + * part_size + part_h) + * part_size + part_w] * trans_std; + + DType wstart = static_cast(pw)* bin_size_w + + roi_start_w; + wstart += trans_x * roi_width; + DType hstart = static_cast(ph) * bin_size_h + + roi_start_h; + hstart += trans_y * roi_height; + + if (top_count[index] <= 0) { + continue; + } + DType diff_val = top_diff[index] / top_count[index]; + const DType* offset_bottom_data = bottom_data + roi_batch_ind * channels * height * width; + DType* offset_bottom_data_diff = bottom_data_diff + roi_batch_ind * channels * height * width; + int gw = floor(static_cast(pw)* group_size / pooled_width); + int gh = floor(static_cast(ph)* group_size / pooled_height); + gw = min(max(gw, 0), group_size - 1); + gh = min(max(gh, 0), group_size - 1); + + for (int ih = 0; ih < sample_per_part; ih++) { + for (int iw = 0; iw < sample_per_part; iw++) { + DType w = wstart + iw*sub_bin_size_w; + DType h = hstart + ih*sub_bin_size_h; + // bilinear interpolation + if (w<-0.5 || w>width - 0.5 || h<-0.5 || h>height - 0.5) { + continue; + } + w = min(max(w, 0.), width - 1.); + h = min(max(h, 0.), height - 1.); + int c = (ctop*group_size + gh)*group_size + gw; + // backward on feature + int x0 = floor(w); + int x1 = ceil(w); + int y0 = floor(h); + int y1 = ceil(h); + DType dist_x = w - x0, dist_y = h - y0; + DType q00 = (1 - dist_x)*(1 - dist_y); + DType q01 = (1 - dist_x)*dist_y; + DType q10 = dist_x*(1 - dist_y); + DType q11 = dist_x*dist_y; + int bottom_index_base = c * height *width; + atomicAdd(offset_bottom_data_diff + bottom_index_base + y0*width + x0, q00*diff_val); + atomicAdd(offset_bottom_data_diff + bottom_index_base + y1*width + x0, q01*diff_val); + atomicAdd(offset_bottom_data_diff + bottom_index_base + y0*width + x1, q10*diff_val); + atomicAdd(offset_bottom_data_diff + bottom_index_base + y1*width + x1, q11*diff_val); + + if (no_trans) { + continue; + } + DType U00 = offset_bottom_data[bottom_index_base + y0*width + x0]; + DType U01 = offset_bottom_data[bottom_index_base + y1*width + x0]; + DType U10 = offset_bottom_data[bottom_index_base + y0*width + x1]; + DType U11 = offset_bottom_data[bottom_index_base + y1*width + x1]; + DType diff_x = (U11*dist_y + U10*(1 - dist_y) - U01*dist_y - U00*(1 - dist_y)) + *trans_std*diff_val; + diff_x *= roi_width; + DType diff_y = (U11*dist_x + U01*(1 - dist_x) - U10*dist_x - U00*(1 - dist_x)) + *trans_std*diff_val; + diff_y *= roi_height; + + atomicAdd(bottom_trans_diff + (((n * num_classes + class_id) * 2) + * part_size + part_h) + * part_size + part_w, diff_x); + atomicAdd(bottom_trans_diff + (((n * num_classes + class_id) * 2 + 1) + * part_size + part_h) + * part_size + part_w, diff_y); + } + } + } + } + + + template + inline void DeformablePSROIPoolBackwardAcc(const Tensor &in_grad, + const Tensor &trans_grad, + const Tensor &out_grad, + const Tensor &data, + const Tensor &bbox, + const Tensor &trans, + const Tensor &top_count, + const bool no_trans, + const float spatial_scale, + const int output_dim, + const int group_size, + const int pooled_size, + const int part_size, + const int sample_per_part, + const float trans_std) { + // LOG(INFO) << "DeformablePSROIPoolBackward"; + const DType *top_diff = out_grad.dptr_; + const DType *bottom_data = data.dptr_; + const DType *bottom_rois = bbox.dptr_; + const DType *bottom_trans = no_trans ? NULL : trans.dptr_; + DType *bottom_data_diff = in_grad.dptr_; + DType *bottom_trans_diff = no_trans ? NULL : trans_grad.dptr_; + const DType *top_count_data = top_count.dptr_; + const int count = out_grad.shape_.Size(); + const int num_rois = bbox.size(0); + const int channels = in_grad.size(1); + const int height = in_grad.size(2); + const int width = in_grad.size(3); + const int pooled_height = pooled_size; + const int pooled_width = pooled_size; + const int num_classes = no_trans ? 1 : trans_grad.size(1) / 2; + const int channels_each_class = no_trans ? output_dim : output_dim / num_classes; + + cudaStream_t stream = Stream::GetStream(in_grad.stream_); + DeformablePSROIPoolBackwardAccKernel << > >( + count, top_diff, top_count_data, num_rois, spatial_scale, channels, height, width, + pooled_height, pooled_width, output_dim, bottom_data_diff, bottom_trans_diff, + bottom_data, bottom_rois, bottom_trans, no_trans, trans_std, sample_per_part, + group_size, part_size, num_classes, channels_each_class); + DeformablePSROIPOOLING_CUDA_CHECK(cudaPeekAtLastError()); + } + +} // namespace cuda + + template + inline void DeformablePSROIPoolForward(const Tensor &out, + const Tensor &data, + const Tensor &bbox, + const Tensor &trans, + const Tensor &top_count, + const bool no_trans, + const float spatial_scale, + const int output_dim, + const int group_size, + const int pooled_size, + const int part_size, + const int sample_per_part, + const float trans_std) { + cuda::DeformablePSROIPoolForward(out, data, bbox, trans, top_count, no_trans, spatial_scale, + output_dim, group_size, pooled_size, part_size, sample_per_part, trans_std); + } + + template + inline void DeformablePSROIPoolBackwardAcc(const Tensor &in_grad, + const Tensor &trans_grad, + const Tensor &out_grad, + const Tensor &data, + const Tensor &bbox, + const Tensor &trans, + const Tensor &top_count, + const bool no_trans, + const float spatial_scale, + const int output_dim, + const int group_size, + const int pooled_size, + const int part_size, + const int sample_per_part, + const float trans_std) { + cuda::DeformablePSROIPoolBackwardAcc(in_grad, trans_grad, out_grad, data, bbox, trans, + top_count, no_trans, spatial_scale, output_dim, group_size, pooled_size, part_size, + sample_per_part, trans_std); + } + +} // namespace mshadow + + +namespace mxnet { +namespace op { + + template<> + Operator* CreateOp(DeformablePSROIPoolingParam param, int dtype) { + Operator* op = NULL; + MSHADOW_REAL_TYPE_SWITCH(dtype, DType, { + op = new DeformablePSROIPoolingOp(param); + }); + return op; + } + +} // namespace op +} // namespace mxnet diff --git a/src/operator/contrib/multi_proposal-inl.h b/src/operator/contrib/multi_proposal-inl.h new file mode 100644 index 000000000000..48f3535b5eab --- /dev/null +++ b/src/operator/contrib/multi_proposal-inl.h @@ -0,0 +1,301 @@ +/*! + * Copyright (c) 2015 by Contributors + * Copyright (c) 2017 Microsoft + * Licensed under The Apache-2.0 License [see LICENSE for details] + * \file multi_proposal-inl.h + * \brief MultiProposal Operator + * \author Piotr Teterwak, Bing Xu, Jian Guo, Xizhou Zhu +*/ +#ifndef MXNET_OPERATOR_CONTRIB_MULTI_PROPOSAL_INL_H_ +#define MXNET_OPERATOR_CONTRIB_MULTI_PROPOSAL_INL_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../operator_common.h" +#include "../mshadow_op.h" + +// extend NumericalParam +namespace mxnet { +namespace op { + +/*! +* \brief structure for numerical tuple input +* \tparam VType data type of param +*/ +template +struct NumericalParam { + NumericalParam() {} + explicit NumericalParam(VType *begin, VType *end) { + int32_t size = static_cast(end - begin); + info.resize(size); + for (int i = 0; i < size; ++i) { + info[i] = *(begin + i); + } + } + inline size_t ndim() const { + return info.size(); + } + std::vector info; +}; + +template +inline std::istream &operator>>(std::istream &is, NumericalParam ¶m) { + while (true) { + char ch = is.get(); + if (ch == '(') break; + if (!isspace(ch)) { + is.setstate(std::ios::failbit); + return is; + } + } + VType idx; + std::vector tmp; + // deal with empty case + size_t pos = is.tellg(); + char ch = is.get(); + if (ch == ')') { + param.info = tmp; + return is; + } + is.seekg(pos); + // finish deal + while (is >> idx) { + tmp.push_back(idx); + char ch; + do { + ch = is.get(); + } while (isspace(ch)); + if (ch == ',') { + while (true) { + ch = is.peek(); + if (isspace(ch)) { + is.get(); continue; + } + if (ch == ')') { + is.get(); break; + } + break; + } + if (ch == ')') break; + } else if (ch == ')') { + break; + } else { + is.setstate(std::ios::failbit); + return is; + } + } + param.info = tmp; + return is; +} + +template +inline std::ostream &operator<<(std::ostream &os, const NumericalParam ¶m) { + os << '('; + for (index_t i = 0; i < param.info.size(); ++i) { + if (i != 0) os << ','; + os << param.info[i]; + } + // python style tuple + if (param.info.size() == 1) os << ','; + os << ')'; + return os; +} + +} // namespace op +} // namespace mxnet + +namespace mxnet { +namespace op { + +namespace proposal { +enum MultiProposalOpInputs {kClsProb, kBBoxPred, kImInfo}; +enum MultiProposalOpOutputs {kOut, kScore}; +enum MultiProposalForwardResource {kTempResource}; +} // proposal + +struct MultiProposalParam : public dmlc::Parameter { + int rpn_pre_nms_top_n; + int rpn_post_nms_top_n; + float threshold; + int rpn_min_size; + NumericalParam scales; + NumericalParam ratios; + int feature_stride; + bool output_score; + bool iou_loss; + DMLC_DECLARE_PARAMETER(MultiProposalParam) { + float tmp[] = {0, 0, 0, 0}; + DMLC_DECLARE_FIELD(rpn_pre_nms_top_n).set_default(6000) + .describe("Number of top scoring boxes to keep after applying NMS to RPN proposals"); + DMLC_DECLARE_FIELD(rpn_post_nms_top_n).set_default(300) + .describe("Overlap threshold used for non-maximum" + "suppresion(suppress boxes with IoU >= this threshold"); + DMLC_DECLARE_FIELD(threshold).set_default(0.7) + .describe("NMS value, below which to suppress."); + DMLC_DECLARE_FIELD(rpn_min_size).set_default(16) + .describe("Minimum height or width in proposal"); + tmp[0] = 4.0f; tmp[1] = 8.0f; tmp[2] = 16.0f; tmp[3] = 32.0f; + DMLC_DECLARE_FIELD(scales).set_default(NumericalParam(tmp, tmp + 4)) + .describe("Used to generate anchor windows by enumerating scales"); + tmp[0] = 0.5f; tmp[1] = 1.0f; tmp[2] = 2.0f; + DMLC_DECLARE_FIELD(ratios).set_default(NumericalParam(tmp, tmp + 3)) + .describe("Used to generate anchor windows by enumerating ratios"); + DMLC_DECLARE_FIELD(feature_stride).set_default(16) + .describe("The size of the receptive field each unit in the convolution layer of the rpn," + "for example the product of all stride's prior to this layer."); + DMLC_DECLARE_FIELD(output_score).set_default(false) + .describe("Add score to outputs"); + DMLC_DECLARE_FIELD(iou_loss).set_default(false) + .describe("Usage of IoU Loss"); + } +}; + +template +Operator *CreateOp(MultiProposalParam param); + +#if DMLC_USE_CXX11 +class MultiProposalProp : public OperatorProperty { + public: + void Init(const std::vector >& kwargs) override { + param_.Init(kwargs); + } + + std::map GetParams() const override { + return param_.__DICT__(); + } + + bool InferShape(std::vector *in_shape, + std::vector *out_shape, + std::vector *aux_shape) const override { + using namespace mshadow; + CHECK_EQ(in_shape->size(), 3) << "Input:[cls_prob, bbox_pred, im_info]"; + const TShape &dshape = in_shape->at(proposal::kClsProb); + if (dshape.ndim() == 0) return false; + Shape<4> bbox_pred_shape; + bbox_pred_shape = Shape4(dshape[0], dshape[1] * 2, dshape[2], dshape[3]); + SHAPE_ASSIGN_CHECK(*in_shape, proposal::kBBoxPred, + bbox_pred_shape); + Shape<2> im_info_shape; + im_info_shape = Shape2(dshape[0], 3); + SHAPE_ASSIGN_CHECK(*in_shape, proposal::kImInfo, im_info_shape); + out_shape->clear(); + // output + out_shape->push_back(Shape2(dshape[0] * param_.rpn_post_nms_top_n, 5)); + // score + out_shape->push_back(Shape2(dshape[0] * param_.rpn_post_nms_top_n, 1)); + return true; + } + + OperatorProperty* Copy() const override { + auto ptr = new MultiProposalProp(); + ptr->param_ = param_; + return ptr; + } + + std::string TypeString() const override { + return "_contrib_MultiProposal"; + } + + std::vector ForwardResource( + const std::vector &in_shape) const override { + return {ResourceRequest::kTempSpace}; + } + + std::vector DeclareBackwardDependency( + const std::vector &out_grad, + const std::vector &in_data, + const std::vector &out_data) const override { + return {}; + } + + int NumVisibleOutputs() const override { + if (param_.output_score) { + return 2; + } else { + return 1; + } + } + + int NumOutputs() const override { + return 2; + } + + std::vector ListArguments() const override { + return {"cls_prob", "bbox_pred", "im_info"}; + } + + std::vector ListOutputs() const override { + return {"output", "score"}; + } + + Operator* CreateOperator(Context ctx) const override; + + private: + MultiProposalParam param_; +}; // class MultiProposalProp + +#endif // DMLC_USE_CXX11 +} // namespace op +} // namespace mxnet + +//======================== +// Anchor Generation Utils +//======================== +namespace mxnet { +namespace op { +namespace utils { + +inline void _MakeAnchor(float w, + float h, + float x_ctr, + float y_ctr, + std::vector *out_anchors) { + out_anchors->push_back(x_ctr - 0.5f * (w - 1.0f)); + out_anchors->push_back(y_ctr - 0.5f * (h - 1.0f)); + out_anchors->push_back(x_ctr + 0.5f * (w - 1.0f)); + out_anchors->push_back(y_ctr + 0.5f * (h - 1.0f)); + out_anchors->push_back(0.0f); +} + +inline void _Transform(float scale, + float ratio, + const std::vector& base_anchor, + std::vector *out_anchors) { + float w = base_anchor[2] - base_anchor[1] + 1.0f; + float h = base_anchor[3] - base_anchor[1] + 1.0f; + float x_ctr = base_anchor[0] + 0.5 * (w - 1.0f); + float y_ctr = base_anchor[1] + 0.5 * (h - 1.0f); + float size = w * h; + float size_ratios = std::floor(size / ratio); + float new_w = std::floor(std::sqrt(size_ratios) + 0.5f) * scale; + float new_h = std::floor((new_w / scale * ratio) + 0.5f) * scale; + + _MakeAnchor(new_w, new_h, x_ctr, + y_ctr, out_anchors); +} + +// out_anchors must have shape (n, 5), where n is ratios.size() * scales.size() +inline void GenerateAnchors(const std::vector& base_anchor, + const std::vector& ratios, + const std::vector& scales, + std::vector *out_anchors) { + for (size_t j = 0; j < ratios.size(); ++j) { + for (size_t k = 0; k < scales.size(); ++k) { + _Transform(scales[k], ratios[j], base_anchor, out_anchors); + } + } +} + +} // namespace utils +} // namespace op +} // namespace mxnet + +#endif // MXNET_OPERATOR_CONTRIB_MULTI_PROPOSAL_INL_H_ diff --git a/src/operator/contrib/multi_proposal.cc b/src/operator/contrib/multi_proposal.cc new file mode 100644 index 000000000000..c8f75eaec547 --- /dev/null +++ b/src/operator/contrib/multi_proposal.cc @@ -0,0 +1,63 @@ +/*! + * Copyright (c) 2017 Microsoft + * Licensed under The Apache-2.0 License [see LICENSE for details] + * \file multi_proposal.cc + * \brief + * \author Xizhou Zhu +*/ + +#include "./multi_proposal-inl.h" + + +namespace mxnet { +namespace op { + +template +class MultiProposalOp : public Operator{ + public: + explicit MultiProposalOp(MultiProposalParam param) { + this->param_ = param; + } + + virtual void Forward(const OpContext &ctx, + const std::vector &in_data, + const std::vector &req, + const std::vector &out_data, + const std::vector &aux_states) { + LOG(FATAL) << "not implemented"; + } + + virtual void Backward(const OpContext &ctx, + const std::vector &out_grad, + const std::vector &in_data, + const std::vector &out_data, + const std::vector &req, + const std::vector &in_grad, + const std::vector &aux_states) { + LOG(FATAL) << "not implemented"; + } + + private: + MultiProposalParam param_; +}; // class MultiProposalOp + +template<> +Operator *CreateOp(MultiProposalParam param) { + return new MultiProposalOp(param); +} + +Operator* MultiProposalProp::CreateOperator(Context ctx) const { + DO_BIND_DISPATCH(CreateOp, param_); +} + +DMLC_REGISTER_PARAMETER(MultiProposalParam); + +MXNET_REGISTER_OP_PROPERTY(_contrib_MultiProposal, MultiProposalProp) +.describe("Generate region proposals via RPN") +.add_argument("cls_score", "NDArray-or-Symbol", "Score of how likely proposal is object.") +.add_argument("bbox_pred", "NDArray-or-Symbol", "BBox Predicted deltas from anchors for proposals") +.add_argument("im_info", "NDArray-or-Symbol", "Image size and scale.") +.add_arguments(MultiProposalParam::__FIELDS__()); + +} // namespace op +} // namespace mxnet diff --git a/src/operator/contrib/multi_proposal.cu b/src/operator/contrib/multi_proposal.cu new file mode 100644 index 000000000000..052d777d5fac --- /dev/null +++ b/src/operator/contrib/multi_proposal.cu @@ -0,0 +1,593 @@ +/*! + * Copyright (c) 2015 by Contributors + * Copyright (c) 2017 Microsoft + * Licensed under The Apache-2.0 License [see LICENSE for details] + * \file multi_proposal.cu + * \brief MultiProposal Operator + * \author Shaoqing Ren, Xizhou Zhu, Jian Guo +*/ +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "../operator_common.h" +#include "../mshadow_op.h" +#include "./multi_proposal-inl.h" + +#define DIVUP(m, n) ((m) / (n) + ((m) % (n) > 0)) + +#define FRCNN_CUDA_CHECK(condition) \ + /* Code block avoids redefinition of cudaError_t error */ \ + do { \ + cudaError_t error = condition; \ + CHECK_EQ(error, cudaSuccess) << " " << cudaGetErrorString(error); \ +} while (0) + +namespace mshadow { +namespace cuda { +namespace multi_proposal { + +// scores are (b, 2 * anchor, h, w) +// workspace_proposals are (b, h * w * anchor, 5) +// w defines "x" and h defines "y" +// count should be total anchors numbers, h * w * anchors +template +__global__ void ProposalGridKernel(const int count, + const int num_anchors, + const int height, + const int width, + const int feature_stride, + const Dtype* scores, + Dtype* workspace_proposals) { + for (int index = blockIdx.x * blockDim.x + threadIdx.x; + index < count; + index += blockDim.x * gridDim.x) { + int a = index % num_anchors; + int w = (index / num_anchors) % width; + int h = (index / num_anchors / width) % height; + int b = index / num_anchors / width / height; + + workspace_proposals[index * 5 + 0] = workspace_proposals[a * 5 + 0] + w * feature_stride; + workspace_proposals[index * 5 + 1] = workspace_proposals[a * 5 + 1] + h * feature_stride; + workspace_proposals[index * 5 + 2] = workspace_proposals[a * 5 + 2] + w * feature_stride; + workspace_proposals[index * 5 + 3] = workspace_proposals[a * 5 + 3] + h * feature_stride; + workspace_proposals[index * 5 + 4] = + scores[((b * (2 * num_anchors) + a + num_anchors) * height + h) * width + w]; + } +} + +// boxes are (b, h * w * anchor, 5) +// deltas are (b, 4 * anchor, h, w) +// out_pred_boxes are (b, h * w * anchor, 5) +// count should be total anchors numbers, b * h * w * anchors +// in-place write: boxes and out_pred_boxes are the same location +template +__global__ void BBoxPredKernel(const int count, + const int num_anchors, + const int feat_height, + const int feat_width, + const int feature_stride, + const Dtype* im_infos, + const Dtype* boxes, + const Dtype* deltas, + Dtype* out_pred_boxes) { + for (int index = blockIdx.x * blockDim.x + threadIdx.x; + index < count; + index += blockDim.x * gridDim.x) { + int a = index % num_anchors; + int w = (index / num_anchors) % feat_width; + int h = (index / num_anchors / feat_width) % feat_height; + int b = index / num_anchors / feat_width / feat_height; + + float im_height = im_infos[b * 3]; + float im_width = im_infos[b * 3 + 1]; + int real_height = static_cast(im_height / feature_stride); + int real_width = static_cast(im_width / feature_stride); + + float width = boxes[index * 5 + 2] - boxes[index * 5 + 0] + 1.0f; + float height = boxes[index * 5 + 3] - boxes[index * 5 + 1] + 1.0f; + float ctr_x = boxes[index * 5 + 0] + 0.5f * (width - 1.0f); + float ctr_y = boxes[index * 5 + 1] + 0.5f * (height - 1.0f); + + int ba = (b * num_anchors + a); + float dx = deltas[((ba * 4) * feat_height + h) * feat_width + w]; + float dy = deltas[((ba * 4 + 1) * feat_height + h) * feat_width + w]; + float dw = deltas[((ba * 4 + 2) * feat_height + h) * feat_width + w]; + float dh = deltas[((ba * 4 + 3) * feat_height + h) * feat_width + w]; + + float pred_ctr_x = dx * width + ctr_x; + float pred_ctr_y = dy * height + ctr_y; + float pred_w = exp(dw) * width; + float pred_h = exp(dh) * height; + + float pred_x1 = pred_ctr_x - 0.5f * (pred_w - 1.0f); + float pred_y1 = pred_ctr_y - 0.5f * (pred_h - 1.0f); + float pred_x2 = pred_ctr_x + 0.5f * (pred_w - 1.0f); + float pred_y2 = pred_ctr_y + 0.5f * (pred_h - 1.0f); + + pred_x1 = max(min(pred_x1, im_width - 1.0f), 0.0f); + pred_y1 = max(min(pred_y1, im_height - 1.0f), 0.0f); + pred_x2 = max(min(pred_x2, im_width - 1.0f), 0.0f); + pred_y2 = max(min(pred_y2, im_height - 1.0f), 0.0f); + + out_pred_boxes[index * 5 + 0] = pred_x1; + out_pred_boxes[index * 5 + 1] = pred_y1; + out_pred_boxes[index * 5 + 2] = pred_x2; + out_pred_boxes[index * 5 + 3] = pred_y2; + + if (h >= real_height || w >= real_width) { + out_pred_boxes[index * 5 + 4] = -1.0f; + } + } +} + +// boxes are (b, h * w * anchor, 5) +// deltas are (b, 4 * anchor, h, w) +// out_pred_boxes are (b, h * w * anchor, 5) +// count should be total anchors numbers, b * h * w * anchors +// in-place write: boxes and out_pred_boxes are the same location +template +__global__ void IoUPredKernel(const int count, + const int num_anchors, + const int feat_height, + const int feat_width, + const int feature_stride, + const Dtype* im_infos, + const Dtype* boxes, + const Dtype* deltas, + Dtype* out_pred_boxes) { + for (int index = blockIdx.x * blockDim.x + threadIdx.x; + index < count; + index += blockDim.x * gridDim.x) { + int a = index % num_anchors; + int w = (index / num_anchors) % feat_width; + int h = (index / num_anchors / feat_width) % feat_height; + int b = index / num_anchors / feat_width / feat_height; + + float im_height = im_infos[b * 3]; + float im_width = im_infos[b * 3 + 1]; + int real_height = static_cast(im_height / feature_stride); + int real_width = static_cast(im_width / feature_stride); + + float x1 = boxes[index * 5 + 0]; + float y1 = boxes[index * 5 + 1]; + float x2 = boxes[index * 5 + 2]; + float y2 = boxes[index * 5 + 3]; + + int ba = (b * num_anchors + a); + float dx1 = deltas[((ba * 4) * feat_height + h) * feat_width + w]; + float dy1 = deltas[((ba * 4 + 1) * feat_height + h) * feat_width + w]; + float dx2 = deltas[((ba * 4 + 2) * feat_height + h) * feat_width + w]; + float dy2 = deltas[((ba * 4 + 3) * feat_height + h) * feat_width + w]; + + float pred_x1 = max(min(x1 + dx1, im_width - 1.0f), 0.0f); + float pred_y1 = max(min(y1 + dy1, im_height - 1.0f), 0.0f); + float pred_x2 = max(min(x2 + dx2, im_width - 1.0f), 0.0f); + float pred_y2 = max(min(y2 + dy2, im_height - 1.0f), 0.0f); + + out_pred_boxes[index * 5 + 0] = pred_x1; + out_pred_boxes[index * 5 + 1] = pred_y1; + out_pred_boxes[index * 5 + 2] = pred_x2; + out_pred_boxes[index * 5 + 3] = pred_y2; + + if (h >= real_height || w >= real_width) { + out_pred_boxes[index * 5 + 4] = -1.0f; + } + } +} + +// filter box with stride less than rpn_min_size +// filter: set score to zero +// dets (b, n, 5) +template +__global__ void FilterBoxKernel(const int count, + const int count_anchors, + const float original_min_size, + const Dtype* im_infos, + Dtype* dets) { + for (int index = blockIdx.x * blockDim.x + threadIdx.x; + index < count; + index += blockDim.x * gridDim.x) { + int b = index / count_anchors; + float iw = dets[index * 5 + 2] - dets[index * 5 + 0] + 1.0f; + float ih = dets[index * 5 + 3] - dets[index * 5 + 1] + 1.0f; + float min_size = original_min_size * im_infos[b * 3 + 2]; + if (iw < min_size || ih < min_size) { + dets[index * 5 + 0] -= min_size / 2; + dets[index * 5 + 1] -= min_size / 2; + dets[index * 5 + 2] += min_size / 2; + dets[index * 5 + 3] += min_size / 2; + dets[index * 5 + 4] = -1.0f; + } + } +} + +// copy score and init order +// dets (n, 5); score (n, ); order (n, ) +// count should be n (total anchors or proposals) +template +__global__ void CopyScoreKernel(const int count, + const Dtype* dets, + Dtype* score, + int* order) { + for (int index = blockIdx.x * blockDim.x + threadIdx.x; + index < count; + index += blockDim.x * gridDim.x) { + score[index] = dets[index * 5 + 4]; + order[index] = index; + } +} + +// reorder proposals according to order and keep the top_n proposals +// prev_dets (n, 5); order (n, ); dets (n, 5) +// count should be output anchor numbers (top_n) +template +__global__ void ReorderProposalsKernel(const int count, + const Dtype* prev_dets, + const int* order, + Dtype* dets) { + for (int index = blockIdx.x * blockDim.x + threadIdx.x; + index < count; + index += blockDim.x * gridDim.x) { + const int order_i = order[index]; + for (int j = 0; j < 5; j ++) { + dets[index * 5 + j] = prev_dets[order_i * 5 + j]; + } + } +} + +__device__ inline float devIoU(float const * const a, float const * const b) { + float left = max(a[0], b[0]), right = min(a[2], b[2]); + float top = max(a[1], b[1]), bottom = min(a[3], b[3]); + float width = max(right - left + 1, 0.f), height = max(bottom - top + 1, 0.f); + float interS = width * height; + float Sa = (a[2] - a[0] + 1) * (a[3] - a[1] + 1); + float Sb = (b[2] - b[0] + 1) * (b[3] - b[1] + 1); + return interS / (Sa + Sb - interS); +} + +__global__ void nms_kernel(const int n_boxes, const float nms_overlap_thresh, + const float *dev_boxes, uint64_t *dev_mask) { + const int threadsPerBlock = sizeof(uint64_t) * 8; + const int row_start = blockIdx.y; + const int col_start = blockIdx.x; + + // if (row_start > col_start) return; + + const int row_size = + min(n_boxes - row_start * threadsPerBlock, threadsPerBlock); + const int col_size = + min(n_boxes - col_start * threadsPerBlock, threadsPerBlock); + + __shared__ float block_boxes[threadsPerBlock * 5]; + if (threadIdx.x < col_size) { + block_boxes[threadIdx.x * 5 + 0] = + dev_boxes[(threadsPerBlock * col_start + threadIdx.x) * 5 + 0]; + block_boxes[threadIdx.x * 5 + 1] = + dev_boxes[(threadsPerBlock * col_start + threadIdx.x) * 5 + 1]; + block_boxes[threadIdx.x * 5 + 2] = + dev_boxes[(threadsPerBlock * col_start + threadIdx.x) * 5 + 2]; + block_boxes[threadIdx.x * 5 + 3] = + dev_boxes[(threadsPerBlock * col_start + threadIdx.x) * 5 + 3]; + block_boxes[threadIdx.x * 5 + 4] = + dev_boxes[(threadsPerBlock * col_start + threadIdx.x) * 5 + 4]; + } + __syncthreads(); + + if (threadIdx.x < row_size) { + const int cur_box_idx = threadsPerBlock * row_start + threadIdx.x; + const float *cur_box = dev_boxes + cur_box_idx * 5; + int i = 0; + uint64_t t = 0; + int start = 0; + if (row_start == col_start) { + start = threadIdx.x + 1; + } + for (i = start; i < col_size; i++) { + if (devIoU(cur_box, block_boxes + i * 5) > nms_overlap_thresh) { + t |= 1ULL << i; + } + } + const int col_blocks = DIVUP(n_boxes, threadsPerBlock); + dev_mask[cur_box_idx * col_blocks + col_start] = t; + } +} + +void _nms(const mshadow::Tensor& boxes, + const float nms_overlap_thresh, + int *keep, + int *num_out) { + const int threadsPerBlock = sizeof(uint64_t) * 8; + const int boxes_num = boxes.size(0); + const int boxes_dim = boxes.size(1); + + float* boxes_dev = boxes.dptr_; + uint64_t* mask_dev = NULL; + + const int col_blocks = DIVUP(boxes_num, threadsPerBlock); + FRCNN_CUDA_CHECK(cudaMalloc(&mask_dev, + boxes_num * col_blocks * sizeof(uint64_t))); + + dim3 blocks(DIVUP(boxes_num, threadsPerBlock), + DIVUP(boxes_num, threadsPerBlock)); + dim3 threads(threadsPerBlock); + nms_kernel<<>>(boxes_num, + nms_overlap_thresh, + boxes_dev, + mask_dev); + FRCNN_CUDA_CHECK(cudaPeekAtLastError()); + std::vector mask_host(boxes_num * col_blocks); + FRCNN_CUDA_CHECK(cudaMemcpy(&mask_host[0], + mask_dev, + sizeof(uint64_t) * boxes_num * col_blocks, + cudaMemcpyDeviceToHost)); + + std::vector remv(col_blocks); + memset(&remv[0], 0, sizeof(uint64_t) * col_blocks); + + int num_to_keep = 0; + for (int i = 0; i < boxes_num; i++) { + int nblock = i / threadsPerBlock; + int inblock = i % threadsPerBlock; + + if (!(remv[nblock] & (1ULL << inblock))) { + keep[num_to_keep++] = i; + uint64_t *p = &mask_host[0] + i * col_blocks; + for (int j = nblock; j < col_blocks; j++) { + remv[j] |= p[j]; + } + } + } + *num_out = num_to_keep; + + FRCNN_CUDA_CHECK(cudaFree(mask_dev)); +} + +// copy proposals to output +// dets (top_n, 5); keep (top_n, ); out (top_n, ) +// count should be top_n (total anchors or proposals) +template +__global__ void PrepareOutput(const int count, + const Dtype* dets, + const int* keep, + const int out_size, + const int image_index, + Dtype* out, + Dtype* score) { + for (int index = blockIdx.x * blockDim.x + threadIdx.x; + index < count; + index += blockDim.x * gridDim.x) { + out[index * 5] = image_index; + if (index < out_size) { + int keep_i = keep[index]; + for (int j = 0; j < 4; ++j) { + out[index * 5 + j + 1] = dets[keep_i * 5 + j]; + } + score[index] = dets[keep_i * 5 + 4]; + } else { + int keep_i = keep[index % out_size]; + for (int j = 0; j < 4; ++j) { + out[index * 5 + j + 1] = dets[keep_i * 5 + j]; + } + score[index] = dets[keep_i * 5 + 4]; + } + } +} +} // namespace multi_proposal +} // namespace cuda +} // namespace mshadow + +namespace mxnet { +namespace op { + +template +class MultiProposalGPUOp : public Operator{ + public: + explicit MultiProposalGPUOp(MultiProposalParam param) { + this->param_ = param; + } + + virtual void Forward(const OpContext &ctx, + const std::vector &in_data, + const std::vector &req, + const std::vector &out_data, + const std::vector &aux_states) { + using namespace mshadow; + using namespace mshadow::expr; + using namespace mshadow::cuda; + using namespace mshadow::cuda::multi_proposal; + CHECK_EQ(in_data.size(), 3); + CHECK_EQ(out_data.size(), 2); + CHECK_GT(req.size(), 1); + CHECK_EQ(req[proposal::kOut], kWriteTo); + /*CHECK_EQ(in_data[proposal::kClsProb].shape_[0], 1) + << "Sorry, multiple images each device is not implemented.";*/ + + Stream *s = ctx.get_stream(); + + Tensor scores = in_data[proposal::kClsProb].get(s); + Tensor bbox_deltas = in_data[proposal::kBBoxPred].get(s); + Tensor im_info = in_data[proposal::kImInfo].get(s); + + Tensor out = out_data[proposal::kOut].get(s); + Tensor out_score = out_data[proposal::kScore].get(s); + + int num_images = scores.size(0); + int num_anchors = scores.size(1) / 2; + int height = scores.size(2); + int width = scores.size(3); + int count_anchors = num_anchors * height * width; // count of total anchors + int count = num_images * count_anchors; + // set to -1 for max + int rpn_pre_nms_top_n = (param_.rpn_pre_nms_top_n > 0) ? param_.rpn_pre_nms_top_n + : count_anchors; + rpn_pre_nms_top_n = std::min(rpn_pre_nms_top_n, count_anchors); + int rpn_post_nms_top_n = std::min(param_.rpn_post_nms_top_n, rpn_pre_nms_top_n); + + // Generate first anchors based on base anchor + std::vector base_anchor(4); + base_anchor[0] = 0.0; + base_anchor[1] = 0.0; + base_anchor[2] = param_.feature_stride - 1.0; + base_anchor[3] = param_.feature_stride - 1.0; + CHECK_EQ(num_anchors, param_.ratios.info.size() * param_.scales.info.size()); + std::vector anchors; + utils::GenerateAnchors(base_anchor, + param_.ratios.info, + param_.scales.info, + &anchors); + + // Copy generated anchors to GPU + float* workspace_proposals_ptr = NULL; + FRCNN_CUDA_CHECK(cudaMalloc(&workspace_proposals_ptr, + sizeof(float) * num_images * count_anchors * 5)); + Tensor workspace_proposals(workspace_proposals_ptr, + Shape3(num_images, count_anchors, 5)); + FRCNN_CUDA_CHECK(cudaMemcpy(workspace_proposals.dptr_, &anchors[0], + sizeof(float) * anchors.size(), cudaMemcpyHostToDevice)); + + // Copy proposals to a mesh grid + dim3 dimGrid((count + kMaxThreadsPerBlock - 1) / kMaxThreadsPerBlock); + dim3 dimBlock(kMaxThreadsPerBlock); + CheckLaunchParam(dimGrid, dimBlock, "ProposalGrid"); + ProposalGridKernel<<>>( + count, num_anchors, height, width, param_.feature_stride, + scores.dptr_, workspace_proposals.dptr_); + FRCNN_CUDA_CHECK(cudaPeekAtLastError()); + + // Transform anchors and bbox_deltas into bboxes + CheckLaunchParam(dimGrid, dimBlock, "BBoxPred"); + if (param_.iou_loss) { + IoUPredKernel<<>>( + count, num_anchors, height, width, param_.feature_stride, im_info.dptr_, + workspace_proposals.dptr_, bbox_deltas.dptr_, workspace_proposals.dptr_); + } else { + BBoxPredKernel<<>>( + count, num_anchors, height, width, param_.feature_stride, im_info.dptr_, + workspace_proposals.dptr_, bbox_deltas.dptr_, workspace_proposals.dptr_); + } + FRCNN_CUDA_CHECK(cudaPeekAtLastError()); + + // filter boxes with less than rpn_min_size + CheckLaunchParam(dimGrid, dimBlock, "FilterBox"); + FilterBoxKernel<<>>( + count, count_anchors, param_.rpn_min_size, im_info.dptr_, workspace_proposals.dptr_); + FRCNN_CUDA_CHECK(cudaPeekAtLastError()); + + + + dimGrid = dim3((count_anchors + kMaxThreadsPerBlock - 1) / kMaxThreadsPerBlock); + dimBlock = dim3(kMaxThreadsPerBlock); + // Copy score to a continuous memory + float* score_ptr = NULL; + FRCNN_CUDA_CHECK(cudaMalloc(&score_ptr, sizeof(float) * count_anchors)); + Tensor score(score_ptr, Shape1(count_anchors)); + int* order_ptr = NULL; + FRCNN_CUDA_CHECK(cudaMalloc(&order_ptr, sizeof(int) * count_anchors)); + Tensor order(order_ptr, Shape1(count_anchors)); + + float* workspace_ordered_proposals_ptr = NULL; + FRCNN_CUDA_CHECK(cudaMalloc(&workspace_ordered_proposals_ptr, + sizeof(float) * rpn_pre_nms_top_n * 5)); + Tensor workspace_ordered_proposals(workspace_ordered_proposals_ptr, + Shape2(rpn_pre_nms_top_n, 5)); + + int* keep; + FRCNN_CUDA_CHECK(cudaMalloc(&keep, sizeof(int) * rpn_pre_nms_top_n)); + + for (int b = 0; b < num_images; b++) { + CheckLaunchParam(dimGrid, dimBlock, "CopyScore"); + CopyScoreKernel << > >( + count_anchors, workspace_proposals.dptr_ + b * count_anchors * 5, + score.dptr_, order.dptr_); + FRCNN_CUDA_CHECK(cudaPeekAtLastError()); + + // argsort score, save order + thrust::stable_sort_by_key(thrust::device, + score.dptr_, + score.dptr_ + score.size(0), + order.dptr_, + thrust::greater()); + FRCNN_CUDA_CHECK(cudaPeekAtLastError()); + + // Reorder proposals according to order + + dimGrid.x = (rpn_pre_nms_top_n + kMaxThreadsPerBlock - 1) / kMaxThreadsPerBlock; + CheckLaunchParam(dimGrid, dimBlock, "ReorderProposals"); + ReorderProposalsKernel << > >( + rpn_pre_nms_top_n, workspace_proposals.dptr_ + b * count_anchors * 5, + order.dptr_, workspace_ordered_proposals.dptr_); + FRCNN_CUDA_CHECK(cudaPeekAtLastError()); + + // perform nms + std::vector _keep(workspace_ordered_proposals.size(0)); + int out_size = 0; + _nms(workspace_ordered_proposals, + param_.threshold, + &_keep[0], + &out_size); + + // copy nms result to gpu + FRCNN_CUDA_CHECK(cudaMemcpy(keep, &_keep[0], sizeof(int) * _keep.size(), + cudaMemcpyHostToDevice)); + + // copy results after nms + dimGrid.x = (rpn_post_nms_top_n + kMaxThreadsPerBlock - 1) / kMaxThreadsPerBlock; + CheckLaunchParam(dimGrid, dimBlock, "PrepareOutput"); + PrepareOutput << > >( + rpn_post_nms_top_n, workspace_ordered_proposals.dptr_, keep, out_size, b, + out.dptr_ + b * rpn_post_nms_top_n * 5, out_score.dptr_ + b * rpn_post_nms_top_n); + FRCNN_CUDA_CHECK(cudaPeekAtLastError()); + } + // free temporary memory + FRCNN_CUDA_CHECK(cudaFree(keep)); + FRCNN_CUDA_CHECK(cudaFree(workspace_ordered_proposals_ptr)); + FRCNN_CUDA_CHECK(cudaFree(workspace_proposals_ptr)); + FRCNN_CUDA_CHECK(cudaFree(score_ptr)); + FRCNN_CUDA_CHECK(cudaFree(order_ptr)); + } + + virtual void Backward(const OpContext &ctx, + const std::vector &out_grad, + const std::vector &in_data, + const std::vector &out_data, + const std::vector &req, + const std::vector &in_grad, + const std::vector &aux_states) { + using namespace mshadow; + using namespace mshadow::expr; + CHECK_EQ(in_grad.size(), 3); + + Stream *s = ctx.get_stream(); + Tensor gscores = in_grad[proposal::kClsProb].get(s); + Tensor gbbox = in_grad[proposal::kBBoxPred].get(s); + Tensor ginfo = in_grad[proposal::kImInfo].get(s); + + // can not assume the grad would be zero + Assign(gscores, req[proposal::kClsProb], 0); + Assign(gbbox, req[proposal::kBBoxPred], 0); + Assign(ginfo, req[proposal::kImInfo], 0); + } + + private: + MultiProposalParam param_; +}; // class MultiProposalGPUOp + +template<> +Operator* CreateOp(MultiProposalParam param) { + return new MultiProposalGPUOp(param); +} +} // namespace op +} // namespace mxnet diff --git a/src/operator/contrib/nn/deformable_im2col.cuh b/src/operator/contrib/nn/deformable_im2col.cuh new file mode 100644 index 000000000000..d9e7b970ca84 --- /dev/null +++ b/src/operator/contrib/nn/deformable_im2col.cuh @@ -0,0 +1,525 @@ +/*! + ******************* BEGIN Caffe Copyright Notice and Disclaimer **************** + * + * COPYRIGHT + * + * All contributions by the University of California: + * Copyright (c) 2014-2017 The Regents of the University of California (Regents) + * All rights reserved. + * + * All other contributions: + * Copyright (c) 2014-2017, the respective contributors + * All rights reserved. + * + * Caffe uses a shared copyright model: each contributor holds copyright over + * their contributions to Caffe. The project versioning records all such + * contribution and copyright details. If a contributor wants to further mark + * their specific copyright on a particular contribution, they should indicate + * their copyright solely in the commit message of the change when it is + * committed. + * + * LICENSE + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * CONTRIBUTION AGREEMENT + * + * By contributing to the BVLC/caffe repository through pull-request, comment, + * or otherwise, the contributor releases their content to the + * license and copyright terms herein. + * + ***************** END Caffe Copyright Notice and Disclaimer ******************** + * + * Copyright (c) 2017 Microsoft + * Licensed under The Apache-2.0 License [see LICENSE for details] + * \file deformable_im2col.cuh + * \brief Function definitions of converting an image to + * column matrix based on kernel, padding, dilation, and offset. + * These functions are mainly used in deformable convolution operators. + * \ref: https://arxiv.org/abs/1703.06211 + * \author Yuwen Xiong, Haozhi Qi, Jifeng Dai + */ + +#ifndef MXNET_OPERATOR_CONTRIB_NN_DEFORMABLE_IM2COL_CUH_ +#define MXNET_OPERATOR_CONTRIB_NN_DEFORMABLE_IM2COL_CUH_ + +#include +#include +#include +#include +#include +#include "../../mxnet_op.h" +#include "../../../common/cuda_utils.h" + + + +namespace mxnet { +namespace op { + +template +__device__ DType deformable_im2col_bilinear(const DType* bottom_data, const int data_width, + const int height, const int width, DType h, DType w) { + + int h_low = floor(h); + int w_low = floor(w); + int h_high; + int w_high; + if (h_low >= height - 1) { + h_high = h_low = height - 1; + h = (DType)h_low; + } + else { + h_high = h_low + 1; + } + + if (w_low >= width - 1) { + w_high = w_low = width - 1; + w = (DType)w_low; + } + else { + w_high = w_low + 1; + } + + DType lh = h - h_low; + DType lw = w - w_low; + DType hh = 1 - lh, hw = 1 - lw; + + DType v1 = bottom_data[h_low * data_width + w_low]; + DType v2 = bottom_data[h_low * data_width + w_high]; + DType v3 = bottom_data[h_high * data_width + w_low]; + DType v4 = bottom_data[h_high * data_width + w_high]; + DType w1 = hh * hw, w2 = hh * lw, w3 = lh * hw, w4 = lh * lw; + + DType val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); + return val; +} + + +template +__device__ DType get_gradient_weight(DType argmax_h, DType argmax_w, + const int h, const int w, const int height, const int width) { + + if (argmax_h < 0 || argmax_h > height || argmax_w < 0 || argmax_w > width) { + //empty + return 0; + } + + argmax_h = max(argmax_h, (DType)0.0f); + argmax_w = max(argmax_w, (DType)0.0f); + + int argmax_h_low = (int)argmax_h; + int argmax_w_low = (int)argmax_w; + int argmax_h_high; + int argmax_w_high; + if (argmax_h_low >= height - 1) { + argmax_h_high = argmax_h_low = height - 1; + argmax_h = (DType)argmax_h_low; + } else { + argmax_h_high = argmax_h_low + 1; + } + if (argmax_w_low >= width - 1) + { + argmax_w_high = argmax_w_low = width - 1; + argmax_w = (DType)argmax_w_low; + } else { + argmax_w_high = argmax_w_low + 1; + } + DType weight = 0; + if (h == argmax_h_low) { + if (w == argmax_w_low) { + weight = (h + 1 - argmax_h) * (w + 1 - argmax_w); + } else if (w == argmax_w_high) { + weight = (h + 1 - argmax_h) * (argmax_w + 1 - w); + } + } else if (h == argmax_h_high) { + if (w == argmax_w_low) { + weight = (argmax_h + 1 - h) * (w + 1 - argmax_w); + } else if (w == argmax_w_high) { + weight = (argmax_h + 1 - h) * (argmax_w + 1 - w); + } + } + return weight; +} + + +template +__device__ DType get_coordinate_weight(DType argmax_h, DType argmax_w, + const int height, const int width, const DType* im_data, + const int data_width, const int bp_dir) { + + if (argmax_h < 0 || argmax_h > height || argmax_w < 0 || argmax_w > width) + { + //empty + return 0; + } + + if (argmax_h < 0) argmax_h = 0; + if (argmax_w < 0) argmax_w = 0; + + int argmax_h_low = (int)argmax_h; + int argmax_w_low = (int)argmax_w; + int argmax_h_high; + int argmax_w_high; + if (argmax_h_low >= height - 1) { + argmax_h_high = argmax_h_low = height - 1; + argmax_h = (DType)argmax_h_low; + } else { + argmax_h_high = argmax_h_low + 1; + } + if (argmax_w_low >= width - 1) { + argmax_w_high = argmax_w_low = width - 1; + argmax_w = (DType)argmax_w_low; + } else { + argmax_w_high = argmax_w_low + 1; + } + DType weight = 0; + + if (bp_dir == 0) { + weight += -1 * (argmax_w_low + 1 - argmax_w) * im_data[argmax_h_low * data_width + argmax_w_low]; + weight += -1 * (argmax_w - argmax_w_low) * im_data[argmax_h_low * data_width + argmax_w_high]; + weight += (argmax_w_low + 1 - argmax_w) * im_data[argmax_h_high * data_width + argmax_w_low]; + weight += (argmax_w - argmax_w_low) * im_data[argmax_h_high * data_width + argmax_w_high]; + } else if (bp_dir == 1) { + weight += -1 * (argmax_h_low + 1 - argmax_h) * im_data[argmax_h_low * data_width + argmax_w_low]; + weight += (argmax_h_low + 1 - argmax_h) * im_data[argmax_h_low * data_width + argmax_w_high]; + weight += -1 * (argmax_h - argmax_h_low) * im_data[argmax_h_high * data_width + argmax_w_low]; + weight += (argmax_h - argmax_h_low) * im_data[argmax_h_high * data_width + argmax_w_high]; + } + + return weight; +} + + +/*! + * \brief deformable_im2col gpu kernel. + * DO NOT call this directly. Use wrapper function im2col() instead; + */ +template +__global__ void deformable_im2col_gpu_kernel(const int n, const DType* data_im, const DType* data_offset, + const int height, const int width, const int kernel_h, const int kernel_w, + const int pad_h, const int pad_w, + const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int channel_per_deformable_group, + const int height_col, const int width_col, + DType* data_col) { + CUDA_KERNEL_LOOP(index, n) { + // index index of output matrix + const int w_col = index % width_col; + const int h_col = (index / width_col) % height_col; + const int c_im = (index / width_col) / height_col; + const int c_col = c_im * kernel_h * kernel_w; + + // compute deformable group index + const int deformable_group_index = c_im / channel_per_deformable_group; + + const int h_in = h_col * stride_h - pad_h; + const int w_in = w_col * stride_w - pad_w; + DType* data_col_ptr = data_col + (c_col * height_col + h_col) * width_col + w_col; + const DType* data_im_ptr = data_im + (c_im * height + h_in) * width + w_in; + const DType* data_offset_ptr = data_offset + deformable_group_index * 2 * kernel_h * kernel_w * height_col * width_col; + + + for (int i = 0; i < kernel_h; ++i) { + for (int j = 0; j < kernel_w; ++j) { + const int data_offset_h_ptr = ((2 * (i * kernel_w + j)) * height_col + h_col) * width_col + w_col; + const int data_offset_w_ptr = ((2 * (i * kernel_w + j) + 1) * height_col + h_col) * width_col + w_col; + const DType offset_h = data_offset_ptr[data_offset_h_ptr]; + const DType offset_w = data_offset_ptr[data_offset_w_ptr]; + DType val = static_cast(0); + const DType h_im = h_in + i * dilation_h + offset_h; + const DType w_im = w_in + j * dilation_w + offset_w; + if (h_im >= 0 && w_im >= 0 && h_im < height && w_im < width) { + const DType map_h = i * dilation_h + offset_h; + const DType map_w = j * dilation_w + offset_w; + const int cur_height = height - h_in; + const int cur_width = width - w_in; + val = deformable_im2col_bilinear(data_im_ptr, width, cur_height, cur_width, map_h, map_w); + } + *data_col_ptr = val; + data_col_ptr += height_col * width_col; + } + } + } +} + + + + + + +/*!\brief + * cpu function of deformable_im2col algorithm + * \param s device stream + * \param data_im pointer of an image (C, H, W, ...) in the image batch + * \param data_offset pointer of offset (C, H, W, ...) in the offset batch + * \param im_shape input image shape in dimensions (N, C, H, W,) + * \param col_shape column buffer shape (#channels, output_im_height, output_im_width, ...) + * \param kernel_shape kernel filter shape + * \param pad pad shape + * \param stride stride shape + * \param dilation dilation shape + * \param deformable_group #offset group that deformable convolution use + * \param data_col column buffer pointer + */ +template +inline void deformable_im2col(mshadow::Stream* s, + const DType* data_im, const DType* data_offset, + const TShape& im_shape, const TShape& col_shape, const TShape& kernel_shape, + const TShape& pad, const TShape& stride, const TShape& dilation, + const uint32_t deformable_group, DType* data_col) { + // num_axes should be smaller than block size + index_t num_spatial_axes = kernel_shape.ndim(); + CHECK_LT(num_spatial_axes, mshadow::cuda::kBaseThreadNum); + index_t channel_per_deformable_group = im_shape[1] / deformable_group; + index_t num_kernels = im_shape[1] * col_shape.ProdShape(1, col_shape.ndim()); + using namespace mxnet_op; + switch (num_spatial_axes) { + case 2: + deformable_im2col_gpu_kernel // NOLINT_NEXT_LINE(whitespace/operators) + <<::GetStream(s)>>>( + num_kernels, data_im, data_offset, im_shape[2], im_shape[3], kernel_shape[0], kernel_shape[1], + pad[0], pad[1], stride[0], stride[1], dilation[0], dilation[1], channel_per_deformable_group, + col_shape[1], col_shape[2], data_col); + MSHADOW_CUDA_POST_KERNEL_CHECK(deformable_im2col_gpu_kernel); + break; + default: + LOG(FATAL) << "im2col_nd_gpu does not support computation with " + << num_spatial_axes << " spatial axes"; + } +} + + +/*! +* \brief deformable_col2im gpu kernel. +* \brief DO NOT call this directly. Use wrapper function deformable_col2im() instead; +*/ +template +__global__ void deformable_col2im_gpu_kernel(const int n, const DType* data_col, const DType* data_offset, + const int channels, const int height, const int width, + const int kernel_h, const int kernel_w, + const int pad_h, const int pad_w, + const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int channel_per_deformable_group, + const int height_col, const int width_col, + DType* grad_im, OpReqType req) { + CUDA_KERNEL_LOOP(index, n) { + const int j = (index / width_col / height_col) % kernel_w; + const int i = (index / width_col / height_col / kernel_w) % kernel_h; + const int c = index / width_col / height_col / kernel_w / kernel_h; + // compute the start and end of the output + + const int deformable_group_index = c / channel_per_deformable_group; + + int w_out = index % width_col; + int h_out = (index / width_col) % height_col; + int w_in = w_out * stride_w - pad_w; + int h_in = h_out * stride_h - pad_h; + + const DType* data_offset_ptr = data_offset + deformable_group_index * 2 * kernel_h * kernel_w * height_col * width_col; + const int data_offset_h_ptr = ((2 * (i * kernel_w + j)) * height_col + h_out) * width_col + w_out; + const int data_offset_w_ptr = ((2 * (i * kernel_w + j) + 1) * height_col + h_out) * width_col + w_out; + const DType offset_h = data_offset_ptr[data_offset_h_ptr]; + const DType offset_w = data_offset_ptr[data_offset_w_ptr]; + const DType cur_inv_h_data = h_in + i * dilation_h + offset_h; + const DType cur_inv_w_data = w_in + j * dilation_w + offset_w; + + const DType cur_top_grad = data_col[index]; + const int cur_h = (int)cur_inv_h_data; + const int cur_w = (int)cur_inv_w_data; + for (int dy = -2; dy <= 2; dy++) { + for (int dx = -2; dx <= 2; dx++) { + if (cur_h + dy >= 0 && cur_h + dy < height && + cur_w + dx >= 0 && cur_w + dx < width && + abs(cur_inv_h_data - (cur_h + dy)) < 1 && + abs(cur_inv_w_data - (cur_w + dx)) < 1 + ) { + int cur_bottom_grad_pos = (c * height + cur_h + dy) * width + cur_w + dx; + DType weight = get_gradient_weight(cur_inv_h_data, cur_inv_w_data, cur_h + dy, cur_w + dx, height, width); + atomicAdd(grad_im + cur_bottom_grad_pos, weight * cur_top_grad); + } + } + } + } +} + + +/*!\brief + * gpu function of deformable_col2im algorithm + * \param s device stream + * \param data_col start pointer of the column buffer to be filled + * \param data_offset pointer of offset (C, H, W, ...) in the offset batch + * \param im_shape input image shape in dimensions (N, C, H, W,) + * \param col_shape column buffer shape + * \param kernel_shape kernel filter shape + * \param pad pad shape + * \param stride stride shape + * \param dilation dilation shape + * \param deformable_group #offset group that deformable convolution use + * \param grad_im pointer of a image (C, H, W,...) in the image batch + */ +template +inline void deformable_col2im(mshadow::Stream* s, + const DType* data_col, const DType* data_offset, + const TShape& im_shape, const TShape& col_shape, const TShape& kernel_shape, + const TShape& pad, const TShape& stride, + const TShape& dilation, const uint32_t deformable_group, + DType* grad_im, OpReqType req) { + index_t num_spatial_axes = kernel_shape.ndim(); + index_t im_size = im_shape.ProdShape(1, im_shape.ndim()); + index_t channel_per_deformable_group = im_shape[1] / deformable_group; + index_t num_kernels = col_shape.ProdShape(0, col_shape.ndim()); + // num_axes should be smaller than block size + CHECK_LT(num_spatial_axes, mshadow::cuda::kBaseThreadNum); + using namespace mxnet_op; + switch (num_spatial_axes) { + case 2: + // To avoid involving atomic operations, we will launch one kernel per + // bottom dimension, and then in the kernel add up the top dimensions. + // NOLINT_NEXT_LINE(whitespace/operators) + deformable_col2im_gpu_kernel<<::GetStream(s)>>>( + num_kernels, data_col, data_offset, im_shape[1], im_shape[2], im_shape[3], + kernel_shape[0], kernel_shape[1], pad[0], pad[1], stride[0], stride[1], + dilation[0], dilation[1], channel_per_deformable_group, col_shape[1], col_shape[2], grad_im, req); + MSHADOW_CUDA_POST_KERNEL_CHECK(deformable_col2im_gpu_kernel); + break; + default: + LOG(FATAL) << "col2im_nd_gpu does not support computation with " + << num_spatial_axes << " spatial axes"; + } +} + + +/*! + * \brief deformable_col2im_coord gpu kernel. + * \brief DO NOT call this directly. Use wrapper function deformable_col2im_coord() instead; + */ +template +__global__ void deformable_col2im_coord_gpu_kernel(const int n, const DType* data_col, + const DType* data_im, const DType* data_offset, + const int channels, const int height, const int width, + const int kernel_h, const int kernel_w, + const int pad_h, const int pad_w, + const int stride_h, const int stride_w, + const int dilation_h, const int dilation_w, + const int channel_per_deformable_group, + const int height_col, const int width_col, + DType* grad_offset, OpReqType req) { + CUDA_KERNEL_LOOP(index, n) { + DType val = 0; + int w = index % width_col; + int h = (index / width_col) % height_col; + int c = index / width_col / height_col; + // compute the start and end of the output + + const int deformable_group_index = c / (2 * kernel_h * kernel_w); + const int col_step = kernel_h * kernel_w; + int cnt = 0; + const DType* data_col_ptr = data_col + deformable_group_index * channel_per_deformable_group * width_col * height_col; + const DType* data_im_ptr = data_im + deformable_group_index * channel_per_deformable_group / kernel_h / kernel_w * height * width; + const DType* data_offset_ptr = data_offset + deformable_group_index * 2 * kernel_h * kernel_w * height_col * width_col; + + const int offset_c = c - deformable_group_index * 2 * kernel_h * kernel_w; + + for (int col_c = (offset_c / 2); col_c < channel_per_deformable_group; col_c += col_step) { + const int col_pos = ((col_c * height_col) + h) * width_col + w; + const int bp_dir = offset_c % 2; + + int j = (col_pos / width_col / height_col) % kernel_w; + int i = (col_pos / width_col / height_col / kernel_w) % kernel_h; + int w_out = col_pos % width_col; + int h_out = (col_pos / width_col) % height_col; + int w_in = w_out * stride_w - pad_w; + int h_in = h_out * stride_h - pad_h; + const int data_offset_h_ptr = (((2 * (i * kernel_w + j)) * height_col + h_out) * width_col + w_out); + const int data_offset_w_ptr = (((2 * (i * kernel_w + j) + 1) * height_col + h_out) * width_col + w_out); + const DType offset_h = data_offset_ptr[data_offset_h_ptr]; + const DType offset_w = data_offset_ptr[data_offset_w_ptr]; + DType inv_h = h_in + i * dilation_h + offset_h; + DType inv_w = w_in + j * dilation_w + offset_w; + if (inv_h < 0 || inv_w < 0 || inv_h >= height || inv_w >= width) { + inv_h = inv_w = -1; + } + const DType weight = get_coordinate_weight( + inv_h, inv_w, + height, width, data_im_ptr + cnt * height * width, width, bp_dir); + val += weight * data_col_ptr[col_pos]; + cnt += 1; + } + + grad_offset[index] = val; + } +} + +/*!\brief + * gpu function of deformable_col2im_coord algorithm + * \param s device stream + * \param data_col start pointer of the column buffer to be filled + * \param data_im pointer of an image (C, H, W, ...) in the image batch + * \param data_offset pointer of offset (C, H, W, ...) in the offset batch + * \param im_shape input image shape in dimensions (N, C, H, W,) + * \param col_shape column buffer shape + * \param kernel_shape kernel filter shape + * \param pad pad shape + * \param stride stride shape + * \param dilation dilation shape + * \param deformable_group #offset group that deformable convolution use + * \param grad_offset pointer of the offset (C, H, W,...) in the offset batch + */ +template +inline void deformable_col2im_coord(mshadow::Stream* s, + const DType* data_col, const DType* data_im, const DType* data_offset, const TShape& im_shape, + const TShape& col_shape, const TShape& kernel_shape, + const TShape& pad, const TShape& stride, + const TShape& dilation, const uint32_t deformable_group, DType* grad_offset, OpReqType req) { + index_t num_spatial_axes = kernel_shape.ndim(); + index_t num_kernels = col_shape[1] * col_shape[2] * 2 * kernel_shape[0] * kernel_shape[1] * deformable_group; + index_t channel_per_deformable_group = col_shape[0] / deformable_group; + // num_axes should be smaller than block size + CHECK_LT(num_spatial_axes, mshadow::cuda::kBaseThreadNum); + using namespace mxnet_op; + switch (num_spatial_axes) { + case 2: + // To avoid involving atomic operations, we will launch one kernel per + // bottom dimension, and then in the kernel add up the top dimensions. + // NOLINT_NEXT_LINE(whitespace/operators) + + deformable_col2im_coord_gpu_kernel << ::GetStream(s) >> >( + num_kernels, data_col, data_im, data_offset, im_shape[1], im_shape[2], im_shape[3], + kernel_shape[0], kernel_shape[1], pad[0], pad[1], stride[0], stride[1], + dilation[0], dilation[1], channel_per_deformable_group, col_shape[1], col_shape[2], grad_offset, req); + MSHADOW_CUDA_POST_KERNEL_CHECK(deformable_col2im_gpu_kernel); + break; + default: + LOG(FATAL) << "col2im_nd_gpu does not support computation with " + << num_spatial_axes << " spatial axes"; + } +} + + +} // namespace op +} // namespace mxnet + +#endif // MXNET_OPERATOR_CONTRIB_NN_DEFORMABLE_IM2COL_CUH_ diff --git a/src/operator/contrib/nn/deformable_im2col.h b/src/operator/contrib/nn/deformable_im2col.h new file mode 100644 index 000000000000..9d6180034c71 --- /dev/null +++ b/src/operator/contrib/nn/deformable_im2col.h @@ -0,0 +1,157 @@ +/*! + ******************* BEGIN Caffe Copyright Notice and Disclaimer **************** + * + * COPYRIGHT + * + * All contributions by the University of California: + * Copyright (c) 2014-2017 The Regents of the University of California (Regents) + * All rights reserved. + * + * All other contributions: + * Copyright (c) 2014-2017, the respective contributors + * All rights reserved. + * + * Caffe uses a shared copyright model: each contributor holds copyright over + * their contributions to Caffe. The project versioning records all such + * contribution and copyright details. If a contributor wants to further mark + * their specific copyright on a particular contribution, they should indicate + * their copyright solely in the commit message of the change when it is + * committed. + * + * LICENSE + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * CONTRIBUTION AGREEMENT + * + * By contributing to the BVLC/caffe repository through pull-request, comment, + * or otherwise, the contributor releases their content to the + * license and copyright terms herein. + * + ***************** END Caffe Copyright Notice and Disclaimer ******************** + * + * Copyright (c) 2017 Microsoft + * Licensed under The Apache-2.0 License [see LICENSE for details] + * \file deformable_im2col.h + * \brief Function definitions of converting an image to + * column matrix based on kernel, padding, dilation, and offset. + * These functions are mainly used in deformable convolution operators. + * \ref: https://arxiv.org/abs/1703.06211 + * \author Yuwen Xiong, Haozhi Qi, Jifeng Dai + */ + +#ifndef MXNET_OPERATOR_CONTRIB_NN_DEFORMABLE_IM2COL_H_ +#define MXNET_OPERATOR_CONTRIB_NN_DEFORMABLE_IM2COL_H_ + +#include +#include +#include +#include +#include "../../mxnet_op.h" + +namespace mxnet { +namespace op { + +/*!\brief + * cpu function of deformable_im2col algorithm + * \param s device stream + * \param data_im pointer of an image (C, H, W, ...) in the image batch + * \param data_offset pointer of offset (C, H, W, ...) in the offset batch + * \param im_shape input image shape in dimensions (N, C, H, W,) + * \param col_shape column buffer shape (#channels, output_im_height, output_im_width, ...) + * \param kernel_shape kernel filter shape + * \param pad pad shape + * \param stride stride shape + * \param dilation dilation shape + * \param deformable_group #offset group that deformable convolution use + * \param data_col column buffer pointer + */ +template +inline void deformable_im2col(mshadow::Stream* s, + const DType* data_im, const DType* data_offset, + const TShape& im_shape, const TShape& col_shape, const TShape& kernel_shape, + const TShape& pad, const TShape& stride, const TShape& dilation, + const uint32_t deformable_group, DType* data_col) { + if (2 == kernel_shape.ndim()) { + LOG(FATAL) << "only implemented in GPU"; + } else { + LOG(FATAL) << "not implemented"; + } +} + + +/*!\brief + * cpu function of deformable_col2im algorithm + * \param s device stream + * \param data_col start pointer of the column buffer to be filled + * \param data_offset pointer of offset (C, H, W, ...) in the offset batch + * \param im_shape input image shape in dimensions (N, C, H, W,) + * \param col_shape column buffer shape + * \param kernel_shape kernel filter shape + * \param pad pad shape + * \param stride stride shape + * \param dilation dilation shape + * \param deformable_group #offset group that deformable convolution use + * \param grad_im pointer of a image (C, H, W,...) in the image batch + */ +template +inline void deformable_col2im(mshadow::Stream* s, + const DType* data_col, const DType* data_offset, + const TShape& im_shape, const TShape& col_shape, const TShape& kernel_shape, + const TShape& pad, const TShape& stride, + const TShape& dilation, const uint32_t deformable_group, + DType* grad_im, OpReqType req) { + index_t num_spatial_axes = kernel_shape.ndim(); + LOG(FATAL) << "only implemented in GPU"; +} + + +/*!\brief + * cpu function of deformable_col2im_coord algorithm + * \param s device stream + * \param data_col start pointer of the column buffer to be filled + * \param data_im pointer of an image (C, H, W, ...) in the image batch + * \param data_offset pointer of offset (C, H, W, ...) in the offset batch + * \param im_shape input image shape in dimensions (N, C, H, W,) + * \param col_shape column buffer shape + * \param kernel_shape kernel filter shape + * \param pad pad shape + * \param stride stride shape + * \param dilation dilation shape + * \param deformable_group #offset group that deformable convolution use + * \param grad_offset pointer of the offset (C, H, W,...) in the offset batch + */ + +template +inline void deformable_col2im_coord(mshadow::Stream* s, + const DType* data_col, const DType* data_im, const DType* data_offset, const TShape& im_shape, + const TShape& col_shape, const TShape& kernel_shape, + const TShape& pad, const TShape& stride, + const TShape& dilation, const uint32_t deformable_group, DType* grad_offset, OpReqType req) { + LOG(FATAL) << "only implemented in GPU"; +} + +} // namespace op +} // namespace mxnet +#ifdef __CUDACC__ +#include "./deformable_im2col.cuh" +#endif +#endif // MXNET_OPERATOR_CONTRIB_NN_DEFORMABLE_IM2COL_H_ diff --git a/src/operator/contrib/psroi_pooling-inl.h b/src/operator/contrib/psroi_pooling-inl.h new file mode 100644 index 000000000000..3a3a9c34927c --- /dev/null +++ b/src/operator/contrib/psroi_pooling-inl.h @@ -0,0 +1,222 @@ +/*! + * Copyright (c) 2017 by Contributors + * Copyright (c) 2017 Microsoft + * Licensed under The Apache-2.0 License [see LICENSE for details] + * \file psroi_pooling-inl.h + * \brief psroi pooling operator and symbol + * \author Yi Li, Tairui Chen, Guodong Zhang, Haozhi Qi, Jifeng Dai +*/ +#ifndef MXNET_OPERATOR_CONTRIB_PSROI_POOLING_INL_H_ +#define MXNET_OPERATOR_CONTRIB_PSROI_POOLING_INL_H_ + +#include +#include +#include +#include +#include +#include +#include +#include "../mshadow_op.h" +#include "../operator_common.h" + + +namespace mxnet { +namespace op { + +// Declare enumeration of input order to make code more intuitive. +// These enums are only visible within this header +namespace psroipool { +enum PSROIPoolingOpInputs {kData, kBox}; +enum PSROIPoolingOpOutputs {kOut}; +} // psroipool + +struct PSROIPoolingParam : public dmlc::Parameter { + // TShape pooled_size; + float spatial_scale; + int output_dim; + int pooled_size; + int group_size; + DMLC_DECLARE_PARAMETER(PSROIPoolingParam) { + DMLC_DECLARE_FIELD(spatial_scale).set_range(0.0, 1.0) + .describe("Ratio of input feature map height (or w) to raw image height (or w). " + "Equals the reciprocal of total stride in convolutional layers"); + DMLC_DECLARE_FIELD(output_dim).describe("fix output dim"); + DMLC_DECLARE_FIELD(pooled_size).describe("fix pooled size"); + DMLC_DECLARE_FIELD(group_size).set_default(0).describe("fix group size"); + } +}; + +template +class PSROIPoolingOp : public Operator { + public: + explicit PSROIPoolingOp(PSROIPoolingParam p) { + this->param_ = p; + } + + virtual void Forward(const OpContext &ctx, + const std::vector &in_data, + const std::vector &req, + const std::vector &out_data, + const std::vector &aux_args) { + using namespace mshadow; + CHECK_EQ(in_data.size(), 2); + CHECK_EQ(out_data.size(), 1); + CHECK_EQ(out_data[psroipool::kOut].shape_[0], in_data[psroipool::kBox].shape_[0]); + Stream *s = ctx.get_stream(); + + Tensor data = in_data[psroipool::kData].get(s); + Tensor bbox = in_data[psroipool::kBox].get(s); + Tensor out = out_data[psroipool::kOut].get(s); + CHECK_EQ(data.CheckContiguous(), true); + CHECK_EQ(bbox.CheckContiguous(), true); + CHECK_EQ(out.CheckContiguous(), true); + out = -FLT_MAX; + PSROIPoolForward(out, data, bbox, param_.spatial_scale, param_.output_dim, param_.group_size); + } + + virtual void Backward(const OpContext &ctx, + const std::vector &out_grad, + const std::vector &in_data, + const std::vector &out_data, + const std::vector &req, + const std::vector &in_grad, + const std::vector &aux_args) { + using namespace mshadow; + CHECK_EQ(in_data.size(), 2); + CHECK_EQ(out_data.size(), 1); + CHECK_EQ(out_grad[psroipool::kOut].shape_[0], in_data[psroipool::kBox].shape_[0]); + CHECK_NE(req[psroipool::kData], kWriteInplace) << + "ROIPooling: Backward doesn't support kWriteInplace."; + CHECK_NE(req[psroipool::kBox], kWriteInplace) << + "ROIPooling: Backward doesn't support kWriteInplace."; + Stream *s = ctx.get_stream(); + + Tensor grad_out = out_grad[psroipool::kOut].get(s); + Tensor bbox = in_data[psroipool::kBox].get(s); + Tensor grad_in = in_grad[psroipool::kData].get(s); + Tensor grad_roi = in_grad[psroipool::kBox].get(s); + + CHECK_EQ(grad_out.CheckContiguous(), true); + CHECK_EQ(bbox.CheckContiguous(), true); + CHECK_EQ(grad_in.CheckContiguous(), true); + + if (kAddTo == req[psroipool::kData] || kWriteTo == req[psroipool::kData]) { + if (kWriteTo == req[psroipool::kData]) { + grad_in = 0.0f; + } + PSROIPoolBackwardAcc(grad_in, grad_out, bbox, param_.spatial_scale, + param_.output_dim, param_.group_size); + } + if (kWriteTo == req[psroipool::kBox]) { + grad_roi = 0.0f; + } + } + + private: + PSROIPoolingParam param_; +}; // class PSROIPoolingOp + +// Decalre Factory function, used for dispatch specialization +template +Operator* CreateOp(PSROIPoolingParam param, int dtype); + +#if DMLC_USE_CXX11 +class PSROIPoolingProp : public OperatorProperty { + public: + std::vector ListArguments() const override { + return {"data", "rois"}; + } + + std::vector ListOutputs() const override { + return {"output"}; + } + + int NumOutputs() const override { + return 1; + } + + int NumVisibleOutputs() const override { + return 1; + } + + void Init(const std::vector >& kwargs) override { + param_.Init(kwargs); + if (param_.group_size == 0) { + param_.group_size = param_.pooled_size; + } + } + + std::map GetParams() const override { + return param_.__DICT__(); + } + + bool InferShape(std::vector *in_shape, + std::vector *out_shape, + std::vector *aux_shape) const override { + using namespace mshadow; + CHECK_EQ(in_shape->size(), 2) << "Input:[data, rois]"; + + // data: [batch_size, c, h, w] + TShape dshape = in_shape->at(psroipool::kData); + CHECK_EQ(dshape.ndim(), 4) << "data should be a 4D tensor"; + + // bbox: [num_rois, 5] + TShape bshape = in_shape->at(psroipool::kBox); + CHECK_EQ(bshape.ndim(), 2) << "bbox should be a 2D tensor of shape [batch, 5]"; + CHECK_EQ(bshape[1], 5) << "bbox should be a 2D tensor of shape [batch, 5]"; + + // out: [num_rois, c, pooled_h, pooled_w] + out_shape->clear(); + out_shape->push_back( + Shape4(bshape[0], param_.output_dim, param_.pooled_size, param_.pooled_size)); + return true; + } + + bool InferType(std::vector *in_type, + std::vector *out_type, + std::vector *aux_type) const override { + CHECK_EQ(in_type->size(), 2); + int dtype = (*in_type)[0]; + CHECK_EQ(dtype, (*in_type)[1]); + CHECK_NE(dtype, -1) << "Input must have specified type"; + + out_type->clear(); + out_type->push_back(dtype); + return true; + } + + OperatorProperty* Copy() const override { + PSROIPoolingProp* psroi_pooling_sym = new PSROIPoolingProp(); + psroi_pooling_sym->param_ = this->param_; + return psroi_pooling_sym; + } + + std::string TypeString() const override { + return "_contrib_PSROIPooling"; + } + + // decalre dependency and inplace optimization options + std::vector DeclareBackwardDependency( + const std::vector &out_grad, + const std::vector &in_data, + const std::vector &out_data) const override { + return {out_grad[psroipool::kOut], in_data[psroipool::kBox]}; + } + + + Operator* CreateOperator(Context ctx) const override { + LOG(FATAL) << "Not Implemented."; + return NULL; + } + + Operator* CreateOperatorEx(Context ctx, std::vector *in_shape, + std::vector *in_type) const override; + + + private: + PSROIPoolingParam param_; +}; // class PSROIPoolingProp +#endif +} // namespace op +} // namespace mxnet +#endif // MXNET_OPERATOR_CONTRIB_PSROI_POOLING_INL_H_ diff --git a/src/operator/contrib/psroi_pooling.cc b/src/operator/contrib/psroi_pooling.cc new file mode 100644 index 000000000000..ad25aec8eee8 --- /dev/null +++ b/src/operator/contrib/psroi_pooling.cc @@ -0,0 +1,80 @@ +/*! + * Copyright (c) 2017 by Contributors + * Copyright (c) 2017 Microsoft + * Licensed under The Apache-2.0 License [see LICENSE for details] + * \file psroi_pooling.cc + * \brief psroi pooling operator + * \author Yi Li, Tairui Chen, Guodong Zhang, Haozhi Qi, Jifeng Dai +*/ +#include "./psroi_pooling-inl.h" +#include +#include +#include +#include +#include + +using std::max; +using std::min; +using std::floor; +using std::ceil; + +namespace mshadow { +template +inline void PSROIPoolForward(const Tensor &out, + const Tensor &data, + const Tensor &bbox, + const float spatial_scale_, + const int output_dim_, + const int group_size_) { + // NOT_IMPLEMENTED; + return; +} + +template +inline void PSROIPoolBackwardAcc(const Tensor &in_grad, + const Tensor &out_grad, + const Tensor &bbox, + const float spatial_scale_, + const int output_dim_, + const int group_size_) { + // NOT_IMPLEMENTED; + return; +} +} // namespace mshadow + +namespace mxnet { +namespace op { + +template<> +Operator *CreateOp(PSROIPoolingParam param, int dtype) { + Operator* op = NULL; + MSHADOW_REAL_TYPE_SWITCH(dtype, DType, { + op = new PSROIPoolingOp(param); + }); + return op; +} + +Operator *PSROIPoolingProp::CreateOperatorEx(Context ctx, std::vector *in_shape, + std::vector *in_type) const { + std::vector out_shape, aux_shape; + std::vector out_type, aux_type; + CHECK(InferType(in_type, &out_type, &aux_type)); + CHECK(InferShape(in_shape, &out_shape, &aux_shape)); + DO_BIND_DISPATCH(CreateOp, param_, in_type->at(0)); +} + +DMLC_REGISTER_PARAMETER(PSROIPoolingParam); + +MXNET_REGISTER_OP_PROPERTY(_contrib_PSROIPooling, PSROIPoolingProp) +.describe("Performs region-of-interest pooling on inputs. Resize bounding box coordinates by " +"spatial_scale and crop input feature maps accordingly. The cropped feature maps are pooled " +"by max pooling to a fixed size output indicated by pooled_size. batch_size will change to " +"the number of region bounding boxes after PSROIPooling") +.add_argument("data", "Symbol", "Input data to the pooling operator, a 4D Feature maps") +.add_argument("rois", "Symbol", "Bounding box coordinates, a 2D array of " +"[[batch_index, x1, y1, x2, y2]]. (x1, y1) and (x2, y2) are top left and down right corners " +"of designated region of interest. batch_index indicates the index of corresponding image " +"in the input data") +.add_arguments(PSROIPoolingParam::__FIELDS__()); +} // namespace op +} // namespace mxnet diff --git a/src/operator/contrib/psroi_pooling.cu b/src/operator/contrib/psroi_pooling.cu new file mode 100644 index 000000000000..962c874c6d1a --- /dev/null +++ b/src/operator/contrib/psroi_pooling.cu @@ -0,0 +1,260 @@ +/*! + * Copyright (c) 2017 by Contributors + * Copyright (c) 2017 Microsoft + * Licensed under The Apache-2.0 License [see LICENSE for details] + * \file psroi_pooling.cu + * \brief psroi pooling operator + * \author Yi Li, Tairui Chen, Guodong Zhang, Haozhi Qi, Jifeng Dai +*/ +#include "./psroi_pooling-inl.h" +#include +#include +#include +#include +#include "../../common/cuda_utils.h" +#include "../mxnet_op.h" + +#define PSROIPOOLING_CUDA_CHECK(condition) \ + /* Code block avoids redefinition of cudaError_t error */ \ + do { \ + cudaError_t error = condition; \ + CHECK_EQ(error, cudaSuccess) << " " << cudaGetErrorString(error); \ + } while (0) +#define CUDA_KERNEL_LOOP(i, n) \ +for (int i = blockIdx.x * blockDim.x + threadIdx.x; \ + i < (n); \ + i += blockDim.x * gridDim.x) + +namespace mshadow { +namespace cuda { + +template +__global__ void PSROIPoolForwardKernel( + const int count, + const DType* bottom_data, + const DType spatial_scale, + const int channels, + const int height, const int width, + const int pooled_height, const int pooled_width, + const DType* bottom_rois, + const int output_dim, + const int group_size, + DType* top_data) { + CUDA_KERNEL_LOOP(index, count) { + // The output is in order (n, ctop, ph, pw) + int pw = index % pooled_width; + int ph = (index / pooled_width) % pooled_height; + int ctop = (index / pooled_width / pooled_height) % output_dim; + int n = index / pooled_width / pooled_height / output_dim; + + // [start, end) interval for spatial sampling + const DType* offset_bottom_rois = bottom_rois + n * 5; + int roi_batch_ind = offset_bottom_rois[0]; + DType roi_start_w = static_cast(round(offset_bottom_rois[1])) * spatial_scale; + DType roi_start_h = static_cast(round(offset_bottom_rois[2])) * spatial_scale; + DType roi_end_w = static_cast(round(offset_bottom_rois[3]) + 1.) * spatial_scale; + DType roi_end_h = static_cast(round(offset_bottom_rois[4]) + 1.) * spatial_scale; + + // Force too small ROIs to be 1x1 + DType roi_width = max(roi_end_w - roi_start_w, 0.1); // avoid 0 + DType roi_height = max(roi_end_h - roi_start_h, 0.1); + + // Compute w and h at bottom + DType bin_size_h = roi_height / static_cast(pooled_height); + DType bin_size_w = roi_width / static_cast(pooled_width); + + int hstart = floor(static_cast(ph) * bin_size_h + + roi_start_h); + int wstart = floor(static_cast(pw)* bin_size_w + + roi_start_w); + int hend = ceil(static_cast(ph + 1) * bin_size_h + + roi_start_h); + int wend = ceil(static_cast(pw + 1) * bin_size_w + + roi_start_w); + // Add roi offsets and clip to input boundaries + hstart = min(max(hstart, 0), height); + hend = min(max(hend, 0), height); + wstart = min(max(wstart, 0), width); + wend = min(max(wend, 0), width); + bool is_empty = (hend <= hstart) || (wend <= wstart); + + int gw = floor(static_cast(pw)* group_size / pooled_width); + int gh = floor(static_cast(ph)* group_size / pooled_height); + gw = min(max(gw, 0), group_size - 1); + gh = min(max(gh, 0), group_size - 1); + int c = (ctop*group_size + gh)*group_size + gw; + + const DType* offset_bottom_data = bottom_data + (roi_batch_ind * channels + c) * height * width; + DType out_sum = 0; + for (int h = hstart; h < hend; ++h) { + for (int w = wstart; w < wend; ++w) { + int bottom_index = h*width + w; + out_sum += offset_bottom_data[bottom_index]; + } + } + + DType bin_area = (hend - hstart)*(wend - wstart); + top_data[index] = is_empty? (DType)0. : out_sum/bin_area; + } +} + +template +inline void PSROIPoolForward(const Tensor &out, + const Tensor &data, + const Tensor &bbox, + const float spatial_scale, + const int output_dim_, + const int group_size_) { + const DType *bottom_data = data.dptr_; + const DType *bottom_rois = bbox.dptr_; + DType *top_data = out.dptr_; + const int count = out.shape_.Size(); + const int channels = data.size(1); + const int height = data.size(2); + const int width = data.size(3); + const int pooled_height = out.size(2); + const int pooled_width = out.size(3); + cudaStream_t stream = Stream::GetStream(out.stream_); + PSROIPoolForwardKernel << > >( + count, bottom_data, spatial_scale, channels, height, width, + pooled_height, pooled_width, bottom_rois, output_dim_, group_size_, top_data); + PSROIPOOLING_CUDA_CHECK(cudaPeekAtLastError()); +} + + +template +__global__ void PSROIPoolBackwardAccKernel( + const int count, + const DType* top_diff, + const int num_rois, + const DType spatial_scale, + const int channels, + const int height, const int width, + const int pooled_height, const int pooled_width, + const int group_size, + const int output_dim, + DType* bottom_diff, + const DType* bottom_rois) { + CUDA_KERNEL_LOOP(index, count) { + // The output is in order (n, ctop, ph, pw) + int pw = index % pooled_width; + int ph = (index / pooled_width) % pooled_height; + int ctop = (index / pooled_width / pooled_height) % output_dim; + int n = index / pooled_width / pooled_height / output_dim; + + // [start, end) interval for spatial sampling + const DType* offset_bottom_rois = bottom_rois + n * 5; + int roi_batch_ind = offset_bottom_rois[0]; + DType roi_start_w = static_cast(round(offset_bottom_rois[1])) * spatial_scale; + DType roi_start_h = static_cast(round(offset_bottom_rois[2])) * spatial_scale; + DType roi_end_w = static_cast(round(offset_bottom_rois[3]) + 1.) * spatial_scale; + DType roi_end_h = static_cast(round(offset_bottom_rois[4]) + 1.) * spatial_scale; + + // Force too small ROIs to be 1x1 + DType roi_width = max(roi_end_w - roi_start_w, 0.1); // avoid 0 + DType roi_height = max(roi_end_h - roi_start_h, 0.1); + + // Compute w and h at bottom + DType bin_size_h = roi_height / static_cast(pooled_height); + DType bin_size_w = roi_width / static_cast(pooled_width); + + int hstart = floor(static_cast(ph)* bin_size_h + + roi_start_h); + int wstart = floor(static_cast(pw)* bin_size_w + + roi_start_w); + int hend = ceil(static_cast(ph + 1) * bin_size_h + + roi_start_h); + int wend = ceil(static_cast(pw + 1) * bin_size_w + + roi_start_w); + // Add roi offsets and clip to input boundaries + hstart = min(max(hstart, 0), height); + hend = min(max(hend, 0), height); + wstart = min(max(wstart, 0), width); + wend = min(max(wend, 0), width); + bool is_empty = (hend <= hstart) || (wend <= wstart); + + // Compute c at bottom + int gw = floor(static_cast(pw)* group_size / pooled_width); + int gh = floor(static_cast(ph)* group_size / pooled_height); + gw = min(max(gw, 0), group_size - 1); + gh = min(max(gh, 0), group_size - 1); + int c = (ctop*group_size + gh)*group_size + gw; + DType* offset_bottom_diff = bottom_diff + (roi_batch_ind * channels + c) * height * width; + DType bin_area = (hend - hstart)*(wend - wstart); + DType diff_val = is_empty ? (DType)0. : top_diff[index] / bin_area; + for (int h = hstart; h < hend; ++h) { + for (int w = wstart; w < wend; ++w) { + int bottom_index = h*width + w; + atomicAdd(offset_bottom_diff + bottom_index, diff_val); + } + } + } +} + + +template +inline void PSROIPoolBackwardAcc(const Tensor &in_grad, + const Tensor &out_grad, + const Tensor &bbox, + const float spatial_scale, + const int output_dim_, + const int group_size_) { + // LOG(INFO) << "PSROIPoolBackward"; + const DType *top_diff = out_grad.dptr_; + const DType *bottom_rois = bbox.dptr_; + DType *bottom_diff = in_grad.dptr_; + const int count = out_grad.shape_.Size(); + const int num_rois = bbox.size(0); + const int channels = in_grad.size(1); + const int height = in_grad.size(2); + const int width = in_grad.size(3); + const int pooled_height = out_grad.size(2); + const int pooled_width = out_grad.size(3); + cudaStream_t stream = Stream::GetStream(in_grad.stream_); + PSROIPoolBackwardAccKernel << > >( + count, top_diff, num_rois, spatial_scale, channels, height, width, + pooled_height, pooled_width, group_size_, output_dim_, bottom_diff, bottom_rois); + PSROIPOOLING_CUDA_CHECK(cudaPeekAtLastError()); +} + +} // namespace cuda + +template +inline void PSROIPoolForward(const Tensor &out, + const Tensor &data, + const Tensor &bbox, + const float spatial_scale, + const int output_dim_, + const int group_size_) { + cuda::PSROIPoolForward(out, data, bbox, spatial_scale, output_dim_, group_size_); +} + +template +inline void PSROIPoolBackwardAcc(const Tensor &in_grad, + const Tensor &out_grad, + const Tensor &bbox, + const float spatial_scale, + const int output_dim_, + const int group_size_) { + cuda::PSROIPoolBackwardAcc(in_grad, out_grad, bbox, spatial_scale, output_dim_, group_size_); +} + +} // namespace mshadow + + +namespace mxnet { +namespace op { + +template<> +Operator* CreateOp(PSROIPoolingParam param, int dtype) { + Operator* op = NULL; + MSHADOW_REAL_TYPE_SWITCH(dtype, DType, { + op = new PSROIPoolingOp(param); + }); + return op; +} + +} // namespace op +} // namespace mxnet diff --git a/tests/python/gpu/test_operator_gpu.py b/tests/python/gpu/test_operator_gpu.py index 48e44133216b..4b884f523789 100644 --- a/tests/python/gpu/test_operator_gpu.py +++ b/tests/python/gpu/test_operator_gpu.py @@ -1093,6 +1093,166 @@ def test_unfuse(): check_rnn_consistency(fused, stack) check_rnn_consistency(stack, fused) +def test_psroipooling_with_type(): + np.random.seed(1234) + arg_params = { + 'psroipool_rois': np.array([[0, 10, 22, 161, 173], [0, 20, 15, 154, 160]])} + + # plain psroipooling + sym = mx.contrib.sym.PSROIPooling(spatial_scale=0.0625, output_dim=2, pooled_size=3, name='psroipool') + ctx_list = [{'ctx': mx.gpu(0), + 'psroipool_data': (1, 18, 14, 14), + 'psroipool_rois': (2, 5), + 'type_dict': {'psroipool_data': np.float64, 'psroipool_rois': np.float64}}, + {'ctx': mx.gpu(0), + 'psroipool_data': (1, 18, 14, 14), + 'psroipool_rois': (2, 5), + 'type_dict': {'psroipool_data': np.float32, 'psroipool_rois': np.float32}}, + {'ctx': mx.gpu(0), + 'psroipool_data': (1, 18, 14, 14), + 'psroipool_rois': (2, 5), + 'type_dict': {'psroipool_data': np.float16, 'psroipool_rois': np.float16}}, + ] + + check_consistency(sym, ctx_list, grad_req={'psroipool_data': 'write', + 'psroipool_rois': 'null'}, arg_params=arg_params) + +def test_deformable_psroipooling_with_type(): + np.random.seed(1234) + arg_params = { + 'deformable_psroipool_rois': np.array([[0, 10, 22, 161, 173], [0, 20, 15, 154, 160]])} + + # deformable psroipooling + sym = mx.contrib.sym.DeformablePSROIPooling(spatial_scale=0.0625, sample_per_part=4, group_size=3, pooled_size=3, + output_dim=2, trans_std=0.1, no_trans=False, name='deformable_psroipool') + + ctx_list = [{'ctx': mx.gpu(0), + 'deformable_psroipool_data': (1, 18, 14, 14), + 'deformable_psroipool_rois': (2, 5), + 'deformable_psroipool_trans': (2, 4, 3, 3), + 'type_dict': {'deformable_psroipool_data': np.float64, 'deformable_psroipool_rois': np.float64, + 'deformable_psroipool_trans': np.float64}}, + {'ctx': mx.gpu(0), + 'deformable_psroipool_data': (1, 18, 14, 14), + 'deformable_psroipool_rois': (2, 5), + 'deformable_psroipool_trans': (2, 4, 3, 3), + 'type_dict': {'deformable_psroipool_data': np.float32, 'deformable_psroipool_rois': np.float32, + 'deformable_psroipool_trans': np.float32}}, + {'ctx': mx.gpu(0), + 'deformable_psroipool_data': (1, 18, 14, 14), + 'deformable_psroipool_rois': (2, 5), + 'deformable_psroipool_trans': (2, 4, 3, 3), + 'type_dict': {'deformable_psroipool_data': np.float16, 'deformable_psroipool_rois': np.float16, + 'deformable_psroipool_trans': np.float16}}, + ] + + check_consistency(sym, ctx_list, grad_req={'deformable_psroipool_data': 'write', + 'deformable_psroipool_rois': 'null', + 'deformable_psroipool_trans': 'write'}, arg_params=arg_params) + +def test_deformable_convolution_with_type(): + np.random.seed(1234) + sym = mx.contrib.sym.DeformableConvolution(num_filter=3, kernel=(3,3), name='deformable_conv') + # since atomicAdd does not support fp16 (which deformable conv uses in backward), we do not test fp16 here + ctx_list = [{'ctx': mx.gpu(0), + 'deformable_conv_data': (2, 2, 10, 10), + 'deformable_conv_offset': (2, 18, 8, 8), + 'type_dict': {'deformable_conv_data': np.float64, 'deformable_conv_offset': np.float64}}, + {'ctx': mx.gpu(0), + 'deformable_conv_data': (2, 2, 10, 10), + 'deformable_conv_offset': (2, 18, 8, 8), + 'type_dict': {'deformable_conv_data': np.float32, 'deformable_conv_offset': np.float32}}, + # {'ctx': mx.gpu(0), + # 'deformable_conv_data': (2, 2, 10, 10), + # 'deformable_conv_offset': (2, 18, 8, 8), + # 'type_dict': {'deformable_conv_data': np.float16, 'deformable_conv_offset': np.float16}}, + ] + # wider tolerance needed for true-fp16 NCHW test above + tol = {np.dtype(np.float16): 0.5, + np.dtype(np.float32): 1e-3, + np.dtype(np.float64): 1e-5, + np.dtype(np.uint8): 0, + np.dtype(np.int32): 0} + check_consistency(sym, ctx_list, tol=tol) + # test ability to turn off training on bias + check_consistency(sym, ctx_list, grad_req={'deformable_conv_data': 'write', + 'deformable_conv_offset': 'write', + 'deformable_conv_weight': 'write', + 'deformable_conv_bias': 'null'}, tol=tol) +def test_deformable_convolution_options(): + # 2D convolution + + # Pad > 0 + # since atomicAdd does not support fp16 (which deformable conv uses in backward), we do not test fp16 here + ctx_list = [{'ctx': mx.gpu(0), + 'deformable_conv_data': (2, 2, 7, 7), + 'deformable_conv_offset': (2, 18, 7, 7), + 'type_dict': {'deformable_conv_data': np.float64, 'deformable_conv_offset': np.float64}}, + {'ctx': mx.gpu(0), + 'deformable_conv_data': (2, 2, 7, 7), + 'deformable_conv_offset': (2, 18, 7, 7), + 'type_dict': {'deformable_conv_data': np.float32, 'deformable_conv_offset': np.float32}}, + # {'ctx': mx.gpu(0), + # 'deformable_conv_data': (2, 2, 7, 7), + # 'deformable_offset': (2, 18, 7, 7), + # 'type_dict': {'deformable_conv_data': np.float16, 'deformable_offset': np.float16}}, + ] + sym = mx.contrib.sym.DeformableConvolution(num_filter=3, kernel=(3,3), pad=(1,1), name='deformable_conv') + check_consistency(sym, ctx_list) + + # Stride > 1 + # since atomicAdd does not support fp16 (which deformable conv uses in backward), we do not test fp16 here + ctx_list = [{'ctx': mx.gpu(0), + 'deformable_conv_data': (2, 2, 7, 7), + 'deformable_conv_offset': (2, 18, 3, 3), + 'type_dict': {'deformable_conv_data': np.float64, 'deformable_conv_offset': np.float64}}, + {'ctx': mx.gpu(0), + 'deformable_conv_data': (2, 2, 7, 7), + 'deformable_conv_offset': (2, 18, 3, 3), + 'type_dict': {'deformable_conv_data': np.float32, 'deformable_conv_offset': np.float32}}, + # {'ctx': mx.gpu(0), + # 'deformable_conv_data': (2, 2, 7, 7), + # 'deformable_conv_offset': (2, 18, 3, 3), + # 'type_dict': {'deformable_conv_data': np.float16, 'deformable_offset': np.float16}}, + ] + sym = mx.contrib.sym.DeformableConvolution(num_filter=3, kernel=(3,3), stride=(2,2), name='deformable_conv') + check_consistency(sym, ctx_list) + + # Dilate > 1 + # since atomicAdd does not support fp16 (which deformable conv uses in backward), we do not test fp16 here + ctx_list = [{'ctx': mx.gpu(0), + 'deformable_conv_data': (2, 2, 7, 7), + 'deformable_conv_offset': (2, 18, 3, 3), + 'type_dict': {'deformable_conv_data': np.float64, 'deformable_conv_offset': np.float64}}, + {'ctx': mx.gpu(0), + 'deformable_conv_data': (2, 2, 7, 7), + 'deformable_conv_offset': (2, 18, 3, 3), + 'type_dict': {'deformable_conv_data': np.float32, 'deformable_conv_offset': np.float32}}, + # {'ctx': mx.gpu(0), + # 'deformable_conv_data': (2, 2, 7, 7), + # 'deformable_conv_offset': (2, 18, 3, 3), + # 'type_dict': {'deformable_conv_data': np.float16, 'deformable_offset': np.float16}}, + ] + sym = mx.contrib.sym.DeformableConvolution(num_filter=3, kernel=(3,3), dilate=(2,2), name='deformable_conv') + check_consistency(sym, ctx_list) + + # Deformable group > 1 + # since atomicAdd does not support fp16 (which deformable conv uses in backward), we do not test fp16 here + ctx_list = [{'ctx': mx.gpu(0), + 'deformable_conv_data': (2, 2, 7, 7), + 'deformable_conv_offset': (2, 36, 5, 5), + 'type_dict': {'deformable_conv_data': np.float64, 'deformable_conv_offset': np.float64}}, + {'ctx': mx.gpu(0), + 'deformable_conv_data': (2, 2, 7, 7), + 'deformable_conv_offset': (2, 36, 5, 5), + 'type_dict': {'deformable_conv_data': np.float32, 'deformable_conv_offset': np.float32}}, + # {'ctx': mx.gpu(0), + # 'deformable_conv_data': (2, 2, 7, 7), + # 'deformable_conv_offset': (2, 36, 5, 5), + # 'type_dict': {'deformable_conv_data': np.float16, 'deformable_offset': np.float16}}, + ] + sym = mx.contrib.sym.DeformableConvolution(num_filter=4, kernel=(3,3), num_deformable_group=2, + name='deformable_conv') def test_residual_fused(): cell = mx.rnn.ResidualCell( mx.rnn.FusedRNNCell(50, num_layers=3, mode='lstm', @@ -1144,3 +1304,8 @@ def test_residual_fused(): test_take_with_type() test_bilinear_sampler_with_type() test_grid_generator_with_type() + test_psroipooling_with_type() + test_deformable_psroipooling_with_type() + test_deformable_convolution_options() + test_deformable_convolution_with_type() + diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py index 79795e9386b6..2cdac27c3edb 100644 --- a/tests/python/unittest/test_operator.py +++ b/tests/python/unittest/test_operator.py @@ -2,6 +2,7 @@ import numpy as np import mxnet as mx import random +import itertools from numpy.testing import assert_allclose from mxnet.test_utils import * @@ -3160,6 +3161,101 @@ def create_operator(self, ctx, shapes, dtypes): check_numeric_gradient(op, [x]) +def test_psroipooling(): + for num_rois in [1, 2]: + for num_classes, num_group in itertools.product([2, 3], [2, 3]): + for image_height, image_width in itertools.product([168, 224], [168, 224]): + for grad_nodes in [['im_data']]: + spatial_scale = 0.0625 + feat_height = np.int(image_height * spatial_scale) + feat_width = np.int(image_width * spatial_scale) + im_data = np.random.rand(1, num_classes*num_group*num_group, feat_height, feat_width) + rois_data = np.zeros([num_rois, 5]) + rois_data[:, [1,3]] = np.sort(np.random.rand(num_rois, 2)*(image_width-1)) + rois_data[:, [2,4]] = np.sort(np.random.rand(num_rois, 2)*(image_height-1)) + + im_data_var = mx.symbol.Variable(name="im_data") + rois_data_var = mx.symbol.Variable(name="rois_data") + op = mx.contrib.sym.PSROIPooling(data=im_data_var, rois=rois_data_var, spatial_scale=spatial_scale, + group_size=num_group, pooled_size=num_group, + output_dim=num_classes, name='test_op') + rtol, atol = 1e-2, 1e-4 + # By now we only have gpu implementation + if mx.Context.default_ctx.device_type == 'gpu': + check_numeric_gradient(op, [im_data, rois_data], rtol=rtol, atol=atol, + grad_nodes=grad_nodes, ctx=mx.gpu(0)) + +def test_deformable_convolution(): + for num_batch in [1, 2]: + for num_channel_data, num_deformable_group in itertools.product([4, 8], [1, 2]): + for input_height, input_width in itertools.product([5, 6], [5, 6]): + for dilate in [(1, 1), (2, 2)]: + for grad_nodes in [['im_data'], ['offset_data']]: + output_height = input_height + output_width = input_width + im_data = np.random.rand(num_batch, num_channel_data, input_height, input_width) + offset_data = \ + np.random.rand(num_batch, num_deformable_group * 3 * 3 * 2, output_height, output_width)\ + * 0.8 + 0.1 + + weight = np.random.normal(0, 0.001, (num_channel_data, num_channel_data, 3, 3)) + bias = np.zeros(num_channel_data) + + im_data_var = mx.symbol.Variable(name="im_data") + offset_data_var = mx.symbol.Variable(name="offset_data") + weight_var = mx.symbol.Variable(name="weight") + bias_var = mx.symbol.Variable(name="bias") + op = mx.contrib.sym.DeformableConvolution(name='test_op', data=im_data_var, + offset=offset_data_var, + weight=weight_var, bias=bias_var, + num_filter=num_channel_data, pad=dilate, + kernel=(3, 3), stride=(1, 1), dilate=dilate, + num_deformable_group=num_deformable_group) + if grad_nodes[0] == 'offset_data': + # wider tolerance needed for coordinate differential + rtol, atol = 1.0, 1e-2 + else: + rtol, atol = 0.05, 1e-4 + # By now we only have gpu implementation + if mx.Context.default_ctx.device_type == 'gpu': + check_numeric_gradient(op, [im_data, offset_data, weight, bias], rtol=rtol, atol=atol, + grad_nodes=grad_nodes, ctx=mx.gpu(0)) + + +def test_deformable_psroipooling(): + for num_rois in [1, 2]: + for num_classes, num_group in itertools.product([2, 3], [2, 3]): + for image_height, image_width in itertools.product([168, 224], [168, 224]): + for grad_nodes in [['im_data'], ['offset_data']]: + spatial_scale = 0.0625 + feat_height = np.int(image_height * spatial_scale) + feat_width = np.int(image_width * spatial_scale) + im_data = np.random.rand(1, num_classes*num_group*num_group, feat_height, feat_width) + rois_data = np.zeros([num_rois, 5]) + rois_data[:, [1,3]] = np.sort(np.random.rand(num_rois, 2)*(image_width-1)) + rois_data[:, [2,4]] = np.sort(np.random.rand(num_rois, 2)*(image_height-1)) + offset_data = np.random.rand(num_rois, 2*num_classes, num_group, num_group) * 0.1 + + im_data_var = mx.symbol.Variable(name="im_data") + rois_data_var = mx.symbol.Variable(name="rois_data") + offset_data_var = mx.symbol.Variable(name="offset_data") + op = mx.contrib.sym.DeformablePSROIPooling(data=im_data_var, rois=rois_data_var, + trans=offset_data_var, spatial_scale=spatial_scale, + sample_per_part=4, group_size=num_group, + pooled_size=num_group, output_dim=num_classes, + trans_std=0.1, no_trans=False, name='test_op') + if grad_nodes[0] == 'offset_data': + # wider tolerance needed for coordinate differential + rtol, atol = 1.0, 1e-2 + else: + rtol, atol = 1e-2, 1e-4 + # By now we only have gpu implementation + if mx.Context.default_ctx.device_type == 'gpu': + check_numeric_gradient(op, [im_data, rois_data, offset_data], rtol=rtol, atol=atol, + grad_nodes=grad_nodes, ctx=mx.gpu(0)) + + + def test_laop(): # Temporarily disabled until lapack is enabled by default return @@ -3409,7 +3505,7 @@ def test_laop(): if grad_check == 1: check_numeric_gradient(test_sumlogdiag, [a]) - + if __name__ == '__main__': import nose nose.runmodule() From 56bd806c0c0dac5ee9673737d6d6386ecd96ce53 Mon Sep 17 00:00:00 2001 From: "Joshua Z. Zhang" Date: Sat, 17 Jun 2017 11:50:08 -0700 Subject: [PATCH 083/834] add verbose option to initializer (#6712) * add verbose option to initializer change verbosity name fix typo fix default fix lint remove lambda function for pickle * use print func directly --- python/mxnet/initializer.py | 45 +++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/python/mxnet/initializer.py b/python/mxnet/initializer.py index 708419950950..8a287d17374a 100755 --- a/python/mxnet/initializer.py +++ b/python/mxnet/initializer.py @@ -5,11 +5,13 @@ import logging import warnings import json +from math import sqrt import numpy as np from .base import string_types from .ndarray import NDArray, load from . import random from . import registry +from . import ndarray # inherit str for backward compatibility class InitDesc(str): @@ -35,6 +37,44 @@ class Initializer(object): """The base class of an initializer.""" def __init__(self, **kwargs): self._kwargs = kwargs + self._verbose = False + self._print_func = None + + def set_verbosity(self, verbose=False, print_func=None): + """Switch on/off verbose mode + + Parameters + ---------- + verbose : bool + switch on/off verbose mode + print_func : function + A function that computes statistics of initialized arrays. + Takes an `NDArray` and returns an `str`. Defaults to mean + absolute value str((|x|/size(x)).asscalar()). + """ + self._verbose = verbose + if print_func is None: + def asum_stat(x): + """returns |x|/size(x), async execution.""" + return str((ndarray.norm(x)/sqrt(x.size)).asscalar()) + print_func = asum_stat + self._print_func = print_func + return self + + def _verbose_print(self, desc, init, arr): + """Internal verbose print function + + Parameters + ---------- + desc : InitDesc or str + name of the array + init : str + initializer pattern + arr : NDArray + initialized array + """ + if self._verbose and self._print_func: + logging.info('Initialized %s as %s: %s', desc, init, self._print_func(arr)) def dumps(self): """Saves the initializer to string @@ -79,17 +119,22 @@ def __call__(self, desc, arr): if init: # when calling Variable initializer create(init)._init_weight(desc, arr) + self._verbose_print(desc, init, arr) else: # register nnvm::FSetInputVariableAttrs in the backend for new patterns # don't add new cases here. if desc.endswith('weight'): self._init_weight(desc, arr) + self._verbose_print(desc, 'weight', arr) elif desc.endswith('bias'): self._init_bias(desc, arr) + self._verbose_print(desc, 'bias', arr) elif desc.endswith('gamma'): self._init_gamma(desc, arr) + self._verbose_print(desc, 'gamma', arr) elif desc.endswith('beta'): self._init_beta(desc, arr) + self._verbose_print(desc, 'beta', arr) else: self._init_default(desc, arr) From 2debc9120f15814b7cc8fb2d43adf4d74025073d Mon Sep 17 00:00:00 2001 From: Soonhwan-Kwon Date: Sun, 18 Jun 2017 15:53:45 +0900 Subject: [PATCH 084/834] fix bug in optimizer because state is not synced with weight context (#6731) * fix bug in optimizer because state is not synced with weight context when load_optimizer_states is true * removed trailing whitespace in optimizer * fix if to elif in logical flow of sync_state_context function * fix sync_state_context as recursive * fix style in sync_state_context --- python/mxnet/optimizer.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/python/mxnet/optimizer.py b/python/mxnet/optimizer.py index 1f7b1d3aed1b..201393b5d625 100644 --- a/python/mxnet/optimizer.py +++ b/python/mxnet/optimizer.py @@ -751,6 +751,7 @@ def update(self, index, weight, grad, state): @register class Test(Optimizer): + """The Test optimizer""" def __init__(self, **kwargs): super(Test, self).__init__(**kwargs) @@ -771,16 +772,35 @@ class Updater(object): def __init__(self, optimizer): self.optimizer = optimizer self.states = {} + self.states_synced = {} def __call__(self, index, grad, weight): """Updates weight given gradient and index.""" if index not in self.states: self.states[index] = self.optimizer.create_state(index, weight) + self.states_synced[index] = True + elif not self.states_synced[index]: + self.states[index] = \ + self.sync_state_context(self.states[index], weight.context) + self.states_synced[index] = True self.optimizer.update(index, weight, grad, self.states[index]) + def sync_state_context(self, state, context): + if isinstance(state, NDArray): + return state.as_in_context(context) + elif isinstance(state, (tuple, list)): + synced_state = (self.sync_state_context(i, context) for i in state) + if isinstance(state, tuple): + return tuple(synced_state) + else: + return list(synced_state) + else: + return state + def set_states(self, states): """Sets updater states.""" self.states = pickle.loads(states) + self.states_synced = dict.fromkeys(self.states.keys(), False) def get_states(self): """Gets updater states.""" From 423490c03e6ee43aa4ad9980932d8af85010410b Mon Sep 17 00:00:00 2001 From: moin Date: Sun, 18 Jun 2017 14:23:29 -0700 Subject: [PATCH 085/834] enable use of lapack by default (#6704) * enable use of lapack by default * corrected search paths for lapack lib --- CMakeLists.txt | 1 + Makefile | 29 +++++++++++++++++++++----- include/mxnet/c_lapack_api.h | 6 ++++-- make/config.mk | 11 ++++++---- make/osx.mk | 8 +++---- make/pip_linux_cpu.mk | 12 ++++++----- tests/python/unittest/test_operator.py | 2 -- 7 files changed, 47 insertions(+), 22 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 73c55313acf9..4edecda47b6d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,6 +13,7 @@ mxnet_option(USE_OPENCV "Build with OpenCV support" ON) mxnet_option(USE_OPENMP "Build with Openmp support" ON) mxnet_option(USE_CUDA "Build with CUDA support" ON) mxnet_option(USE_CUDNN "Build with cudnn support" ON) # one could set CUDNN_ROOT for search path +mxnet_option(USE_LAPACK "Build with lapack support" ON) mxnet_option(USE_MKL_IF_AVAILABLE "Use MKL if found" ON) mxnet_option(USE_MKLML_MKL "Use MKLML variant of MKL (if MKL found)" ON IF USE_MKL_IF_AVAILABLE AND UNIX AND (NOT APPLE)) mxnet_option(USE_MKL_EXPERIMENTAL "Use experimental MKL (if MKL enabled and found)" OFF) diff --git a/Makefile b/Makefile index 98eaba00231c..7731e7e24d2a 100644 --- a/Makefile +++ b/Makefile @@ -106,16 +106,35 @@ else endif endif -# lapack settings. +# verify existence of separate lapack library when using blas/openblas/atlas +# switch off lapack support in case it can't be found +# issue covered with this +# - for Ubuntu 14.04 or lower, lapack is not automatically installed with openblas +# - for Ubuntu, installing atlas will not automatically install the atlas provided lapack library +# silently switching lapack off instead of letting the build fail because of backward compatibility ifeq ($(USE_LAPACK), 1) -ifeq ($(USE_BLAS),$(filter $(USE_BLAS),openblas apple atlas mkl)) - CFLAGS += -DMXNET_USE_LAPACK +ifeq ($(USE_BLAS),$(filter $(USE_BLAS),blas openblas atlas)) +ifeq (,$(wildcard /lib/liblapack.a)) +ifeq (,$(wildcard /usr/lib/liblapack.a)) +ifeq (,$(wildcard $(USE_LAPACK_PATH)/liblapack.a)) + USE_LAPACK = 0 +endif +endif endif -ifeq ($(USE_BLAS),$(filter $(USE_BLAS),openblas atlas mkl)) - LDFLAGS += -llapack endif endif +# lapack settings. +ifeq ($(USE_LAPACK), 1) + ifneq ($(USE_LAPACK_PATH), ) + LDFLAGS += -L$(USE_LAPACK_PATH) + endif + ifeq ($(USE_BLAS),$(filter $(USE_BLAS),blas openblas atlas)) + LDFLAGS += -llapack + endif + CFLAGS += -DMXNET_USE_LAPACK +endif + ifeq ($(USE_CUDNN), 1) CFLAGS += -DMSHADOW_USE_CUDNN=1 LDFLAGS += -lcudnn diff --git a/include/mxnet/c_lapack_api.h b/include/mxnet/c_lapack_api.h index 2e3ff49760b0..28f34ee20d54 100644 --- a/include/mxnet/c_lapack_api.h +++ b/include/mxnet/c_lapack_api.h @@ -71,9 +71,11 @@ inline char loup(char uplo, bool invert) { return invert ? (uplo == 'U' ? 'L' : MXNET_LAPACK_CWRAPPER1(dpotri, double) #else + // use pragma message instead of warning - #pragma message("Warning: lapack usage not enabled, linalg-operators will be not available." \ - " Build with USE_LAPACK=1 to get lapack functionalities.") + #pragma message("Warning: lapack usage not enabled, linalg-operators will not be available." \ + " Ensure that lapack library is installed and build with USE_LAPACK=1 to get lapack" \ + " functionalities.") // Define compilable stubs. #define MXNET_LAPACK_CWRAPPER1(func, dtype) \ diff --git a/make/config.mk b/make/config.mk index 5a0c64c62598..58668fb33d8e 100644 --- a/make/config.mk +++ b/make/config.mk @@ -65,10 +65,6 @@ USE_OPENCV = 1 # use openmp for parallelization USE_OPENMP = 1 -# whether use lapack during compilation -# only effective when compiled with blas versions openblas/apple/atlas/mkl -USE_LAPACK = 0 - # MKL ML Library for Intel CPU/Xeon Phi # Please refer to MKL_README.md for details @@ -97,6 +93,13 @@ else USE_BLAS = atlas endif +# whether use lapack during compilation +# only effective when compiled with blas versions openblas/apple/atlas/mkl +USE_LAPACK = 1 + +# path to lapack library in case of a non-standard installation +USE_LAPACK_PATH = + # add path to intel library, you may need it for MKL, if you did not add the path # to environment variable USE_INTEL_PATH = NONE diff --git a/make/osx.mk b/make/osx.mk index 01e50c2e76bb..7823b072a2ad 100644 --- a/make/osx.mk +++ b/make/osx.mk @@ -62,14 +62,14 @@ USE_OPENCV = 1 # use openmp for parallelization USE_OPENMP = 0 -# whether use lapack during compilation -# only effective when compiled with blas versions openblas/apple/atlas/mkl -USE_LAPACK = 0 - # choose the version of blas you want to use # can be: mkl, blas, atlas, openblas USE_BLAS = apple +# whether use lapack during compilation +# only effective when compiled with blas versions openblas/apple/atlas/mkl +USE_LAPACK = 1 + # add path to intel library, you may need it for MKL, if you did not add the path # to environment variable USE_INTEL_PATH = NONE diff --git a/make/pip_linux_cpu.mk b/make/pip_linux_cpu.mk index d7e5fa862246..01bc2702ebb7 100644 --- a/make/pip_linux_cpu.mk +++ b/make/pip_linux_cpu.mk @@ -29,16 +29,18 @@ ADD_CFLAGS += -Ldeps/lib -Ideps/include # matrix computation libraries for CPU/GPU #--------------------------------------------- -# whether use lapack during compilation -# only effective when compiled with blas versions openblas/apple/atlas/mkl -# you can disable it, however, you will not be able to use linalg-operators -USE_LAPACK = 0 - # choose the version of blas you want to use # can be: mkl, blas, atlas, openblas # in default use atlas for linux while apple for osx USE_BLAS=openblas +# whether use lapack during compilation +# only effective when compiled with blas versions openblas/apple/atlas/mkl +USE_LAPACK = 1 + +# path to lapack library in case of a non-standard installation +USE_LAPACK_PATH = + # whether use opencv during compilation # you can disable it, however, you will not able to use # imbin iterator diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py index 2cdac27c3edb..db50052ee60a 100644 --- a/tests/python/unittest/test_operator.py +++ b/tests/python/unittest/test_operator.py @@ -3257,8 +3257,6 @@ def test_deformable_psroipooling(): def test_laop(): - # Temporarily disabled until lapack is enabled by default - return # Currently no support for GPU. Will be added soon # so keep these tests here in this file and activate From 46e5dfb8f4142a724bb0f258199b5fd2b4725814 Mon Sep 17 00:00:00 2001 From: Hitoshi Mitake Date: Mon, 19 Jun 2017 16:23:24 +0900 Subject: [PATCH 086/834] example/svm_mnist: use module instead of model (#6738) Current example/svm_mnist is based on model, a deprecated API that causes a warning like this: ``` svm_mnist.py:71: DeprecationWarning: mxnet.model.FeedForward has been deprecated. Please use mxnet.mod.Module instead. ``` So this commit lets the example use module and removes the warning message. --- example/svm_mnist/svm_mnist.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/example/svm_mnist/svm_mnist.py b/example/svm_mnist/svm_mnist.py index d2925e94dfb3..177a927a4548 100644 --- a/example/svm_mnist/svm_mnist.py +++ b/example/svm_mnist/svm_mnist.py @@ -62,18 +62,22 @@ # Here we instatiate and fit the model for our data # The article actually suggests using 400 epochs, # But I reduced to 10, for convinience -model = mx.model.FeedForward( - ctx = mx.cpu(0), # Run on CPU 0 +mod = mx.mod.Module( + context = mx.cpu(0), # Run on CPU 0 symbol = mlp, # Use the network we just defined - num_epoch = 10, # Train for 10 epochs - learning_rate = 0.1, # Learning rate - momentum = 0.9, # Momentum for SGD with momentum - wd = 0.00001, # Weight decay for regularization - ) -model.fit( - X=train_iter, # Training data set + label_names = ['svm_label'], +) +mod.fit( + train_data=train_iter, eval_data=test_iter, # Testing data set. MXNet computes scores on test set every epoch - batch_end_callback = mx.callback.Speedometer(batch_size, 200)) # Logging module to print out progress + batch_end_callback = mx.callback.Speedometer(batch_size, 200), # Logging module to print out progress + num_epoch = 10, # Train for 10 epochs + optimizer_params = { + 'learning_rate': 0.1, # Learning rate + 'momentum': 0.9, # Momentum for SGD with momentum + 'wd': 0.00001, # Weight decay for regularization + }, +) # Uncomment to view an example # plt.imshow((X_show[0].reshape((28,28))*255).astype(np.uint8), cmap='Greys_r') @@ -81,4 +85,4 @@ # print 'Result:', model.predict(X_test[0:1])[0].argmax() # Now it prints how good did the network did for this configuration -print('Accuracy:', model.score(test_iter)*100, '%') \ No newline at end of file +print('Accuracy:', mod.score(test_iter, mx.metric.Accuracy())[0][1]*100, '%') From c281bd0d4274135ee4c3ce9718bba06b74e556fc Mon Sep 17 00:00:00 2001 From: Leonard Date: Tue, 20 Jun 2017 00:56:16 +0900 Subject: [PATCH 087/834] Remove executable bit from header and cu files in src/operator (#6748) --- src/operator/batch_norm-inl.h | 0 src/operator/batch_norm.cu | 0 src/operator/batch_norm_v1-inl.h | 0 src/operator/batch_norm_v1.cu | 0 src/operator/cudnn_batch_norm-inl.h | 0 src/operator/cudnn_batch_norm.cu | 0 src/operator/cudnn_lrn-inl.h | 0 src/operator/lrn-inl.h | 0 src/operator/lrn.cu | 0 src/operator/operator_common.h | 0 src/operator/optimizer_op-inl.h | 0 11 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 src/operator/batch_norm-inl.h mode change 100755 => 100644 src/operator/batch_norm.cu mode change 100755 => 100644 src/operator/batch_norm_v1-inl.h mode change 100755 => 100644 src/operator/batch_norm_v1.cu mode change 100755 => 100644 src/operator/cudnn_batch_norm-inl.h mode change 100755 => 100644 src/operator/cudnn_batch_norm.cu mode change 100755 => 100644 src/operator/cudnn_lrn-inl.h mode change 100755 => 100644 src/operator/lrn-inl.h mode change 100755 => 100644 src/operator/lrn.cu mode change 100755 => 100644 src/operator/operator_common.h mode change 100755 => 100644 src/operator/optimizer_op-inl.h diff --git a/src/operator/batch_norm-inl.h b/src/operator/batch_norm-inl.h old mode 100755 new mode 100644 diff --git a/src/operator/batch_norm.cu b/src/operator/batch_norm.cu old mode 100755 new mode 100644 diff --git a/src/operator/batch_norm_v1-inl.h b/src/operator/batch_norm_v1-inl.h old mode 100755 new mode 100644 diff --git a/src/operator/batch_norm_v1.cu b/src/operator/batch_norm_v1.cu old mode 100755 new mode 100644 diff --git a/src/operator/cudnn_batch_norm-inl.h b/src/operator/cudnn_batch_norm-inl.h old mode 100755 new mode 100644 diff --git a/src/operator/cudnn_batch_norm.cu b/src/operator/cudnn_batch_norm.cu old mode 100755 new mode 100644 diff --git a/src/operator/cudnn_lrn-inl.h b/src/operator/cudnn_lrn-inl.h old mode 100755 new mode 100644 diff --git a/src/operator/lrn-inl.h b/src/operator/lrn-inl.h old mode 100755 new mode 100644 diff --git a/src/operator/lrn.cu b/src/operator/lrn.cu old mode 100755 new mode 100644 diff --git a/src/operator/operator_common.h b/src/operator/operator_common.h old mode 100755 new mode 100644 diff --git a/src/operator/optimizer_op-inl.h b/src/operator/optimizer_op-inl.h old mode 100755 new mode 100644 From 5126bd50ce03350b973a9423557c469868e34d31 Mon Sep 17 00:00:00 2001 From: David Charte Date: Mon, 19 Jun 2017 17:59:19 +0200 Subject: [PATCH 088/834] Fix typos in installation guide (#6746) "python" -> "R" in R validation examples --- docs/get_started/install.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/get_started/install.md b/docs/get_started/install.md index 8f31d337ed22..edeb912ad4ef 100644 --- a/docs/get_started/install.md +++ b/docs/get_started/install.md @@ -1357,7 +1357,7 @@ array([[ 3., 3., 3.],
-Run a short *MXNet* python program to create a 2X3 matrix of ones, multiply each element in the matrix by 2 followed by adding 1. We expect the output to be a 2X3 matrix with all elements being 3. +Run a short *MXNet* R program to create a 2X3 matrix of ones, multiply each element in the matrix by 2 followed by adding 1. We expect the output to be a 2X3 matrix with all elements being 3. ```r library(mxnet) @@ -1376,7 +1376,7 @@ b
-Run a short *MXNet* python program to create a 2X3 matrix of ones *a* on a *GPU*, multiply each element in the matrix by 2 followed by adding 1. We expect the output to be a 2X3 matrix with all elements being 3. We use *mx.gpu()*, to set *MXNet* context to be GPUs. +Run a short *MXNet* R program to create a 2X3 matrix of ones *a* on a *GPU*, multiply each element in the matrix by 2 followed by adding 1. We expect the output to be a 2X3 matrix with all elements being 3. We use *mx.gpu()*, to set *MXNet* context to be GPUs. ```r library(mxnet) From 36b6fe55784f0b72a6dc685951854119ef956fba Mon Sep 17 00:00:00 2001 From: Zehao Shi Date: Mon, 19 Jun 2017 11:01:14 -0500 Subject: [PATCH 089/834] Fix pad example error (#6745) * Fix a spelling mistake. * FIX pad example --- src/operator/pad.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/operator/pad.cc b/src/operator/pad.cc index febd5207ff81..ded48c99f608 100644 --- a/src/operator/pad.cc +++ b/src/operator/pad.cc @@ -701,7 +701,7 @@ Example:: [ 20. 20. 21. 22. 22.] [ 20. 20. 21. 22. 22.]]]] - pad(x, mode="constant", constant_value=0, pad_width=(0,0,0,0,2,2,1,1)) = + pad(x, mode="constant", constant_value=0, pad_width=(0,0,0,0,1,1,1,1)) = [[[[ 0. 0. 0. 0. 0.] [ 0. 1. 2. 3. 0.] From 9d0508e2cd3ee6509c233566429c62eb573b63d0 Mon Sep 17 00:00:00 2001 From: Jonas Rauber Date: Mon, 19 Jun 2017 18:01:43 +0200 Subject: [PATCH 090/834] Update vgg.py (#6744) --- example/image-classification/symbols/vgg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/image-classification/symbols/vgg.py b/example/image-classification/symbols/vgg.py index a7ec3860784f..82bc5b7afa65 100644 --- a/example/image-classification/symbols/vgg.py +++ b/example/image-classification/symbols/vgg.py @@ -43,7 +43,7 @@ def get_symbol(num_classes, **kwargs): relu5_1 = mx.symbol.Activation(data=conv5_1, act_type="relu", name="relu5_1") conv5_2 = mx.symbol.Convolution( data=relu5_1, kernel=(3, 3), pad=(1, 1), num_filter=512, name="conv5_2") - relu5_2 = mx.symbol.Activation(data=conv5_2, act_type="relu", name="conv1_2") + relu5_2 = mx.symbol.Activation(data=conv5_2, act_type="relu", name="relu5_2") pool5 = mx.symbol.Pooling( data=relu5_2, pool_type="max", kernel=(2, 2), stride=(2,2), name="pool5") # group 6 From 82e8bda9761e28f09ade2202843cacff011655e7 Mon Sep 17 00:00:00 2001 From: Jonas Rauber Date: Mon, 19 Jun 2017 18:02:07 +0200 Subject: [PATCH 091/834] Update vgg.py (#6742) --- example/image-classification/symbols/vgg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/image-classification/symbols/vgg.py b/example/image-classification/symbols/vgg.py index 82bc5b7afa65..8dae74f1c94f 100644 --- a/example/image-classification/symbols/vgg.py +++ b/example/image-classification/symbols/vgg.py @@ -6,7 +6,7 @@ import mxnet as mx def get_symbol(num_classes, **kwargs): - ## define alexnet + ## define VGG11 data = mx.symbol.Variable(name="data") # group 1 conv1_1 = mx.symbol.Convolution(data=data, kernel=(3, 3), pad=(1, 1), num_filter=64, name="conv1_1") From 4d79edaca97945da7400b258bcca4a0d5e86d1af Mon Sep 17 00:00:00 2001 From: Stefan Date: Mon, 19 Jun 2017 15:38:02 -0700 Subject: [PATCH 092/834] Enable support for float64 dot operator. (#6756) * added float64 support for dot operator, incl unit tests * minor change in dot operator unit test * minor bug fix in dot operator unit test --- src/operator/tensor/matrix_op-inl.h | 414 +++++++++++++------------ tests/python/unittest/test_operator.py | 188 ++++++----- 2 files changed, 313 insertions(+), 289 deletions(-) diff --git a/src/operator/tensor/matrix_op-inl.h b/src/operator/tensor/matrix_op-inl.h index 9f48e0cf306c..a36bbbc8b0da 100644 --- a/src/operator/tensor/matrix_op-inl.h +++ b/src/operator/tensor/matrix_op-inl.h @@ -374,52 +374,52 @@ void DotForward_(const nnvm::NodeAttrs& attrs, << "Binary function only support input/output with the same type"; CHECK_EQ(outputs[0].type_flag_, inputs[1].type_flag_) << "Binary function only support input/output with the same type"; - CHECK_EQ(outputs[0].type_flag_, kFloat32) - << "dot only support 32 bit float so far"; - - if (inputs[0].ndim() == 1 && inputs[1].ndim() == 1) { - CHECK_NE(req[0], kAddTo) << "AddTo not yet suported"; - Tensor out = outputs[0].get(s); - VectorDot(out, - inputs[0].get(s), - inputs[1].get(s)); - } else { - int ma, na, mb, nb, m, n; - if (param.transpose_a) { - ma = inputs[0].size(0); - na = inputs[0].Size()/ma; - m = na; - } else { - na = inputs[0].size(inputs[0].ndim()-1); - ma = inputs[0].Size()/na; - m = ma; - } - if (param.transpose_b) { - nb = inputs[1].size(inputs[1].ndim()-1); - mb = inputs[1].Size()/nb; - n = mb; - } else { - mb = inputs[1].size(0); - nb = inputs[1].Size()/mb; - n = nb; - } - - Tensor input0 = - inputs[0].get_with_shape(Shape2(ma, na), s); - Tensor input1 = - inputs[1].get_with_shape(Shape2(mb, nb), s); - Tensor out = - outputs[0].get_with_shape(Shape2(m, n), s); - if (param.transpose_a && param.transpose_b) { - ASSIGN_DISPATCH(out, req[0], dot(input0.T(), input1.T())); - } else if (!param.transpose_a && param.transpose_b) { - ASSIGN_DISPATCH(out, req[0], dot(input0, input1.T())); - } else if (param.transpose_a && !param.transpose_b) { - ASSIGN_DISPATCH(out, req[0], dot(input0.T(), input1)); + CHECK(outputs[0].type_flag_ == kFloat32 || outputs[0].type_flag_ == kFloat64) + << "dot only supports float32 and float64"; + MSHADOW_TYPE_SWITCH(outputs[0].type_flag_, DType, { + if (inputs[0].ndim() == 1 && inputs[1].ndim() == 1) { + CHECK_NE(req[0], kAddTo) << "AddTo not yet suported"; + Tensor out = outputs[0].get(s); + VectorDot(out, + inputs[0].get(s), + inputs[1].get(s)); } else { - ASSIGN_DISPATCH(out, req[0], dot(input0, input1)); + int ma, na, mb, nb, m, n; + if (param.transpose_a) { + ma = inputs[0].size(0); + na = inputs[0].Size()/ma; + m = na; + } else { + na = inputs[0].size(inputs[0].ndim()-1); + ma = inputs[0].Size()/na; + m = ma; + } + if (param.transpose_b) { + nb = inputs[1].size(inputs[1].ndim()-1); + mb = inputs[1].Size()/nb; + n = mb; + } else { + mb = inputs[1].size(0); + nb = inputs[1].Size()/mb; + n = nb; + } + Tensor input0 = + inputs[0].get_with_shape(Shape2(ma, na), s); + Tensor input1 = + inputs[1].get_with_shape(Shape2(mb, nb), s); + Tensor out = + outputs[0].get_with_shape(Shape2(m, n), s); + if (param.transpose_a && param.transpose_b) { + ASSIGN_DISPATCH(out, req[0], dot(input0.T(), input1.T())); + } else if (!param.transpose_a && param.transpose_b) { + ASSIGN_DISPATCH(out, req[0], dot(input0, input1.T())); + } else if (param.transpose_a && !param.transpose_b) { + ASSIGN_DISPATCH(out, req[0], dot(input0.T(), input1)); + } else { + ASSIGN_DISPATCH(out, req[0], dot(input0, input1)); + } } - } + }); } template @@ -434,74 +434,76 @@ void DotBackward_(const nnvm::NodeAttrs& attrs, Stream *s = ctx.get_stream(); CHECK_NE(req[0], kWriteInplace); CHECK_NE(req[1], kWriteInplace); - - if (inputs[1].ndim() == 1 && inputs[2].ndim() == 1) { - Tensor mout_grad = inputs[0].get(s); - Tensor mlhs_data = inputs[1].get(s); - Tensor mrhs_data = inputs[2].get(s); - Tensor mlhs_grad = outputs[0].get(s); - Tensor mrhs_grad = outputs[1].get(s); - ASSIGN_DISPATCH(mrhs_grad, req[1], - broadcast_scalar(mout_grad, mlhs_data.shape_) * mlhs_data); - ASSIGN_DISPATCH(mlhs_grad, req[0], - broadcast_scalar(mout_grad, mlhs_data.shape_) * mrhs_data); - } else { - int ma, na, mb, nb, m, n; - if (param.transpose_a) { - ma = outputs[0].size(0); - na = outputs[0].Size()/ma; - m = na; - } else { - na = outputs[0].size(outputs[0].ndim()-1); - ma = outputs[0].Size()/na; - m = ma; - } - if (param.transpose_b) { - nb = outputs[1].size(outputs[1].ndim()-1); - mb = outputs[1].Size()/nb; - n = mb; - } else { - mb = outputs[1].size(0); - nb = outputs[1].Size()/mb; - n = nb; - } - - Tensor mout_grad = - inputs[0].get_with_shape(Shape2(m, n), s); - Tensor mlhs_data = - inputs[1].get_with_shape(Shape2(ma, na), s); - Tensor mrhs_data = - inputs[2].get_with_shape(Shape2(mb, nb), s); - Tensor mlhs_grad = - outputs[0].get_with_shape(Shape2(ma, na), s); - Tensor mrhs_grad = - outputs[1].get_with_shape(Shape2(mb, nb), s); - if (param.transpose_a && param.transpose_b) { - // Gradient of z = dot(x.T, y.T) - // dy = dot(x, dz).T = dot(dz.T, x.T) - // dx = dot(dz, y).T = dot(y.T, dz.T) - ASSIGN_DISPATCH(mrhs_grad, req[1], dot(mout_grad.T(), mlhs_data.T())); - ASSIGN_DISPATCH(mlhs_grad, req[0], dot(mrhs_data.T(), mout_grad.T())); - } else if (!param.transpose_a && param.transpose_b) { - // Gradient of z = dot(x, y.T) - // dy = dot(x.T, dz).T = dot(dz.T, x) - // dx = dot(dz, y) - ASSIGN_DISPATCH(mrhs_grad, req[1], dot(mout_grad.T(), mlhs_data)); - ASSIGN_DISPATCH(mlhs_grad, req[0], dot(mout_grad, mrhs_data)); - } else if (param.transpose_a && !param.transpose_b) { - // Gradient of z = dot(x.T, y) - // dy = dot(x, dz) - // dx = dot(dz, y.T).T = dot(y, dz.T) - ASSIGN_DISPATCH(mrhs_grad, req[1], dot(mlhs_data, mout_grad)); - ASSIGN_DISPATCH(mlhs_grad, req[0], dot(mrhs_data, mout_grad.T())); + CHECK(outputs[0].type_flag_ == kFloat32 || outputs[0].type_flag_ == kFloat64) + << "dot only supports float32 and float64"; + MSHADOW_TYPE_SWITCH(outputs[0].type_flag_, DType, { + if (inputs[1].ndim() == 1 && inputs[2].ndim() == 1) { + Tensor mout_grad = inputs[0].get(s); + Tensor mlhs_data = inputs[1].get(s); + Tensor mrhs_data = inputs[2].get(s); + Tensor mlhs_grad = outputs[0].get(s); + Tensor mrhs_grad = outputs[1].get(s); + ASSIGN_DISPATCH(mrhs_grad, req[1], + broadcast_scalar(mout_grad, mlhs_data.shape_) * mlhs_data); + ASSIGN_DISPATCH(mlhs_grad, req[0], + broadcast_scalar(mout_grad, mlhs_data.shape_) * mrhs_data); } else { - // Gradient of z = dot(x, y) - // dy = dot(x.T, dz) - // dx = dot(dz, y.T) - ASSIGN_DISPATCH(mrhs_grad, req[1], dot(mlhs_data.T(), mout_grad)); - ASSIGN_DISPATCH(mlhs_grad, req[0], dot(mout_grad, mrhs_data.T())); + int ma, na, mb, nb, m, n; + if (param.transpose_a) { + ma = outputs[0].size(0); + na = outputs[0].Size()/ma; + m = na; + } else { + na = outputs[0].size(outputs[0].ndim()-1); + ma = outputs[0].Size()/na; + m = ma; + } + if (param.transpose_b) { + nb = outputs[1].size(outputs[1].ndim()-1); + mb = outputs[1].Size()/nb; + n = mb; + } else { + mb = outputs[1].size(0); + nb = outputs[1].Size()/mb; + n = nb; + } + Tensor mout_grad = + inputs[0].get_with_shape(Shape2(m, n), s); + Tensor mlhs_data = + inputs[1].get_with_shape(Shape2(ma, na), s); + Tensor mrhs_data = + inputs[2].get_with_shape(Shape2(mb, nb), s); + Tensor mlhs_grad = + outputs[0].get_with_shape(Shape2(ma, na), s); + Tensor mrhs_grad = + outputs[1].get_with_shape(Shape2(mb, nb), s); + if (param.transpose_a && param.transpose_b) { + // Gradient of z = dot(x.T, y.T) + // dy = dot(x, dz).T = dot(dz.T, x.T) + // dx = dot(dz, y).T = dot(y.T, dz.T) + ASSIGN_DISPATCH(mrhs_grad, req[1], dot(mout_grad.T(), mlhs_data.T())); + ASSIGN_DISPATCH(mlhs_grad, req[0], dot(mrhs_data.T(), mout_grad.T())); + } else if (!param.transpose_a && param.transpose_b) { + // Gradient of z = dot(x, y.T) + // dy = dot(x.T, dz).T = dot(dz.T, x) + // dx = dot(dz, y) + ASSIGN_DISPATCH(mrhs_grad, req[1], dot(mout_grad.T(), mlhs_data)); + ASSIGN_DISPATCH(mlhs_grad, req[0], dot(mout_grad, mrhs_data)); + } else if (param.transpose_a && !param.transpose_b) { + // Gradient of z = dot(x.T, y) + // dy = dot(x, dz) + // dx = dot(dz, y.T).T = dot(y, dz.T) + ASSIGN_DISPATCH(mrhs_grad, req[1], dot(mlhs_data, mout_grad)); + ASSIGN_DISPATCH(mlhs_grad, req[0], dot(mrhs_data, mout_grad.T())); + } else { + // Gradient of z = dot(x, y) + // dy = dot(x.T, dz) + // dx = dot(dz, y.T) + ASSIGN_DISPATCH(mrhs_grad, req[1], dot(mlhs_data.T(), mout_grad)); + ASSIGN_DISPATCH(mlhs_grad, req[0], dot(mout_grad, mrhs_data.T())); + } } - } + }); } inline bool DotShape(const nnvm::NodeAttrs& attrs, @@ -553,6 +555,7 @@ void BatchDotForward_(const nnvm::NodeAttrs& attrs, const std::vector& inputs, const std::vector& req, const std::vector& outputs) { + using namespace mshadow; using namespace mshadow::expr; mshadow::Stream *s = ctx.get_stream(); const DotParam& param = nnvm::get(attrs.parsed); @@ -560,33 +563,34 @@ void BatchDotForward_(const nnvm::NodeAttrs& attrs, << "Binary function only support input/output with the same type"; CHECK_EQ(outputs[0].type_flag_, inputs[1].type_flag_) << "Binary function only support input/output with the same type"; - CHECK_EQ(outputs[0].type_flag_, mshadow::kFloat32) - << "dot only support 32 bit float so far"; - - mshadow::Tensor out = outputs[0].get(s); - mshadow::Tensor mlhs = inputs[0].get(s); - mshadow::Tensor mrhs = inputs[1].get(s); - mshadow::Tensor workspace = - ctx.requested[0].get_space_typed(mshadow::Shape1(3 * out.size(0)), s); - if (kNullOp != req[0]) { - if (param.transpose_a && param.transpose_b) { - mshadow::BatchGEMM(out, mlhs, mrhs, 1.0f, - (kAddTo == req[0]) ? 1.0f : 0.0f, - workspace); - } else if (!param.transpose_a && param.transpose_b) { - mshadow::BatchGEMM(out, mlhs, mrhs, 1.0f, - (kAddTo == req[0]) ? 1.0f : 0.0f, - workspace); - } else if (param.transpose_a && !param.transpose_b) { - mshadow::BatchGEMM(out, mlhs, mrhs, 1.0f, - (kAddTo == req[0]) ? 1.0f : 0.0f, - workspace); - } else { - mshadow::BatchGEMM(out, mlhs, mrhs, 1.0f, - (kAddTo == req[0]) ? 1.0f : 0.0f, - workspace); + CHECK(outputs[0].type_flag_ == kFloat32 || outputs[0].type_flag_ == kFloat64) + << "dot only supports float32 and float64"; + MSHADOW_TYPE_SWITCH(outputs[0].type_flag_, DType, { + mshadow::Tensor out = outputs[0].get(s); + mshadow::Tensor mlhs = inputs[0].get(s); + mshadow::Tensor mrhs = inputs[1].get(s); + mshadow::Tensor workspace = + ctx.requested[0].get_space_typed(mshadow::Shape1(3 * out.size(0)), s); + if (kNullOp != req[0]) { + if (param.transpose_a && param.transpose_b) { + mshadow::BatchGEMM(out, mlhs, mrhs, (DType)1.0f, + (kAddTo == req[0]) ? (DType)1.0f : (DType)0.0f, + workspace); + } else if (!param.transpose_a && param.transpose_b) { + mshadow::BatchGEMM(out, mlhs, mrhs, (DType)1.0f, + (kAddTo == req[0]) ? (DType)1.0f : (DType)0.0f, + workspace); + } else if (param.transpose_a && !param.transpose_b) { + mshadow::BatchGEMM(out, mlhs, mrhs, (DType)1.0f, + (kAddTo == req[0]) ? (DType)1.0f : (DType)0.0f, + workspace); + } else { + mshadow::BatchGEMM(out, mlhs, mrhs, (DType)1.0f, + (kAddTo == req[0]) ? (DType)1.0f : (DType)0.0f, + workspace); + } } - } + }); } template @@ -595,79 +599,83 @@ void BatchDotBackward_(const nnvm::NodeAttrs& attrs, const std::vector& inputs, const std::vector& req, const std::vector& outputs) { + using namespace mshadow; using namespace mshadow::expr; mshadow::Stream *s = ctx.get_stream(); const DotParam& param = nnvm::get(attrs.parsed); CHECK_NE(req[1], kWriteInplace); CHECK_NE(req[0], kWriteInplace); - - mshadow::Tensor mout_grad = inputs[0].get(s); - mshadow::Tensor mlhs_data = inputs[1].get(s); - mshadow::Tensor mrhs_data = inputs[2].get(s); - mshadow::Tensor mlhs_grad = outputs[0].get(s); - mshadow::Tensor mrhs_grad = outputs[1].get(s); - mshadow::Tensor workspace = - ctx.requested[0].get_space_typed( - mshadow::Shape2(2, 3 * mout_grad.size(0)), s); - mshadow::Tensor rhs_workspace = workspace[0]; - mshadow::Tensor lhs_workspace = workspace[1]; - if (param.transpose_a && param.transpose_b) { - // Gradient of z = dot(x.T, y.T) - // dy = dot(x, dz).T = dot(dz.T, x.T) - // dx = dot(dz, y).T = dot(y.T, dz.T) - if (kNullOp != req[1]) { - mshadow::BatchGEMM(mrhs_grad, mout_grad, mlhs_data, 1.0f, - (kAddTo == req[1]) ? 1.0f : 0.0f, - rhs_workspace); - } - if (kNullOp != req[0]) { - mshadow::BatchGEMM(mlhs_grad, mrhs_data, mout_grad, 1.0f, - (kAddTo == req[0]) ? 1.0f : 0.0f, - lhs_workspace); - } - } else if (!param.transpose_a && param.transpose_b) { - // Gradient of z = dot(x, y.T) - // dy = dot(x.T, dz).T = dot(dz.T, x) - // dx = dot(dz, y) - if (kNullOp != req[1]) { - mshadow::BatchGEMM(mrhs_grad, mout_grad, mlhs_data, 1.0f, - (kAddTo == req[1]) ? 1.0f : 0.0f, - rhs_workspace); - } - if (kNullOp != req[0]) { - mshadow::BatchGEMM(mlhs_grad, mout_grad, mrhs_data, 1.0f, - (kAddTo == req[0]) ? 1.0f : 0.0f, - lhs_workspace); - } - } else if (param.transpose_a && !param.transpose_b) { - // Gradient of z = dot(x.T, y) - // dy = dot(x, dz) - // dx = dot(dz, y.T).T = dot(y, dz.T) - if (kNullOp != req[1]) { - mshadow::BatchGEMM(mrhs_grad, mlhs_data, mout_grad, 1.0f, - (kAddTo == req[1]) ? 1.0f : 0.0f, - rhs_workspace); - } - if (kNullOp != req[0]) { - mshadow::BatchGEMM(mlhs_grad, mrhs_data, mout_grad, 1.0f, - (kAddTo == req[0]) ? 1.0f : 0.0f, - lhs_workspace); - } - } else { - // Gradient of z = dot(x, y) - // dy = dot(x.T, dz) - // dx = dot(dz, y.T) - if (kNullOp != req[1]) { - mshadow::BatchGEMM(mrhs_grad, mlhs_data, mout_grad, 1.0f, - (kAddTo == req[1]) ? 1.0f : 0.0f, - rhs_workspace); - } - if (kNullOp != req[0]) { - mshadow::BatchGEMM(mlhs_grad, mout_grad, mrhs_data, 1.0f, - (kAddTo == req[0]) ? 1.0f : 0.0f, - lhs_workspace); + CHECK(outputs[0].type_flag_ == kFloat32 || outputs[0].type_flag_ == kFloat64) + << "dot only supports float32 and float64"; + MSHADOW_TYPE_SWITCH(outputs[0].type_flag_, DType, { + mshadow::Tensor mout_grad = inputs[0].get(s); + mshadow::Tensor mlhs_data = inputs[1].get(s); + mshadow::Tensor mrhs_data = inputs[2].get(s); + mshadow::Tensor mlhs_grad = outputs[0].get(s); + mshadow::Tensor mrhs_grad = outputs[1].get(s); + mshadow::Tensor workspace = + ctx.requested[0].get_space_typed( + mshadow::Shape2(2, 3 * mout_grad.size(0)), s); + mshadow::Tensor rhs_workspace = workspace[0]; + mshadow::Tensor lhs_workspace = workspace[1]; + if (param.transpose_a && param.transpose_b) { + // Gradient of z = dot(x.T, y.T) + // dy = dot(x, dz).T = dot(dz.T, x.T) + // dx = dot(dz, y).T = dot(y.T, dz.T) + if (kNullOp != req[1]) { + mshadow::BatchGEMM(mrhs_grad, mout_grad, mlhs_data, (DType)1.0f, + (kAddTo == req[1]) ? (DType)1.0f : (DType)0.0f, + rhs_workspace); + } + if (kNullOp != req[0]) { + mshadow::BatchGEMM(mlhs_grad, mrhs_data, mout_grad, (DType)1.0f, + (kAddTo == req[0]) ? (DType)1.0f : (DType)0.0f, + lhs_workspace); + } + } else if (!param.transpose_a && param.transpose_b) { + // Gradient of z = dot(x, y.T) + // dy = dot(x.T, dz).T = dot(dz.T, x) + // dx = dot(dz, y) + if (kNullOp != req[1]) { + mshadow::BatchGEMM(mrhs_grad, mout_grad, mlhs_data, (DType)1.0f, + (kAddTo == req[1]) ? (DType)1.0f : (DType)0.0f, + rhs_workspace); + } + if (kNullOp != req[0]) { + mshadow::BatchGEMM(mlhs_grad, mout_grad, mrhs_data, (DType)1.0f, + (kAddTo == req[0]) ? (DType)1.0f : (DType)0.0f, + lhs_workspace); + } + } else if (param.transpose_a && !param.transpose_b) { + // Gradient of z = dot(x.T, y) + // dy = dot(x, dz) + // dx = dot(dz, y.T).T = dot(y, dz.T) + if (kNullOp != req[1]) { + mshadow::BatchGEMM(mrhs_grad, mlhs_data, mout_grad, (DType)1.0f, + (kAddTo == req[1]) ? (DType)1.0f : (DType)0.0f, + rhs_workspace); + } + if (kNullOp != req[0]) { + mshadow::BatchGEMM(mlhs_grad, mrhs_data, mout_grad, (DType)1.0f, + (kAddTo == req[0]) ? (DType)1.0f : (DType)0.0f, + lhs_workspace); + } + } else { + // Gradient of z = dot(x, y) + // dy = dot(x.T, dz) + // dx = dot(dz, y.T) + if (kNullOp != req[1]) { + mshadow::BatchGEMM(mrhs_grad, mlhs_data, mout_grad, (DType)1.0f, + (kAddTo == req[1]) ? (DType)1.0f : (DType)0.0f, + rhs_workspace); + } + if (kNullOp != req[0]) { + mshadow::BatchGEMM(mlhs_grad, mout_grad, mrhs_data, (DType)1.0f, + (kAddTo == req[0]) ? (DType)1.0f : (DType)0.0f, + lhs_workspace); + } } - } + }); } inline bool BatchDotShape(const nnvm::NodeAttrs& attrs, diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py index db50052ee60a..fece5414dbc3 100644 --- a/tests/python/unittest/test_operator.py +++ b/tests/python/unittest/test_operator.py @@ -1532,102 +1532,118 @@ def test_stn(): def test_dot(ctx=default_context()): np.random.seed(1234) + dtypes = ['float32', 'float64'] # Test normal dot. - for m in range(1, 5): - for k in range(1, 5): - for n in range(1, 5): - a_npy = np.random.normal(0, 1, (m, k)) - b_npy = np.random.normal(0, 1, (k, n)) - c_npy = np.empty((m, n)) - ograd_npy = np.random.normal(0, 1, (m, n)) - agrad_npy = np.empty((m, k)) - bgrad_npy = np.empty((k, n)) - c_npy[:, :] = np.dot(a_npy[:, :], b_npy[:, :]) - bgrad_npy[:, :] = np.dot(a_npy[:, :].T, ograd_npy[:, :]) - agrad_npy[:, :] = np.dot(ograd_npy[:, :], b_npy[:, :].T) - a = mx.sym.Variable('a') - b = mx.sym.Variable('b') - c = mx.sym.dot(a, b) - exe = c.simple_bind(ctx=ctx, a=a_npy.shape, b=b_npy.shape) - outputs = exe.forward(is_train=True, a=a_npy, b=b_npy) - assert_almost_equal(outputs[0].asnumpy(), c_npy, rtol=1e-3) - exe.backward(out_grads=[mx.nd.array(ograd_npy, ctx=exe._ctx)]) - assert_almost_equal(exe.grad_dict['a'].asnumpy(), agrad_npy, rtol=1e-3) - assert_almost_equal(exe.grad_dict['b'].asnumpy(), bgrad_npy, rtol=1e-3) + for data_type in dtypes: + for m in range(1, 5): + for k in range(1, 5): + for n in range(1, 5): + a_npy = np.random.normal(0, 1, (m, k)) + a_npy = a_npy.astype(data_type) + b_npy = np.random.normal(0, 1, (k, n)) + b_npy = b_npy.astype(data_type) + c_npy = np.empty((m, n), dtype=data_type) + ograd_npy = np.random.normal(0, 1, (m, n)) + ograd_npy = ograd_npy.astype(data_type) + agrad_npy = np.empty((m, k), dtype=data_type) + bgrad_npy = np.empty((k, n), dtype=data_type) + c_npy[:, :] = np.dot(a_npy[:, :], b_npy[:, :]) + bgrad_npy[:, :] = np.dot(a_npy[:, :].T, ograd_npy[:, :]) + agrad_npy[:, :] = np.dot(ograd_npy[:, :], b_npy[:, :].T) + a = mx.sym.Variable('a', dtype=data_type) + b = mx.sym.Variable('b', dtype=data_type) + c = mx.sym.dot(a, b) + exe = c.simple_bind(ctx=ctx, a=a_npy.shape, b=b_npy.shape) + outputs = exe.forward(is_train=True, a=a_npy, b=b_npy) + assert_almost_equal(outputs[0].asnumpy(), c_npy, rtol=1e-3) + exe.backward(out_grads=[mx.nd.array(ograd_npy, mx.cpu())]) + assert_almost_equal(exe.grad_dict['a'].asnumpy(), agrad_npy, rtol=1e-3) + assert_almost_equal(exe.grad_dict['b'].asnumpy(), bgrad_npy, rtol=1e-3) # Test dot with transpose flag using gradient checker. - def dot_sym(): - x = mx.sym.Variable('x') - y = mx.sym.Variable('y') + def dot_sym(data_type): + x = mx.sym.Variable('x', dtype=data_type) + y = mx.sym.Variable('y', dtype=data_type) return mx.sym.dot(x, y) - def dot_sym_xT(): - x = mx.sym.Variable('x') - y = mx.sym.Variable('y') + def dot_sym_xT(data_type): + x = mx.sym.Variable('x', dtype=data_type) + y = mx.sym.Variable('y', dtype=data_type) return mx.sym.dot(x, y, transpose_a=True) - def dot_sym_yT(): - x = mx.sym.Variable('x') - y = mx.sym.Variable('y') + def dot_sym_yT(data_type): + x = mx.sym.Variable('x', dtype=data_type) + y = mx.sym.Variable('y', dtype=data_type) return mx.sym.dot(x, y, transpose_b=True) - def dot_sym_xT_yT(): - x = mx.sym.Variable('x') - y = mx.sym.Variable('y') + def dot_sym_xT_yT(data_type): + x = mx.sym.Variable('x', dtype=data_type) + y = mx.sym.Variable('y', dtype=data_type) return mx.sym.dot(x, y, transpose_a=True, transpose_b=True) - for ashape, bshape in [((3, 4), (4, 5)), ((2,3,4), (4, 5, 6))]: - m1_npy = np.random.uniform(-1, 1, ashape) - m2_npy = np.random.uniform(-1, 1, bshape) - check_numeric_gradient(dot_sym(), [m1_npy, m2_npy], numeric_eps=1e-1, rtol=2e-2, atol=1e-3) - check_numeric_gradient(dot_sym_xT(), [m1_npy.T, m2_npy], numeric_eps=1e-1, rtol=2e-2, atol=1e-3) - check_numeric_gradient(dot_sym_yT(), [m1_npy, m2_npy.T], numeric_eps=1e-1, rtol=2e-2, atol=1e-3) - check_numeric_gradient(dot_sym_xT_yT(), [m1_npy.T, m2_npy.T], numeric_eps=1e-1, rtol=2e-2, atol=1e-3) + for data_type in dtypes: + for ashape, bshape in [((3, 4), (4, 5)), ((2, 3, 4), (4, 5, 6))]: + m1_npy = np.random.uniform(-1, 1, ashape) + m1_npy = m1_npy.astype(data_type) + m2_npy = np.random.uniform(-1, 1, bshape) + m2_npy = m2_npy.astype(data_type) + check_numeric_gradient(dot_sym(data_type), [m1_npy, m2_npy], numeric_eps=1e-1, rtol=2e-2, atol=1e-3) + check_numeric_gradient(dot_sym_xT(data_type), [m1_npy.T, m2_npy], numeric_eps=1e-1, rtol=2e-2, atol=1e-3) + check_numeric_gradient(dot_sym_yT(data_type), [m1_npy, m2_npy.T], numeric_eps=1e-1, rtol=2e-2, atol=1e-3) + check_numeric_gradient(dot_sym_xT_yT(data_type), [m1_npy.T, m2_npy.T], numeric_eps=1e-1, rtol=2e-2, atol=1e-3) def test_batch_dot(): - for batch_size in range(1, 5): - for m in range(1, 5): - for k in range(1, 5): - for n in range(1, 5): - transpose_a = (np.random.rand() > 0.5) - transpose_b = (np.random.rand() > 0.5) - a_npy = np.random.normal(0, 1, (batch_size, m, k)) - b_npy = np.random.normal(0, 1, (batch_size, k, n)) - c_npy = np.empty((batch_size, m, n)) - ograd_npy = np.random.normal(0, 1, (batch_size, m, n)) - agrad_npy = np.empty((batch_size, m, k)) - bgrad_npy = np.empty((batch_size, k, n)) - a_init_grad_npy = np.random.normal(size=(batch_size, m, k)) - b_init_grad_npy = np.random.normal(size=(batch_size, k, n)) - for i in range(batch_size): - c_npy[i, :, :] = np.dot(a_npy[i, :, :], b_npy[i, :, :]) - bgrad_npy[i, :, :] = np.dot(a_npy[i, :, :].T, ograd_npy[i, :, :]) - agrad_npy[i, :, :] = np.dot(ograd_npy[i, :, :], b_npy[i, :, :].T) - a = mx.sym.Variable('a') - b = mx.sym.Variable('b') - c = mx.sym.batch_dot(a, b, transpose_a=transpose_a, transpose_b=transpose_b) - if transpose_a: - a_npy = np.transpose(a_npy, axes=(0, 2, 1)) - agrad_npy = np.transpose(agrad_npy, axes=(0, 2, 1)) - a_init_grad_npy = np.transpose(a_init_grad_npy, axes=(0, 2, 1)) - if transpose_b: - b_npy = np.transpose(b_npy, axes=(0, 2, 1)) - bgrad_npy = np.transpose(bgrad_npy, axes=(0, 2, 1)) - b_init_grad_npy = np.transpose(b_init_grad_npy, axes=(0, 2, 1)) - exe = c.simple_bind(ctx=default_context(), - a=a_npy.shape, b=b_npy.shape, grad_req='write') - exe_add = c.simple_bind(ctx=default_context(), - a=a_npy.shape, b=b_npy.shape, grad_req='add') - exe_add.grad_dict['a'][:] = a_init_grad_npy - exe_add.grad_dict['b'][:] = b_init_grad_npy - outputs = exe.forward(is_train=True, a=a_npy, b=b_npy) - assert_almost_equal(outputs[0].asnumpy(), c_npy, rtol=1e-3, atol=1e-4) - exe.backward(out_grads=[mx.nd.array(ograd_npy, ctx=exe._ctx)]) - assert_almost_equal(exe.grad_dict['a'].asnumpy(), agrad_npy, rtol=1e-3, atol=1e-4) - assert_almost_equal(exe.grad_dict['b'].asnumpy(), bgrad_npy, rtol=1e-3, atol=1e-4) - exe_add.forward(is_train=True, a=a_npy, b=b_npy) - exe_add.backward(out_grads=[mx.nd.array(ograd_npy, ctx=exe._ctx)]) - assert_almost_equal(exe_add.grad_dict['a'].asnumpy(), - agrad_npy + a_init_grad_npy, rtol=1e-3, atol=1e-4) - assert_almost_equal(exe_add.grad_dict['b'].asnumpy(), - bgrad_npy + b_init_grad_npy, rtol=1e-3, atol=1e-4) + dtypes = ['float32', 'float64'] + + for data_type in dtypes: + for batch_size in range(1, 5): + for m in range(1, 5): + for k in range(1, 5): + for n in range(1, 5): + transpose_a = (np.random.rand() > 0.5) + transpose_b = (np.random.rand() > 0.5) + a_npy = np.random.normal(0, 1, (batch_size, m, k)) + a_npy = a_npy.astype(data_type) + b_npy = np.random.normal(0, 1, (batch_size, k, n)) + b_npy = b_npy.astype(data_type) + c_npy = np.empty((batch_size, m, n), dtype=data_type) + ograd_npy = np.random.normal(0, 1, (batch_size, m, n)) + ograd_npy = ograd_npy.astype(data_type) + agrad_npy = np.empty((batch_size, m, k), dtype=data_type) + bgrad_npy = np.empty((batch_size, k, n), dtype=data_type) + a_init_grad_npy = np.random.normal(size=(batch_size, m, k)) + a_init_grad_npy = a_npy.astype(data_type) + b_init_grad_npy = np.random.normal(size=(batch_size, k, n)) + b_init_grad_npy = b_npy.astype(data_type) + for i in range(batch_size): + c_npy[i, :, :] = np.dot(a_npy[i, :, :], b_npy[i, :, :]) + bgrad_npy[i, :, :] = np.dot(a_npy[i, :, :].T, ograd_npy[i, :, :]) + agrad_npy[i, :, :] = np.dot(ograd_npy[i, :, :], b_npy[i, :, :].T) + a = mx.sym.Variable('a', dtype=data_type) + b = mx.sym.Variable('b', dtype=data_type) + c = mx.sym.batch_dot(a, b, transpose_a=transpose_a, transpose_b=transpose_b) + if transpose_a: + a_npy = np.transpose(a_npy, axes=(0, 2, 1)) + agrad_npy = np.transpose(agrad_npy, axes=(0, 2, 1)) + a_init_grad_npy = np.transpose(a_init_grad_npy, axes=(0, 2, 1)) + if transpose_b: + b_npy = np.transpose(b_npy, axes=(0, 2, 1)) + bgrad_npy = np.transpose(bgrad_npy, axes=(0, 2, 1)) + b_init_grad_npy = np.transpose(b_init_grad_npy, axes=(0, 2, 1)) + exe = c.simple_bind(ctx=default_context(), + a=a_npy.shape, b=b_npy.shape, grad_req='write') + exe_add = c.simple_bind(ctx=default_context(), + a=a_npy.shape, b=b_npy.shape, grad_req='add') + exe_add.grad_dict['a'][:] = a_init_grad_npy + exe_add.grad_dict['b'][:] = b_init_grad_npy + outputs = exe.forward(is_train=True, a=a_npy, b=b_npy) + assert_almost_equal(outputs[0].asnumpy(), c_npy, rtol=1e-3, atol=1e-4) + exe.backward(out_grads=[mx.nd.array(ograd_npy, ctx=exe._ctx)]) + assert_almost_equal(exe.grad_dict['a'].asnumpy(), agrad_npy, rtol=1e-3, atol=1e-4) + assert_almost_equal(exe.grad_dict['b'].asnumpy(), bgrad_npy, rtol=1e-3, atol=1e-4) + exe_add.forward(is_train=True, a=a_npy, b=b_npy) + exe_add.backward(out_grads=[mx.nd.array(ograd_npy, ctx=exe._ctx)]) + assert_almost_equal(exe_add.grad_dict['a'].asnumpy(), + agrad_npy + a_init_grad_npy, rtol=1e-3, atol=1e-4) + assert_almost_equal(exe_add.grad_dict['b'].asnumpy(), + bgrad_npy + b_init_grad_npy, rtol=1e-3, atol=1e-4) def get_correlation(data1,data2,kernel_size,max_displacement,stride1,stride2,pad_size,is_multiply): From 63392cff28ebb2c83998239e72eec1fb0aed5785 Mon Sep 17 00:00:00 2001 From: Yuwen Xiong Date: Tue, 20 Jun 2017 11:18:21 +0800 Subject: [PATCH 093/834] Update README.md (#6754) --- example/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/example/README.md b/example/README.md index cd765affd567..12ada4d0ceef 100644 --- a/example/README.md +++ b/example/README.md @@ -74,6 +74,8 @@ If you want to contribute to this list and the examples, please open a new pull * [Fast Neural Style in Scala](https://github.com/Ldpe2G/DeepLearningForFun/tree/master/Mxnet-Scala/FastNeuralStyle) by [Ldpe2G](https://github.com/Ldpe2G) * [LSTM Human Activity Recognition](https://github.com/Ldpe2G/DeepLearningForFun/tree/master/Mxnet-Scala/HumanActivityRecognition) by [Ldpe2G](https://github.com/Ldpe2G) * [Visual Question Answering](https://github.com/liuzhi136/Visual-Question-Answering) by [liuzhi136](https://github.com/liuzhi136) +* [Deformable ConvNets](https://arxiv.org/abs/1703.06211) ([github](https://github.com/msracver/Deformable-ConvNets)) by [MSRACVer](https://github.com/msracver) + ### IPython Notebooks ----------------- From 63ac7932688a3e479006cbb0eafed59d4c6ad137 Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Mon, 19 Jun 2017 23:59:40 -0700 Subject: [PATCH 094/834] mod (#6698) * mod * fix include * fix cpplint * fix R * fix cpp header * return NAN instead of 0 and remove warning for mod * fix cpp header * fix build for half * fix half * fix half * 0 grad for mod for now * add override for half2 * fix half2_t * updated backward * fix registration * mod, working backward compatible with numpy * update mshadow * fix lint * fix half2 neg * fix scala --- R-package/R/symbol.R | 12 ++ R-package/src/ndarray.cc | 13 ++ cpp-package/include/mxnet-cpp/ndarray.h | 16 ++ cpp-package/include/mxnet-cpp/ndarray.hpp | 18 ++ cpp-package/include/mxnet-cpp/op_suppl.h | 14 ++ cpp-package/include/mxnet-cpp/symbol.h | 3 + cpp-package/include/mxnet-cpp/symbol.hpp | 7 + docs/api/python/ndarray.md | 4 + docs/api/python/symbol.md | 3 + mshadow | 2 +- python/mxnet/ndarray.py | 75 +++++++ python/mxnet/symbol.py | 30 +++ .../main/scala/ml/dmlc/mxnet/NDArray.scala | 24 +++ .../src/main/scala/ml/dmlc/mxnet/Symbol.scala | 10 + src/ndarray/ndarray_function.h | 4 + src/operator/mshadow_op.h | 183 +++++++++++++++++- .../elemwise_binary_broadcast_op_basic.cc | 33 ++++ .../elemwise_binary_broadcast_op_basic.cu | 7 + .../tensor/elemwise_binary_op_basic.cc | 16 ++ .../tensor/elemwise_binary_op_basic.cu | 7 + .../tensor/elemwise_binary_scalar_op_basic.cc | 20 ++ .../tensor/elemwise_binary_scalar_op_basic.cu | 12 ++ tests/python/unittest/test_operator.py | 112 +++++++---- 23 files changed, 583 insertions(+), 42 deletions(-) diff --git a/R-package/R/symbol.R b/R-package/R/symbol.R index d2fd67bc45c0..091a6468c1a1 100644 --- a/R-package/R/symbol.R +++ b/R-package/R/symbol.R @@ -158,4 +158,16 @@ init.symbol.methods <- function() { setMethod("/", signature(e1 = "Rcpp_MXSymbol", e2 = "numeric"), function(e1, e2) { mx.varg.symbol.internal.DivScalar(list(e1, scalar = e2)) }) + setMethod("%%", signature(e1 = "Rcpp_MXSymbol", e2 = "Rcpp_MXSymbol"), function(e1, e2) { + mx.varg.symbol.internal.Mod(list(e1, e2)) + }) + setMethod("%%", signature(e1 = "Rcpp_MXSymbol", e2 = "numeric"), function(e1, e2) { + mx.varg.symbol.internal.ModScalar(list(e1, scalar = e2)) + }) + setMethod("%/%", signature(e1 = "Rcpp_MXSymbol", e2 = "Rcpp_MXSymbol"), function(e1, e2) { + mx.varg.symbol.internal.Mod(list(e1, e2)) + }) + setMethod("%/%", signature(e1 = "Rcpp_MXSymbol", e2 = "numeric"), function(e1, e2) { + mx.varg.symbol.internal.ModScalar(list(e1, scalar = e2)) + }) } diff --git a/R-package/src/ndarray.cc b/R-package/src/ndarray.cc index 335e5d47b486..c2bfe0c945a6 100644 --- a/R-package/src/ndarray.cc +++ b/R-package/src/ndarray.cc @@ -541,6 +541,9 @@ NDArray::RObjectType DispatchOps(SEXP op, SEXP lhs, SEXP rhs) { static OpHandle div = NDArrayFunction::FindHandle("_div"); static OpHandle div_scalar = NDArrayFunction::FindHandle("_div_scalar"); static OpHandle rdiv_scalar = NDArrayFunction::FindHandle("_rdiv_scalar"); + static OpHandle mod = NDArrayFunction::FindHandle("_mod"); + static OpHandle mod_scalar = NDArrayFunction::FindHandle("_mod_scalar"); + static OpHandle rmod_scalar = NDArrayFunction::FindHandle("_rmod_scalar"); // parse the arguments std::string values[2]; NDArrayHandle handles[2]; @@ -591,6 +594,16 @@ NDArray::RObjectType DispatchOps(SEXP op, SEXP lhs, SEXP rhs) { } break; } + case '%': { + if (lhs_nd && rhs_nd) { + out = BinaryOp(mod, handles); + } else if (lhs_nd && !rhs_nd) { + out = BinaryScalarOp(mod_scalar, handles[0], values[1]); + } else { + out = BinaryScalarOp(rmod_scalar, handles[1], values[0]); + } + break; + } default: { RLOG_FATAL << "Operator " << sop << "not supported for MXNDArray"; } diff --git a/cpp-package/include/mxnet-cpp/ndarray.h b/cpp-package/include/mxnet-cpp/ndarray.h index 58376a8ef6da..f908b4ff38eb 100644 --- a/cpp-package/include/mxnet-cpp/ndarray.h +++ b/cpp-package/include/mxnet-cpp/ndarray.h @@ -145,10 +145,12 @@ class NDArray { NDArray operator-(mx_float scalar); NDArray operator*(mx_float scalar); NDArray operator/(mx_float scalar); + NDArray operator%(mx_float scalar); NDArray operator+(const NDArray &); NDArray operator-(const NDArray &); NDArray operator*(const NDArray &); NDArray operator/(const NDArray &); + NDArray operator%(const NDArray &); /*! * \brief set all the elements in ndarray to be scalar * \param scalar the scalar to set @@ -184,6 +186,13 @@ class NDArray { */ NDArray &operator/=(mx_float scalar); /*! + * \brief elementwise modulo from current ndarray + * this mutate the current NDArray + * \param scalar the data to subtract + * \return reference of self + */ + NDArray &operator%=(mx_float scalar); + /*! * \brief elementwise add to current space * this mutate the current NDArray * \param src the data to add @@ -211,6 +220,13 @@ class NDArray { * \return reference of self */ NDArray &operator/=(const NDArray &src); + /*! + * \brief elementwise modulo from current ndarray + * this mutate the current NDArray + * \param src the data to subtract + * \return reference of self + */ + NDArray &operator%=(const NDArray &src); NDArray ArgmaxChannel(); /*! * \brief Do a synchronize copy from a continugous CPU memory region. diff --git a/cpp-package/include/mxnet-cpp/ndarray.hpp b/cpp-package/include/mxnet-cpp/ndarray.hpp index 69d1082bf8fa..6157a6600cb4 100644 --- a/cpp-package/include/mxnet-cpp/ndarray.hpp +++ b/cpp-package/include/mxnet-cpp/ndarray.hpp @@ -93,6 +93,11 @@ inline NDArray NDArray::operator/(mx_float scalar) { Operator("_div_scalar")(*this, scalar).Invoke(ret); return ret; } +inline NDArray NDArray::operator%(mx_float scalar) { + NDArray ret; + Operator("_mod_scalar")(*this, scalar).Invoke(ret); + return ret; +} inline NDArray NDArray::operator+(const NDArray &rhs) { NDArray ret; Operator("_plus")(*this, rhs).Invoke(ret); @@ -113,6 +118,11 @@ inline NDArray NDArray::operator/(const NDArray &rhs) { Operator("_div")(*this, rhs).Invoke(ret); return ret; } +inline NDArray NDArray::operator%(const NDArray &rhs) { + NDArray ret; + Operator("_mod")(*this, rhs).Invoke(ret); + return ret; +} inline NDArray &NDArray::operator=(mx_float scalar) { Operator("_set_value")(scalar).Invoke(*this); return *this; @@ -133,6 +143,10 @@ inline NDArray &NDArray::operator/=(mx_float scalar) { Operator("_div_scalar")(*this, scalar).Invoke(*this); return *this; } +inline NDArray &NDArray::operator%=(mx_float scalar) { + Operator("_mod_scalar")(*this, scalar).Invoke(*this); + return *this; +} inline NDArray &NDArray::operator+=(const NDArray &rhs) { Operator("_plus")(*this, rhs).Invoke(*this); return *this; @@ -149,6 +163,10 @@ inline NDArray &NDArray::operator/=(const NDArray &rhs) { Operator("_div")(*this, rhs).Invoke(*this); return *this; } +inline NDArray &NDArray::operator%=(const NDArray &rhs) { + Operator("_mod")(*this, rhs).Invoke(*this); + return *this; +} inline NDArray NDArray::ArgmaxChannel() { NDArray ret; diff --git a/cpp-package/include/mxnet-cpp/op_suppl.h b/cpp-package/include/mxnet-cpp/op_suppl.h index 9381a1ecade9..c40449cc9f89 100644 --- a/cpp-package/include/mxnet-cpp/op_suppl.h +++ b/cpp-package/include/mxnet-cpp/op_suppl.h @@ -35,6 +35,10 @@ inline Symbol _Div(Symbol lhs, Symbol rhs) { return Operator("_Div")(lhs, rhs) .CreateSymbol(); } +inline Symbol _Mod(Symbol lhs, Symbol rhs) { + return Operator("_Mod")(lhs, rhs) + .CreateSymbol(); +} inline Symbol _Power(Symbol lhs, Symbol rhs) { return Operator("_Power")(lhs, rhs) .CreateSymbol(); @@ -77,6 +81,16 @@ inline Symbol _RDivScalar(mx_float scalar, Symbol rhs) { .SetParam("scalar", scalar) .CreateSymbol(); } +inline Symbol _ModScalar(Symbol lhs, mx_float scalar) { + return Operator("_ModScalar")(lhs) + .SetParam("scalar", scalar) + .CreateSymbol(); +} +inline Symbol _RModScalar(mx_float scalar, Symbol rhs) { + return Operator("_RModScalar")(rhs) + .SetParam("scalar", scalar) + .CreateSymbol(); +} inline Symbol _PowerScalar(Symbol lhs, mx_float scalar) { return Operator("_PowerScalar")(lhs) .SetParam("scalar", scalar) diff --git a/cpp-package/include/mxnet-cpp/symbol.h b/cpp-package/include/mxnet-cpp/symbol.h index 03a8409f8087..e853c2617ea4 100644 --- a/cpp-package/include/mxnet-cpp/symbol.h +++ b/cpp-package/include/mxnet-cpp/symbol.h @@ -72,11 +72,13 @@ class Symbol { Symbol operator-(const Symbol &rhs) const; Symbol operator*(const Symbol &rhs) const; Symbol operator/(const Symbol &rhs) const; + Symbol operator%(const Symbol &rhs) const; Symbol operator+(mx_float scalar) const; Symbol operator-(mx_float scalar) const; Symbol operator*(mx_float scalar) const; Symbol operator/(mx_float scalar) const; + Symbol operator%(mx_float scalar) const; Symbol Copy() const; /*! * \brief construct a variable Symbol @@ -252,6 +254,7 @@ Symbol operator+(mx_float lhs, const Symbol &rhs); Symbol operator-(mx_float lhs, const Symbol &rhs); Symbol operator*(mx_float lhs, const Symbol &rhs); Symbol operator/(mx_float lhs, const Symbol &rhs); +Symbol operator%(mx_float lhs, const Symbol &rhs); } // namespace cpp } // namespace mxnet #endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_SYMBOL_H_ diff --git a/cpp-package/include/mxnet-cpp/symbol.hpp b/cpp-package/include/mxnet-cpp/symbol.hpp index 40108325d594..26962ba5c99b 100644 --- a/cpp-package/include/mxnet-cpp/symbol.hpp +++ b/cpp-package/include/mxnet-cpp/symbol.hpp @@ -38,6 +38,7 @@ inline Symbol Symbol::operator+(const Symbol &rhs) const { return _Plus(*this, r inline Symbol Symbol::operator-(const Symbol &rhs) const { return _Minus(*this, rhs); } inline Symbol Symbol::operator*(const Symbol &rhs) const { return _Mul(*this, rhs); } inline Symbol Symbol::operator/(const Symbol &rhs) const { return _Div(*this, rhs); } +inline Symbol Symbol::operator%(const Symbol &rhs) const { return _Mod(*this, rhs); } inline Symbol Symbol::operator+(mx_float scalar) const { return _PlusScalar(*this, scalar); } @@ -50,6 +51,9 @@ inline Symbol Symbol::operator*(mx_float scalar) const { inline Symbol Symbol::operator/(mx_float scalar) const { return _DivScalar(*this, scalar); } +inline Symbol Symbol::operator%(mx_float scalar) const { + return _ModScalar(*this, scalar); +} inline Symbol Symbol::operator[](int index) { SymbolHandle out; MXSymbolGetOutput(GetHandle(), index, &out); @@ -337,6 +341,9 @@ inline Symbol operator*(mx_float lhs, const Symbol &rhs) { return rhs * lhs; } inline Symbol operator/(mx_float lhs, const Symbol &rhs) { return mxnet::cpp::_RDivScalar(lhs, rhs); } +inline Symbol operator%(mx_float lhs, const Symbol &rhs) { + return mxnet::cpp::_RModScalar(lhs, rhs); +} } // namespace cpp } // namespace mxnet diff --git a/docs/api/python/ndarray.md b/docs/api/python/ndarray.md index 2581c2c3354b..a782b910e656 100644 --- a/docs/api/python/ndarray.md +++ b/docs/api/python/ndarray.md @@ -120,6 +120,8 @@ In the rest of this document, we first overview the methods provided by the NDArray.__mul__ NDArray.__div__ NDArray.__rdiv__ + NDArray.__mod__ + NDArray.__rmod__ NDArray.__pow__ ``` @@ -133,6 +135,7 @@ In the rest of this document, we first overview the methods provided by the NDArray.__isub__ NDArray.__imul__ NDArray.__idiv__ + NDArray.__imod__ ``` ### Comparison operators @@ -259,6 +262,7 @@ In the rest of this document, we first overview the methods provided by the negative multiply divide + modulo dot batch_dot add_n diff --git a/docs/api/python/symbol.md b/docs/api/python/symbol.md index 14ed06b9db9b..f99bee2bd79b 100644 --- a/docs/api/python/symbol.md +++ b/docs/api/python/symbol.md @@ -86,6 +86,8 @@ Composite multiple symbols into a new one by an operator. Symbol.__mul__ Symbol.__div__ Symbol.__rdiv__ + Symbol.__mod__ + Symbol.__rmod__ Symbol.__pow__ ``` @@ -249,6 +251,7 @@ Composite multiple symbols into a new one by an operator. broadcast_sub broadcast_mul broadcast_div + broadcast_mod negative dot batch_dot diff --git a/mshadow b/mshadow index eda261eef135..8db65bd081c7 160000 --- a/mshadow +++ b/mshadow @@ -1 +1 @@ -Subproject commit eda261eef135a51e7388e680b295996d18d4e4d1 +Subproject commit 8db65bd081c7e243028ace93ef0acc9efc4383ba diff --git a/python/mxnet/ndarray.py b/python/mxnet/ndarray.py index 8900843f5937..9ec4d47bbb81 100644 --- a/python/mxnet/ndarray.py +++ b/python/mxnet/ndarray.py @@ -206,6 +206,25 @@ def __rtruediv__(self, other): def __itruediv__(self, other): return self.__idiv__(other) + def __mod__(self, other): + """x.__mod__(y) <=> x%y <=> mx.nd.modulo(x, y) """ + return modulo(self, other) + + def __rmod__(self, other): + """x.__rmod__(y) <=> y%x <=> mx.nd.modulo(y, x) """ + return modulo(other, self) + + def __imod__(self, other): + """x.__rmod__(y) <=> x%=y """ + if not self.writable: + raise ValueError('trying to take modulo from a readonly NDArray') + if isinstance(other, NDArray): + return broadcast_mod(self, other, out=self) + elif isinstance(other, numeric_types): + return _internal._mod_scalar(self, float(other), out=self) + else: + raise TypeError('type %s not supported' % str(type(other))) + def __pow__(self, other): """x.__pow__(y) <=> x**y <=> mx.nd.power(x,y) """ return power(self, other) @@ -1516,6 +1535,62 @@ def divide(lhs, rhs): _internal._rdiv_scalar) # pylint: enable= no-member, protected-access +def modulo(lhs, rhs): + """Returns element-wise modulo of the input arrays with broadcasting. + + Equivalent to ``lhs % rhs`` and ``mx.nd.broadcast_mod(lhs, rhs)``. + + .. note:: + + If the corresponding dimensions of two arrays have the same size or one of them has size 1, + then the arrays are broadcastable to a common shape. + + Parameters + ---------- + lhs : scalar or array + First array in modulo. + rhs : scalar or array + Second array in modulo. + The arrays to be taken modulo. If ``lhs.shape != rhs.shape``, they must be + broadcastable to a common shape. + + Returns + ------- + NDArray + The element-wise modulo of the input arrays. + + Examples + -------- + >>> x = mx.nd.ones((2,3))*6 + >>> y = mx.nd.ones((2,1))*4 + >>> x.asnumpy() + array([[ 6., 6., 6.], + [ 6., 6., 6.]], dtype=float32) + >>> y.asnumpy() + array([[ 4.], + [ 4.]], dtype=float32) + >>> x%5 + + >>> (x%5).asnumpy() + array([[ 1., 1., 1.], + [ 1., 1., 1.]], dtype=float32) + >>> (x%y).asnumpy() + array([[ 2., 2., 2.], + [ 2., 2., 2.]], dtype=float32) + >>> mx.nd.modulo(x,y).asnumpy() + array([[ 2., 2., 2.], + [ 2., 2., 2.]], dtype=float32) + """ + # pylint: disable= no-member, protected-access + return _ufunc_helper( + lhs, + rhs, + broadcast_mod, + operator.mod, + _internal._mod_scalar, + _internal._rmod_scalar) + # pylint: enable= no-member, protected-access + def power(base, exp): """Returns result of first array elements raised to powers from second array, element-wise with broadcasting. diff --git a/python/mxnet/symbol.py b/python/mxnet/symbol.py index 14203e59862d..bd0aca65f521 100644 --- a/python/mxnet/symbol.py +++ b/python/mxnet/symbol.py @@ -172,6 +172,36 @@ def __rdiv__(self, other): else: raise TypeError('type %s not supported' % str(type(other))) + def __mod__(self, other): + """x.__mod__(y) <=> x%y + + Scalar input is supported. + Broadcasting is not supported. Use `broadcast_mod` instead. """ + if isinstance(other, Symbol): + return _internal._Mod(self, other) + if isinstance(other, Number): + return _internal._ModScalar(self, scalar=other) + else: + raise TypeError('type %s not supported' % str(type(other))) + + def __rmod__(self, other): + """x.__rmod__(y) <=> y%x + + Only `NDArray` is supported for now. + + Example usage: + ---------- + >>> x = mx.nd.ones((2,3))*3 + >>> y = mx.nd.ones((2,3)) + >>> x.__rmod__(y).asnumpy() + array([[ 1., 1., 1., + [ 1., 1., 1., dtype=float32) + """ + if isinstance(other, Number): + return _internal._RModScalar(self, scalar=other) + else: + raise TypeError('type %s not supported' % str(type(other))) + def __truediv__(self, other): return self.__div__(other) diff --git a/scala-package/core/src/main/scala/ml/dmlc/mxnet/NDArray.scala b/scala-package/core/src/main/scala/ml/dmlc/mxnet/NDArray.scala index 49eea3dc9d05..94ce0086f6e4 100644 --- a/scala-package/core/src/main/scala/ml/dmlc/mxnet/NDArray.scala +++ b/scala-package/core/src/main/scala/ml/dmlc/mxnet/NDArray.scala @@ -880,6 +880,30 @@ class NDArray private[mxnet](private[mxnet] val handle: NDArrayHandle, NDArray.lesserEqual(this, other) } + def %(other: NDArray): NDArray = { + NDArray.genericNDArrayFunctionInvoke("_mod", Seq(this, other)) + } + + def %(other: Float): NDArray = { + NDArray.genericNDArrayFunctionInvoke("_mod_scalar", Seq(this, other)) + } + + def %=(other: NDArray): NDArray = { + if (!writable) { + throw new IllegalArgumentException("trying to take modulo from a readonly NDArray") + } + NDArray.genericNDArrayFunctionInvoke("_mod", Seq(this, other), Map("out" -> this)) + this + } + + def %=(other: Float): NDArray = { + if (!writable) { + throw new IllegalArgumentException("trying to take modulo from a readonly NDArray") + } + NDArray.genericNDArrayFunctionInvoke("_mod_scalar", Seq(this, other), Map("out" -> this)) + this + } + /** * Return a copied flat java array of current array (row-major). * @return A copy of array content. diff --git a/scala-package/core/src/main/scala/ml/dmlc/mxnet/Symbol.scala b/scala-package/core/src/main/scala/ml/dmlc/mxnet/Symbol.scala index de60e472e76c..4e8d4c2bd9f9 100644 --- a/scala-package/core/src/main/scala/ml/dmlc/mxnet/Symbol.scala +++ b/scala-package/core/src/main/scala/ml/dmlc/mxnet/Symbol.scala @@ -85,6 +85,11 @@ class Symbol private(private[mxnet] val handle: SymbolHandle) { def <=(other: Symbol): Symbol = Symbol.lesserEqual(this, other) def <=[@specialized(Int, Float, Double) V](other: V): Symbol = Symbol.lesserEqual(this, other) + def %(other: Symbol): Symbol = Symbol.createFromListedSymbols("_Mod")(Array(this, other)) + def %[@specialized(Int, Float, Double) V](other: V): Symbol = { + Symbol.createFromListedSymbols("_ModScalar")(Array(this), Map("scalar" -> other.toString)) + } + override def clone(): Symbol = { val clonedHandle = new SymbolHandleRef checkCall(_LIB.mxSymbolCopy(handle, clonedHandle)) @@ -1236,6 +1241,11 @@ class SymbolConversions[@specialized(Int, Float, Double) V](val value: V) { def <=(other: Symbol): Symbol = { other >= value } + + def %(other: Symbol): Symbol = { + Symbol.createFromListedSymbols("_RModScalar")( + Array(other), Map("scalar" -> value.toString)) + } } trait SymbolGenerator { diff --git a/src/ndarray/ndarray_function.h b/src/ndarray/ndarray_function.h index 00dd3d0e959a..479f6f99f07a 100644 --- a/src/ndarray/ndarray_function.h +++ b/src/ndarray/ndarray_function.h @@ -41,6 +41,10 @@ struct Div : public BinaryBase { typedef mshadow::op::div mshadow_op; }; +struct Mod : public BinaryBase { + typedef op::mshadow_op::mod mshadow_op; +}; + struct ClipMin : public BinaryBase { struct mshadow_op { template diff --git a/src/operator/mshadow_op.h b/src/operator/mshadow_op.h index 7e950c980e53..c63739ba5085 100644 --- a/src/operator/mshadow_op.h +++ b/src/operator/mshadow_op.h @@ -8,8 +8,13 @@ #define MXNET_OPERATOR_MSHADOW_OP_H_ #include +#include #include "special_functions-inl.h" +#ifdef __CUDACC__ +#include +#endif + namespace mxnet { namespace op { namespace mshadow_op { @@ -24,14 +29,14 @@ using std::isnan; struct identity { template MSHADOW_XINLINE static DType Map(DType a) { - return DType(a); + return a; } }; struct identity_grad { template MSHADOW_XINLINE static DType Map(DType a) { - return DType(DType(1.0f)); + return DType(1.0f); } }; @@ -434,15 +439,15 @@ struct abs { struct sign { template MSHADOW_XINLINE static DType Map(DType a) { - if (a < 0.0f) return DType(-DType(1.0f)); - if (a > 0.0f) return DType(DType(1.0f)); - return DType(DType(0.0f)); + if (a < 0.0f) return DType(-1.0f); + if (a > 0.0f) return DType(1.0f); + return DType(0.0f); } }; struct sign_grad { template MSHADOW_XINLINE static DType Map(DType a) { - return DType(DType(0.0f)); + return DType(0.0f); } }; /*! \brief used for generate element of power */ @@ -664,6 +669,172 @@ struct rdiv_grad { } }; +struct mod { + template + MSHADOW_XINLINE static DType Map(DType a, DType b) { + if (b == DType(0)) { + return DType(0); + } else if (b < DType(0)) { + if (a < DType(0)) { + return DType(-::fmod(-a, -b)); + } else { + return DType(::fmod(a, -b) + (::fmod(a, -b) != DType(0) ? b : DType(0))); + } + } else { + if (a < DType(0)) { + return DType(-::fmod(-a, b) + (::fmod(-a, b) != DType(0) ? b : DType(0))); + } else { + return DType(::fmod(a, b)); + } + } + } +}; +#ifdef __CUDACC__ +template<> +MSHADOW_XINLINE mshadow::half::half2_t mod::Map + (mshadow::half::half2_t a, + mshadow::half::half2_t b) { + return a%b; +} +#endif + +struct mod_grad { + template + MSHADOW_XINLINE static DType Map(DType a, DType b) { + return DType(0); + } +}; +template<> +MSHADOW_XINLINE double mod_grad::Map(double a, double b) { + return 1.0f; +} +template<> +MSHADOW_XINLINE float mod_grad::Map(float a, float b) { + return 1.0f; +} +#ifdef __CUDACC__ +template<> +MSHADOW_XINLINE mshadow::half::half_t mod_grad::Map + (mshadow::half::half_t a, + mshadow::half::half_t b) { + return mshadow::half::half_t(1.0f); +} +template<> +MSHADOW_XINLINE mshadow::half::half2_t mod_grad::Map + (mshadow::half::half2_t a, + mshadow::half::half2_t b) { + mshadow::half::half2_t result = mshadow::half::half2_t(); +#if MSHADOW_CUDA_HALF2 + result.half2_ = ::__float2half2_rn(1.0f); +#else + result.half_t2[0] = mshadow::half::half_t(0.0f); + result.half_t2[1] = mshadow::half::half_t(1.0f); +#endif + return result; +} +#endif + +struct mod_rgrad { + template + MSHADOW_XINLINE static DType Map(DType a, DType b) { + return DType(0); + } +}; +template<> +MSHADOW_XINLINE double mod_rgrad::Map(double a, double b) { + return -::floor(a/b); +} +template<> +MSHADOW_XINLINE float mod_rgrad::Map(float a, float b) { + return -::floorf(a/b); +} +#ifdef __CUDACC__ +template<> +MSHADOW_XINLINE mshadow::half::half_t mod_rgrad::Map + (mshadow::half::half_t a, + mshadow::half::half_t b) { + return mshadow::half::half_t(-::floorf(static_cast(a/b))); +} +template<> +MSHADOW_XINLINE mshadow::half::half2_t mod_rgrad::Map + (mshadow::half::half2_t a, + mshadow::half::half2_t b) { +#if MSHADOW_CUDA_HALF2 + return mshadow::half::half2_t(__hneg2(::h2floor((a/b).half2_))); +#else + return mshadow::half::half2_t(mshadow::half::half_t(-::floorf( + static_cast(a.half_t2[0]/b.half_t2[0]))), + mshadow::half::half_t(-::floorf( + static_cast(a.half_t2[1]/b.half_t2[1])))); +#endif +} +#endif + +struct rmod { + template + MSHADOW_XINLINE static DType Map(DType a, DType b) { + if (a == DType(0)) { + return DType(0); + } else if (a < DType(0)) { + if (b < DType(0)) { + return DType(-::fmod(-b, -a)); + } else { + return DType(::fmod(b, -a) + (::fmod(b, -a) != DType(0) ? a : DType(0))); + } + } else { + if (b < DType(0)) { + return DType(-::fmod(-b, a) + (::fmod(-b, a) != DType(0) ? a : DType(0))); + } else { + return DType(::fmod(b, a)); + } + } + } +}; +#ifdef __CUDACC__ +template<> +MSHADOW_XINLINE mshadow::half::half2_t rmod::Map + (mshadow::half::half2_t a, + mshadow::half::half2_t b) { + return b%a; +} +#endif + +struct rmod_grad { + template + MSHADOW_XINLINE static DType Map(DType a, DType b) { + return DType(0); + } +}; +template<> +MSHADOW_XINLINE double rmod_grad::Map(double a, double b) { + return -::floor(b/a); +} +template<> +MSHADOW_XINLINE float rmod_grad::Map(float a, float b) { + return -::floorf(b/a); +} +#ifdef __CUDACC__ +template<> +MSHADOW_XINLINE mshadow::half::half_t rmod_grad::Map + (mshadow::half::half_t a, + mshadow::half::half_t b) { + return mshadow::half::half_t(-::floorf(static_cast(b/a))); +} +template<> +MSHADOW_XINLINE mshadow::half::half2_t rmod_grad::Map + (mshadow::half::half2_t a, + mshadow::half::half2_t b) { +#if MSHADOW_CUDA_HALF2 + return mshadow::half::half2_t(::__hneg2(::h2floor((b/a).half2_))); +#else + return mshadow::half::half2_t(mshadow::half::half_t(-::floorf( + static_cast(b.half_t2[0]/a.half_t2[0]))), + mshadow::half::half_t(-::floorf( + static_cast(b.half_t2[1]/a.half_t2[1])))); +#endif +} +#endif + struct clip { template MSHADOW_XINLINE static DType Map(DType x, DType bound) { diff --git a/src/operator/tensor/elemwise_binary_broadcast_op_basic.cc b/src/operator/tensor/elemwise_binary_broadcast_op_basic.cc index 0d0a1d8b5df0..27a4b5f25c82 100644 --- a/src/operator/tensor/elemwise_binary_broadcast_op_basic.cc +++ b/src/operator/tensor/elemwise_binary_broadcast_op_basic.cc @@ -153,5 +153,38 @@ NNVM_REGISTER_OP(_backward_broadcast_div) .set_attr("FCompute", BinaryBroadcastBackwardUseIn); +MXNET_OPERATOR_REGISTER_BINARY_BROADCAST(broadcast_mod) +.describe(R"code(Returns element-wise modulo of the input arrays with broadcasting. + +Example:: + + x = [[ 8., 8., 8.], + [ 8., 8., 8.]] + + y = [[ 2.], + [ 3.]] + + broadcast_mod(x, y) = [[ 0., 0., 0.], + [ 2., 2., 2.]] + +)code" ADD_FILELINE) +.set_attr("FCompute", BinaryBroadcastCompute) +.set_attr("FGradient", ElemwiseGradUseIn{"_backward_broadcast_mod"}); + +NNVM_REGISTER_OP(_backward_broadcast_mod) +.set_num_inputs(3) +.set_num_outputs(2) +.set_attr("TIsBackward", true) +.set_attr("FInplaceOption", + [](const NodeAttrs& attrs){ + return std::vector >{{0, 1}}; + }) +.set_attr("FResourceRequest", + [](const NodeAttrs& attrs) { + return std::vector{ResourceRequest::kTempSpace}; + }) +.set_attr("FCompute", BinaryBroadcastBackwardUseIn); + } // namespace op } // namespace mxnet diff --git a/src/operator/tensor/elemwise_binary_broadcast_op_basic.cu b/src/operator/tensor/elemwise_binary_broadcast_op_basic.cu index f23d3d0cbad8..ef0e679d6166 100644 --- a/src/operator/tensor/elemwise_binary_broadcast_op_basic.cu +++ b/src/operator/tensor/elemwise_binary_broadcast_op_basic.cu @@ -37,5 +37,12 @@ NNVM_REGISTER_OP(_backward_broadcast_div) .set_attr("FCompute", BinaryBroadcastBackwardUseIn); +NNVM_REGISTER_OP(broadcast_mod) +.set_attr("FCompute", BinaryBroadcastCompute); + +NNVM_REGISTER_OP(_backward_broadcast_mod) +.set_attr("FCompute", BinaryBroadcastBackwardUseIn); + } // namespace op } // namespace mxnet diff --git a/src/operator/tensor/elemwise_binary_op_basic.cc b/src/operator/tensor/elemwise_binary_op_basic.cc index be4c1d88e983..1f363a114375 100644 --- a/src/operator/tensor/elemwise_binary_op_basic.cc +++ b/src/operator/tensor/elemwise_binary_op_basic.cc @@ -78,5 +78,21 @@ NNVM_REGISTER_OP(_backward_div) .set_attr("FCompute", BinaryBackwardUseIn); +MXNET_OPERATOR_REGISTER_BINARY(_mod) +.add_alias("_Mod") +.set_attr("FCompute", BinaryCompute) +.set_attr("FGradient", ElemwiseGradUseIn{"_backward_mod"}); + +NNVM_REGISTER_OP(_backward_mod) +.set_num_inputs(3) +.set_num_outputs(2) +.set_attr("TIsBackward", true) +.set_attr("FInplaceOption", + [](const NodeAttrs& attrs){ + return std::vector >{{0, 1}}; + }) +.set_attr("FCompute", BinaryBackwardUseIn); + } // namespace op } // namespace mxnet diff --git a/src/operator/tensor/elemwise_binary_op_basic.cu b/src/operator/tensor/elemwise_binary_op_basic.cu index ff432380d6d1..6355c4e5cf01 100644 --- a/src/operator/tensor/elemwise_binary_op_basic.cu +++ b/src/operator/tensor/elemwise_binary_op_basic.cu @@ -40,5 +40,12 @@ NNVM_REGISTER_OP(_backward_div) .set_attr("FCompute", BinaryBackwardUseInWithHalf2); +NNVM_REGISTER_OP(_mod) +.set_attr("FCompute", BinaryComputeWithHalf2); + +NNVM_REGISTER_OP(_backward_mod) +.set_attr("FCompute", BinaryBackwardUseInWithHalf2); + } // namespace op } // namespace mxnet diff --git a/src/operator/tensor/elemwise_binary_scalar_op_basic.cc b/src/operator/tensor/elemwise_binary_scalar_op_basic.cc index ddbba4d10f2c..bd0b5335e3ae 100644 --- a/src/operator/tensor/elemwise_binary_scalar_op_basic.cc +++ b/src/operator/tensor/elemwise_binary_scalar_op_basic.cc @@ -44,5 +44,25 @@ MXNET_OPERATOR_REGISTER_BINARY(_backward_rdiv_scalar) .set_attr_parser([](NodeAttrs* attrs) {attrs->parsed = std::stod(attrs->dict["scalar"]);}) .set_attr("FCompute", BinaryScalarBackward); +MXNET_OPERATOR_REGISTER_BINARY_SCALAR(_mod_scalar) +.set_attr("FCompute", BinaryScalarCompute) +.set_attr("FGradient", ElemwiseGradUseIn{"_backward_mod_scalar"}) +.add_alias("_ModScalar"); + +MXNET_OPERATOR_REGISTER_BINARY(_backward_mod_scalar) +.add_argument("scalar", "float", "scalar value") +.set_attr_parser([](NodeAttrs* attrs) {attrs->parsed = std::stod(attrs->dict["scalar"]);}) +.set_attr("FCompute", BinaryScalarBackward); + +MXNET_OPERATOR_REGISTER_BINARY_SCALAR(_rmod_scalar) +.set_attr("FCompute", BinaryScalarCompute) +.set_attr("FGradient", ElemwiseGradUseIn{"_backward_rmod_scalar"}) +.add_alias("_RModScalar"); + +MXNET_OPERATOR_REGISTER_BINARY(_backward_rmod_scalar) +.add_argument("scalar", "float", "scalar value") +.set_attr_parser([](NodeAttrs* attrs) {attrs->parsed = std::stod(attrs->dict["scalar"]);}) +.set_attr("FCompute", BinaryScalarBackward); + } // namespace op } // namespace mxnet diff --git a/src/operator/tensor/elemwise_binary_scalar_op_basic.cu b/src/operator/tensor/elemwise_binary_scalar_op_basic.cu index 356b34901670..ae19aa8a72f6 100644 --- a/src/operator/tensor/elemwise_binary_scalar_op_basic.cu +++ b/src/operator/tensor/elemwise_binary_scalar_op_basic.cu @@ -30,5 +30,17 @@ NNVM_REGISTER_OP(_rdiv_scalar) NNVM_REGISTER_OP(_backward_rdiv_scalar) .set_attr("FCompute", BinaryScalarBackward); +NNVM_REGISTER_OP(_mod_scalar) +.set_attr("FCompute", BinaryScalarCompute); + +NNVM_REGISTER_OP(_backward_mod_scalar) +.set_attr("FCompute", BinaryScalarBackward); + +NNVM_REGISTER_OP(_rmod_scalar) +.set_attr("FCompute", BinaryScalarCompute); + +NNVM_REGISTER_OP(_backward_rmod_scalar) +.set_attr("FCompute", BinaryScalarBackward); + } // namespace op } // namespace mxnet diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py index fece5414dbc3..55c1d2488d6e 100644 --- a/tests/python/unittest/test_operator.py +++ b/tests/python/unittest/test_operator.py @@ -994,23 +994,41 @@ def gen_broadcast_data(idx): def gen_broadcast_data_int(idx): d = gen_broadcast_data(idx); - return [np.round(d[0]*100), np.round(d[1]*100)] + return [np.round(d[0]*100).astype(int), np.round(d[1]*100).astype(int)] def gen_binary_data(dummy): ndim = np.random.randint(1, 6) shape = np.random.randint(1, 6, size=(ndim,)) return [np.random.random(shape), np.random.random(shape)] -def check_binary_op_forward(symbol, baseline, gen_data): +def gen_binary_data_int(dummy): + d = gen_binary_data(dummy); + return [np.round(d[0]*100).astype(int), np.round(d[1]*100).astype(int)] + +def check_binary_op_forward(symbol, baseline, gen_data, rtol=1e-3, atol=1e-5): sample_num = 200 for i in range(sample_num): d = gen_data(i) x = baseline(d[0], d[1]) y = symbol.bind(default_context(), args={'a': mx.nd.array(d[0]), 'b' : mx.nd.array(d[1])}) y.forward(is_train=True) - assert_allclose(x, y.outputs[0].asnumpy(), rtol=1e-3, atol=1e-5) - -def check_binary_op_backward(symbol, baseline, gen_data): + y = y.outputs[0].asnumpy() + idx = np.abs(x-y) > atol+rtol*np.abs(x) + if idx.any(): + print('found precision problem') + d[0] = np.broadcast_to(d[0], x.shape) + d[1] = np.broadcast_to(d[1], x.shape) + print('a: {}'.format(d[0][idx])) + print('b: {}'.format(d[1][idx])) + import struct + print('a hex: {}'.format(struct.pack('d', d[0][idx]).encode('hex'))) + print('b hex: {}'.format(struct.pack('d', np.broadcast_to(d[1], x.shape)[idx]).encode('hex'))) + print('in baseline(a, b): {}'.format(x[idx])) + print('in symbol(a, b): {}'.format(y[idx])) + print('diff: {}'.format(np.abs(x-y)[idx] - atol-rtol*np.abs(x)[idx])) + assert_allclose(y, x, rtol=rtol, atol=atol) + +def check_binary_op_backward(symbol, baseline, gen_data, rtol=1e-3, atol=1e-5): sample_num = 200 for i in range(sample_num): d = gen_data(i) @@ -1033,8 +1051,8 @@ def reduce_op(shape, x): args_grad=[y_1, y_2]) y.forward(is_train=True) y.backward([mx.nd.array(out)]) - assert_allclose(x_1, y_1.asnumpy(), rtol=1e-3, atol=1e-5) - assert_allclose(x_2, y_2.asnumpy(), rtol=1e-3, atol=1e-5) + assert_allclose(y_1.asnumpy(), x_1, rtol=rtol, atol=atol) + assert_allclose(y_2.asnumpy(), x_2, rtol=rtol, atol=atol) def test_binary_op(): a = mx.sym.Variable('a') @@ -1060,6 +1078,16 @@ def test_bdiv(a, b): check_binary_op_forward(c, lambda a, b: a / b, gen_binary_data) check_binary_op_backward(c, lambda g_out, a, b: (g_out / b, - g_out * a / (b * b)), gen_binary_data) + def test_bmod(a, b): + c = a % b + check_binary_op_forward(c, lambda a, b: a % b, gen_binary_data) + check_binary_op_backward(c, lambda g_out, a, b: (g_out, - g_out * (a // b)), gen_binary_data) + + def test_bmod_int(a, b): + c = mx.sym.cast(a, dtype='int32') % mx.sym.cast(b, dtype='int32') + check_binary_op_forward(c, lambda a, b: a % b, gen_binary_data_int) + check_binary_op_backward(c, lambda g_out, a, b: (np.zeros_like(a), np.zeros_like(b)), gen_binary_data_int) + def test_bpow(a, b): c = a ** b check_binary_op_forward(c, lambda a, b: a ** b, gen_binary_data) @@ -1075,6 +1103,8 @@ def test_bneq(a, b): test_bminus(a, b) test_bmul(a, b) test_bdiv(a, b) + test_bmod(a, b) + test_bmod_int(a, b) test_bpow(a, b) test_bneq(a, b) @@ -1102,6 +1132,16 @@ def test_bdiv(a, b): check_binary_op_forward(c, lambda a, b: a / b, gen_broadcast_data) check_binary_op_backward(c, lambda g_out, a, b: (g_out / b, - g_out * a / (b * b)), gen_broadcast_data) + def test_bmod(a, b): + c = mx.sym.broadcast_mod(a, b) + check_binary_op_forward(c, lambda a, b: a % b, gen_broadcast_data, atol=1) + check_binary_op_backward(c, lambda g_out, a, b: (g_out, - g_out * (a // b)), gen_broadcast_data, atol=1) + + def test_bmod_int(a, b): + c = mx.sym.broadcast_mod(mx.sym.cast(a, dtype='int32'), mx.sym.cast(b, dtype='int32')) + check_binary_op_forward(c, lambda a, b: a % b, gen_broadcast_data_int) + check_binary_op_backward(c, lambda g_out, a, b: (np.zeros_like(a), np.zeros_like(b)), gen_broadcast_data_int) + def test_bpow(a, b): c = mx.sym.broadcast_power(a, b) check_binary_op_forward(c, lambda a, b: a ** b, gen_broadcast_data) @@ -1117,6 +1157,8 @@ def test_bequal(a, b): test_bminus(a, b) test_bmul(a, b) test_bdiv(a, b) + test_bmod(a, b) + test_bmod_int(a, b) test_bpow(a, b) test_bequal(a, b) @@ -3276,7 +3318,7 @@ def test_laop(): # Currently no support for GPU. Will be added soon # so keep these tests here in this file and activate - # gpu-testing when it is ready. + # gpu-testing when it is ready. dev = default_context() if dev.device_type == 'gpu': return @@ -3293,37 +3335,37 @@ def test_laop(): shape2 = (3, 2) shape3 = (3, 3) shape4 = (2, 2) - #Ensure that ithis tests don't get changed by other calls to random. + #Ensure that ithis tests don't get changed by other calls to random. np.random.seed(42) - data_in1 = np.random.uniform(1, 10, shape1) - data_in2 = np.random.uniform(1, 10, shape2) - data_in3 = np.random.uniform(1, 10, shape3) - data_in4 = np.random.uniform(1, 10, shape4) + data_in1 = np.random.uniform(1, 10, shape1) + data_in2 = np.random.uniform(1, 10, shape2) + data_in3 = np.random.uniform(1, 10, shape3) + data_in4 = np.random.uniform(1, 10, shape4) # Check all transpositions of gemm operator. - data_in1_t = np.transpose(data_in1) - data_in2_t = np.transpose(data_in2) + data_in1_t = np.transpose(data_in1) + data_in2_t = np.transpose(data_in2) res_gemm = 4*np.dot(data_in1,data_in2)+7*data_in4 - test_gemm = mx.sym.linalg_gemm(data1, data2, data3, alpha = 4, beta = 7) + test_gemm = mx.sym.linalg_gemm(data1, data2, data3, alpha = 4, beta = 7) check_symbolic_forward(test_gemm, [data_in1, data_in2, data_in4], [res_gemm]) if grad_check == 1: check_numeric_gradient(test_gemm, [data_in1, data_in2, data_in4], numeric_eps=1e-3, rtol=1e-1, atol=1e-1) res_gemm = 4*np.dot(data_in1_t,data_in2_t)+7*data_in3 - test_gemm = mx.sym.linalg_gemm(data1, data2, data3, alpha = 4, beta = 7, transpose_a = 1, transpose_b = 1) + test_gemm = mx.sym.linalg_gemm(data1, data2, data3, alpha = 4, beta = 7, transpose_a = 1, transpose_b = 1) check_symbolic_forward(test_gemm, [data_in1, data_in2, data_in3], [res_gemm]) if grad_check == 1: check_numeric_gradient(test_gemm, [data_in1, data_in2, data_in3], numeric_eps=1e-3, rtol=1e-1, atol=1e-1) res_gemm = 4*np.dot(data_in1_t,data_in1)+7*data_in3 - test_gemm = mx.sym.linalg_gemm(data1, data2, data3, alpha = 4, beta = 7, transpose_a = 1) + test_gemm = mx.sym.linalg_gemm(data1, data2, data3, alpha = 4, beta = 7, transpose_a = 1) check_symbolic_forward(test_gemm, [data_in1, data_in1, data_in3], [res_gemm]) if grad_check == 1: check_numeric_gradient(test_gemm, [data_in1, data_in1, data_in3], numeric_eps=1e-3, rtol=1e-1, atol=1e-1) res_gemm = 4*np.dot(data_in1,data_in1_t)+7*data_in4 - test_gemm = mx.sym.linalg_gemm(data1, data2, data3, alpha = 4, beta = 7, transpose_b = 1) + test_gemm = mx.sym.linalg_gemm(data1, data2, data3, alpha = 4, beta = 7, transpose_b = 1) check_symbolic_forward(test_gemm, [data_in1, data_in1, data_in4], [res_gemm]) if grad_check == 1: check_numeric_gradient(test_gemm, [data_in1, data_in1, data_in4], numeric_eps=1e-3, rtol=1e-1, atol=1e-1) - # Check batch of gemm. + # Check batch of gemm. a = np.tile(np.array(data_in1).flatten(),3) a = np.reshape(a,(3,1,2,3)) b = np.tile(np.array(data_in2).flatten(),3) @@ -3333,34 +3375,34 @@ def test_laop(): r = 4*np.dot(data_in1,data_in2)+7*data_in4 r = np.tile(r.flatten(),3) r = np.reshape(r,(3,1,2,2)) - test_gemm = mx.sym.linalg_gemm(data1, data2, data3, alpha = 4, beta = 7) + test_gemm = mx.sym.linalg_gemm(data1, data2, data3, alpha = 4, beta = 7) check_symbolic_forward(test_gemm, [a, b, c], [r]) if grad_check == 1: check_numeric_gradient(test_gemm, [a, b, c], numeric_eps=1e-3, rtol=1e-1, atol=1e-1) - # Check gemm2 operator same way as gemm. + # Check gemm2 operator same way as gemm. res_gemm = 4*np.dot(data_in1,data_in2) - test_gemm = mx.sym.linalg_gemm2(data1, data2, alpha = 4) + test_gemm = mx.sym.linalg_gemm2(data1, data2, alpha = 4) check_symbolic_forward(test_gemm, [data_in1, data_in2], [res_gemm]) if grad_check == 1: check_numeric_gradient(test_gemm, [data_in1, data_in2], numeric_eps=1e-3, rtol=1e-1, atol=1e-1) res_gemm = 4*np.dot(data_in1_t, data_in2_t) - test_gemm = mx.sym.linalg_gemm2(data1, data2, alpha = 4, transpose_a = 1, transpose_b = 1) + test_gemm = mx.sym.linalg_gemm2(data1, data2, alpha = 4, transpose_a = 1, transpose_b = 1) check_symbolic_forward(test_gemm, [data_in1, data_in2], [res_gemm]) if grad_check == 1: check_numeric_gradient(test_gemm, [data_in1, data_in2], numeric_eps=1e-3, rtol=1e-1, atol=1e-1) res_gemm = 4*np.dot(data_in1_t,data_in1) - test_gemm = mx.sym.linalg_gemm2(data1, data2, alpha = 4, transpose_a = 1) + test_gemm = mx.sym.linalg_gemm2(data1, data2, alpha = 4, transpose_a = 1) check_symbolic_forward(test_gemm, [data_in1, data_in1], [res_gemm]) if grad_check == 1: check_numeric_gradient(test_gemm, [data_in1, data_in1], numeric_eps=1e-3, rtol=1e-1, atol=1e-1) res_gemm = 4*np.dot(data_in1,data_in1_t) - test_gemm = mx.sym.linalg_gemm2(data1, data2, alpha = 4, transpose_b = 1) + test_gemm = mx.sym.linalg_gemm2(data1, data2, alpha = 4, transpose_b = 1) check_symbolic_forward(test_gemm, [data_in1, data_in1], [res_gemm]) if grad_check == 1: check_numeric_gradient(test_gemm, [data_in1, data_in1], numeric_eps=1e-3, rtol=1e-1, atol=1e-1) - # Check batch of gemm2. + # Check batch of gemm2. a = np.tile(np.array(data_in1).flatten(),3) a = np.reshape(a,(3,1,2,3)) b = np.tile(np.array(data_in2).flatten(),3) @@ -3368,12 +3410,12 @@ def test_laop(): r = 4*np.dot(data_in1,data_in2) r = np.tile(r.flatten(),3) r = np.reshape(r,(3,1,2,2)) - test_gemm = mx.sym.linalg_gemm2(data1, data2, alpha = 4) + test_gemm = mx.sym.linalg_gemm2(data1, data2, alpha = 4) check_symbolic_forward(test_gemm, [a, b], [r]) if grad_check == 1: check_numeric_gradient(test_gemm, [a, b], numeric_eps=1e-3, rtol=1e-1, atol=1e-1) - # Now test all the other operators. + # Now test all the other operators. # Tests with trivial 1x1 matrices. shape = (4, 4, 1, 1 ) @@ -3404,7 +3446,7 @@ def test_laop(): if grad_check == 1: check_numeric_gradient(test_trmm, [trian_in,data_in], atol = 0.02, rtol = 2.0) # test sumlogdiag - res_sumlogdiag = np.reshape(np.log(data_in),(4,4)) + res_sumlogdiag = np.reshape(np.log(data_in),(4,4)) test_sumlogdiag = mx.sym.linalg_sumlogdiag(data1) check_symbolic_forward(test_sumlogdiag, [data_in], [res_sumlogdiag]) if grad_check == 1: @@ -3417,9 +3459,9 @@ def test_laop(): inv = [ 2.98333, 0.01667, 2.65, -0.83333, 0.01667, 0.05, 0.05, 0, 2.65, 0.05, 2.5, -0.75, -0.83333, 0, -0.75, 0.25 ] ident = [ 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1 ] - # Tests for numeric gradients for potrf/potri/trmm/trsm are suppressed by default - # as they are very volatile and may often report false negatives which - # have to be excluded by manual inspection. + # Tests for numeric gradients for potrf/potri/trmm/trsm are suppressed by default + # as they are very volatile and may often report false negatives which + # have to be excluded by manual inspection. grad_check = 0 # test potrf @@ -3430,7 +3472,7 @@ def test_laop(): check_symbolic_forward(test_potrf, [a], [r]) if grad_check == 1: check_numeric_gradient(test_potrf, [a], numeric_eps=1e-3, rtol=1e-2, atol=1e-1) - + #test potri a = np.tile(np.array(trian),3) a = np.reshape(a,(3,1,4,4)) @@ -3450,7 +3492,7 @@ def test_laop(): check_symbolic_forward(test_trsm, [a,b], [r]) if grad_check == 1: check_numeric_gradient(test_trsm, [a,b], numeric_eps=1e-3, rtol=1e-2, atol=1e-1) - + test_trsm2 = mx.sym.linalg_trsm(data1,data2,alpha = -2, rightside = 1, transpose = 1) r = -2*np.reshape(np.array(trian),(4,4)) r = np.reshape(np.tile(np.reshape(r,(16)),3),(3,1,4,4)) From 94d8e18ad293e374d137e05f3e42575b4574b226 Mon Sep 17 00:00:00 2001 From: formath Date: Wed, 21 Jun 2017 06:31:58 +0800 Subject: [PATCH 095/834] set __layout__ attribute for FusedRNNCell output states (#6747) * set __layout__ attr for FucedRNNCell output states * right usage not that in desc --- python/mxnet/rnn/rnn_cell.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/mxnet/rnn/rnn_cell.py b/python/mxnet/rnn/rnn_cell.py index c00f8a39d8c3..d0505f87ac40 100644 --- a/python/mxnet/rnn/rnn_cell.py +++ b/python/mxnet/rnn/rnn_cell.py @@ -672,11 +672,15 @@ def unroll(self, length, inputs, begin_state=None, layout='NTC', merge_outputs=N mode=self._mode, name=self._prefix+'rnn', **states) + attr = {'__layout__' : 'LNC'} if not self._get_next_state: outputs, states = rnn, [] elif self._mode == 'lstm': + rnn[1]._set_attr(**attr) + rnn[2]._set_attr(**attr) outputs, states = rnn[0], [rnn[1], rnn[2]] else: + rnn[1]._set_attr(**attr) outputs, states = rnn[0], [rnn[1]] if axis == 1: From 3384d206a20faaae164ea9b7af6c4ff8e89baa69 Mon Sep 17 00:00:00 2001 From: Leonard Date: Wed, 21 Jun 2017 07:32:39 +0900 Subject: [PATCH 096/834] Reformulate some comments in executor_manager and executor_group (#6740) --- python/mxnet/executor_manager.py | 4 ++-- python/mxnet/module/executor_group.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/python/mxnet/executor_manager.py b/python/mxnet/executor_manager.py index 4361d75cd4bd..0fb9eb3baade 100644 --- a/python/mxnet/executor_manager.py +++ b/python/mxnet/executor_manager.py @@ -30,7 +30,7 @@ def _split_input_slice(batch_size, work_load_list): Raises ------ ValueError - If there are two many splits such that some slice can be empty. + In case of too many splits, leading to some empty slices. """ total_work_load = sum(work_load_list) batch_num_list = [round(work_load * batch_size / total_work_load) @@ -44,7 +44,7 @@ def _split_input_slice(batch_size, work_load_list): begin = int(min((end, batch_size))) end = int(min((begin + batch_num, batch_size))) if begin >= end: - raise ValueError('Too many slices such that some splits are empty') + raise ValueError('Too many slices. Some splits are empty.') slices.append(slice(begin, end)) return slices diff --git a/python/mxnet/module/executor_group.py b/python/mxnet/module/executor_group.py index ce71fa3ad4e9..7e2e5e26bb35 100755 --- a/python/mxnet/module/executor_group.py +++ b/python/mxnet/module/executor_group.py @@ -111,8 +111,8 @@ class DataParallelExecutorGroup(object): shared_group : DataParallelExecutorGroup Defaults to ``None``. This is used in bucketing. When not ``None``, it should be a executor group corresponding to a different bucket. In other words, it will correspond to a different - symbol but with the same set of parameters (e.g. unrolled RNNs with different lengths). - In this case, many memory will be shared. + symbol with the same set of parameters (e.g. unrolled RNNs with different lengths). + In this case the memory regions of the parameters will be shared. logger : Logger Default is `logging`. fixed_param_names: list of str From 3c38aed9fa91656287dfd3ccbc30355cec70fbb4 Mon Sep 17 00:00:00 2001 From: Roshani Nagmote Date: Tue, 20 Jun 2017 15:38:26 -0700 Subject: [PATCH 097/834] Env variables doc modified (#6755) * Env variables doc modified * correction after review * correction after review * addressed some of the review comments * addressed review comments --- docs/how_to/env_var.md | 111 ++++++++++++++++++++++++++--------------- 1 file changed, 71 insertions(+), 40 deletions(-) diff --git a/docs/how_to/env_var.md b/docs/how_to/env_var.md index 40423b55b5ee..cb993192bc7f 100644 --- a/docs/how_to/env_var.md +++ b/docs/how_to/env_var.md @@ -3,85 +3,116 @@ Environment Variables MXNet has several settings that you can change with environment variables. Typically, you wouldn't need to change these settings, but they are listed here for reference. +For example, you can set these environment variables in Linux or macOS as follows: +``` +export MXNET_GPU_WORKER_NTHREADS=3 +``` + ## Set the Number of Threads -* MXNET_GPU_WORKER_NTHREADS (default=2) - - The maximum number of threads that do the computation job on each GPU. -* MXNET_GPU_COPY_NTHREADS (default=1) - - The maximum number of threads that do the memory copy job on each GPU. -* MXNET_CPU_WORKER_NTHREADS (default=1) - - The maximum number of threads that do the CPU computation job. -* MXNET_CPU_PRIORITY_NTHREADS (default=4) - - The number of threads given to prioritized CPU jobs. -* MXNET_CPU_NNPACK_NTHREADS (default=4) - - The number of threads used for NNPACK. +* MXNET_GPU_WORKER_NTHREADS + - Values: Int ```(default=2)``` + - The maximum number of threads to use on each GPU. This parameter is used to parallelize the computation within a single GPU card. +* MXNET_GPU_COPY_NTHREADS + - Values: Int ```(default=1)``` + - The maximum number of concurrent threads that do the memory copy job on each GPU. +* MXNET_CPU_WORKER_NTHREADS + - Values: Int ```(default=1)``` + - The maximum number of scheduling threads on CPU. It specifies how many operators can be run in parallel. +* MXNET_CPU_PRIORITY_NTHREADS + - Values: Int ```(default=4)``` + - The number of threads given to prioritized CPU jobs. +* MXNET_CPU_NNPACK_NTHREADS + - Values: Int ```(default=4)``` + - The number of threads used for NNPACK. NNPACK package aims to provide high-performance implementations of some layers for multi-core CPUs. Checkout [NNPACK](http://mxnet.io/how_to/nnpack.html) to know more about it. ## Memory Options -* MXNET_EXEC_ENABLE_INPLACE (default=true) - - Whether to enable in-place optimization in symbolic execution. -* NNVM_EXEC_MATCH_RANGE (default=16) - - The rough matching scale in the symbolic execution memory allocator. +* MXNET_EXEC_ENABLE_INPLACE + - Values: true or false ```(default=true)``` + - Whether to enable in-place optimization in symbolic execution. Checkout [in-place optimization](http://mxnet.io/architecture/note_memory.html#in-place-operations) to know more about it. +* NNVM_EXEC_MATCH_RANGE + - Values: Int ```(default=16)``` + - The approximate matching scale in the symbolic execution memory allocator. - Set this to 0 if you don't want to enable memory sharing between graph nodes(for debugging purposes). -* MXNET_EXEC_NUM_TEMP (default=1) - - The maximum number of temp workspaces to allocate to each device. + - This variable has impact on the result of memory planning. So, MXNet sweep between [1, NNVM_EXEC_MATCH_RANGE], and selects the best value. +* MXNET_EXEC_NUM_TEMP + - Values: Int ```(default=1)``` + - The maximum number of temporary workspaces to allocate to each device. This controls space replicas and in turn reduces the memory usage. - Setting this to a small number can save GPU memory. It will also likely decrease the level of parallelism, which is usually acceptable. -* MXNET_GPU_MEM_POOL_RESERVE (default=5) + - MXNet internally uses graph coloring algorithm to [optimize memory consumption](http://mxnet.io/architecture/note_memory.html). + - This parameter is also used to get number of matching colors in graph and in turn how much parallelism one can get in each GPU. Color based match usually costs more memory but also enables more parallelism. +* MXNET_GPU_MEM_POOL_RESERVE + - Values: Int ```(default=5)``` - The percentage of GPU memory to reserve for things other than the GPU array, such as kernel launch or cudnn handle space. - If you see a strange out-of-memory error from the kernel launch, after multiple iterations, try setting this to a larger value. ## Engine Type -* MXNET_ENGINE_TYPE (default=ThreadedEnginePerDevice) +* MXNET_ENGINE_TYPE + - Values: String ```(default=ThreadedEnginePerDevice)``` - The type of underlying execution engine of MXNet. - Choices: - - NaiveEngine: A very simple engine that uses the master thread to do computation. + - NaiveEngine: A very simple engine that uses the master thread to do the computation synchronously. Setting this engine disables multi-threading. You can use this type for debugging in case of any error. Backtrace will give you the series of calls that lead to the error. Remember to set MXNET_ENGINE_TYPE back to empty after debugging. - ThreadedEngine: A threaded engine that uses a global thread pool to schedule jobs. - - ThreadedEnginePerDevice: A threaded engine that allocates thread per GPU. + - ThreadedEnginePerDevice: A threaded engine that allocates thread per GPU and executes jobs asynchronously. ## Execution Options -* MXNET_EXEC_BULK_EXEC_INFERENCE (default=1) +* MXNET_EXEC_BULK_EXEC_INFERENCE + - Values: 0(false) or 1(true) ```(default=1)``` - If set to `1`, during inference MXNet executes the entire computation graph in bulk mode, which reduces kernel launch gaps in between symbolic operators. -* MXNET_EXEC_BULK_EXEC_TRAIN (default=1) +* MXNET_EXEC_BULK_EXEC_TRAIN + - Values: 0(false) or 1(true) ```(default=1)``` - If set to `1`, during training MXNet executes the computation graph as several subgraphs in bulk mode. -* MXNET_EXEC_BULK_EXEC_MAX_NODE_TRAIN (default=15) +* MXNET_EXEC_BULK_EXEC_MAX_NODE_TRAIN + - Values: Int ```(default=15)``` - The maximum number of nodes in the subgraph executed in bulk during training(not inference). Setting this to a larger number may reduce the degree of parallelism for multi-GPU training. ## Control the Data Communication -* MXNET_KVSTORE_REDUCTION_NTHREADS (default=4) +* MXNET_KVSTORE_REDUCTION_NTHREADS + - Values: Int ```(default=4)``` - The number of CPU threads used for summing big arrays. -* MXNET_KVSTORE_BIGARRAY_BOUND (default=1e6) - - The minimum size of a "big array." - - When the array size is bigger than this threshold, MXNET_KVSTORE_REDUCTION_NTHREADS threads are used for reduction. -* MXNET_ENABLE_GPU_P2P (default=1) - - If true, MXNet tries to use GPU peer-to-peer communication, if available, - when kvstore's type is `device` +* MXNET_KVSTORE_BIGARRAY_BOUND + - Values: Int ```(default=1000000)``` + - The minimum size of a "big array". + - When the array size is bigger than this threshold, MXNET_KVSTORE_REDUCTION_NTHREADS threads are used for reduction. + - This parameter is also used as a load balancer in kvstore. It controls when to partition a single weight to all the servers. If the size of a single weight is less than MXNET_KVSTORE_BIGARRAY_BOUND then, it is sent to a single randomly picked server otherwise it is partitioned to all the servers. +* MXNET_ENABLE_GPU_P2P + - Values: 0(false) or 1(true) ```(default=1)``` + - If true, MXNet tries to use GPU peer-to-peer communication, if available on your device, + when kvstore's type is `device`. ## Memonger -* MXNET_BACKWARD_DO_MIRROR (default=0) - - whether do `mirror` during training for saving device memory. - - when set to `1`, then during forward propagation, graph executor will `mirror` some layer's feature map and drop others, but it will re-compute this dropped feature maps when needed. `MXNET_BACKWARD_DO_MIRROR=1` will save 30%~50% of device memory, but retains about 95% of running speed. - - one extension of `mirror` in MXNet is called [memonger technology](https://arxiv.org/abs/1604.06174), it will only use O(sqrt(N)) memory at 75% running speed. +* MXNET_BACKWARD_DO_MIRROR + - Values: 0(false) or 1(true) ```(default=0)``` + - MXNet uses mirroring concept to save memory. Normally backward pass needs some forward input and it is stored in memory but you can choose to release this saved input and recalculate it in backward pass when needed. This basically trades off the computation for memory consumption. + - This parameter decides whether to do `mirror` during training for saving device memory. + - When set to `1`, during forward propagation, graph executor will `mirror` some layer's feature map and drop others, but it will re-compute this dropped feature maps when needed. + - `MXNET_BACKWARD_DO_MIRROR=1` will save 30%~50% of device memory, but retains about 95% of running speed. + - One extension of `mirror` in MXNet is called [memonger technology](https://arxiv.org/abs/1604.06174), it will only use O(sqrt(N)) memory at 75% running speed. Checkout the code [here](https://github.com/dmlc/mxnet-memonger). ## Control the profiler When USE_PROFILER is enabled in Makefile or CMake, the following environments can be used to profile the application without changing code. Execution options may affect the granularity of profiling result. If you need profiling result of every operator, please set MXNET_EXEC_BULK_EXEC_INFERENCE and MXNET_EXEC_BULK_EXEC_TRAIN to 0. -* MXNET_PROFILER_AUTOSTART (default=0) +* MXNET_PROFILER_AUTOSTART + - Values: 0(false) or 1(true) ```(default=0)``` - Set to 1, MXNet starts the profiler automatically. The profiling result is stored into profile.json in the working directory. -* MXNET_PROFILER_MODE (default=0) +* MXNET_PROFILER_MODE + - Values: 0(false) or 1(true) ```(default=0)``` - If set to '0', profiler records the events of the symbolic operators. - If set to '1', profiler records the events of all operators. ## Other Environment Variables -* MXNET_CUDNN_AUTOTUNE_DEFAULT (default=0) - - The default value of cudnn_tune for convolution layers. - - Auto tuning is turn off by default. For benchmarking, set this to 1 to turn it on by default. +* MXNET_CUDNN_AUTOTUNE_DEFAULT + - Values: 0(false) or 1(true) ```(default=0)``` + - The default value of cudnn auto tunning for convolution layers. + - Auto tuning is turned off by default. For benchmarking, set this to 1 to turn it on by default. Settings for Minimum Memory Usage --------------------------------- @@ -92,4 +123,4 @@ Settings for More GPU Parallelism --------------------------------- - Set ```MXNET_GPU_WORKER_NTHREADS``` to a larger number (e.g., 2) - To reduce memory usage, consider setting ```MXNET_EXEC_NUM_TEMP```. -- This might not speed things up, especially for image applications, because GPU is usually fully utilized even with serialized jobs. + - This might not speed things up, especially for image applications, because GPU is usually fully utilized even with serialized jobs. From 0df68e8c60dee9060389ab7c6540e86ba20d5f57 Mon Sep 17 00:00:00 2001 From: Young Jin Kim Date: Tue, 20 Jun 2017 15:39:29 -0700 Subject: [PATCH 098/834] MKL 1D ReLU fix (#6700) * Use the latest MKLML release * Add 1D relu in the MKL code pass * Fix a dimension error in relu backward function --- prepare_mkl.sh | 6 +++--- src/operator/mkl/mkl_relu-inl.h | 18 ++++++++++++++++-- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/prepare_mkl.sh b/prepare_mkl.sh index ac799b5cd74c..a320c44320f0 100755 --- a/prepare_mkl.sh +++ b/prepare_mkl.sh @@ -57,10 +57,10 @@ MXNET_ROOT=`dirname $0` USE_MKLML=0 # NOTE: if you update the following line, please also update the dockerfile at # tests/ci_build/Dockerfile.mkl -VERSION_MATCH=20170210 -ARCHIVE_BASENAME=mklml_lnx_2017.0.2.20170209.tgz +VERSION_MATCH=20170425 +ARCHIVE_BASENAME=mklml_lnx_2018.0.20170425.tgz MKL_CONTENT_DIR=`echo $ARCHIVE_BASENAME | rev | cut -d "." -f 2- | rev` -MKLURL="https://github.com/dmlc/web-data/raw/master/mxnet/mklml-release/$ARCHIVE_BASENAME" +MKLURL="https://github.com/01org/mkl-dnn/releases/download/v0.7/$ARCHIVE_BASENAME" # there are diffrent MKL lib to be used for GCC and for ICC reg='^[0-9]+$' VERSION_LINE=`GetVersionName $MKLROOT` diff --git a/src/operator/mkl/mkl_relu-inl.h b/src/operator/mkl/mkl_relu-inl.h index b77d18d0042c..8d7ab5e1e2db 100644 --- a/src/operator/mkl/mkl_relu-inl.h +++ b/src/operator/mkl/mkl_relu-inl.h @@ -100,7 +100,13 @@ class MKLReluOp : public Operator { Stream *s = ctx.get_stream(); Tensor data; Tensor out; - if (in_data[activation::kData].ndim() == 2) { + if (in_data[activation::kData].ndim() == 1) { + Shape<4> dshape = Shape4(in_data[activation::kData].shape_[0], 1, 1, 1); + data = mkl_experimental_direct_get_with_shape( + in_data[activation::kData], dshape, s); + out = mkl_experimental_direct_get_with_shape( + out_data[activation::kOut], dshape, s); + } else if (in_data[activation::kData].ndim() == 2) { Shape<4> dshape = Shape4(in_data[activation::kData].shape_[0], in_data[activation::kData].shape_[1], 1, 1); data = mkl_experimental_direct_get_with_shape( @@ -197,7 +203,15 @@ class MKLReluOp : public Operator { Tensor m_out_data; Tensor m_in_grad; - if (out_grad[activation::kOut].ndim() == 2) { + if (out_grad[activation::kOut].ndim() == 1) { + Shape<4> dshape = Shape4(out_grad[activation::kOut].shape_[0], 1, 1, 1); + m_out_grad = mkl_experimental_direct_get_with_shape( + out_grad[activation::kOut], dshape, s); + m_out_data = mkl_experimental_direct_get_with_shape( + out_data[activation::kOut], dshape, s); + m_in_grad = mkl_experimental_direct_get_with_shape( + in_grad[activation::kData], dshape, s); + } else if (out_grad[activation::kOut].ndim() == 2) { Shape<4> dshape = Shape4(out_grad[activation::kOut].shape_[0], out_grad[activation::kOut].shape_[1], 1, 1); m_out_grad = mkl_experimental_direct_get_with_shape( From 3ceb6d2f91121d5ffa5b81f435e8bcfcc1a75792 Mon Sep 17 00:00:00 2001 From: Eric Junyuan Xie Date: Tue, 20 Jun 2017 20:33:39 -0700 Subject: [PATCH 099/834] refactor cachedop to specialize (#6735) revert cachedop for perl fix t st imt qprove error message ci fix fix fix --- include/mxnet/c_api.h | 31 +-- perl-package/AI-MXNet/lib/AI/MXNet.pm | 1 - .../AI-MXNet/lib/AI/MXNet/CachedOp.pm | 41 ---- perl-package/AI-MXNet/lib/AI/MXNet/NDArray.pm | 1 - .../AI-MXNet/lib/AI/MXNet/NDArray/Base.pm | 38 ---- perl-package/AI-MXNet/lib/AI/MXNet/Symbol.pm | 1 - .../AI-MXNet/lib/AI/MXNet/Symbol/Base.pm | 14 -- perl-package/AI-MXNet/t/test_ndarray.t | 16 +- perl-package/AI-MXNet/t/test_symbol.t | 22 +-- perl-package/AI-MXNetCAPI/mxnet.i | 41 ---- perl-package/AI-MXNetCAPI/mxnet_typemaps.i | 5 +- python/mxnet/_ctypes/common.py | 30 --- python/mxnet/_ctypes/ndarray.py | 74 ++++--- python/mxnet/_ctypes/symbol.py | 16 -- python/mxnet/cython/base.pyi | 70 +------ python/mxnet/cython/ndarray.pyx | 100 ++++++---- python/mxnet/cython/symbol.pyx | 16 -- python/mxnet/ndarray.py | 8 +- python/mxnet/symbol.py | 34 +++- src/c_api/c_api_ndarray.cc | 185 ++++++++++++------ src/c_api/c_api_symbolic.cc | 16 -- src/initialize.cc | 2 +- tests/python/unittest/test_ndarray.py | 10 +- tests/python/unittest/test_operator.py | 6 +- tests/python/unittest/test_symbol.py | 13 -- 25 files changed, 292 insertions(+), 499 deletions(-) delete mode 100644 perl-package/AI-MXNet/lib/AI/MXNet/CachedOp.pm delete mode 100644 python/mxnet/_ctypes/common.py diff --git a/include/mxnet/c_api.h b/include/mxnet/c_api.h index 90270f776456..a0e842c21765 100644 --- a/include/mxnet/c_api.h +++ b/include/mxnet/c_api.h @@ -585,24 +585,20 @@ MXNET_DLL int MXAutogradBackward(mx_uint num_output, /*! * \brief create cached operator */ -MXNET_DLL int MXCachedCreateOp(AtomicSymbolCreator creator, - int num_inputs, - int num_params, - const char **param_keys, - const char **param_vals, +MXNET_DLL int MXCreateCachedOp(SymbolHandle handle, CachedOpHandle *out); /*! * \brief free cached operator */ -MXNET_DLL int MXCachedFree(CachedOpHandle handle); +MXNET_DLL int MXFreeCachedOp(CachedOpHandle handle); /*! * \brief invoke cached operator */ -MXNET_DLL int MXCachedInvoke(CachedOpHandle handle, - int num_inputs, - NDArrayHandle *inputs, - int *num_outputs, - NDArrayHandle **outputs); +MXNET_DLL int MXInvokeCachedOp(CachedOpHandle handle, + int num_inputs, + NDArrayHandle *inputs, + int *num_outputs, + NDArrayHandle **outputs); //-------------------------------------------- // Part 3: symbolic configuration generation //-------------------------------------------- @@ -670,19 +666,6 @@ MXNET_DLL int MXSymbolCreateAtomicSymbol(AtomicSymbolCreator creator, const char **keys, const char **vals, SymbolHandle *out); -/*! - * \brief Create an AtomicSymbol from cached op. - * \param handle cached node attribute. - * \param name name of new symbol. - * \param num_args the number of symbol arguments - * \param args symbol arguments - * \return 0 when success, -1 when failure happens - */ -MXNET_DLL int MXCachedCreateSymbol(CachedOpHandle handle, - const char* name, - mx_uint num_args, - SymbolHandle* args, - SymbolHandle* out); /*! * \brief Create a Variable Symbol. * \param name name of the variable diff --git a/perl-package/AI-MXNet/lib/AI/MXNet.pm b/perl-package/AI-MXNet/lib/AI/MXNet.pm index 530b6eca23a4..41bb1a18b493 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet.pm @@ -3,7 +3,6 @@ use v5.14.0; use strict; use warnings; use AI::MXNet::Base; -use AI::MXNet::CachedOp; use AI::MXNet::Callback; use AI::MXNet::NDArray; use AI::MXNet::Symbol; diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/CachedOp.pm b/perl-package/AI-MXNet/lib/AI/MXNet/CachedOp.pm deleted file mode 100644 index bec3f5029c33..000000000000 --- a/perl-package/AI-MXNet/lib/AI/MXNet/CachedOp.pm +++ /dev/null @@ -1,41 +0,0 @@ -package AI::MXNet::CachedOp; - -=head1 NAME - - AI::MXNet::CachedOp - A wrapper around CachedOpHandle -=cut - -use strict; -use warnings; -use AI::MXNet::Base; -use Mouse; - -has 'op' => (is => 'ro', isa => 'Str', required => 1); -has 'handle' => (is => 'ro', isa => 'CachedOpHandle', required => 1); -around BUILDARGS => sub { - my $orig = shift; - my $class = shift; - my ($op, $num_input, %kwargs) = @_; - for my $key (keys %kwargs) - { - $kwargs{ $key } = "(" .join(", ", @{ $kwargs{ $key } }) .")" - if ref $kwargs{ $key } eq 'ARRAY'; - } - my $AtomicSymbolCreator = check_call(AI::NNVMCAPI::GetOpHandle($op)); - my $handle = check_call( - AI::MXNetCAPI::CachedCreateOp( - $AtomicSymbolCreator, - $num_input, - scalar(keys %kwargs), - \%kwargs - ) - ); - return $class->$orig(op => $op, handle => $handle); -}; - -sub DEMOLISH -{ - check_call(AI::MXNetCAPI::CachedFree(shift->handle)); -} - -1; \ No newline at end of file diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/NDArray.pm b/perl-package/AI-MXNet/lib/AI/MXNet/NDArray.pm index 68c4e7061ec3..53579b2f1caf 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/NDArray.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/NDArray.pm @@ -1372,7 +1372,6 @@ method backward(Maybe[AI::MXNet::NDArray] $out_grad=, Bool $retain_graph=0) ) } -method CachedOp(@args) { AI::MXNet::CachedOp->new(@args) } my $lvalue_methods = join "\n", map {"use attributes 'AI::MXNet::NDArray', \\&AI::MXNet::NDArray::$_, 'lvalue';"} qw/at slice aspdl asmpdl reshape copy sever T astype as_in_context copyto empty zero ones full diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/NDArray/Base.pm b/perl-package/AI-MXNet/lib/AI/MXNet/NDArray/Base.pm index 0c48336c2aae..7fb6d0e61110 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/NDArray/Base.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/NDArray/Base.pm @@ -140,44 +140,6 @@ method _init_ndarray_module() } } -method invoke( - AI::MXNet::CachedOp $cached_op, - ArrayRef[AI::MXNet::NDArray] $args, - Maybe[AI::MXNet::NDArray|ArrayRef[AI::MXNet::NDArray]] $out=, - Maybe[Str] $name= -) -{ - my $original_output; - if(defined $out) - { - $original_output = $out; - if(not ref($out) eq 'ARRAY') - { - $out = [$out]; - } - } - else - { - $out = []; - } - my $output = check_call( - AI::MXNetCAPI::CachedInvoke( - $cached_op->handle, - scalar(@$args), - [map { $_->handle } @$args], - [map { $_->handle } @$out] - ) - ); - return $original_output if defined $original_output; - if(@$output == 1) - { - return $self->new(handle => $output->[0]); - } - else - { - return [map { $self->new(handle => $_) } @$output]; - } -} __PACKAGE__->_init_ndarray_module; diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Symbol.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Symbol.pm index eec32640953c..e22e4189721a 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Symbol.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Symbol.pm @@ -1347,7 +1347,6 @@ method arange(Index :$start=0, Index :$stop=, Num :$step=1.0, Index :$repeat=1, }); } -method CachedOp(@args) { AI::MXNet::CachedOp->new(@args) } sub _parse_arguments { diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Symbol/Base.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Symbol/Base.pm index 3eaee237bed0..69ff952eca1a 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Symbol/Base.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Symbol/Base.pm @@ -167,20 +167,6 @@ method _init_symbol_module() } } -method invoke(AI::MXNet::CachedOp $cached_op, ArrayRef[AI::MXNet::Symbol] $args, Maybe[Str] $name=) -{ - my $hint = lc($cached_op->op); - $name = AI::MXNet::Symbol::NameManager->current->get($name, $hint); - my $handle = check_call( - AI::MXNetCAPI::CachedCreateSymbol( - $cached_op->handle, - $name, - scalar(@$args), - [map { $_->handle } @$args] - ) - ); - return $self->new(handle => $handle); -} __PACKAGE__->_init_symbol_module; diff --git a/perl-package/AI-MXNet/t/test_ndarray.t b/perl-package/AI-MXNet/t/test_ndarray.t index 53e5749d00f6..d4e1a4d074b8 100644 --- a/perl-package/AI-MXNet/t/test_ndarray.t +++ b/perl-package/AI-MXNet/t/test_ndarray.t @@ -2,7 +2,7 @@ use strict; use warnings; use AI::MXNet qw(mx); use AI::MXNet::TestUtils qw(almost_equal); -use Test::More tests => 9; +use Test::More tests => 8; sub test_ndarray_reshape { @@ -36,17 +36,6 @@ sub test_moveaxis is_deeply($X->moveaxis(2, 0)->shape, [3, 2, 2]); } -sub test_cached -{ - my $op = mx->nd->CachedOp('Convolution', 3, kernel=>[3, 3], num_filter=>10); - my $data = mx->nd->ones([3, 4, 10, 10]); - my $weight = mx->nd->ones([10, 4, 3, 3]); - my $bias = mx->nd->ones([10]); - my $o1 = mx->nd->invoke($op, [$data, $weight, $bias]); - $bias .= 2; - my $o2 = mx->nd->invoke($op, [$data, $weight, $bias]); - ok(almost_equal($o2->aspdl, $o1->aspdl + 1)); -} sub test_output { @@ -64,5 +53,4 @@ sub test_output test_ndarray_reshape(); test_moveaxis(); -test_cached(); -test_output(); \ No newline at end of file +test_output(); diff --git a/perl-package/AI-MXNet/t/test_symbol.t b/perl-package/AI-MXNet/t/test_symbol.t index bf9e90598929..f21a14cf78ff 100644 --- a/perl-package/AI-MXNet/t/test_symbol.t +++ b/perl-package/AI-MXNet/t/test_symbol.t @@ -1,6 +1,6 @@ use strict; use warnings; -use Test::More tests => 102; +use Test::More tests => 98; use AI::MXNet qw(mx); use AI::MXNet::TestUtils qw(mlp2 conv check_consistency zip assert enumerate); use Storable qw(freeze thaw); @@ -221,24 +221,6 @@ sub test_load_000800 test_load_000800(); -sub test_cached -{ - my $op = mx->sym->CachedOp('Convolution', 3, kernel=>[3, 3], num_filter=>10); - my $data = mx->sym->var('data'); - my $weight = mx->sym->var('weight'); - my $bias = mx->sym->var('bias'); - my $out = mx->sym->invoke($op, [$data, $weight, $bias], 'conv'); - is_deeply($out->list_arguments, ['data', 'weight', 'bias']); - is_deeply($out->list_outputs, ['conv_output']); - { - local($mx::NameManager) = mx->name->Prefix('test_'); - is(mx->sym->invoke($op, [$data, $weight, $bias])->name,'test_convolution0'); - is(mx->sym->invoke($op, [$data, $weight, $bias])->name, 'test_convolution1'); - } -} - -test_cached(); - __DATA__ { "nodes": [ @@ -427,4 +409,4 @@ __DATA__ ], "arg_nodes": [0, 1, 2, 5, 6, 9, 10, 12, 13, 15], "heads": [[16, 0]] -} \ No newline at end of file +} diff --git a/perl-package/AI-MXNetCAPI/mxnet.i b/perl-package/AI-MXNetCAPI/mxnet.i index 295832eb24dc..d0705d5acc72 100644 --- a/perl-package/AI-MXNetCAPI/mxnet.i +++ b/perl-package/AI-MXNetCAPI/mxnet.i @@ -119,7 +119,6 @@ static void ExecutorMonitor_callback(const char* name, NDArrayHandle handle, voi SWIG_TypeClientData(SWIGTYPE_p_MXKVStore, (void *)"KVStoreHandle"); SWIG_TypeClientData(SWIGTYPE_p_MXRecordIO, (void *)"RecordIOHandle"); SWIG_TypeClientData(SWIGTYPE_p_MXRtc, (void *)"RtcHandle"); - SWIG_TypeClientData(SWIGTYPE_p_MXCachedOp, (void *)"CachedOpHandle"); %} /*! \brief manually define unsigned int */ @@ -151,8 +150,6 @@ typedef MXKVStore *KVStoreHandle; typedef MXRecordIO *RecordIOHandle; /*! \brief handle to MXRtc*/ typedef MXRtc *RtcHandle; -/*! \brief handle to cached operator */ -typedef MXCachedOp *CachedOpHandle; typedef void (*ExecutorMonitorCallback)(const char*, NDArrayHandle, @@ -628,30 +625,6 @@ int MXAutogradBackward(mx_uint num_output, NDArrayHandle* in, int retain_graph); -/*! - * \brief create cached operator - */ -int MXCachedCreateOp(AtomicSymbolCreator in, - int num_inputs, - int num_params, - const char **keys, - const char **vals, - CachedOpHandle *out); - -/*! - * \brief free cached operator - */ -int MXCachedFree(CachedOpHandle handle); - -/*! - * \brief invoke cached operator - */ -int MXCachedInvoke(CachedOpHandle handle, - int num_inputs, - NDArrayHandle *in, - int *out_size, - NDArrayHandle** out_array); - //-------------------------------------------- // Part 3: symbolic configuration generation //-------------------------------------------- @@ -719,20 +692,6 @@ int MXSymbolCreateAtomicSymbol(AtomicSymbolCreator in, const char **keys, const char **vals, SymbolHandle *out); -/*! - * \brief Create an AtomicSymbol from cached op. - * \param handle cached node attribute. - * \param name name of new symbol. - * \param num_args the number of symbol arguments - * \param args symbol arguments - * \return 0 when success, -1 when failure happens - */ -int MXCachedCreateSymbol(CachedOpHandle handle, - const char* name, - mx_uint num_args, - SymbolHandle* in, - SymbolHandle* out); - /*! * \brief Create a Variable Symbol. * \param name name of the variable diff --git a/perl-package/AI-MXNetCAPI/mxnet_typemaps.i b/perl-package/AI-MXNetCAPI/mxnet_typemaps.i index 8574647512f5..792f8472d05a 100644 --- a/perl-package/AI-MXNetCAPI/mxnet_typemaps.i +++ b/perl-package/AI-MXNetCAPI/mxnet_typemaps.i @@ -311,13 +311,12 @@ (DataIterHandle *out) (ExecutorHandle temp), (KVStoreHandle *out) (KVStoreHandle temp), (RecordIOHandle *out) (RecordIOHandle temp), - (RtcHandle *out) (RtcHandle temp), - (CachedOpHandle *out) (CachedOpHandle temp) + (RtcHandle *out) (RtcHandle temp) { $1 = &temp; } %typemap(argout) (NDArrayHandle *out), (FunctionHandle* out), (SymbolHandle *out), (ExecutorHandle *out), (DataIterHandle *out), - (KVStoreHandle *out), (RecordIOHandle *out), (RtcHandle *out) (RtcHandle temp), (CachedOpHandle *out) (CachedOpHandle temp) + (KVStoreHandle *out), (RecordIOHandle *out), (RtcHandle *out) (RtcHandle temp) { if(!result) { diff --git a/python/mxnet/_ctypes/common.py b/python/mxnet/_ctypes/common.py deleted file mode 100644 index 24e2048eee4c..000000000000 --- a/python/mxnet/_ctypes/common.py +++ /dev/null @@ -1,30 +0,0 @@ -# coding: utf-8 -"""Common code between symbolic and ndarray.""" -from __future__ import absolute_import as _abs - -import ctypes - -from ..base import _LIB -from ..base import c_array, c_str -from ..base import OpHandle, CachedOpHandle -from ..base import check_call - - -class CachedOp(object): - """Cached operator handle.""" - __slots__ = ["handle", "op"] - def __init__(self, op, num_input, **kwargs): - self.op = op - op_handle = OpHandle() - check_call(_LIB.NNGetOpHandle(c_str(op), ctypes.byref(op_handle))) - self.handle = CachedOpHandle() - check_call(_LIB.MXCachedCreateOp( - op_handle, - ctypes.c_int(num_input), - ctypes.c_int(len(kwargs)), - c_array(ctypes.c_char_p, [c_str(key) for key in kwargs]), - c_array(ctypes.c_char_p, [c_str(str(val)) for val in kwargs.values()]), - ctypes.byref(self.handle))) - - def __del__(self): - check_call(_LIB.MXCachedFree(self.handle)) diff --git a/python/mxnet/_ctypes/ndarray.py b/python/mxnet/_ctypes/ndarray.py index a678e1726f02..396c57a41dfb 100644 --- a/python/mxnet/_ctypes/ndarray.py +++ b/python/mxnet/_ctypes/ndarray.py @@ -10,10 +10,9 @@ from ..base import _LIB from ..base import c_array, py_str, c_str, mx_uint, _Null -from ..base import NDArrayHandle, OpHandle +from ..base import NDArrayHandle, OpHandle, CachedOpHandle from ..base import check_call from ..ndarray_doc import _build_doc -from .common import CachedOp class NDArrayBase(object): @@ -81,31 +80,48 @@ def _imperative_invoke(handle, ndargs, keys, vals, out): for i in range(num_output.value)] -def invoke(cached_op, args, out=None, name=None): # pylint: disable=unused-argument - """ctypes implementation of imperative invoke wrapper""" - if out is not None: - original_output = out - if isinstance(out, NDArrayBase): - out = (out,) - num_output = ctypes.c_int(len(out)) - output_vars = c_array(NDArrayHandle, [i.handle for i in out]) - output_vars = ctypes.cast(output_vars, ctypes.POINTER(NDArrayHandle)) - else: - original_output = None - output_vars = ctypes.POINTER(NDArrayHandle)() - num_output = ctypes.c_int(0) - - check_call(_LIB.MXCachedInvoke( - cached_op.handle, - ctypes.c_int(len(args)), - c_array(NDArrayHandle, [arr.handle for arr in args]), - ctypes.byref(num_output), - ctypes.byref(output_vars))) +class CachedOp(object): + """Cached operator handle.""" + __slots__ = ["handle"] + def __init__(self, sym): + self.handle = CachedOpHandle() + check_call(_LIB.MXCreateCachedOp( + sym.handle, + ctypes.byref(self.handle))) - if original_output is not None: - return original_output - if num_output.value == 1: - return _ndarray_cls(ctypes.cast(output_vars[0], NDArrayHandle)) - else: - return [_ndarray_cls(ctypes.cast(output_vars[i], NDArrayHandle)) - for i in range(num_output.value)] + def __del__(self): + check_call(_LIB.MXFreeCachedOp(self.handle)) + + def __call__(self, *args, **kwargs): + """ctypes implementation of imperative invoke wrapper""" + out = kwargs.pop('out', None) + if out is not None: + original_output = out + if isinstance(out, NDArrayBase): + out = (out,) + num_output = ctypes.c_int(len(out)) + output_vars = c_array(NDArrayHandle, [i.handle for i in out]) + output_vars = ctypes.cast(output_vars, ctypes.POINTER(NDArrayHandle)) + else: + original_output = None + output_vars = ctypes.POINTER(NDArrayHandle)() + num_output = ctypes.c_int(0) + if kwargs: + raise TypeError( + "CachedOp.__call__ got unexpected keyword argument(s): " + \ + ', '.join(kwargs.keys())) + + check_call(_LIB.MXInvokeCachedOp( + self.handle, + ctypes.c_int(len(args)), + c_array(NDArrayHandle, [arr.handle for arr in args]), + ctypes.byref(num_output), + ctypes.byref(output_vars))) + + if original_output is not None: + return original_output + if num_output.value == 1: + return _ndarray_cls(ctypes.cast(output_vars[0], NDArrayHandle)) + else: + return [_ndarray_cls(ctypes.cast(output_vars[i], NDArrayHandle)) + for i in range(num_output.value)] diff --git a/python/mxnet/_ctypes/symbol.py b/python/mxnet/_ctypes/symbol.py index 9026b20cd7db..5cbff551cf55 100644 --- a/python/mxnet/_ctypes/symbol.py +++ b/python/mxnet/_ctypes/symbol.py @@ -8,8 +8,6 @@ from ..base import c_array, c_str, mx_uint from ..base import SymbolHandle from ..base import check_call -from ..name import NameManager -from .common import CachedOp # pylint: disable=unused-import _symbol_cls = None @@ -102,20 +100,6 @@ def _set_symbol_class(cls): _symbol_cls = cls -def invoke(cached_op, args, name=None): - """Call cached symbolic operator""" - ret = SymbolHandle() - hint = cached_op.op.lower() - name = c_str(NameManager.current.get(name, hint)) - check_call(_LIB.MXCachedCreateSymbol( - cached_op.handle, - name, - mx_uint(len(args)), - c_array(SymbolHandle, [s.handle for s in args]), - ctypes.byref(ret))) - return _symbol_cls(ret) - - def _symbol_creator(handle, args, kwargs, keys, vals, name): sym_handle = SymbolHandle() check_call(_LIB.MXSymbolCreateAtomicSymbol( diff --git a/python/mxnet/cython/base.pyi b/python/mxnet/cython/base.pyi index 651258135ef3..d73e1a7d0194 100644 --- a/python/mxnet/cython/base.pyi +++ b/python/mxnet/cython/base.pyi @@ -99,75 +99,11 @@ cdef extern from "mxnet/c_api.h": const char **param_keys, const char **param_vals); int MXNDArrayFree(NDArrayHandle handle); - int MXCachedCreateOp(OpHandle creator, - int num_inputs, - int num_params, - const char **param_keys, - const char **param_vals, + int MXCreateCachedOp(SymbolHandle handle, CachedOpHandle *out); - int MXCachedFree(CachedOpHandle handle); - int MXCachedInvoke(CachedOpHandle handle, + int MXFreeCachedOp(CachedOpHandle handle); + int MXInvokeCachedOp(CachedOpHandle handle, int num_inputs, NDArrayHandle *inputs, int *num_outputs, NDArrayHandle **outputs); - int MXCachedCreateSymbol(CachedOpHandle handle, - const char* name, - unsigned num_args, - SymbolHandle* args, - SymbolHandle* out); - - -cdef class CachedOp: - """Cached operator handle.""" - cdef CachedOpHandle chandle - cdef string cop - - cdef _set_handle(self, handle): - cdef unsigned long long ptr - if handle is None: - self.chandle = NULL - else: - ptr = handle.value - self.chandle = (ptr) - - property handle: - def __get__(self): - if self.chandle == NULL: - return None - else: - return _ctypes.cast(self.chandle, _ctypes.c_void_p) - def __set__(self, value): - self._set_handle(value) - - property op: - def __get__(self): - return py_str(self.cop.c_str()) - def __set__(self, value): - self.cop = c_str(value) - - def __init__(self, op, num_input, **kwargs): - cdef OpHandle op_handle - cdef vector[string] ckeys - cdef vector[string] cvals - - self.op = op - CALL(NNGetOpHandle(self.cop.c_str(), &op_handle)) - - for k, v in kwargs.items(): - ckeys.push_back(c_str(k)) - cvals.push_back(c_str(str(v))) - - cdef vector[const char*] param_keys = SVec2Ptr(ckeys) - cdef vector[const char*] param_vals = SVec2Ptr(cvals) - - CALL(MXCachedCreateOp( - op_handle, - num_input, - len(kwargs), - CBeginPtr(param_keys), - CBeginPtr(param_vals), - &self.chandle)) - - def __del__(self): - CALL(MXCachedFree(self.chandle)) diff --git a/python/mxnet/cython/ndarray.pyx b/python/mxnet/cython/ndarray.pyx index 24e37b54c7be..a861ae661b45 100644 --- a/python/mxnet/cython/ndarray.pyx +++ b/python/mxnet/cython/ndarray.pyx @@ -61,46 +61,76 @@ cdef NewArray(NDArrayHandle handle): return nd -def invoke(cached_op, args, out=None, name=None): - """ctypes implementation of imperative invoke wrapper""" - cdef vector[NDArrayHandle] ndvars - cdef vector[NDArrayHandle] output_vars - cdef NDArrayHandle* p_output_vars - cdef NDArrayHandle ret_handle - cdef int num_output - - for i in args: - ndvars.push_back((i).chandle) +cdef class CachedOp: + """Cached operator handle.""" + cdef CachedOpHandle chandle - original_output = None - if out is not None: - original_output = out - if isinstance(out, NDArrayBase): - output_vars.push_back((out).chandle) + cdef _set_handle(self, handle): + cdef unsigned long long ptr + if handle is None: + self.chandle = NULL else: - for i in out: - output_vars.push_back((i).chandle) + ptr = handle.value + self.chandle = (ptr) - num_output = output_vars.size() - if output_vars.size() == 0: - output_vars.resize(1) - p_output_vars = NULL - else: - p_output_vars = &output_vars[0] + property handle: + def __get__(self): + if self.chandle == NULL: + return None + else: + return _ctypes.cast(self.chandle, _ctypes.c_void_p) + def __set__(self, value): + self._set_handle(value) - CALL(MXCachedInvoke( - (cached_op).chandle, - len(args), - &ndvars[0] if ndvars.size() != 0 else NULL, - &num_output, - &p_output_vars)) + def __init__(self, sym): + cdef unsigned long long ptr = sym.handle.value + CALL(MXCreateCachedOp( + (ptr), + &self.chandle)) + + def __del__(self): + CALL(MXFreeCachedOp(self.chandle)) + + def __call__(self, *args, out=None): + """ctypes implementation of imperative invoke wrapper""" + cdef vector[NDArrayHandle] ndvars + cdef vector[NDArrayHandle] output_vars + cdef NDArrayHandle* p_output_vars + cdef NDArrayHandle ret_handle + cdef int num_output + + for i in args: + ndvars.push_back((i).chandle) + + original_output = None + if out is not None: + original_output = out + if isinstance(out, NDArrayBase): + output_vars.push_back((out).chandle) + else: + for i in out: + output_vars.push_back((i).chandle) - if original_output is not None: - return original_output - if num_output == 1: - return NewArray(p_output_vars[0]) - else: - return tuple(NewArray(p_output_vars[i]) for i in range(num_output)) + num_output = output_vars.size() + if output_vars.size() == 0: + output_vars.resize(1) + p_output_vars = NULL + else: + p_output_vars = &output_vars[0] + + CALL(MXInvokeCachedOp( + (self).chandle, + len(args), + &ndvars[0] if ndvars.size() != 0 else NULL, + &num_output, + &p_output_vars)) + + if original_output is not None: + return original_output + if num_output == 1: + return NewArray(p_output_vars[0]) + else: + return tuple(NewArray(p_output_vars[i]) for i in range(num_output)) def _imperative_invoke(handle, ndargs, keys, vals, out): diff --git a/python/mxnet/cython/symbol.pyx b/python/mxnet/cython/symbol.pyx index e8787fba77a3..aea0aa9f4809 100644 --- a/python/mxnet/cython/symbol.pyx +++ b/python/mxnet/cython/symbol.pyx @@ -79,22 +79,6 @@ cdef NewSymbol(SymbolHandle handle): return sym -def invoke(cached_op, args, name=None): - cdef SymbolHandle ret - cdef vector[SymbolHandle] sym_args - hint = cached_op.op.lower() - cdef string cname = c_str(NameManager.current.get(name, hint)) - for i in args: - sym_args.push_back((i).chandle) - CALL(MXCachedCreateSymbol( - (cached_op).chandle, - cname.c_str(), - len(args), - &sym_args[0] if sym_args.size() != 0 else NULL, - &ret)) - return NewSymbol(ret) - - def _symbol_creator(handle, args, kwargs, keys, vals, name): cdef unsigned long long ihandle = handle cdef OpHandle chandle = ihandle diff --git a/python/mxnet/ndarray.py b/python/mxnet/ndarray.py index 9ec4d47bbb81..31b7d7cfb944 100644 --- a/python/mxnet/ndarray.py +++ b/python/mxnet/ndarray.py @@ -32,18 +32,18 @@ try: if int(_os.environ.get("MXNET_ENABLE_CYTHON", True)) == 0: from ._ctypes.ndarray import NDArrayBase, _set_ndarray_class - from ._ctypes.ndarray import invoke, CachedOp, _imperative_invoke + from ._ctypes.ndarray import CachedOp, _imperative_invoke elif _sys.version_info >= (3, 0): from ._cy3.ndarray import NDArrayBase, _set_ndarray_class, _imperative_invoke - from ._cy3.ndarray import invoke, CachedOp, _imperative_invoke + from ._cy3.ndarray import CachedOp, _imperative_invoke else: from ._cy2.ndarray import NDArrayBase, _set_ndarray_class, _imperative_invoke - from ._cy2.ndarray import invoke, CachedOp, _imperative_invoke + from ._cy2.ndarray import CachedOp, _imperative_invoke except ImportError: if int(_os.environ.get("MXNET_ENFORCE_CYTHON", False)) != 0: raise ImportError("Cython Module cannot be loaded but MXNET_ENFORCE_CYTHON=1") from ._ctypes.ndarray import NDArrayBase, _set_ndarray_class, _imperative_invoke - from ._ctypes.ndarray import invoke, CachedOp, _imperative_invoke + from ._ctypes.ndarray import CachedOp, _imperative_invoke # pylint: enable=unused-import # pylint: disable= no-member diff --git a/python/mxnet/symbol.py b/python/mxnet/symbol.py index bd0aca65f521..ec0eed76fd19 100644 --- a/python/mxnet/symbol.py +++ b/python/mxnet/symbol.py @@ -29,18 +29,18 @@ try: if int(_os.environ.get("MXNET_ENABLE_CYTHON", True)) == 0: from ._ctypes.symbol import SymbolBase, _set_symbol_class - from ._ctypes.symbol import CachedOp, invoke, _symbol_creator # pylint: disable=unused-import + from ._ctypes.symbol import _symbol_creator # pylint: disable=unused-import elif _sys.version_info >= (3, 0): from ._cy3.symbol import SymbolBase, _set_symbol_class - from ._cy3.symbol import CachedOp, invoke, _symbol_creator # pylint: disable=unused-import + from ._cy3.symbol import _symbol_creator # pylint: disable=unused-import else: from ._cy2.symbol import SymbolBase, _set_symbol_class - from ._cy2.symbol import CachedOp, invoke, _symbol_creator # pylint: disable=unused-import + from ._cy2.symbol import _symbol_creator # pylint: disable=unused-import except ImportError: if int(_os.environ.get("MXNET_ENFORCE_CYTHON", False)) != 0: raise ImportError("Cython Module cannot be loaded but MXNET_ENFORCE_CYTHON=1") from ._ctypes.symbol import SymbolBase, _set_symbol_class - from ._ctypes.symbol import CachedOp, invoke, _symbol_creator # pylint: disable=unused-import + from ._ctypes.symbol import _symbol_creator # pylint: disable=unused-import _GRAD_REQ_MAP = {'null': 0, 'write': 1, 'add': 3} @@ -735,7 +735,7 @@ def list_auxiliary_states(self): Returns ------- - aux_states : list of string + aux_states : list of str List of the auxiliary states in input symbol. Notes @@ -751,6 +751,30 @@ def list_auxiliary_states(self): self.handle, ctypes.byref(size), ctypes.byref(sarr))) return [py_str(sarr[i]) for i in range(size.value)] + def list_inputs(self): + """Lists all arguments and auxiliary states of this Symbol. + + Returns + ------- + inputs : list of str + List of all inputs. + + Examples + -------- + >>> bn = mx.sym.BatchNorm(name='bn') + >>> bn.list_arguments() + ['bn_data', 'bn_gamma', 'bn_beta'] + >>> bn.list_auxiliary_states() + ['bn_moving_mean', 'bn_moving_var'] + >>> bn.list_inputs() + ['bn_data', 'bn_gamma', 'bn_beta', 'bn_moving_mean', 'bn_moving_var'] + """ + size = ctypes.c_uint() + sarr = ctypes.POINTER(ctypes.c_char_p)() + check_call(_LIB.NNSymbolListInputNames( + self.handle, 0, ctypes.byref(size), ctypes.byref(sarr))) + return [py_str(sarr[i]) for i in range(size.value)] + def infer_type(self, *args, **kwargs): """Infers the type of all arguments and all outputs, given the known types for some arguments. diff --git a/src/c_api/c_api_ndarray.cc b/src/c_api/c_api_ndarray.cc index 0be1d3574dd9..dfdd46b6aa90 100644 --- a/src/c_api/c_api_ndarray.cc +++ b/src/c_api/c_api_ndarray.cc @@ -103,31 +103,43 @@ void SetNDInputsOutputs(const nnvm::Op* op, void SetContext(Context* p_ctx, const nnvm::NodeAttrs& attrs, - const int& num_inputs, const std::vector& ndinputs, - const int& infered_num_outputs, - const std::vector& ndoutputs) { + const std::vector& ndoutputs, + const Context& default_ctx) { Context& ctx = *p_ctx; - if (num_inputs) { + if (ndinputs.size()) { ctx = ndinputs[0].ctx(); - } else if (infered_num_outputs && !ndoutputs[0].is_none()) { + for (size_t i = 1; i < ndinputs.size(); ++i) { + CHECK_EQ(ndinputs[i].ctx().dev_mask(), ctx.dev_mask()) + << "All inputs must live on the same context. " + << "But the first argument is on " + << (ctx.dev_mask() == gpu::kDevMask ? "GPU" : "CPU") + << " while the " << i+1 << "-th argument is on " + << (ndinputs[i].ctx().dev_mask() == gpu::kDevMask ? "GPU" : "CPU"); + } + } else if (ndoutputs.size() && !ndoutputs[0].is_none()) { ctx = ndoutputs[0].ctx(); } else if (attrs.dict.find("ctx") != attrs.dict.end()) { ctx = Context::FromString(attrs.dict.at("ctx")); } else { - ctx = Context::CPU(); + ctx = default_ctx; } // Pinned context doesn't propagate if (ctx.dev_type == Context::kCPUPinned) { ctx = Context::CPU(); } +#if !MXNET_USE_CUDA + if (ctx.dev_mask() == gpu::kDevMask) { + LOG(INFO) << "GPU support is disabled. Compile MXNet with " + << "USE_CUDA=1 to enable GPU support."; + } +#endif // MXNET_USE_CUDA } void SetShapeType(const nnvm::Op* op, const nnvm::NodeAttrs& attrs, const Context& ctx, const std::vector& ndinputs, - const int& infered_num_outputs, std::vector* p_ndoutputs) { std::vector& ndoutputs = *p_ndoutputs; static auto& infershape = nnvm::Op::GetAttr("FInferShape"); @@ -148,7 +160,7 @@ void SetShapeType(const nnvm::Op* op, CHECK(infershape.count(op)) << "Operator " << op->name << " is missing FInferShape attribute"; CHECK(infershape[op](attrs, &in_shapes, &out_shapes)); - CHECK_EQ(out_shapes.size(), static_cast(infered_num_outputs)); + CHECK_EQ(out_shapes.size(), ndoutputs.size()); // infer type std::vector& in_types = ret->arg_types; @@ -165,9 +177,9 @@ void SetShapeType(const nnvm::Op* op, CHECK(infertype.count(op)) << "Operator " << op->name << " is missing FInferType attribute"; CHECK(infertype[op](attrs, &in_types, &out_types)); - CHECK_EQ(out_types.size(), static_cast(infered_num_outputs)); + CHECK_EQ(out_types.size(), ndoutputs.size()); - for (int i = 0; i < infered_num_outputs; ++i) { + for (size_t i = 0; i < ndoutputs.size(); ++i) { if (ndoutputs[i].is_none()) { ndoutputs[i] = NDArray(out_shapes[i], ctx, true, out_types[i]); } else { @@ -322,35 +334,28 @@ void PushOperator(std::shared_ptr opr, 0, PROFILER_MESSAGE(op->name.c_str())); } -void ImperativeInvokeImpl(const nnvm::NodeAttrs& attrs, - int num_inputs, - NDArrayHandle *inputs, - int *num_outputs, - NDArrayHandle **outputs) { +void ImperativeInvokeImpl(const Context& default_ctx, + const nnvm::NodeAttrs& attrs, + std::vector* p_ndinputs, + std::vector* p_ndoutputs) { static auto& fcpu = nnvm::Op::GetAttr("FCompute"); static auto& fgpu = nnvm::Op::GetAttr("FCompute"); static auto& ndfunc = nnvm::Op::GetAttr("FNDArrayFunction"); static auto& createop = nnvm::Op::GetAttr("FCreateLayerOp"); MXAPIThreadLocalEntry *ret = MXAPIThreadLocalStore::Get(); - NDArray** outarray = *reinterpret_cast(outputs); - const nnvm::Op *op = attrs.op; - int infered_num_outputs; - int num_visible_outputs; - SetNumOutputs(op, attrs, num_inputs, - &infered_num_outputs, &num_visible_outputs); + const nnvm::Op *op = attrs.op; + std::vector& ndinputs = *p_ndinputs; + std::vector& ndoutputs = *p_ndoutputs; - std::vector ndinputs, ndoutputs; - SetNDInputsOutputs(op, &ndinputs, &ndoutputs, num_inputs, inputs, - num_outputs, infered_num_outputs, num_visible_outputs, outarray); if (ndfunc.count(op)) { ndfunc[op](attrs, ndinputs, &ndoutputs); } else { // TODO(piiswrong): infer ctx Context ctx; - SetContext(&ctx, attrs, num_inputs, ndinputs, infered_num_outputs, ndoutputs); - SetShapeType(op, attrs, ctx, ndinputs, infered_num_outputs, &ndoutputs); + SetContext(&ctx, attrs, ndinputs, ndoutputs, default_ctx); + SetShapeType(op, attrs, ctx, ndinputs, &ndoutputs); std::vector read_vars, write_vars; std::vector requested; @@ -383,22 +388,8 @@ void ImperativeInvokeImpl(const nnvm::NodeAttrs& attrs, requested, auxidx, ndinputs, ndoutputs); } else { LOG(FATAL) - << "Operator " << op->name - << " cannot be run; requires at least one of" - << " FCompute, NDArrayFunction, FCreateOperator be registered"; - } - } - - if (outarray == nullptr) { - ret->ret_handles.clear(); - for (int i = 0; i < num_visible_outputs; ++i) { - ret->ret_handles.push_back( - reinterpret_cast(new NDArray(std::move(ndoutputs[i])))); - } - *outputs = dmlc::BeginPtr(ret->ret_handles); - } else { - for (int i = 0; i < *num_outputs; ++i) { - *outarray[i] = std::move(ndoutputs[i]); + << "Operator " << op->name << " is not implemented for " + << (ctx.dev_mask() == gpu::kDevMask ? "GPU." : "CPU."); } } } @@ -412,46 +403,114 @@ int MXImperativeInvoke(AtomicSymbolCreator creator, const char **param_keys, const char **param_vals) { const nnvm::Op* op = static_cast(creator); + MXAPIThreadLocalEntry *ret = MXAPIThreadLocalStore::Get(); + NDArray** outarray = *reinterpret_cast(outputs); API_BEGIN(); nnvm::NodeAttrs attrs; SetOpAttrs(op, &attrs, num_inputs, num_params, param_keys, param_vals); - ImperativeInvokeImpl(attrs, num_inputs, inputs, num_outputs, outputs); + + int infered_num_outputs; + int num_visible_outputs; + SetNumOutputs(op, attrs, num_inputs, &infered_num_outputs, &num_visible_outputs); + + std::vector ndinputs, ndoutputs; + SetNDInputsOutputs(op, &ndinputs, &ndoutputs, num_inputs, inputs, + num_outputs, infered_num_outputs, num_visible_outputs, outarray); + + ImperativeInvokeImpl(Context::CPU(), attrs, &ndinputs, &ndoutputs); + + if (outarray == nullptr) { + ret->ret_handles.clear(); + for (int i = 0; i < num_visible_outputs; ++i) { + ret->ret_handles.push_back( + reinterpret_cast(new NDArray(std::move(ndoutputs[i])))); + } + *outputs = dmlc::BeginPtr(ret->ret_handles); + } else { + for (int i = 0; i < *num_outputs; ++i) { + *outarray[i] = std::move(ndoutputs[i]); + } + } API_END(); } -int MXCachedCreateOp(AtomicSymbolCreator creator, - int num_inputs, - int num_params, - const char **param_keys, - const char **param_vals, +int MXCreateCachedOp(SymbolHandle handle, CachedOpHandle *out) { - const nnvm::Op* op = static_cast(creator); + nnvm::Symbol* sym = static_cast(handle); API_BEGIN(); - nnvm::NodeAttrs *attrs = new nnvm::NodeAttrs; - SetOpAttrs(op, attrs, num_inputs, num_params, param_keys, param_vals); - *out = attrs; + nnvm::Graph *g = new nnvm::Graph; + g->outputs = sym->outputs; + auto vars = sym->ListInputs(nnvm::Symbol::kAll); + CHECK_GE(vars.size(), 1) << "CachedOp must have at least 1 input."; + g->attrs["vars"] = std::make_shared(std::move(vars)); + *out = g; API_END(); } -int MXCachedFree(CachedOpHandle handle) { - nnvm::NodeAttrs *attrs = static_cast(handle); - +int MXFreeCachedOp(CachedOpHandle handle) { + nnvm::Graph *g = static_cast(handle); API_BEGIN(); - delete attrs; + delete g; API_END(); } -int MXCachedInvoke(CachedOpHandle handle, - int num_inputs, - NDArrayHandle *inputs, - int *num_outputs, - NDArrayHandle **outputs) { - nnvm::NodeAttrs *attrs = static_cast(handle); +int MXInvokeCachedOp(CachedOpHandle handle, + int num_inputs, + NDArrayHandle *inputs, + int *num_outputs, + NDArrayHandle **outputs) { + nnvm::Graph *g = static_cast(handle); + MXAPIThreadLocalEntry *ret = MXAPIThreadLocalStore::Get(); + NDArray** outarray = *reinterpret_cast(outputs); API_BEGIN(); - ImperativeInvokeImpl(*attrs, num_inputs, inputs, num_outputs, outputs); + const std::vector& vars = + g->GetAttr >("vars"); + const nnvm::IndexedGraph& idx = g->indexed_graph(); + CHECK_EQ(static_cast(num_inputs), vars.size()) + << "Actually number of inputs differs from expected number of inputs"; + Context default_ctx = static_cast(inputs[0])->ctx(); + + std::vector buff(idx.num_node_entries()); + for (size_t i = 0; i < vars.size(); ++i) { + buff[idx.entry_id(idx.node_id(vars[i].get()), 0)] = + *static_cast(inputs[i]); + } + + for (size_t i = 0; i < idx.num_nodes(); ++i) { + const nnvm::IndexedGraph::Node& node = idx[i]; + if (node.source->attrs.op == nullptr) continue; + std::vector in; + in.reserve(node.inputs.size()); + for (const auto& j : node.inputs) { + in.emplace_back(buff[idx.entry_id(j)]); + } + std::vector out(node.source->num_outputs()); + ImperativeInvokeImpl(default_ctx, node.source->attrs, &in, &out); + + for (size_t j = 0; j < node.source->num_outputs(); ++j) { + buff[idx.entry_id(i, j)] = std::move(out[j]); + } + } + + if (outarray == nullptr) { + ret->ret_handles.clear(); + for (const auto& i : idx.outputs()) { + ret->ret_handles.push_back( + reinterpret_cast( + new NDArray(std::move(buff[idx.entry_id(i)])))); + } + *num_outputs = idx.outputs().size(); + *outputs = dmlc::BeginPtr(ret->ret_handles); + } else { + CHECK_EQ(static_cast(*num_outputs), idx.outputs().size()) + << "Specifed number of output differs from expected number of outputs"; + for (size_t i = 0; i < idx.outputs().size(); ++i) { + *outarray[i] = std::move(buff[idx.entry_id(idx.outputs()[i])]); + } + } API_END(); } diff --git a/src/c_api/c_api_symbolic.cc b/src/c_api/c_api_symbolic.cc index cad9e604df60..d3603e94b2a1 100644 --- a/src/c_api/c_api_symbolic.cc +++ b/src/c_api/c_api_symbolic.cc @@ -124,22 +124,6 @@ int MXSymbolCreateAtomicSymbol(AtomicSymbolCreator creator, API_END_HANDLE_ERROR(delete s;); } -int MXCachedCreateSymbol(CachedOpHandle handle, - const char* name, - mx_uint num_args, - SymbolHandle* args, - SymbolHandle* out) { - nnvm::Symbol *s = new nnvm::Symbol(); - const nnvm::NodeAttrs *attrs = static_cast(handle); - API_BEGIN(); - *s = nnvm::Symbol::CreateFunctor(*attrs); - nnvm::array_view parg( - (nnvm::Symbol**)args, (nnvm::Symbol**)args + num_args); // NOLINT(*) - s->Compose(parg, std::unordered_map(), name); - *out = s; - API_END_HANDLE_ERROR(delete s;) -} - int MXSymbolCreateVariable(const char *name, SymbolHandle *out) { return NNSymbolCreateVariable(name, out); } diff --git a/src/initialize.cc b/src/initialize.cc index d57fec84f72b..c1e897f01a57 100644 --- a/src/initialize.cc +++ b/src/initialize.cc @@ -28,7 +28,7 @@ void segfault_logger(int sig) { } #endif // DMLC_LOG_STACK_TRACE - exit(1); + exit(-1); } class LibraryInitializer { diff --git a/tests/python/unittest/test_ndarray.py b/tests/python/unittest/test_ndarray.py index dd38bdf98606..8c58d3b47a69 100644 --- a/tests/python/unittest/test_ndarray.py +++ b/tests/python/unittest/test_ndarray.py @@ -628,13 +628,17 @@ def test_iter(): def test_cached(): - op = mx.nd.CachedOp('Convolution', 3, kernel=(3, 3), num_filter=10) + sym = mx.sym.Convolution(kernel=(3, 3), num_filter=10) + 2 + op = mx.nd.CachedOp(sym) data = mx.nd.ones((3, 4, 10, 10)) weight = mx.nd.ones((10, 4, 3, 3)) bias = mx.nd.ones((10,)) - o1 = mx.nd.invoke(op, [data, weight, bias]) + o1 = op(data, weight, bias) bias[:] = 2 - o2 = mx.nd.invoke(op, [data, weight, bias]) + o2 = op(data, weight, bias) + assert_almost_equal(o2.asnumpy(), o1.asnumpy()+1) + o2[:] = 0 + op(data, weight, bias, out=o2) assert_almost_equal(o2.asnumpy(), o1.asnumpy()+1) def test_output(): diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py index 55c1d2488d6e..8129a41ee751 100644 --- a/tests/python/unittest/test_operator.py +++ b/tests/python/unittest/test_operator.py @@ -3237,7 +3237,7 @@ def test_psroipooling(): op = mx.contrib.sym.PSROIPooling(data=im_data_var, rois=rois_data_var, spatial_scale=spatial_scale, group_size=num_group, pooled_size=num_group, output_dim=num_classes, name='test_op') - rtol, atol = 1e-2, 1e-4 + rtol, atol = 1e-2, 1e-3 # By now we only have gpu implementation if mx.Context.default_ctx.device_type == 'gpu': check_numeric_gradient(op, [im_data, rois_data], rtol=rtol, atol=atol, @@ -3273,7 +3273,7 @@ def test_deformable_convolution(): # wider tolerance needed for coordinate differential rtol, atol = 1.0, 1e-2 else: - rtol, atol = 0.05, 1e-4 + rtol, atol = 0.05, 1e-3 # By now we only have gpu implementation if mx.Context.default_ctx.device_type == 'gpu': check_numeric_gradient(op, [im_data, offset_data, weight, bias], rtol=rtol, atol=atol, @@ -3306,7 +3306,7 @@ def test_deformable_psroipooling(): # wider tolerance needed for coordinate differential rtol, atol = 1.0, 1e-2 else: - rtol, atol = 1e-2, 1e-4 + rtol, atol = 1e-2, 1e-3 # By now we only have gpu implementation if mx.Context.default_ctx.device_type == 'gpu': check_numeric_gradient(op, [im_data, rois_data, offset_data], rtol=rtol, atol=atol, diff --git a/tests/python/unittest/test_symbol.py b/tests/python/unittest/test_symbol.py index 28fc8a4fc77b..093a8f3a40e0 100644 --- a/tests/python/unittest/test_symbol.py +++ b/tests/python/unittest/test_symbol.py @@ -225,19 +225,6 @@ def test_zero_prop2(): assert False -def test_cached(): - op = mx.sym.CachedOp('Convolution', 3, kernel=(3, 3), num_filter=10) - data = mx.sym.var('data') - weight = mx.sym.var('weight') - bias = mx.sym.var('bias') - out = mx.sym.invoke(op, [data, weight, bias], 'conv') - assert out.list_arguments() == ['data', 'weight', 'bias'] - assert out.list_outputs() == ['conv_output'] - with mx.name.Prefix('test_'): - assert mx.sym.invoke(op, [data, weight, bias]).name == 'test_convolution0' - assert mx.sym.invoke(op, [data, weight, bias]).name == 'test_convolution1' - - if __name__ == '__main__': import nose nose.runmodule() From e4c7bfcf4305fde55aa309782bf007a6c19fd5a6 Mon Sep 17 00:00:00 2001 From: Przemyslaw Tredak Date: Wed, 21 Jun 2017 16:35:39 -0700 Subject: [PATCH 100/834] Multi precision SGD (#6428) * Start of multiprecision * Modified resnet-v1-fp16 and alexnet-fp16 examples * Making linter happy * Making pylint happy * Merging with SGD * Fix * Making pylint happy * Fix * Fixes from review * Testing mp_sgd * Making pylint happy * Debug print * Making pylint happy * Fix for the race condition and removed debug prints * Making linter happy --- example/image-classification/common/fit.py | 3 +- .../symbols/alexnet_fp16.py | 48 +++---- .../symbols/resnet-v1-fp16.py | 30 ++--- python/mxnet/optimizer.py | 52 ++++++-- src/operator/elemwise_op_common.h | 27 ++-- src/operator/optimizer_op-inl.h | 104 +++++++++++++++ src/operator/optimizer_op.cc | 34 +++++ src/operator/optimizer_op.cu | 6 + tests/python/unittest/test_optimizer.py | 118 ++++++++++++------ 9 files changed, 310 insertions(+), 112 deletions(-) diff --git a/example/image-classification/common/fit.py b/example/image-classification/common/fit.py index 6d79385cb6ff..82bcde401336 100755 --- a/example/image-classification/common/fit.py +++ b/example/image-classification/common/fit.py @@ -146,7 +146,8 @@ def fit(args, network, data_loader, **kwargs): 'learning_rate': lr, 'momentum' : args.mom, 'wd' : args.wd, - 'lr_scheduler': lr_scheduler} + 'lr_scheduler': lr_scheduler, + 'multi_precision': True} monitor = mx.mon.Monitor(args.monitor, pattern=".*") if args.monitor > 0 else None diff --git a/example/image-classification/symbols/alexnet_fp16.py b/example/image-classification/symbols/alexnet_fp16.py index 94440812618f..9e7d4dc0d822 100755 --- a/example/image-classification/symbols/alexnet_fp16.py +++ b/example/image-classification/symbols/alexnet_fp16.py @@ -10,10 +10,8 @@ def get_symbol(num_classes, **kwargs): input_data = mx.symbol.Variable(name="data") input_data = mx.symbol.Cast(data=input_data, dtype=np.float16) # stage 1 - weight = mx.symbol.Variable(name='conv1_weight', dtype=np.float32) - bias = mx.symbol.Variable(name='conv1_bias', dtype=np.float32) - weight = mx.symbol.Cast(data=weight, dtype=np.float16) - bias = mx.symbol.Cast(data=bias, dtype=np.float16) + weight = mx.symbol.Variable(name='conv1_weight', dtype=np.float16) + bias = mx.symbol.Variable(name='conv1_bias', dtype=np.float16) conv1 = mx.symbol.Convolution(name='conv1', data=input_data, weight=weight, bias=bias, kernel=(11, 11), stride=(4, 4), num_filter=96) relu1 = mx.symbol.Activation(data=conv1, act_type="relu") @@ -21,62 +19,48 @@ def get_symbol(num_classes, **kwargs): pool1 = mx.symbol.Pooling( data=lrn1, pool_type="max", kernel=(3, 3), stride=(2,2)) # stage 2 - weight = mx.symbol.Variable(name='conv2_weight', dtype=np.float32) - bias = mx.symbol.Variable(name='conv2_bias', dtype=np.float32) - weight = mx.symbol.Cast(data=weight, dtype=np.float16) - bias = mx.symbol.Cast(data=bias, dtype=np.float16) + weight = mx.symbol.Variable(name='conv2_weight', dtype=np.float16) + bias = mx.symbol.Variable(name='conv2_bias', dtype=np.float16) conv2 = mx.symbol.Convolution(name='conv2', data=pool1, weight=weight, bias=bias, kernel=(5, 5), pad=(2, 2), num_filter=256) relu2 = mx.symbol.Activation(data=conv2, act_type="relu") lrn2 = mx.symbol.LRN(data=relu2, alpha=0.0001, beta=0.75, knorm=2, nsize=5) pool2 = mx.symbol.Pooling(data=lrn2, kernel=(3, 3), stride=(2, 2), pool_type="max") # stage 3 - weight = mx.symbol.Variable(name='conv3_weight', dtype=np.float32) - bias = mx.symbol.Variable(name='conv3_bias', dtype=np.float32) - weight = mx.symbol.Cast(data=weight, dtype=np.float16) - bias = mx.symbol.Cast(data=bias, dtype=np.float16) + weight = mx.symbol.Variable(name='conv3_weight', dtype=np.float16) + bias = mx.symbol.Variable(name='conv3_bias', dtype=np.float16) conv3 = mx.symbol.Convolution(name='conv3', data=pool2, weight=weight, bias=bias, kernel=(3, 3), pad=(1, 1), num_filter=384) relu3 = mx.symbol.Activation(data=conv3, act_type="relu") - weight = mx.symbol.Variable(name='conv4_weight', dtype=np.float32) - bias = mx.symbol.Variable(name='conv4_bias', dtype=np.float32) - weight = mx.symbol.Cast(data=weight, dtype=np.float16) - bias = mx.symbol.Cast(data=bias, dtype=np.float16) + weight = mx.symbol.Variable(name='conv4_weight', dtype=np.float16) + bias = mx.symbol.Variable(name='conv4_bias', dtype=np.float16) conv4 = mx.symbol.Convolution(name='conv4', data=relu3, weight=weight, bias=bias, kernel=(3, 3), pad=(1, 1), num_filter=384) relu4 = mx.symbol.Activation(data=conv4, act_type="relu") - weight = mx.symbol.Variable(name='conv5_weight', dtype=np.float32) - bias = mx.symbol.Variable(name='conv5_bias', dtype=np.float32) - weight = mx.symbol.Cast(data=weight, dtype=np.float16) - bias = mx.symbol.Cast(data=bias, dtype=np.float16) + weight = mx.symbol.Variable(name='conv5_weight', dtype=np.float16) + bias = mx.symbol.Variable(name='conv5_bias', dtype=np.float16) conv5 = mx.symbol.Convolution(name='conv5', data=relu4, weight=weight, bias=bias, kernel=(3, 3), pad=(1, 1), num_filter=256) relu5 = mx.symbol.Activation(data=conv5, act_type="relu") pool3 = mx.symbol.Pooling(data=relu5, kernel=(3, 3), stride=(2, 2), pool_type="max") # stage 4 flatten = mx.symbol.Flatten(data=pool3) - weight = mx.symbol.Variable(name='fc1_weight', dtype=np.float32) - bias = mx.symbol.Variable(name='fc1_bias', dtype=np.float32) - weight = mx.symbol.Cast(data=weight, dtype=np.float16) - bias = mx.symbol.Cast(data=bias, dtype=np.float16) + weight = mx.symbol.Variable(name='fc1_weight', dtype=np.float16) + bias = mx.symbol.Variable(name='fc1_bias', dtype=np.float16) fc1 = mx.symbol.FullyConnected(name='fc1', data=flatten, weight=weight, bias=bias, num_hidden=4096) relu6 = mx.symbol.Activation(data=fc1, act_type="relu") dropout1 = mx.symbol.Dropout(data=relu6, p=0.5) # stage 5 - weight = mx.symbol.Variable(name='fc2_weight', dtype=np.float32) - bias = mx.symbol.Variable(name='fc2_bias', dtype=np.float32) - weight = mx.symbol.Cast(data=weight, dtype=np.float16) - bias = mx.symbol.Cast(data=bias, dtype=np.float16) + weight = mx.symbol.Variable(name='fc2_weight', dtype=np.float16) + bias = mx.symbol.Variable(name='fc2_bias', dtype=np.float16) fc2 = mx.symbol.FullyConnected(name='fc2', data=dropout1, weight=weight, bias=bias, num_hidden=4096) relu7 = mx.symbol.Activation(data=fc2, act_type="relu") dropout2 = mx.symbol.Dropout(data=relu7, p=0.5) # stage 6 - weight = mx.symbol.Variable(name='fc3_weight', dtype=np.float32) - bias = mx.symbol.Variable(name='fc3_bias', dtype=np.float32) - weight = mx.symbol.Cast(data=weight, dtype=np.float16) - bias = mx.symbol.Cast(data=bias, dtype=np.float16) + weight = mx.symbol.Variable(name='fc3_weight', dtype=np.float16) + bias = mx.symbol.Variable(name='fc3_bias', dtype=np.float16) fc3 = mx.symbol.FullyConnected(name='fc3', data=dropout2, weight=weight, bias=bias, num_hidden=num_classes) label = mx.symbol.Variable(name='softmax_label') diff --git a/example/image-classification/symbols/resnet-v1-fp16.py b/example/image-classification/symbols/resnet-v1-fp16.py index 91d69dcfb6e5..ae3f3c46fe7b 100755 --- a/example/image-classification/symbols/resnet-v1-fp16.py +++ b/example/image-classification/symbols/resnet-v1-fp16.py @@ -29,20 +29,17 @@ def residual_unit(data, num_filter, stride, dim_match, name, bottle_neck=True, b Workspace used in convolution operator """ if bottle_neck: - weight = mx.symbol.Variable(name=name + '_conv1_weight', dtype=np.float32) - weight = mx.symbol.Cast(data=weight, dtype=np.float16) + weight = mx.symbol.Variable(name=name + '_conv1_weight', dtype=np.float16) conv1 = mx.sym.Convolution(data=data, weight=weight, num_filter=int(num_filter*0.25), kernel=(1,1), stride=stride, pad=(0,0), no_bias=True, workspace=workspace, name=name + '_conv1') bn1 = mx.sym.BatchNorm(data=conv1, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn1') act1 = mx.sym.Activation(data=bn1, act_type='relu', name=name + '_relu1') - weight = mx.symbol.Variable(name=name + '_conv2_weight', dtype=np.float32) - weight = mx.symbol.Cast(data=weight, dtype=np.float16) + weight = mx.symbol.Variable(name=name + '_conv2_weight', dtype=np.float16) conv2 = mx.sym.Convolution(data=act1, weight=weight, num_filter=int(num_filter*0.25), kernel=(3,3), stride=(1,1), pad=(1,1), no_bias=True, workspace=workspace, name=name + '_conv2') bn2 = mx.sym.BatchNorm(data=conv2, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn2') act2 = mx.sym.Activation(data=bn2, act_type='relu', name=name + '_relu2') - weight = mx.symbol.Variable(name=name + '_conv3_weight', dtype=np.float32) - weight = mx.symbol.Cast(data=weight, dtype=np.float16) + weight = mx.symbol.Variable(name=name + '_conv3_weight', dtype=np.float16) conv3 = mx.sym.Convolution(data=act2, weight=weight, num_filter=num_filter, kernel=(1,1), stride=(1,1), pad=(0,0), no_bias=True, workspace=workspace, name=name + '_conv3') bn3 = mx.sym.BatchNorm(data=conv3, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn3') @@ -50,8 +47,7 @@ def residual_unit(data, num_filter, stride, dim_match, name, bottle_neck=True, b if dim_match: shortcut = data else: - weight = mx.symbol.Variable(name=name + '_conv1sc_weight', dtype=np.float32) - weight = mx.symbol.Cast(data=weight, dtype=np.float16) + weight = mx.symbol.Variable(name=name + '_conv1sc_weight', dtype=np.float16) conv1sc = mx.sym.Convolution(data=data, weight=weight, num_filter=num_filter, kernel=(1,1), stride=stride, no_bias=True, workspace=workspace, name=name+'_conv1sc') shortcut = mx.sym.BatchNorm(data=conv1sc, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_sc') @@ -59,14 +55,12 @@ def residual_unit(data, num_filter, stride, dim_match, name, bottle_neck=True, b shortcut._set_attr(mirror_stage='True') return mx.sym.Activation(data=bn3 + shortcut, act_type='relu', name=name + '_relu3') else: - weight = mx.symbol.Variable(name=name + '_conv1_weight', dtype=np.float32) - weight = mx.symbol.Cast(data=weight, dtype=np.float16) + weight = mx.symbol.Variable(name=name + '_conv1_weight', dtype=np.float16) conv1 = mx.sym.Convolution(data=data, weight=weight, num_filter=num_filter, kernel=(3,3), stride=stride, pad=(1,1), no_bias=True, workspace=workspace, name=name + '_conv1') bn1 = mx.sym.BatchNorm(data=conv1, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_bn1') act1 = mx.sym.Activation(data=bn1, act_type='relu', name=name + '_relu1') - weight = mx.symbol.Variable(name=name + '_conv2_weight', dtype=np.float32) - weight = mx.symbol.Cast(data=weight, dtype=np.float16) + weight = mx.symbol.Variable(name=name + '_conv2_weight', dtype=np.float16) conv2 = mx.sym.Convolution(data=act1, weight=weight, num_filter=num_filter, kernel=(3,3), stride=(1,1), pad=(1,1), no_bias=True, workspace=workspace, name=name + '_conv2') bn2 = mx.sym.BatchNorm(data=conv2, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_bn2') @@ -74,8 +68,7 @@ def residual_unit(data, num_filter, stride, dim_match, name, bottle_neck=True, b if dim_match: shortcut = data else: - weight = mx.symbol.Variable(name=name + '_conv1sc_weight', dtype=np.float32) - weight = mx.symbol.Cast(data=weight, dtype=np.float16) + weight = mx.symbol.Variable(name=name + '_conv1sc_weight', dtype=np.float16) conv1sc = mx.sym.Convolution(data=data, weight=weight, num_filter=num_filter, kernel=(1,1), stride=stride, no_bias=True, workspace=workspace, name=name+'_conv1sc') shortcut = mx.sym.BatchNorm(data=conv1sc, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_sc') @@ -105,8 +98,7 @@ def resnet(units, num_stages, filter_list, num_classes, image_shape, bottle_neck data = mx.sym.Variable(name='data') data = mx.symbol.Cast(data=data, dtype=np.float16) (nchannel, height, width) = image_shape - weight = mx.symbol.Variable(name='conv0_weight', dtype=np.float32) - weight = mx.symbol.Cast(data=weight, dtype=np.float16) + weight = mx.symbol.Variable(name='conv0_weight', dtype=np.float16) if height <= 32: # such as cifar10 body = mx.sym.Convolution(data=data, weight=weight, num_filter=filter_list[0], kernel=(3, 3), stride=(1,1), pad=(1, 1), no_bias=True, name="conv0", workspace=workspace) @@ -131,10 +123,8 @@ def resnet(units, num_stages, filter_list, num_classes, image_shape, bottle_neck # Although kernel is not used here when global_pool=True, we should put one pool1 = mx.symbol.Pooling(data=body, global_pool=True, kernel=(7, 7), pool_type='avg', name='pool1') flat = mx.symbol.Flatten(data=pool1) - weight = mx.symbol.Variable(name='fc1_weight', dtype=np.float32) - bias = mx.symbol.Variable(name='fc1_bias', dtype=np.float32) - weight = mx.symbol.Cast(data=weight, dtype=np.float16) - bias = mx.symbol.Cast(data=bias, dtype=np.float16) + weight = mx.symbol.Variable(name='fc1_weight', dtype=np.float16) + bias = mx.symbol.Variable(name='fc1_bias', dtype=np.float16) fc1 = mx.symbol.FullyConnected(data=flat, weight=weight, bias=bias, num_hidden=num_classes, name='fc1') fc1 = mx.symbol.Cast(data=fc1, dtype=np.float32) return mx.symbol.SoftmaxOutput(data=fc1, name='softmax') diff --git a/python/mxnet/optimizer.py b/python/mxnet/optimizer.py index 201393b5d625..3d318003715a 100644 --- a/python/mxnet/optimizer.py +++ b/python/mxnet/optimizer.py @@ -2,8 +2,11 @@ import math import pickle import logging -from .ndarray import NDArray, zeros, clip, sqrt, sign -from .ndarray import sgd_update, sgd_mom_update, adam_update, rmsprop_update, rmspropalex_update +import warnings +import numpy +from .ndarray import NDArray, zeros, clip, sqrt, sign, array +from .ndarray import (sgd_update, sgd_mom_update, adam_update, rmsprop_update, rmspropalex_update, + mp_sgd_update, mp_sgd_mom_update) from .random import normal @@ -323,16 +326,34 @@ class SGD(Optimizer): ---------- momentum : float, optional The momentum value. + multi_precision: bool, optional + Flag to control the internal precision of the optimizer. + ``False`` results in using the same precision as the weights (default), + ``True`` makes internal 32-bit copy of the weights and applies gradients + in 32-bit precision even if actual weights used in the model have lower precision. + Turning this on can improve convergence and accuracy when training with float16. """ - def __init__(self, momentum=0.0, **kwargs): + def __init__(self, momentum=0.0, multi_precision=False, **kwargs): super(SGD, self).__init__(**kwargs) self.momentum = momentum + self.multi_precision = multi_precision def create_state(self, index, weight): - if self.momentum == 0.0: - return None - else: - return zeros(weight.shape, weight.context, dtype=weight.dtype) + momentum = None + weight_master_copy = None + if self.multi_precision and weight.dtype == numpy.float16: + weight_master_copy = array(weight, ctx=weight.context, dtype=numpy.float32) + if self.momentum != 0.0: + momentum = zeros(weight.shape, weight.context, dtype=numpy.float32) + return (momentum, weight_master_copy) + if weight.dtype == numpy.float16 and not self.multi_precision: + warnings.warn("Accumulating with float16 in optimizer can lead to " + "poor accuracy or slow convergence. " + "Consider using multi_precision=True option of the " + "SGD optimizer") + if self.momentum != 0.0: + momentum = zeros(weight.shape, weight.context, dtype=weight.dtype) + return momentum def update(self, index, weight, grad, state): assert(isinstance(weight, NDArray)) @@ -346,13 +367,22 @@ def update(self, index, weight, grad, state): kwargs['momentum'] = self.momentum if self.clip_gradient: kwargs['clip_gradient'] = self.clip_gradient + use_multi_precision = isinstance(state, (list, tuple)) - if state is not None: - sgd_mom_update(weight, grad, state, out=weight, + if not use_multi_precision: + if state is not None: + sgd_mom_update(weight, grad, state, out=weight, + lr=lr, wd=wd, **kwargs) + else: + sgd_update(weight, grad, out=weight, lr=lr, wd=wd, **kwargs) else: - sgd_update(weight, grad, out=weight, - lr=lr, wd=wd, **kwargs) + if state[0] is not None: + mp_sgd_mom_update(weight, grad, state[0], state[1], out=weight, + lr=lr, wd=wd, **kwargs) + else: + mp_sgd_update(weight, grad, state[1], out=weight, + lr=lr, wd=wd, **kwargs) @register class DCASGD(Optimizer): diff --git a/src/operator/elemwise_op_common.h b/src/operator/elemwise_op_common.h index b7e87cf2bc70..9402ac0bf468 100644 --- a/src/operator/elemwise_op_common.h +++ b/src/operator/elemwise_op_common.h @@ -22,33 +22,42 @@ namespace mxnet { namespace op { template + std::string (*attr_string)(const AttrType&), + int n_in = -1, int n_out = -1> inline bool ElemwiseAttr(const nnvm::NodeAttrs& attrs, std::vector *in_attrs, std::vector *out_attrs, const AttrType& none) { AttrType dattr = none; - auto deduce = [&](std::vector *vec, const char *name) { - for (size_t i = 0; i < vec->size(); ++i) { + size_t in_size = in_attrs->size(); + size_t out_size = out_attrs->size(); + if (n_in != -1) + in_size = static_cast(n_in); + if (n_out != -1) + out_size = static_cast(n_out); + + auto deduce = [&](std::vector *vec, size_t size, const char *name) { + for (size_t i = 0; i < size; ++i) { CHECK(assign(&dattr, (*vec)[i])) << "Incompatible attr in node " << attrs.name << " at " << i << "-th " << name << ": " << "expected " << attr_string(dattr) << ", got " << attr_string((*vec)[i]); } }; - deduce(in_attrs, "input"); - if (reverse_infer) deduce(out_attrs, "output"); + deduce(in_attrs, in_size, "input"); + if (reverse_infer) deduce(out_attrs, out_size, "output"); - auto write = [&](std::vector *vec, const char *name) { - for (size_t i = 0; i < vec->size(); ++i) { + auto write = [&](std::vector *vec, size_t size, const char *name) { + for (size_t i = 0; i < size; ++i) { CHECK(assign(&(*vec)[i], dattr)) << "Incompatible attr in node " << attrs.name << " at " << i << "-th " << name << ": " << "expected " << attr_string(dattr) << ", got " << attr_string((*vec)[i]); } }; - write(in_attrs, "input"); - write(out_attrs, "output"); + write(in_attrs, in_size, "input"); + write(out_attrs, out_size, "output"); + if (is_none(dattr)) return false; return true; } diff --git a/src/operator/optimizer_op-inl.h b/src/operator/optimizer_op-inl.h index 85091c008ab4..9f4959350362 100644 --- a/src/operator/optimizer_op-inl.h +++ b/src/operator/optimizer_op-inl.h @@ -153,6 +153,110 @@ inline void SGDMomUpdate(const nnvm::NodeAttrs& attrs, }); } +template +inline bool MP_SGD_InferType(const nnvm::NodeAttrs& attrs, + std::vector *in_attrs, + std::vector *out_attrs) { + CHECK_EQ(in_attrs->size(), static_cast(total_in)) << " in operator " << attrs.name; + CHECK_EQ(out_attrs->size(), static_cast(n_out)) << " in operator " << attrs.name; + for (int i = n_in; i < total_in; ++i) { + TYPE_ASSIGN_CHECK(*in_attrs, i, mshadow::kFloat32); + } + return ElemwiseAttr( + attrs, in_attrs, out_attrs, -1); +} + +struct MP_SGDKernel { + template + MSHADOW_XINLINE static void Map(int i, DType* out_data, const DType* weight_data, + const DType* grad_data, float* weight32, const float param_clip_gradient, + const float param_lr, const float param_wd, const float param_rescale_grad, + const OpReqType req) { + if (param_clip_gradient >= 0.0f) { + float w = weight32[i]; + w = (1.f - param_lr*param_wd)*w - + (param_lr) * mshadow_op::clip::Map(param_rescale_grad*static_cast(grad_data[i]), + param_clip_gradient); + weight32[i] = w; + KERNEL_ASSIGN(out_data[i], req, (DType)w); + } else { + float w = weight32[i]; + w = (1.f-param_lr*param_wd)*w + - (param_lr*param_rescale_grad)*static_cast(grad_data[i]); + weight32[i] = w; + KERNEL_ASSIGN(out_data[i], req, (DType)w); + } + } +}; + +template +inline void MP_SGDUpdate(const nnvm::NodeAttrs& attrs, + const OpContext &ctx, + const std::vector &inputs, + const std::vector &req, + const std::vector &outputs) { + using namespace mxnet_op; + const SGDParam& param = nnvm::get(attrs.parsed); + Stream* s = ctx.get_stream(); + MSHADOW_REAL_TYPE_SWITCH(inputs[0].type_flag_, DType, { + Tensor weight = inputs[0].FlatTo2D(s); + Tensor grad = inputs[1].FlatTo2D(s); + Tensor weight32 = inputs[2].FlatTo2D(s); + Tensor out = outputs[0].FlatTo2D(s); + Kernel::Launch(s, weight.shape_.Size(), out.dptr_, weight.dptr_, + grad.dptr_, weight32.dptr_, param.clip_gradient, + param.lr, param.wd, + param.rescale_grad, req[0]); + }); +} + +struct MP_SGDMomKernel { + template + MSHADOW_XINLINE static void Map(int i, DType* out_data, float* mom_data, + const DType* weight_data, const DType* grad_data, float* weight32, + const float param_clip_gradient, const float param_momentum, const float param_lr, + const float param_wd, const float param_rescale_grad, const OpReqType req) { + float w = weight32[i]; + float mom = mom_data[i]; + if (param_clip_gradient >= 0.0f) { + mom = param_momentum*mom + - param_lr*param_wd*w + - param_lr + *mshadow_op::clip::Map(param_rescale_grad*static_cast(grad_data[i]), + param_clip_gradient); + } else { + mom = param_momentum*mom + - param_lr*param_wd*w + - param_lr*param_rescale_grad*static_cast(grad_data[i]); + } + mom_data[i] = mom; + w = w + mom; + weight32[i] = w; + KERNEL_ASSIGN(out_data[i], req, w); + } +}; + +template +inline void MP_SGDMomUpdate(const nnvm::NodeAttrs& attrs, + const OpContext &ctx, + const std::vector &inputs, + const std::vector &req, + const std::vector &outputs) { + using namespace mxnet_op; + SGDMomParam param = nnvm::get(attrs.parsed); + Stream* s = ctx.get_stream(); + MSHADOW_REAL_TYPE_SWITCH(inputs[0].type_flag_, DType, { + Tensor weight = inputs[0].FlatTo2D(s); + Tensor grad = inputs[1].FlatTo2D(s); + Tensor mom = inputs[2].FlatTo2D(s); + Tensor weight32 = inputs[3].FlatTo2D(s); + Tensor out = outputs[0].FlatTo2D(s); + Kernel::Launch(s, weight.shape_.Size(), out.dptr_, mom.dptr_, + weight.dptr_, grad.dptr_, weight32.dptr_, param.clip_gradient, param.momentum, + param.lr, param.wd, param.rescale_grad, req[0]); + }); +} + struct AdamParam : public dmlc::Parameter { float lr; float beta1; diff --git a/src/operator/optimizer_op.cc b/src/operator/optimizer_op.cc index 9ec6aacaafac..3fdb9c2498fb 100644 --- a/src/operator/optimizer_op.cc +++ b/src/operator/optimizer_op.cc @@ -68,6 +68,40 @@ Where the parameter ``momentum`` is the decay rate of momentum estimates at each .add_argument("mom", "NDArray-or-Symbol", "Momentum") .add_arguments(SGDMomParam::__FIELDS__()); +NNVM_REGISTER_OP(mp_sgd_update) +.describe("Updater function for multi-precision sgd optimizer") +.set_num_inputs(3) +.set_num_outputs(1) +.set_attr_parser(ParamParser) +.set_attr("FInferShape", ElemwiseShape<3, 1>) +.set_attr("FInferType", MP_SGD_InferType<2, 1, 3>) +.set_attr("FCompute", MP_SGDUpdate) +.set_attr("FMutateInputs", + [](const nnvm::NodeAttrs& attrs) { + return std::vector{2}; + }) +.add_argument("weight", "NDArray-or-Symbol", "Weight") +.add_argument("grad", "NDArray-or-Symbol", "gradient") +.add_argument("weight32", "NDArray-or-Symbol", "Weight32") +.add_arguments(SGDParam::__FIELDS__()); + +NNVM_REGISTER_OP(mp_sgd_mom_update) +.describe("Updater function for multi-precision sgd optimizer") +.set_num_inputs(4) +.set_num_outputs(1) +.set_attr_parser(ParamParser) +.set_attr("FInferShape", ElemwiseShape<4, 1>) +.set_attr("FInferType", MP_SGD_InferType<2, 1, 4>) +.set_attr("FMutateInputs", + [](const nnvm::NodeAttrs& attrs) { + return std::vector{2, 3}; + }) +.set_attr("FCompute", MP_SGDMomUpdate) +.add_argument("weight", "NDArray-or-Symbol", "Weight") +.add_argument("grad", "NDArray-or-Symbol", "Gradient") +.add_argument("mom", "NDArray-or-Symbol", "Momentum") +.add_argument("weight32", "NDArray-or-Symbol", "Weight32") +.add_arguments(SGDMomParam::__FIELDS__()); NNVM_REGISTER_OP(adam_update) .describe(R"code(Update function for Adam optimizer. Adam is seen as a generalization diff --git a/src/operator/optimizer_op.cu b/src/operator/optimizer_op.cu index 2b2667ec317b..a30584dd183f 100644 --- a/src/operator/optimizer_op.cu +++ b/src/operator/optimizer_op.cu @@ -15,6 +15,12 @@ NNVM_REGISTER_OP(sgd_update) NNVM_REGISTER_OP(sgd_mom_update) .set_attr("FCompute", SGDMomUpdate); +NNVM_REGISTER_OP(mp_sgd_update) +.set_attr("FCompute", MP_SGDUpdate); + +NNVM_REGISTER_OP(mp_sgd_mom_update) +.set_attr("FCompute", MP_SGDMomUpdate); + NNVM_REGISTER_OP(adam_update) .set_attr("FCompute", AdamUpdate); diff --git a/tests/python/unittest/test_optimizer.py b/tests/python/unittest/test_optimizer.py index 11ca7bed1743..cf7b82eaaa88 100644 --- a/tests/python/unittest/test_optimizer.py +++ b/tests/python/unittest/test_optimizer.py @@ -30,9 +30,9 @@ def test_lr_wd_mult(): assert not mx.test_utils.almost_equal(args1['fc2_weight'], args2['fc2_weight'], 1e-1) -def compare_optimizer(opt1, opt2, shape): - w1 = mx.random.uniform(shape=shape, ctx=default_context()) - g1 = mx.random.uniform(shape=shape, ctx=default_context()) +def compare_optimizer(opt1, opt2, shape, dtype): + w1 = mx.random.uniform(shape=shape, ctx=default_context(), dtype=dtype) + g1 = mx.random.uniform(shape=shape, ctx=default_context(), dtype=dtype) w2 = w1.copyto(default_context()) g2 = g1.copyto(default_context()) @@ -41,22 +41,25 @@ def compare_optimizer(opt1, opt2, shape): state2 = opt2.create_state(0, w2) if state1 is not None and state2 is not None: for s1, s2, in zip(state1, state2): - assert(same(s1.asnumpy(), s2.asnumpy())) + if s1 is not None or s2 is not None: + assert(same(s1.asnumpy(), s2.asnumpy())) opt1.update(0, w1, g1, state1) opt2.update(0, w2, g2, state2) if state1 is not None and state2 is not None: for s1, s2, in zip(state1, state2): - assert_almost_equal(s1.asnumpy(), s2.asnumpy(), rtol=1e-4, atol=1e-5) + if s1 is not None or s2 is not None: + assert_almost_equal(s1.asnumpy(), s2.asnumpy(), rtol=1e-4, atol=1e-5) assert_almost_equal(w1.asnumpy(), w2.asnumpy(), rtol=1e-4, atol=1e-5) # SGD class PySGD(mx.optimizer.Optimizer): """python reference implemenation of sgd""" - def __init__(self, learning_rate=0.01, momentum=0.0, **kwargs): + def __init__(self, learning_rate=0.01, momentum=0.0, multi_precision=False, **kwargs): super(PySGD, self).__init__(learning_rate=learning_rate, **kwargs) self.momentum = momentum + self.multi_precision = multi_precision def create_state(self, index, weight): """Create additional optimizer state: momentum @@ -67,10 +70,18 @@ def create_state(self, index, weight): The weight data """ - if self.momentum == 0.0: - return None + momentum = None + weight_master_copy = None + do_multi_precision = self.multi_precision and weight.dtype == np.float16 + if do_multi_precision: + if self.momentum != 0.0: + momentum = mx.nd.zeros(weight.shape, weight.context, dtype=np.float32) + weight_master_copy = array(weight, ctx=weight.context, dtype=np.float32) + return (momentum, weight_master_copy) else: - return mx.nd.zeros(weight.shape, weight.context, dtype=weight.dtype) + if self.momentum != 0.0: + momentum = mx.nd.zeros(weight.shape, weight.context, dtype=weight.dtype) + return momentum def update(self, index, weight, grad, state): """Update the parameters. @@ -92,43 +103,72 @@ def update(self, index, weight, grad, state): lr = self._get_lr(index) wd = self._get_wd(index) self._update_count(index) - - if self.momentum == 0.0: - if self.clip_gradient is not None: - weight[:] = ((1 - lr*wd)*weight - - lr*mx.nd.clip(grad*self.rescale_grad, -self.clip_gradient, self.clip_gradient)) + use_multi_precision = isinstance(state, list) or isinstance(state, tuple) + + if not use_multi_precision: + if self.momentum == 0.0: + if self.clip_gradient is not None: + weight[:] = ((1 - lr*wd)*weight - + lr*mx.nd.clip(grad*self.rescale_grad, -self.clip_gradient, self.clip_gradient)) + else: + weight[:] = (1 - lr*wd)*weight - lr*self.rescale_grad*grad else: - weight[:] = (1 - lr*wd)*weight - lr*self.rescale_grad*grad + mom = state + if self.clip_gradient is not None: + mom[:] = (self.momentum*mom - lr*wd*weight - + lr*mx.nd.clip(grad*self.rescale_grad, -self.clip_gradient, self.clip_gradient)) + weight += mom + else: + mom[:] = self.momentum*mom - lr*wd*weight - lr*self.rescale_grad*grad + weight += mom else: - mom = state - if self.clip_gradient is not None: - mom[:] = (self.momentum*mom - lr*wd*weight - - lr*mx.nd.clip(grad*self.rescale_grad, -self.clip_gradient, self.clip_gradient)) - weight += mom + grad32 = array(grad, ctx=grad.context, dtype=np.float32) + mom = state[0] + weight32 = state[1] + if self.momentum == 0.0: + if self.clip_gradient is not None: + weight32[:] = ((1 - lr*wd)*weight32 - + lr*mx.nd.clip(grad32*self.rescale_grad, -self.clip_gradient, self.clip_gradient)) + else: + weight32[:] = (1 - lr*wd)*weight32 - lr*self.rescale_grad*grad32 else: - mom[:] = self.momentum*mom - lr*wd*weight - lr*self.rescale_grad*grad - weight += mom + if self.clip_gradient is not None: + mom[:] = (self.momentum*mom - lr*wd*weight32 - + lr*mx.nd.clip(grad32*self.rescale_grad, -self.clip_gradient, self.clip_gradient)) + weight32 += mom + else: + mom[:] = self.momentum*mom - lr*wd*weight32 - lr*self.rescale_grad*grad32 + weight32 += mom + tmp = weight32.astype(weight.dtype) + tmp.copyto(weight) def test_sgd(): mx.random.seed(0) opt1 = PySGD opt2 = mx.optimizer.SGD shape = (3, 4, 5) - kwargs = [{}, {'momentum': 0.9}, - {'clip_gradient': 0.5}, - {'clip_gradient': 0.4, 'rescale_grad': 0.14}, - {'rescale_grad': 0.8}, - {'clip_gradient': 0.5, 'wd': 0.07}, - {'clip_gradient': 0.4, 'rescale_grad': 0.14, 'wd': 0.03}, - {'rescale_grad': 0.8, 'wd': 0.05}, - {'clip_gradient': 0.5, 'momentum': 0.9}, - {'clip_gradient': 0.4, 'rescale_grad': 0.14, 'momentum': 0.9}, - {'rescale_grad': 0.8, 'momentum': 0.9}, - {'clip_gradient': 0.5, 'wd': 0.07, 'momentum': 0.9}, - {'clip_gradient': 0.4, 'rescale_grad': 0.14, 'wd': 0.03, 'momentum': 0.9}, - {'rescale_grad': 0.8, 'wd': 0.05, 'momentum': 0.9}] - for kwarg in kwargs: - compare_optimizer(opt1(**kwarg), opt2(**kwarg), shape) + mom_options = [{}, {'momentum': 0.9}] + cg_options = [{}, {'clip_gradient': 0.4}, {'clip_gradient': 0.5}] + rg_options = [{}, {'rescale_grad': 0.14}, {'rescale_grad': 0.8}] + wd_options = [{}, {'wd': 0.03}, {'wd': 0.05}, {'wd': 0.07}] + mp_options = [{}, {'multi_precision': False}, {'multi_precision': True}] + for dtype in [np.float16, np.float32, np.float64]: + for mom_option in mom_options: + for cg_option in cg_options: + for rg_option in rg_options: + for wd_option in wd_options: + for mp_option in mp_options: + kwarg = {} + kwarg.update(mom_option) + kwarg.update(cg_option) + kwarg.update(rg_option) + kwarg.update(wd_option) + kwarg.update(mp_option) + if (dtype == np.float16 and + ('multi_precision' not in kwarg or + not kwarg['multi_precision'])): + continue + compare_optimizer(opt1(**kwarg), opt2(**kwarg), shape, dtype) # ADAM @@ -208,7 +248,7 @@ def test_adam(): {'clip_gradient': 0.4, 'rescale_grad': 0.14, 'wd': 0.03}, {'rescale_grad': 0.8, 'wd': 0.05}] for kwarg in kwargs: - compare_optimizer(opt1(**kwarg), opt2(**kwarg), shape) + compare_optimizer(opt1(**kwarg), opt2(**kwarg), shape, np.float32) # RMSProp class PyRMSProp(mx.optimizer.Optimizer): @@ -348,7 +388,7 @@ def test_rms(): {'clip_gradient': 0.4, 'rescale_grad': 0.14, 'wd': 0.03, 'centered': True, 'clip_weights': 0.01}, {'rescale_grad': 0.8, 'wd': 0.05, 'centered': True, 'clip_weights': 0.01}] for kwarg in kwargs: - compare_optimizer(opt1(**kwarg), opt2(**kwarg), shape) + compare_optimizer(opt1(**kwarg), opt2(**kwarg), shape, np.float32) if __name__ == '__main__': test_adam() From 0f7bdfa2b33fc9585b1cb41af6e2354d388dec67 Mon Sep 17 00:00:00 2001 From: Rob De Feo Date: Thu, 22 Jun 2017 03:36:15 +0100 Subject: [PATCH 101/834] Fix MacOS install instructions for venv and python. closes #6776 (#6777) --- docs/get_started/install.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/get_started/install.md b/docs/get_started/install.md index edeb912ad4ef..766730cde089 100644 --- a/docs/get_started/install.md +++ b/docs/get_started/install.md @@ -551,8 +551,8 @@ Install *MXNet* with OpenBLAS acceleration. **Step 5** Install [Graphviz](http://www.graphviz.org/). (Optional, needed for graph visualization using `mxnet.viz` package). ```bash -sudo apt-get install graphviz -pip install graphviz +$ brew install graphviz +(mxnet)$ pip install graphviz ``` **Step 6** Validate the installation by running simple *MXNet* code described [here](#validate-mxnet-installation). @@ -590,8 +590,8 @@ $ pip install mxnet **Step 3** Install [Graphviz](http://www.graphviz.org/). (Optional, needed for graph visualization using `mxnet.viz` package). ```bash -sudo apt-get install graphviz -pip install graphviz +$ brew install graphviz +$ pip install graphviz ``` **Step 4** Validate the installation by running simple MXNet code described [here](#validate-mxnet-installation). From bf4d774e2d3fdc33ae748919626d8f5ed86420d4 Mon Sep 17 00:00:00 2001 From: Offer Markovich Date: Thu, 22 Jun 2017 08:06:36 +0300 Subject: [PATCH 102/834] Support LSTM callbacks (#6659) Allow training continuation --- R-package/R/lstm.R | 29 ++++++++++++++++++++++++++++- R-package/R/rnn_model.R | 26 +++++++++++++++++++++++++- 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/R-package/R/lstm.R b/R-package/R/lstm.R index 27c1c2e96eac..622388993c8c 100644 --- a/R-package/R/lstm.R +++ b/R-package/R/lstm.R @@ -181,6 +181,10 @@ lstm.inference.symbol <- function(num.lstm.layer, input.size, #' A number in [0,1) containing the dropout ratio from the last hidden layer to the output layer. #' @param optimizer string, default="sgd" #' The optimization method. +#' @param epoch.end.callback function, optional +#' The callback when iteration ends. +#' @param batch.end.callback function, optional +#' The callback when one mini-batch iteration ends. #' @param ... other parameters passing to \code{mx.lstm}/. #' @return model A trained lstm unrolled model. #' @@ -193,19 +197,29 @@ mx.lstm <- function(train.data, eval.data=NULL, num.round=10, update.period=1, initializer=mx.init.uniform(0.01), dropout=0, optimizer='sgd', + epoch.end.callback=NULL, batch.end.callback=NULL, + model, + arg.params, ...) { # check data and change data into iterator train.data <- check.data(train.data, batch.size, TRUE) eval.data <- check.data(eval.data, batch.size, FALSE) + + # get unrolled lstm symbol - rnn.sym <- lstm.unroll(num.lstm.layer=num.lstm.layer, + if(missing(model)){ + rnn.sym <- lstm.unroll(num.lstm.layer=num.lstm.layer, num.hidden=num.hidden, seq.len=seq.len, input.size=input.size, num.embed=num.embed, num.label=num.label, dropout=dropout) + } else { + rnn.sym=model$symbol + } + init.states.c <- lapply(1:num.lstm.layer, function(i) { state.c <- paste0("l", i, ".init.c") return (state.c) @@ -229,6 +243,17 @@ mx.lstm <- function(train.data, eval.data=NULL, init.states.name=init.states.name, initializer=initializer, dropout=dropout) + # restore states + if (!missing(arg.params)){ + arg.names <- names(model$rnn.exec$ref.arg.arrays) + for (k in names(arg.params)) { + if ((k %in% arg.names) && is.param.name(k) ) { + rnn.input <- list() + rnn.input[[k]] <- arg.params[[k]] + mx.exec.update.arg.arrays(model$rnn.exec, rnn.input, match.name=TRUE) + } + } + } # train lstm model model <- train.rnn( model, train.data, eval.data, @@ -236,6 +261,8 @@ mx.lstm <- function(train.data, eval.data=NULL, update.period=update.period, ctx=ctx, init.states.name=init.states.name, + epoch.end.callback=epoch.end.callback, + batch.end.callback=batch.end.callback, ...) # change model into MXFeedForwardModel model <- list(symbol=model$symbol, arg.params=model$rnn.exec$ref.arg.arrays, aux.params=model$rnn.exec$ref.aux.arrays) diff --git a/R-package/R/rnn_model.R b/R-package/R/rnn_model.R index 86daa7ad8633..b269d0722601 100644 --- a/R-package/R/rnn_model.R +++ b/R-package/R/rnn_model.R @@ -102,8 +102,16 @@ get.label <- function(label, ctx) { train.rnn <- function (model, train.data, eval.data, num.round, update.period, init.states.name, - optimizer='sgd', ctx=mx.ctx.default(), ...) { + optimizer='sgd', ctx=mx.ctx.default(), + epoch.end.callback, + batch.end.callback, + verbose=TRUE, + ...) { m <- model + + model <- list(symbol=model$symbol, arg.params=model$rnn.exec$ref.arg.arrays, + aux.params=model$rnn.exec$ref.aux.arrays) + seq.len <- m$seq.len batch.size <- m$batch.size num.rnn.layer <- m$num.rnn.layer @@ -173,6 +181,11 @@ train.rnn <- function (model, train.data, eval.data, train.nll <- train.nll + calc.nll(as.array(seq.label.probs), batch.size) nbatch <- nbatch + seq.len + + if (!is.null(batch.end.callback)) { + batch.end.callback(iteration, nbatch, environment()) + } + if ((epoch.counter %% log.period) == 0) { message(paste0("Epoch [", epoch.counter, "] Train: NLL=", train.nll / nbatch, @@ -220,6 +233,17 @@ train.rnn <- function (model, train.data, eval.data, "] Val: NLL=", val.nll / nbatch, ", Perp=", exp(val.nll / nbatch))) } + # get the model out + + + epoch_continue <- TRUE + if (!is.null(epoch.end.callback)) { + epoch_continue <- epoch.end.callback(iteration, 0, environment(), verbose = verbose) + } + + if (!epoch_continue) { + break + } } return (m) From 0ba880968de51fb4d4151eba4668a789161181e5 Mon Sep 17 00:00:00 2001 From: Arik Poznanski Date: Thu, 22 Jun 2017 20:04:50 +0300 Subject: [PATCH 103/834] Added script to test caffe converted models layer by layer (#6680) * added script for comparing layer by layer the converter results, weights and outputs * added to contributors list * Refactored test_converter script to run both performance test and layer-by-layer comparison test for known models. Compare login remains in standalone compare_layers script. * fix cv2, and accuracy assertion failed on vgg, increased threshold --- CONTRIBUTORS.md | 2 + python/mxnet/test_utils.py | 2 +- tests/ci_build/Dockerfile.caffe_gpu | 3 +- tools/caffe_converter/.gitignore | 1 + tools/caffe_converter/caffe_proto_utils.py | 171 +++++++++ tools/caffe_converter/compare_layers.py | 328 ++++++++++++++++++ .../caffe_converter/convert_caffe_modelzoo.py | 4 +- tools/caffe_converter/test_converter.py | 45 ++- 8 files changed, 543 insertions(+), 13 deletions(-) create mode 100644 tools/caffe_converter/caffe_proto_utils.py create mode 100644 tools/caffe_converter/compare_layers.py diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 843ad8a056fc..f4cc011059dc 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -137,6 +137,8 @@ List of Contributors * [Roshani Nagmote](https://github.com/Roshrini) * [Chetan Khatri](https://github.com/chetkhatri/) * [James Liu](https://github.com/jamesliu/) +* [Nir Ben-Zvi](https://github.com/nirbenz/) +* [Arik Poznanski](https://github.com/arikpoz/) * [Yuwen Xiong](https://github.com/Orpine/) * [Haozhi Qi](https://github.com/Oh233/) * [Yi Li](https://github.com/liyi14/) diff --git a/python/mxnet/test_utils.py b/python/mxnet/test_utils.py index 3ab44d0917a1..0666e46d930f 100644 --- a/python/mxnet/test_utils.py +++ b/python/mxnet/test_utils.py @@ -946,7 +946,7 @@ def download(url, fname=None, dirname=None, overwrite=False): if fname is None: fname = url.split('/')[-1] if not overwrite and os.path.exists(fname): - logging.info("%s exists, skip to downloada", fname) + logging.info("%s exists, skipping download", fname) return fname if dirname is None: diff --git a/tests/ci_build/Dockerfile.caffe_gpu b/tests/ci_build/Dockerfile.caffe_gpu index c971dfb90bf4..4f6522dab8ef 100644 --- a/tests/ci_build/Dockerfile.caffe_gpu +++ b/tests/ci_build/Dockerfile.caffe_gpu @@ -2,13 +2,14 @@ FROM nvidia/cuda:7.5-cudnn5-devel COPY install/ubuntu_install_core.sh /install/ RUN /install/ubuntu_install_core.sh + COPY install/ubuntu_install_python.sh /install/ RUN /install/ubuntu_install_python.sh RUN apt-get install -y libprotobuf-dev libleveldb-dev \ libsnappy-dev libopencv-dev libhdf5-serial-dev protobuf-compiler \ libatlas-base-dev python-dev libgflags-dev libgoogle-glog-dev liblmdb-dev \ - python-numpy + python-numpy python-opencv RUN apt-get install -y --no-install-recommends libboost-all-dev diff --git a/tools/caffe_converter/.gitignore b/tools/caffe_converter/.gitignore index 0447b0d4ac3a..322dff360126 100644 --- a/tools/caffe_converter/.gitignore +++ b/tools/caffe_converter/.gitignore @@ -1 +1,2 @@ model/ +Cat-hd-wallpapers.jpg diff --git a/tools/caffe_converter/caffe_proto_utils.py b/tools/caffe_converter/caffe_proto_utils.py new file mode 100644 index 000000000000..65b2030fc2c5 --- /dev/null +++ b/tools/caffe_converter/caffe_proto_utils.py @@ -0,0 +1,171 @@ +"""Helper functions for parsing caffe prototxt into a workable DAG +""" + + +def process_network_proto(caffe_root, deploy_proto): + """ + Runs the caffe upgrade tool on the prototxt to create a prototxt in the latest format. + This enable us to work just with latest structures, instead of supporting all the variants + + :param caffe_root: link to caffe root folder, where the upgrade tool is located + :param deploy_proto: name of the original prototxt file + :return: name of new processed prototxt file + """ + processed_deploy_proto = deploy_proto + ".processed" + + from shutil import copyfile + copyfile(deploy_proto, processed_deploy_proto) + + # run upgrade tool on new file name (same output file) + import os + upgrade_tool_command_line = caffe_root + '/build/tools/upgrade_net_proto_text.bin ' \ + + processed_deploy_proto + ' ' + processed_deploy_proto + os.system(upgrade_tool_command_line) + + return processed_deploy_proto + + +class LayerRecord(object): + + def __init__(self, layer_def): + + self.layer_def = layer_def + self.name = layer_def.name + self.type = layer_def.type + + # keep filter, stride and pad + if layer_def.type == 'Convolution': + self.filter = list(layer_def.convolution_param.kernel_size) + if len(self.filter) == 1: + self.filter *= 2 + self.pad = list(layer_def.convolution_param.pad) + if len(self.pad) == 0: + self.pad = [0, 0] + elif len(self.pad) == 1: + self.pad *= 2 + self.stride = list(layer_def.convolution_param.stride) + if len(self.stride) == 0: + self.stride = [1, 1] + elif len(self.stride) == 1: + self.stride *= 2 + + elif layer_def.type == 'Pooling': + self.filter = [layer_def.pooling_param.kernel_size] + if len(self.filter) == 1: + self.filter *= 2 + self.pad = [layer_def.pooling_param.pad] + if len(self.pad) == 0: + self.pad = [0, 0] + elif len(self.pad) == 1: + self.pad *= 2 + self.stride = [layer_def.pooling_param.stride] + if len(self.stride) == 0: + self.stride = [1, 1] + elif len(self.stride) == 1: + self.stride *= 2 + + else: + self.filter = [0, 0] + self.pad = [0, 0] + self.stride = [1, 1] + + # keep tops + self.tops = list(layer_def.top) + + # keep bottoms + self.bottoms = list(layer_def.bottom) + + # list of parent layers + self.parents = [] + + # list of child layers + self.children = [] + + +def read_network_dag(processed_deploy_prototxt): + """ + Reads from the caffe prototxt the network structure + :param processed_deploy_prototxt: name of prototxt to load, preferably the prototxt should + be processed before using a call to process_network_proto() + :return: network_def, layer_name_to_record, top_to_layers + network_def: caffe network structure, gives access to *all* the network information + layer_name_to_record: *ordered* dictionary which maps between layer name and a structure which + describes in a simple form the layer parameters + top_to_layers: dictionary which maps a blob name to an ordered list of layers which output it + when a top is used several times, like in inplace layhers, the list will contain all the layers + by order of appearance + """ + + from caffe.proto import caffe_pb2 + from google.protobuf import text_format + from collections import OrderedDict + + # load prototxt file + network_def = caffe_pb2.NetParameter() + with open(processed_deploy_prototxt, 'r') as proto_file: + text_format.Merge(str(proto_file.read()), network_def) + + # map layer name to layer record + layer_name_to_record = OrderedDict() + for layer_def in network_def.layer: + if (len(layer_def.include) == 0) or \ + (caffe_pb2.TEST in [item.phase for item in layer_def.include]): + + layer_name_to_record[layer_def.name] = LayerRecord(layer_def) + + top_to_layers = dict() + for layer in network_def.layer: + # no specific phase, or TEST phase is specifically asked for + if (len(layer.include) == 0) or (caffe_pb2.TEST in [item.phase for item in layer.include]): + for top in layer.top: + if top not in top_to_layers: + top_to_layers[top] = list() + top_to_layers[top].append(layer.name) + + # find parents and children of all layers + for child_layer_name in layer_name_to_record.keys(): + child_layer_def = layer_name_to_record[child_layer_name] + for bottom in child_layer_def.bottoms: + for parent_layer_name in top_to_layers[bottom]: + if parent_layer_name in layer_name_to_record: + parent_layer_def = layer_name_to_record[parent_layer_name] + if parent_layer_def not in child_layer_def.parents: + child_layer_def.parents.append(parent_layer_def) + if child_layer_def not in parent_layer_def.children: + parent_layer_def.children.append(child_layer_def) + + # update filter, strid, pad for maxout "structures" + for layer_name in layer_name_to_record.keys(): + layer_def = layer_name_to_record[layer_name] + if layer_def.type == 'Eltwise' and \ + len(layer_def.parents) == 1 and \ + layer_def.parents[0].type == 'Slice' and \ + len(layer_def.parents[0].parents) == 1 and \ + layer_def.parents[0].parents[0].type in ['Convolution', 'InnerProduct']: + layer_def.filter = layer_def.parents[0].parents[0].filter + layer_def.stride = layer_def.parents[0].parents[0].stride + layer_def.pad = layer_def.parents[0].parents[0].pad + + return network_def, layer_name_to_record, top_to_layers + + +def read_caffe_mean(caffe_mean_file): + """ + Reads caffe formatted mean file + :param caffe_mean_file: path to caffe mean file, presumably with 'binaryproto' suffix + :return: mean image, converted from BGR to RGB format + """ + + import caffe_parser + import numpy as np + mean_blob = caffe_parser.caffe_pb2.BlobProto() + with open(caffe_mean_file, 'rb') as f: + mean_blob.ParseFromString(f.read()) + + img_mean_np = np.array(mean_blob.data) + img_mean_np = img_mean_np.reshape(mean_blob.channels, mean_blob.height, mean_blob.width) + + # swap channels from Caffe BGR to RGB + img_mean_np[[0, 2], :, :] = img_mean_np[[2, 0], :, :] + + return img_mean_np diff --git a/tools/caffe_converter/compare_layers.py b/tools/caffe_converter/compare_layers.py new file mode 100644 index 000000000000..097d86215515 --- /dev/null +++ b/tools/caffe_converter/compare_layers.py @@ -0,0 +1,328 @@ +"""Test converted models layer by layer +""" +import os +import argparse +import logging +import mxnet as mx +import cv2 +import numpy as np + +logging.basicConfig(level=logging.INFO) + + +def read_image(img_path, image_dims=None, mean=None): + """ + Reads an image from file path or URL, optionally resizing to given image dimensions and + subtracting mean. + :param img_path: path to file, or url to download + :param image_dims: image dimensions to resize to, or None + :param mean: mean file to subtract, or None + :return: loaded image, in RGB format + """ + + import urllib + + filename = img_path.split("/")[-1] + if img_path.startswith('http'): + urllib.urlretrieve(img_path, filename) + img = cv2.imread(filename) + else: + img = cv2.imread(img_path) + + img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) + + if image_dims is not None: + img = cv2.resize(img, image_dims) # resize to image_dims to fit model + img = np.rollaxis(img, 2) # change to (c, h, w) order + img = img[np.newaxis, :] # extend to (n, c, h, w) + if mean is not None: + mean = np.array(mean) + if mean.shape == (3,): + mean = mean[np.newaxis, :, np.newaxis, np.newaxis] # extend to (n, c, 1, 1) + img = img.astype(np.float32) - mean # subtract mean + + return img + + +def _ch_dev(arg_params, aux_params, ctx): + """ + Changes device of given mxnet arguments + :param arg_params: arguments + :param aux_params: auxiliary parameters + :param ctx: new device context + :return: arguments and auxiliary parameters on new device + """ + new_args = dict() + new_auxs = dict() + for k, v in arg_params.items(): + new_args[k] = v.as_in_context(ctx) + for k, v in aux_params.items(): + new_auxs[k] = v.as_in_context(ctx) + return new_args, new_auxs + + +def convert_and_compare_caffe_to_mxnet(image_url, gpu, caffe_prototxt_path, caffe_model_path, + caffe_mean, mean_diff_allowed, max_diff_allowed): + """ + Run the layer comparison on a caffe model, given its prototxt, weights and mean. + The comparison is done by inferring on a given image using both caffe and mxnet model + :param image_url: image file or url to run inference on + :param gpu: gpu to use, -1 for cpu + :param caffe_prototxt_path: path to caffe prototxt + :param caffe_model_path: path to caffe weights + :param caffe_mean: path to caffe mean file + """ + + import caffe + from caffe_proto_utils import read_network_dag, process_network_proto, read_caffe_mean + from convert_model import convert_model + + if isinstance(caffe_mean, str): + caffe_mean = read_caffe_mean(caffe_mean) + elif len(caffe_mean) == 3: + # swap channels from Caffe BGR to RGB + caffe_mean = caffe_mean[::-1] + + # get caffe root location, this is needed to run the upgrade network utility, so we only need + # to support parsing of latest caffe + caffe_root = os.path.dirname(os.path.dirname(caffe.__path__[0])) + caffe_prototxt_path = process_network_proto(caffe_root, caffe_prototxt_path) + + _, layer_name_to_record, top_to_layers = read_network_dag(caffe_prototxt_path) + + caffe.set_mode_cpu() + caffe_net = caffe.Net(caffe_prototxt_path, caffe_model_path, caffe.TEST) + + image_dims = tuple(caffe_net.blobs['data'].shape)[2:4] + + logging.info('getting image %s', image_url) + img_rgb = read_image(image_url, image_dims, caffe_mean) + img_bgr = img_rgb[:, ::-1, :, :] + + caffe_net.blobs['data'].reshape(*img_bgr.shape) + caffe_net.blobs['data'].data[...] = img_bgr + _ = caffe_net.forward() + + # read sym and add all outputs + sym, arg_params, aux_params, _ = convert_model(caffe_prototxt_path, caffe_model_path) + sym = sym.get_internals() + + # now mxnet + if gpu < 0: + ctx = mx.cpu(0) + else: + ctx = mx.gpu(gpu) + + arg_params, aux_params = _ch_dev(arg_params, aux_params, ctx) + arg_params["data"] = mx.nd.array(img_rgb, ctx) + arg_params["prob_label"] = mx.nd.empty((1,), ctx) + exe = sym.bind(ctx, arg_params, args_grad=None, grad_req="null", aux_states=aux_params) + exe.forward(is_train=False) + + compare_layers_from_nets(caffe_net, arg_params, aux_params, exe, layer_name_to_record, + top_to_layers, mean_diff_allowed, max_diff_allowed) + + return + + +def _bfs(root_node, process_node): + """ + Implementation of Breadth-first search (BFS) on caffe network DAG + :param root_node: root node of caffe network DAG + :param process_node: function to run on each node + """ + + from collections import deque + + seen_nodes = set() + next_nodes = deque() + + seen_nodes.add(root_node) + next_nodes.append(root_node) + + while next_nodes: + current_node = next_nodes.popleft() + + # process current node + process_node(current_node) + + for child_node in current_node.children: + if child_node not in seen_nodes: + seen_nodes.add(child_node) + next_nodes.append(child_node) + + +def compare_layers_from_nets(caffe_net, arg_params, aux_params, exe, layer_name_to_record, + top_to_layers, mean_diff_allowed, max_diff_allowed): + """ + Compare layer by layer of a caffe network with mxnet network + :param caffe_net: loaded caffe network + :param arg_params: arguments + :param aux_params: auxiliary parameters + :param exe: mxnet model + :param layer_name_to_record: map between caffe layer and information record + :param top_to_layers: map between caffe blob name to layers which outputs it (including inplace) + :param mean_diff_allowed: mean difference allowed between caffe blob and mxnet blob + :param max_diff_allowed: max difference allowed between caffe blob and mxnet blob + """ + + import re + + log_format = ' {0:<40} {1:<40} {2:<8} {3:>10} {4:>10} {5:<1}' + + compare_layers_from_nets.is_first_convolution = True + + def _compare_blob(caf_blob, mx_blob, caf_name, mx_name, blob_type, note): + diff = np.abs(mx_blob - caf_blob) + diff_mean = diff.mean() + diff_max = diff.max() + logging.info(log_format.format(caf_name, mx_name, blob_type, '%4.5f' % diff_mean, + '%4.5f' % diff_max, note)) + assert diff_mean < mean_diff_allowed + assert diff_max < max_diff_allowed + + def _process_layer_parameters(layer): + + logging.debug('processing layer %s of type %s', layer.name, layer.type) + + normalized_layer_name = re.sub('[-/]', '_', layer.name) + + # handle weight and bias of convolution and fully-connected layers + if layer.name in caffe_net.params and layer.type in ['Convolution', 'InnerProduct']: + + has_bias = len(caffe_net.params[layer.name]) > 1 + + mx_name_weight = '{}_weight'.format(normalized_layer_name) + mx_beta = arg_params[mx_name_weight].asnumpy() + + # first convolution should change from BGR to RGB + if layer.type == 'Convolution' and compare_layers_from_nets.is_first_convolution: + compare_layers_from_nets.is_first_convolution = False + + # swapping BGR of caffe into RGB in mxnet + mx_beta = mx_beta[:, ::-1, :, :] + + caf_beta = caffe_net.params[layer.name][0].data + _compare_blob(caf_beta, mx_beta, layer.name, mx_name_weight, 'weight', '') + + if has_bias: + mx_name_bias = '{}_bias'.format(normalized_layer_name) + mx_gamma = arg_params[mx_name_bias].asnumpy() + caf_gamma = caffe_net.params[layer.name][1].data + _compare_blob(caf_gamma, mx_gamma, layer.name, mx_name_bias, 'bias', '') + + elif layer.name in caffe_net.params and layer.type == 'Scale': + + bn_name = normalized_layer_name.replace('scale', 'bn') + beta_name = '{}_beta'.format(bn_name) + gamma_name = '{}_gamma'.format(bn_name) + + mx_beta = arg_params[beta_name].asnumpy() + caf_beta = caffe_net.params[layer.name][1].data + _compare_blob(caf_beta, mx_beta, layer.name, beta_name, 'mov_mean', '') + + mx_gamma = arg_params[gamma_name].asnumpy() + caf_gamma = caffe_net.params[layer.name][0].data + _compare_blob(caf_gamma, mx_gamma, layer.name, gamma_name, 'mov_var', '') + + elif layer.name in caffe_net.params and layer.type == 'BatchNorm': + + mean_name = '{}_moving_mean'.format(normalized_layer_name) + var_name = '{}_moving_var'.format(normalized_layer_name) + + mx_mean = aux_params[mean_name].asnumpy() + caf_mean = caffe_net.params[layer.name][0].data + _compare_blob(caf_mean, mx_mean, layer.name, mean_name, 'mean', '') + + mx_var = aux_params[var_name].asnumpy() + caf_var = caffe_net.params[layer.name][1].data + _compare_blob(caf_var, mx_var, layer.name, var_name, 'var', + 'expect 1e-04 change due to cudnn eps') + + elif layer.type in ['Input', 'Pooling', 'ReLU', 'Eltwise', 'Softmax', 'LRN', 'Concat', + 'Dropout']: + # no parameters to check for these layers + pass + + else: + logging.warn('No handling for layer %s of type %s, should we ignore it?', layer.name, + layer.type) + + return + + def _process_layer_output(caffe_blob_name): + + logging.debug('processing blob %s', caffe_blob_name) + + # skip blobs not originating from actual layers, e.g. artificial split layers added by caffe + if caffe_blob_name not in top_to_layers: + return + + caf_blob = caffe_net.blobs[caffe_blob_name].data + + # data should change from BGR to RGB + if caffe_blob_name == 'data': + # swapping BGR of caffe into RGB in mxnet + caf_blob = caf_blob[:, ::-1, :, :] + mx_name = 'data' + + else: + # get last layer name which outputs this blob name + last_layer_name = top_to_layers[caffe_blob_name][-1] + normalized_last_layer_name = re.sub('[-/]', '_', last_layer_name) + mx_name = '{}_output'.format(normalized_last_layer_name) + mx_name = mx_name.replace('scale', 'bn') + + if mx_name not in exe.output_dict: + logging.error('mxnet blob %s is missing, time to extend the compare tool..', mx_name) + return + + mx_blob = exe.output_dict[mx_name].asnumpy() + _compare_blob(caf_blob, mx_blob, caffe_blob_name, mx_name, 'output', '') + + return + + # check layer parameters + logging.info('\n***** Network Parameters '.ljust(140, '*')) + logging.info(log_format.format('CAFFE', 'MXNET', 'Type', 'Mean(diff)', 'Max(diff)', 'Note')) + first_layer_name = layer_name_to_record.keys()[0] + _bfs(layer_name_to_record[first_layer_name], _process_layer_parameters) + + # check layer output + logging.info('\n***** Network Outputs '.ljust(140, '*')) + logging.info(log_format.format('CAFFE', 'MXNET', 'Type', 'Mean(diff)', 'Max(diff)', 'Note')) + for caffe_blob_name in caffe_net.blobs.keys(): + _process_layer_output(caffe_blob_name) + + return + + +def main(): + """Entrypoint for compare_layers""" + + parser = argparse.ArgumentParser( + description='Tool for testing caffe to mxnet conversion layer by layer') + parser.add_argument('--image_url', type=str, + default='http://writm.com/wp-content/uploads/2016/08/Cat-hd-wallpapers.jpg', + help='input image to test inference, can be either file path or url') + parser.add_argument('--caffe_prototxt_path', type=str, + default='./model.prototxt', + help='path to caffe prototxt') + parser.add_argument('--caffe_model_path', type=str, + default='./model.caffemodel', + help='path to caffe weights') + parser.add_argument('--caffe_mean', type=str, + default='./model_mean.binaryproto', + help='path to caffe mean file') + parser.add_argument('--mean_diff_allowed', type=int, default=1e-03, + help='mean difference allowed between caffe blob and mxnet blob') + parser.add_argument('--max_diff_allowed', type=int, default=1e-01, + help='max difference allowed between caffe blob and mxnet blob') + parser.add_argument('--gpu', type=int, default=-1, help='the gpu id used for predict') + args = parser.parse_args() + convert_and_compare_caffe_to_mxnet(args.image_url, args.gpu, args.caffe_prototxt_path, + args.caffe_model_path, args.caffe_mean, + args.mean_diff_allowed, args.max_diff_allowed) + +if __name__ == '__main__': + main() diff --git a/tools/caffe_converter/convert_caffe_modelzoo.py b/tools/caffe_converter/convert_caffe_modelzoo.py index 2cb3da9149fe..cb6d3ba7fc48 100644 --- a/tools/caffe_converter/convert_caffe_modelzoo.py +++ b/tools/caffe_converter/convert_caffe_modelzoo.py @@ -80,7 +80,7 @@ def get_model_meta_info(model_name): """returns a dict with model information""" return dict(dict(model_meta_info)[model_name]) -def _download_caffe_model(model_name, meta_info, dst_dir='./model'): +def download_caffe_model(model_name, meta_info, dst_dir='./model'): """Download caffe model into disk by the given meta info """ if not os.path.isdir(dst_dir): os.mkdir(dst_dir) @@ -98,7 +98,7 @@ def _download_caffe_model(model_name, meta_info, dst_dir='./model'): def convert_caffe_model(model_name, meta_info, dst_dir='./model'): """Download, convert and save a caffe model""" - (prototxt, caffemodel, mean) = _download_caffe_model(model_name, meta_info, dst_dir) + (prototxt, caffemodel, mean) = download_caffe_model(model_name, meta_info, dst_dir) model_name = os.path.join(dst_dir, model_name) convert_model(prototxt, caffemodel, model_name) if isinstance(mean, str): diff --git a/tools/caffe_converter/test_converter.py b/tools/caffe_converter/test_converter.py index 96947baf0c2b..c7eb86b2e437 100644 --- a/tools/caffe_converter/test_converter.py +++ b/tools/caffe_converter/test_converter.py @@ -5,16 +5,18 @@ import sys import logging import mxnet as mx -from convert_caffe_modelzoo import convert_caffe_model, get_model_meta_info +from convert_caffe_modelzoo import convert_caffe_model, get_model_meta_info, download_caffe_model +from compare_layers import convert_and_compare_caffe_to_mxnet + curr_path = os.path.abspath(os.path.dirname(__file__)) sys.path.append(os.path.join(curr_path, "../../example/image-classification")) from test_score import download_data # pylint: disable=wrong-import-position from score import score # pylint: disable=wrong-import-position logging.basicConfig(level=logging.DEBUG) -def test_imagenet_model(model_name, val_data, gpus, batch_size): - """test model on imagenet """ - logging.info('test %s', model_name) +def test_imagenet_model_performance(model_name, val_data, gpus, batch_size): + """test model performance on imagenet """ + logging.info('test performance of model: %s', model_name) meta_info = get_model_meta_info(model_name) [model_name, mean] = convert_caffe_model(model_name, meta_info) sym, arg_params, aux_params = mx.model.load_checkpoint(model_name, 0) @@ -25,27 +27,51 @@ def test_imagenet_model(model_name, val_data, gpus, batch_size): mean_args = {'rgb_mean':','.join([str(i) for i in mean])} print(val_data) + gpus_string = '' if gpus[0] == -1 else ','.join([str(i) for i in gpus]) (speed,) = score(model=(sym, arg_params, aux_params), data_val=val_data, label_name='prob_label', metrics=acc, - gpus=gpus, + gpus=gpus_string, batch_size=batch_size, max_num_examples=500, **mean_args) logging.info('speed : %f image/sec', speed) for a in acc: logging.info(a.get()) - assert acc[0].get()[1] > meta_info['top-1-acc'] - 0.03 - assert acc[1].get()[1] > meta_info['top-5-acc'] - 0.03 + max_performance_diff_allowed = 0.03 + assert acc[0].get()[1] > meta_info['top-1-acc'] - max_performance_diff_allowed + assert acc[1].get()[1] > meta_info['top-5-acc'] - max_performance_diff_allowed + + +def test_model_weights_and_outputs(model_name, image_url, gpu): + """ + Run the layer comparison on one of the known caffe models. + :param model_name: available models are listed in convert_caffe_modelzoo.py + :param image_url: image file or url to run inference on + :param gpu: gpu to use, -1 for cpu + """ + + logging.info('test weights and outputs of model: %s', model_name) + meta_info = get_model_meta_info(model_name) + + (prototxt, caffemodel, mean) = download_caffe_model(model_name, meta_info, dst_dir='./model') + convert_and_compare_caffe_to_mxnet(image_url, gpu, prototxt, caffemodel, mean, + mean_diff_allowed=1e-03, max_diff_allowed=1e-01) + + return + def main(): """Entrypoint for test_converter""" parser = argparse.ArgumentParser(description='Test Caffe converter') parser.add_argument('--cpu', action='store_true', help='use cpu?') + parser.add_argument('--image_url', type=str, + default='http://writm.com/wp-content/uploads/2016/08/Cat-hd-wallpapers.jpg', + help='input image to test inference, can be either file path or url') args = parser.parse_args() if args.cpu: - gpus = '' + gpus = [-1] batch_size = 32 else: gpus = mx.test_utils.list_gpus() @@ -56,7 +82,8 @@ def main(): val = download_data() for m in models: - test_imagenet_model(m, val, ','.join([str(i) for i in gpus]), batch_size) + test_model_weights_and_outputs(m, args.image_url, gpus[0]) + test_imagenet_model_performance(m, val, gpus, batch_size) if __name__ == '__main__': main() From 3fb169fcccbe246e872a7577b60e7181f7c1fc48 Mon Sep 17 00:00:00 2001 From: Naveen Swamy Date: Thu, 22 Jun 2017 12:38:46 -0700 Subject: [PATCH 104/834] Update mxnet_channels.md (#6786) Update to request subscription to the Mailing list before asking Slack invites. --- docs/community/mxnet_channels.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/community/mxnet_channels.md b/docs/community/mxnet_channels.md index 3f7a24cd5653..c938bd03465c 100644 --- a/docs/community/mxnet_channels.md +++ b/docs/community/mxnet_channels.md @@ -3,4 +3,4 @@ Converse with the MXNet community via the following channels: - [MXNet Apache mailing list](https://lists.apache.org/list.html?dev@mxnet.apache.org) (dev@mxnet.apache.org): To subscribe, send an email to dev-subscribe@mxnet.apache.org. -- [MXNet Slack channel](https://apache-mxnet.slack.com): To request an invitation to the channel please email: dev@mxnet.apache.org. Note: if you have an email address with apache.org, you do not need an approval to join the MXNet Slack channel. +- [MXNet Slack channel](https://apache-mxnet.slack.com): To request an invitation to the channel please subscribe to the mailing list above and then email: dev@mxnet.apache.org. Note: if you have an email address with apache.org, you do not need an approval to join the MXNet Slack channel. From 073f8030b49b197cdd269da41a025a3b70a514da Mon Sep 17 00:00:00 2001 From: Hessel Tuinhof Date: Thu, 22 Jun 2017 21:39:21 +0200 Subject: [PATCH 105/834] [R] align parameters with original paper (#6781) Original paper proposes for local response normalization layer a k=2. --- example/image-classification/symbol_alexnet.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/example/image-classification/symbol_alexnet.R b/example/image-classification/symbol_alexnet.R index ec768c9adb14..097444335451 100644 --- a/example/image-classification/symbol_alexnet.R +++ b/example/image-classification/symbol_alexnet.R @@ -6,12 +6,12 @@ get_symbol <- function(num_classes = 1000) { conv1 <- mx.symbol.Convolution(data = input_data, kernel = c(11, 11), stride = c(4, 4), num_filter = 96) relu1 <- mx.symbol.Activation(data = conv1, act_type = "relu") pool1 <- mx.symbol.Pooling(data = relu1, pool_type = "max", kernel = c(3, 3), stride = c(2, 2)) - lrn1 <- mx.symbol.LRN(data = pool1, alpha = 0.0001, beta = 0.75, knorm = 1, nsize = 5) + lrn1 <- mx.symbol.LRN(data = pool1, alpha = 0.0001, beta = 0.75, knorm = 2, nsize = 5) # stage 2 conv2 <- mx.symbol.Convolution(data = lrn1, kernel = c(5, 5), pad = c(2, 2), num_filter = 256) relu2 <- mx.symbol.Activation(data = conv2, act_type = "relu") pool2 <- mx.symbol.Pooling(data = relu2, kernel = c(3, 3), stride = c(2, 2), pool_type = "max") - lrn2 <- mx.symbol.LRN(data = pool2, alpha = 0.0001, beta = 0.75, knorm = 1, nsize = 5) + lrn2 <- mx.symbol.LRN(data = pool2, alpha = 0.0001, beta = 0.75, knorm = 2, nsize = 5) # stage 3 conv3 <- mx.symbol.Convolution(data = lrn2, kernel = c(3, 3), pad = c(1, 1), num_filter = 384) relu3 <- mx.symbol.Activation(data = conv3, act_type = "relu") From 61e219732bc71f89cdd2f750bab5ba16f39fbf7f Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Thu, 22 Jun 2017 15:24:38 -0700 Subject: [PATCH 106/834] add -Werror to NVCCFLAGS (#6774) * add -Werror to NVCCFLAGS * add -Werror to NVCCFLAGS * RTLD_LOCAL instead of GLOBAL * fix device/host cross-call warnings --- Makefile | 5 +++-- python/mxnet/base.py | 2 +- src/operator/mshadow_op.h | 24 ++++++++++++++++-------- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 7731e7e24d2a..69527f42d248 100644 --- a/Makefile +++ b/Makefile @@ -45,6 +45,7 @@ CFLAGS = -DMSHADOW_FORCE_STREAM $(WARNFLAGS) ifeq ($(DEV), 1) CFLAGS += -g -Werror + NVCCFLAGS += -Werror cross-execution-space-call endif # CFLAGS for debug @@ -56,9 +57,9 @@ endif CFLAGS += -I$(ROOTDIR)/mshadow/ -I$(ROOTDIR)/dmlc-core/include -fPIC -I$(NNVM_PATH)/include -I$(DLPACK_PATH)/include -Iinclude $(MSHADOW_CFLAGS) LDFLAGS = -pthread $(MSHADOW_LDFLAGS) $(DMLC_LDFLAGS) ifeq ($(DEBUG), 1) - NVCCFLAGS = -std=c++11 -Xcompiler -D_FORCE_INLINES -g -G -O0 -ccbin $(CXX) $(MSHADOW_NVCCFLAGS) + NVCCFLAGS += -std=c++11 -Xcompiler -D_FORCE_INLINES -g -G -O0 -ccbin $(CXX) $(MSHADOW_NVCCFLAGS) else - NVCCFLAGS = -std=c++11 -Xcompiler -D_FORCE_INLINES -O3 -ccbin $(CXX) $(MSHADOW_NVCCFLAGS) + NVCCFLAGS += -std=c++11 -Xcompiler -D_FORCE_INLINES -O3 -ccbin $(CXX) $(MSHADOW_NVCCFLAGS) endif # CFLAGS for profiler diff --git a/python/mxnet/base.py b/python/mxnet/base.py index aeb7ef8ecfd1..8bca2746de9f 100644 --- a/python/mxnet/base.py +++ b/python/mxnet/base.py @@ -41,7 +41,7 @@ class MXNetError(Exception): def _load_lib(): """Load libary by searching possible path.""" lib_path = libinfo.find_lib_path() - lib = ctypes.CDLL(lib_path[0], ctypes.RTLD_GLOBAL) + lib = ctypes.CDLL(lib_path[0], ctypes.RTLD_LOCAL) # DMatrix functions lib.MXGetLastError.restype = ctypes.c_char_p return lib diff --git a/src/operator/mshadow_op.h b/src/operator/mshadow_op.h index c63739ba5085..eac2a2c50d24 100644 --- a/src/operator/mshadow_op.h +++ b/src/operator/mshadow_op.h @@ -676,15 +676,19 @@ struct mod { return DType(0); } else if (b < DType(0)) { if (a < DType(0)) { - return DType(-::fmod(-a, -b)); + return DType(-::fmod(-static_cast(a), -static_cast(b))); } else { - return DType(::fmod(a, -b) + (::fmod(a, -b) != DType(0) ? b : DType(0))); + return DType(::fmod(static_cast(a), -static_cast(b)) + + (::fmod(static_cast(a), -static_cast(b)) != DType(0) + ? b : DType(0))); } } else { if (a < DType(0)) { - return DType(-::fmod(-a, b) + (::fmod(-a, b) != DType(0) ? b : DType(0))); + return DType(-::fmod(-static_cast(a), static_cast(b)) + + (::fmod(-static_cast(a), static_cast(b)) != DType(0) + ? b : DType(0))); } else { - return DType(::fmod(a, b)); + return DType(::fmod(static_cast(a), static_cast(b))); } } } @@ -777,15 +781,19 @@ struct rmod { return DType(0); } else if (a < DType(0)) { if (b < DType(0)) { - return DType(-::fmod(-b, -a)); + return DType(-::fmod(-static_cast(b), -static_cast(a))); } else { - return DType(::fmod(b, -a) + (::fmod(b, -a) != DType(0) ? a : DType(0))); + return DType(::fmod(static_cast(b), -static_cast(a)) + + (::fmod(static_cast(b), -static_cast(a)) != DType(0) + ? a : DType(0))); } } else { if (b < DType(0)) { - return DType(-::fmod(-b, a) + (::fmod(-b, a) != DType(0) ? a : DType(0))); + return DType(-::fmod(-static_cast(b), static_cast(a)) + + (::fmod(-static_cast(b), static_cast(a)) != DType(0) + ? a : DType(0))); } else { - return DType(::fmod(b, a)); + return DType(::fmod(static_cast(b), static_cast(a))); } } } From 77b20c9121e521e7c544f2332c75bea7e68e98a9 Mon Sep 17 00:00:00 2001 From: Chris Olivier Date: Thu, 22 Jun 2017 17:23:19 -0700 Subject: [PATCH 107/834] Deadlock and crashes during shutdown (#6773) * Fix CMake build not linking lapack when enabled * Fix image-classification-predict linking with cmake * Shutdown race condition fixes * set mshadow module commit * move commit for mshadow lint fix * Trigger build * point to HEAD of master, which has latest commits * Set submodule tag * Fix windows build problem * remove unistd.h include * No lapack on Windows * Help MSVC compiler select the correct ThreadPool constructor --- CMakeLists.txt | 7 +- .../predict-cpp/CMakeLists.txt | 23 ++++- mshadow | 2 +- src/common/lazy_alloc_array.h | 98 +++++++++++++++---- src/engine/thread_pool.h | 63 ++++++++++++ src/engine/threaded_engine_perdevice.cc | 98 +++++++++++++------ src/storage/storage.cc | 6 +- tests/CMakeLists.txt | 1 - 8 files changed, 241 insertions(+), 57 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4edecda47b6d..179c420bcf37 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,7 +13,7 @@ mxnet_option(USE_OPENCV "Build with OpenCV support" ON) mxnet_option(USE_OPENMP "Build with Openmp support" ON) mxnet_option(USE_CUDA "Build with CUDA support" ON) mxnet_option(USE_CUDNN "Build with cudnn support" ON) # one could set CUDNN_ROOT for search path -mxnet_option(USE_LAPACK "Build with lapack support" ON) +mxnet_option(USE_LAPACK "Build with lapack support" ON IF NOT MSVC) mxnet_option(USE_MKL_IF_AVAILABLE "Use MKL if found" ON) mxnet_option(USE_MKLML_MKL "Use MKLML variant of MKL (if MKL found)" ON IF USE_MKL_IF_AVAILABLE AND UNIX AND (NOT APPLE)) mxnet_option(USE_MKL_EXPERIMENTAL "Use experimental MKL (if MKL enabled and found)" OFF) @@ -198,12 +198,15 @@ if(USE_OPENMP) set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}") set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}") endif() +elseif(UNIX) + list(APPEND mxnet_LINKER_LIBS pthread) endif() if(USE_LAPACK) add_definitions(-DMXNET_USE_LAPACK=1) + list(APPEND mxnet_LINKER_LIBS lapack) else(USE_LAPACK) - # Workaround for Windows until using new Jenkinsfile. + # Workaround for Windows until using new Jenkinsfile. if(USE_BLAS STREQUAL "open") add_definitions(-DMXNET_USE_LAPACK=1) endif() diff --git a/example/image-classification/predict-cpp/CMakeLists.txt b/example/image-classification/predict-cpp/CMakeLists.txt index 646f907f8f6d..59c98d8ee568 100644 --- a/example/image-classification/predict-cpp/CMakeLists.txt +++ b/example/image-classification/predict-cpp/CMakeLists.txt @@ -3,12 +3,31 @@ if(USE_OPENCV) if(NOT OpenCV_FOUND) # if not OpenCV 3.x, then imgcodecs are not found find_package(OpenCV REQUIRED COMPONENTS core highgui imgproc) endif() + + if(NOT MSVC) + set(UNITTEST_STATIC_LINK ON) + endif() + add_executable(image-classification-predict image-classification-predict.cc) include_directories(SYSTEM ${OpenCV_INCLUDE_DIRS}) - target_link_libraries(image-classification-predict mxnet) + + if(UNITTEST_STATIC_LINK) + target_link_libraries(image-classification-predict + ${BEGIN_WHOLE_ARCHIVE} mxnet_static ${END_WHOLE_ARCHIVE} + dmlc + ${mxnet_LINKER_LIBS} + ) + else() + target_link_libraries(image-classification-predict + dmlc + ${nnvm_LINKER_LIBS} + ${mxnet_LINKER_LIBS} + mxnet + ) + endif() target_link_libraries(image-classification-predict ${OpenCV_LIBS}) if(UNIX) - target_link_libraries(image-classification-predict rt) + target_link_libraries(image-classification-predict rt) endif() list(APPEND mxnet_LINKER_LIBS ${OpenCV_LIBS}) endif() diff --git a/mshadow b/mshadow index 8db65bd081c7..20b54f068c10 160000 --- a/mshadow +++ b/mshadow @@ -1 +1 @@ -Subproject commit 8db65bd081c7e243028ace93ef0acc9efc4383ba +Subproject commit 20b54f068c1035f0319fa5e5bbfb129c450a5256 diff --git a/src/common/lazy_alloc_array.h b/src/common/lazy_alloc_array.h index d3722e557b04..61b81e5ce407 100644 --- a/src/common/lazy_alloc_array.h +++ b/src/common/lazy_alloc_array.h @@ -12,6 +12,7 @@ #include #include #include +#include namespace mxnet { namespace common { @@ -19,6 +20,7 @@ namespace common { template class LazyAllocArray { public: + LazyAllocArray(); /*! * \brief Get element of corresponding index, * if it is not created create by creator @@ -26,7 +28,7 @@ class LazyAllocArray { * \param creator a lambda function to create new element when needed. */ template - inline TElem* Get(int index, FCreate creator); + inline std::shared_ptr Get(int index, FCreate creator); /*! * \brief for each not null element of the array, call fvisit * \param fvisit a function of (size_t, TElem*) @@ -36,53 +38,104 @@ class LazyAllocArray { /*! \brief clear all the allocated elements in array */ inline void Clear(); + void SignalForKill(); + private: + template + class unique_unlock { + public: + explicit unique_unlock(std::unique_lock *lock) + : lock_(lock) { + if (lock_) { + lock_->unlock(); + } + } + ~unique_unlock() { + if (lock_) { + lock_->lock(); + } + } + private: + std::unique_lock *lock_; + }; + /*! \brief the initial size of the array */ static constexpr std::size_t kInitSize = 16; /*! \brief mutex used during creation */ std::mutex create_mutex_; /*! \brief internal data fir initial size */ - std::array, kInitSize> head_; + std::array, kInitSize> head_; /*! \brief overflow array of more elements */ - std::vector > more_; + std::vector > more_; + /*! \brief Signal shutdown of array */ + std::atomic exit_now_; }; +template +inline LazyAllocArray::LazyAllocArray() + : exit_now_(false) { +} + // implementations template template -inline TElem* LazyAllocArray::Get(int index, FCreate creator) { +inline std::shared_ptr LazyAllocArray::Get(int index, FCreate creator) { CHECK_GE(index, 0); size_t idx = static_cast(index); if (idx < kInitSize) { - TElem *ptr = head_[idx].get(); - if (ptr != nullptr) { + std::shared_ptr ptr = head_[idx]; + if (ptr) { return ptr; } else { std::lock_guard lock(create_mutex_); - TElem *ptr = head_[idx].get(); - if (ptr != nullptr) return ptr; - head_[idx].reset(ptr = creator()); - return ptr; + if (!exit_now_.load()) { + std::shared_ptr ptr = head_[idx]; + if (ptr) { + return ptr; + } + ptr = head_[idx] = std::shared_ptr(creator()); + return ptr; + } } } else { std::lock_guard lock(create_mutex_); - idx -= kInitSize; - if (more_.size() <= idx) more_.resize(idx + 1); - TElem *ptr = more_[idx].get(); - if (ptr != nullptr) return ptr; - more_[idx].reset(ptr = creator()); - return ptr; + if (!exit_now_.load()) { + idx -= kInitSize; + if (more_.size() <= idx) { + more_.reserve(idx + 1); + while (more_.size() <= idx) { + more_.push_back(std::shared_ptr(nullptr)); + } + } + std::shared_ptr ptr = more_[idx]; + if (ptr) { + return ptr; + } + ptr = more_[idx] = std::shared_ptr(creator()); + return ptr; + } } + return nullptr; } template inline void LazyAllocArray::Clear() { - std::lock_guard lock(create_mutex_); + std::unique_lock lock(create_mutex_); + exit_now_.store(true); + // Currently, head_ and more_ never get smaller, so it's safe to + // iterate them outside of the lock. The loops should catch + // any growth which might happen when create_mutex_ is unlocked for (size_t i = 0; i < head_.size(); ++i) { - head_[i].reset(nullptr); + std::shared_ptr p = head_[i]; + head_[i] = std::shared_ptr(nullptr); + unique_unlock unlocker(&lock); + p = std::shared_ptr(nullptr); } for (size_t i = 0; i < more_.size(); ++i) { - more_[i].reset(nullptr); + std::shared_ptr p = more_[i]; + more_[i] = std::shared_ptr(nullptr); + unique_unlock unlocker(&lock); + p = std::shared_ptr(nullptr); } } @@ -101,6 +154,13 @@ inline void LazyAllocArray::ForEach(FVisit fvisit) { } } } + +template +inline void LazyAllocArray::SignalForKill() { + std::lock_guard lock(create_mutex_); + exit_now_.store(true); +} + } // namespace common } // namespace mxnet #endif // MXNET_COMMON_LAZY_ALLOC_ARRAY_H_ diff --git a/src/engine/thread_pool.h b/src/engine/thread_pool.h index b88cddaa29c5..060f4734a675 100644 --- a/src/engine/thread_pool.h +++ b/src/engine/thread_pool.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include "mxnet/base.h" @@ -19,6 +20,42 @@ namespace engine { */ class ThreadPool { public: + /*! \brief Simple manually-signalled event gate which remains open */ + class SimpleEvent { + public: + SimpleEvent() + : signaled_(false) {} + void wait() { + std::unique_lock lock(mutex_); + if (!signaled_) { + condition_variable_.wait(lock); + } + } + void signal() { + signaled_ = true; + std::unique_lock lk(mutex_); + condition_variable_.notify_all(); + } + + /*! \brief Signal event upon destruction, even for exceptions (RAII) */ + struct SetReadyOnDestroy { + explicit inline SetReadyOnDestroy(std::shared_ptr event) + : event_(event) { + } + inline ~SetReadyOnDestroy() { + if (event_) { + event_->signal(); + } + } + std::shared_ptr event_; + }; + + private: + std::mutex mutex_; + std::condition_variable condition_variable_; + std::atomic signaled_; + }; + /*! * \brief Constructor takes function to run. * \param size size of the thread pool. @@ -30,6 +67,19 @@ class ThreadPool { i = std::thread(func); } } + explicit ThreadPool(size_t size, + std::function ready)> func, + const bool wait) + : worker_threads_(size) { + for (auto& i : worker_threads_) { + std::shared_ptr ptr = std::make_shared(); + ready_events_.emplace_back(ptr); + i = std::thread(func, ptr); + } + if (wait) { + WaitForReady(); + } + } ~ThreadPool() noexcept(false) { for (auto&& i : worker_threads_) { i.join(); @@ -37,10 +87,23 @@ class ThreadPool { } private: + /*! + * \brief Wait for all started threads to signal that they're ready + */ + void WaitForReady() { + for (std::shared_ptr ptr : ready_events_) { + ptr->wait(); + } + } + /*! * \brief Worker threads. */ std::vector worker_threads_; + /*! + * \brief Startup synchronization objects + */ + std::list> ready_events_; /*! * \brief Disallow default construction. */ diff --git a/src/engine/threaded_engine_perdevice.cc b/src/engine/threaded_engine_perdevice.cc index a0731683cef7..2b333d60647a 100644 --- a/src/engine/threaded_engine_perdevice.cc +++ b/src/engine/threaded_engine_perdevice.cc @@ -33,18 +33,18 @@ class ThreadedEnginePerDevice : public ThreadedEngine { ThreadedEnginePerDevice() noexcept(false) { gpu_worker_nthreads_ = common::GetNumThreadPerGPU(); - gpu_copy_nthreads_ = dmlc::GetEnv("MXNET_GPU_COPY_NTHREADS", 1); cpu_worker_nthreads_ = dmlc::GetEnv("MXNET_CPU_WORKER_NTHREADS", 1); // create CPU task int cpu_priority_nthreads = dmlc::GetEnv("MXNET_CPU_PRIORITY_NTHREADS", 4); cpu_priority_worker_.reset(new ThreadWorkerBlock()); cpu_priority_worker_->pool.reset(new ThreadPool( - cpu_priority_nthreads, [this] { + cpu_priority_nthreads, [this]() { this->CPUWorker(cpu_priority_worker_.get()); })); // GPU tasks will be created lazily } ~ThreadedEnginePerDevice() noexcept(false) { + SignalQueuesForKill(); gpu_normal_workers_.Clear(); gpu_copy_workers_.Clear(); cpu_normal_workers_.Clear(); @@ -70,13 +70,17 @@ class ThreadedEnginePerDevice : public ThreadedEngine { } else { int dev_id = ctx.dev_id; int nthread = cpu_worker_nthreads_; + auto ptr = cpu_normal_workers_.Get(dev_id, [this, dev_id, nthread]() { auto blk = new ThreadWorkerBlock(); blk->pool.reset(new ThreadPool(nthread, [this, blk] () { this->CPUWorker(blk); })); return blk; - })->task_queue.Push(opr_block, opr_block->priority); + }); + if (ptr) { + ptr->task_queue.Push(opr_block, opr_block->priority); + } } } else { CHECK_EQ(ctx.dev_mask(), gpu::kDevMask); @@ -87,21 +91,34 @@ class ThreadedEnginePerDevice : public ThreadedEngine { int nthread = gpu_worker_nthreads_; int dev_id = ctx.dev_id; if (is_copy) { + auto ptr = gpu_copy_workers_.Get(dev_id, [this, dev_id, is_copy, nthread]() { auto blk = new ThreadWorkerBlock(); - blk->pool.reset(new ThreadPool(nthread, [this, dev_id, is_copy, blk] () { - this->GPUWorker(dev_id, is_copy, blk); - })); + blk->pool.reset(new ThreadPool( + nthread, + [this, dev_id, is_copy, blk] + (std::shared_ptr ready_event) { + this->GPUWorker(dev_id, is_copy, blk, ready_event); + }, true)); return blk; - })->task_queue.Push(opr_block, opr_block->priority); + }); + if (ptr) { + ptr->task_queue.Push(opr_block, opr_block->priority); + } } else { - gpu_normal_workers_.Get(dev_id, [this, dev_id, is_copy, nthread]() { + auto ptr = gpu_normal_workers_.Get(dev_id, [this, dev_id, is_copy, nthread]() { auto blk = new ThreadWorkerBlock(); - blk->pool.reset(new ThreadPool(nthread, [this, dev_id, is_copy, blk] () { - this->GPUWorker(dev_id, is_copy, blk); - })); + blk->pool.reset(new ThreadPool( + nthread, + [this, dev_id, is_copy, blk] + (std::shared_ptr ready_event) { + this->GPUWorker(dev_id, is_copy, blk, ready_event); + }, true)); return blk; - })->task_queue.Push(opr_block, opr_block->priority); + }); + if (ptr) { + ptr->task_queue.Push(opr_block, opr_block->priority); + } } } } @@ -115,17 +132,16 @@ class ThreadedEnginePerDevice : public ThreadedEngine { dmlc::ConcurrentBlockingQueue task_queue; // thread pool that works on this task std::unique_ptr pool; + // constructor + ThreadWorkerBlock() = default; // destructor - ~ThreadWorkerBlock() noexcept(false) { - task_queue.SignalForKill(); - } + ~ThreadWorkerBlock() noexcept(false) {} }; + /*! \brief number of concurrent thread cpu worker uses */ int cpu_worker_nthreads_; /*! \brief number of concurrent thread each gpu worker uses */ int gpu_worker_nthreads_; - /*! \brief number of concurrent thread each gpu copy worker uses */ - int gpu_copy_nthreads_; // cpu worker common::LazyAllocArray > cpu_normal_workers_; // cpu priority worker @@ -143,18 +159,22 @@ class ThreadedEnginePerDevice : public ThreadedEngine { template inline void GPUWorker(int dev_id, bool is_copy_worker, - ThreadWorkerBlock *block) { - #if MXNET_USE_CUDA - // allocate stream - mshadow::SetDevice(dev_id); - RunContext run_ctx; + ThreadWorkerBlock *block, + std::shared_ptr ready_event) { +#if MXNET_USE_CUDA mshadow::Stream *stream; - if (is_copy_worker) { - stream = mshadow::NewStream(false, false); - } else { - stream = mshadow::NewStream(true, MXNET_USE_CUDNN != 0); - } - run_ctx.stream = stream; + RunContext run_ctx; + do { + ThreadPool::SimpleEvent::SetReadyOnDestroy setReady(ready_event); + // allocate stream + mshadow::SetDevice(dev_id); + if (is_copy_worker) { + stream = mshadow::NewStream(false, false); + } else { + stream = mshadow::NewStream(true, MXNET_USE_CUDNN != 0); + } + run_ctx.stream = stream; + } while (false); // execute task OprBlock* opr_block; auto* task_queue = &(block->task_queue); @@ -163,7 +183,9 @@ class ThreadedEnginePerDevice : public ThreadedEngine { } // Catch exception for CUDA driver shutdown MSHADOW_CATCH_ERROR(mshadow::DeleteStream(stream)); - #endif +#else + ready_event->signal(); +#endif } /*! * \brief CPU worker that performs operations on CPU. @@ -180,6 +202,24 @@ class ThreadedEnginePerDevice : public ThreadedEngine { this->ExecuteOprBlock(run_ctx, opr_block); } } + +/*! \brief Signal a single queue for shutdown */ + template + static inline void SignalQueueForKill(common::LazyAllocArray *array) { + array->ForEach([](size_t i, Object *block) { + block->task_queue.SignalForKill(); + }); + } + + /*! Signal all queues for shutdown */ + void SignalQueuesForKill() { + SignalQueueForKill(&gpu_normal_workers_); + SignalQueueForKill(&gpu_copy_workers_); + SignalQueueForKill(&cpu_normal_workers_); + if (cpu_priority_worker_) { + cpu_priority_worker_->task_queue.SignalForKill(); + } + } }; Engine *CreateThreadedEnginePerDevice() { diff --git a/src/storage/storage.cc b/src/storage/storage.cc index 1418216c7a76..997f033b927b 100644 --- a/src/storage/storage.cc +++ b/src/storage/storage.cc @@ -59,7 +59,7 @@ Storage::Handle StorageImpl::Alloc(size_t size, Context ctx) { hd.ctx = ctx; hd.size = size; auto&& device = storage_managers_.at(ctx.dev_type); - storage::StorageManager *manager = device.Get( + std::shared_ptr manager = device.Get( ctx.dev_id, [ctx]() { storage::StorageManager *ptr = nullptr; switch (ctx.dev_type) { @@ -95,7 +95,7 @@ Storage::Handle StorageImpl::Alloc(size_t size, Context ctx) { void StorageImpl::Free(Storage::Handle handle) { const Context &ctx = handle.ctx; auto&& device = storage_managers_.at(ctx.dev_type); - storage::StorageManager *manager = device.Get( + std::shared_ptr manager = device.Get( ctx.dev_id, []() { LOG(FATAL) << "Cannot Free space to a device you have not allocated"; return nullptr; @@ -107,7 +107,7 @@ void StorageImpl::Free(Storage::Handle handle) { void StorageImpl::DirectFree(Storage::Handle handle) { const Context &ctx = handle.ctx; auto&& device = storage_managers_.at(ctx.dev_type); - storage::StorageManager *manager = device.Get( + std::shared_ptr manager = device.Get( ctx.dev_id, []() { LOG(FATAL) << "Cannot Free space to a device you have not allocated"; return nullptr; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7b7f283b82d4..418d40e9eb8a 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -39,7 +39,6 @@ if(GTEST_FOUND) ${mxnet_LINKER_LIBS} ) else() - message(STATUS " OpenBLAS_LIB: ${OpenBLAS_LIB}") target_link_libraries(${PROJECT_NAME}_unit_tests ${GTEST_LIBRARY} rt From 1621b76005c9148c14d6acd57ccb2583ab7d308c Mon Sep 17 00:00:00 2001 From: Haibin Lin Date: Thu, 22 Jun 2017 17:28:58 -0700 Subject: [PATCH 108/834] support str key type in kvstore (#6765) * update kvstore unit test * update model/module.py * fix lint * remove int keys in kvstore * update cast to str function * remove _cast_to_str_keys * fix lint * always cast to str --- include/mxnet/c_api.h | 41 ++++++++ include/mxnet/kvstore.h | 27 ++++++ python/mxnet/kvstore.py | 68 +++++++------ python/mxnet/model.py | 24 +++-- python/mxnet/module/module.py | 5 +- src/c_api/c_api.cc | 47 +++++++++ src/kvstore/kvstore.cc | 1 - src/kvstore/kvstore_local.h | 46 +++++++++ tests/python/unittest/test_kvstore.py | 133 +++++++++++++++----------- 9 files changed, 289 insertions(+), 103 deletions(-) diff --git a/include/mxnet/c_api.h b/include/mxnet/c_api.h index a0e842c21765..b8f8411353bf 100644 --- a/include/mxnet/c_api.h +++ b/include/mxnet/c_api.h @@ -1311,6 +1311,19 @@ MXNET_DLL int MXKVStoreInit(KVStoreHandle handle, const int* keys, NDArrayHandle* vals); +/*! + * \brief Init a list of (key,value) pairs in kvstore, where each key is a string + * \param handle handle to the kvstore + * \param num the number of key-value pairs + * \param keys the list of keys + * \param vals the list of values + * \return 0 when success, -1 when failure happens + */ +MXNET_DLL int MXKVStoreInitEx(KVStoreHandle handle, + mx_uint num, + const char** keys, + NDArrayHandle* vals); + /*! * \brief Push a list of (key,value) pairs to kvstore * \param handle handle to the kvstore @@ -1325,6 +1338,20 @@ MXNET_DLL int MXKVStorePush(KVStoreHandle handle, const int* keys, NDArrayHandle* vals, int priority); +/*! + * \brief Push a list of (key,value) pairs to kvstore, where each key is a string + * \param handle handle to the kvstore + * \param num the number of key-value pairs + * \param keys the list of keys + * \param vals the list of values + * \param priority the priority of the action + * \return 0 when success, -1 when failure happens + */ +MXNET_DLL int MXKVStorePushEx(KVStoreHandle handle, + mx_uint num, + const char** keys, + NDArrayHandle* vals, + int priority); /*! * \brief pull a list of (key, value) pairs from the kvstore * \param handle handle to the kvstore @@ -1339,6 +1366,20 @@ MXNET_DLL int MXKVStorePull(KVStoreHandle handle, const int* keys, NDArrayHandle* vals, int priority); +/*! + * \brief pull a list of (key, value) pairs from the kvstore, where each key is a string + * \param handle handle to the kvstore + * \param num the number of key-value pairs + * \param keys the list of keys + * \param vals the list of values + * \param priority the priority of the action + * \return 0 when success, -1 when failure happens + */ +MXNET_DLL int MXKVStorePullEx(KVStoreHandle handle, + mx_uint num, + const char** keys, + NDArrayHandle* vals, + int priority); /*! * \brief user-defined updater for the kvstore * It's this updater's responsibility to delete \a recv and \a local diff --git a/include/mxnet/kvstore.h b/include/mxnet/kvstore.h index dafaf1bf9cab..a77f653d492c 100644 --- a/include/mxnet/kvstore.h +++ b/include/mxnet/kvstore.h @@ -63,6 +63,13 @@ class KVStore { */ virtual void Init(const std::vector& keys, const std::vector& values) = 0; + /*! + * \brief Initialize a list of key-value pair to the store. + * \param keys a list of unique keys in string format + * \param values a list of values + */ + virtual void Init(const std::vector& str_keys, + const std::vector& values) = 0; /*! * \brief push a list of key-value pairs into the store * @@ -102,6 +109,16 @@ class KVStore { virtual void Push(const std::vector& keys, const std::vector& values, int priority = 0) = 0; + + /*! + * \brief push a list of key-value pairs into the store + * \param keys the list of keys in string format + * \param values the list of values + * \param priority Priority of the action. + */ + virtual void Push(const std::vector& str_keys, + const std::vector& values, + int priority = 0) = 0; /*! * \brief pull a list of key-value pairs from the store * @@ -128,6 +145,16 @@ class KVStore { virtual void Pull(const std::vector& keys, const std::vector& values, int priority = 0) = 0; + /*! + * \brief pull a list of key-value pairs from the store + * \param keys the list of keys in string format + * \param values the list of buffers for the pulled data, they should be preallocated + * \param priority Priority of the action. + */ + virtual void Pull(const std::vector& str_keys, + const std::vector& values, + int priority = 0) = 0; + /** * \brief the prototype of user-defined updater diff --git a/python/mxnet/kvstore.py b/python/mxnet/kvstore.py index ab07421caffd..10b83b04db97 100644 --- a/python/mxnet/kvstore.py +++ b/python/mxnet/kvstore.py @@ -11,30 +11,26 @@ from . import optimizer as opt def _ctype_key_value(keys, vals): - """ - Returns ctype arrays for the key-value args. For internal use. - """ - if isinstance(keys, int): - if isinstance(vals, NDArray): - return (c_array(ctypes.c_int, [keys]), - c_array(NDArrayHandle, [vals.handle])) - else: - for value in vals: - assert(isinstance(value, NDArray)) - return (c_array(ctypes.c_int, [keys] * len(vals)), - c_array(NDArrayHandle, [value.handle for value in vals])) - else: + if isinstance(keys, (tuple, list)): assert(len(keys) == len(vals)) - for k in keys: - assert(isinstance(k, int)) c_keys = [] c_vals = [] for key, val in zip(keys, vals): c_key_i, c_val_i = _ctype_key_value(key, val) c_keys += c_key_i c_vals += c_val_i - return (c_array(ctypes.c_int, c_keys), c_array(NDArrayHandle, c_vals)) - + return (c_array(ctypes.c_char_p, c_keys), c_array(NDArrayHandle, c_vals)) + names = [] + keys = str(keys) + if isinstance(vals, NDArray): + names.append(c_str(keys)) + return (c_array(ctypes.c_char_p, names), + c_array(NDArrayHandle, [vals.handle])) + else: + for value in vals: + assert(isinstance(value, NDArray)) + return (c_array(ctypes.c_char_p, [c_str(keys)] * len(vals)), + c_array(NDArrayHandle, [value.handle for value in vals])) def _updater_wrapper(updater): """A wrapper for the user-defined handle.""" @@ -74,7 +70,7 @@ def init(self, key, value): Parameters ---------- - key : int or sequence of int + key : str or sequence of str The keys. value : NDArray or sequence of NDArray Values corresponding to the keys. @@ -84,20 +80,19 @@ def init(self, key, value): >>> # init a single key-value pair >>> shape = (2,3) >>> kv = mx.kv.create('local') - >>> kv.init(3, mx.nd.ones(shape)*2) + >>> kv.init('3', mx.nd.ones(shape)*2) >>> a = mx.nd.zeros(shape) - >>> kv.pull(3, out=a) + >>> kv.pull('3', out=a) >>> print a.asnumpy() [[ 2. 2. 2.] [ 2. 2. 2.]] >>> # init a list of key-value pairs - >>> keys = [5, 7, 9] + >>> keys = ['5', '7', '9'] >>> kv.init(keys, [mx.nd.ones(shape)]*len(keys)) """ ckeys, cvals = _ctype_key_value(key, value) - check_call(_LIB.MXKVStoreInit( - self.handle, mx_uint(len(ckeys)), ckeys, cvals)) + check_call(_LIB.MXKVStoreInitEx(self.handle, mx_uint(len(ckeys)), ckeys, cvals)) def push(self, key, value, priority=0): """ Pushes a single or a sequence of key-value pairs into the store. @@ -110,7 +105,7 @@ def push(self, key, value, priority=0): Parameters ---------- - key : int or list of int + key : str or list of str Keys. value : NDArray or list of NDArray or list of list of NDArray @@ -124,8 +119,8 @@ def push(self, key, value, priority=0): Examples -------- >>> # push a single key-value pair - >>> kv.push(3, mx.nd.ones(shape)*8) - >>> kv.pull(3, out=a) # pull out the value + >>> kv.push('3', mx.nd.ones(shape)*8) + >>> kv.pull('3', out=a) # pull out the value >>> print a.asnumpy() [[ 8. 8. 8.] [ 8. 8. 8.]] @@ -133,8 +128,8 @@ def push(self, key, value, priority=0): >>> # aggregate the value and the push >>> gpus = [mx.gpu(i) for i in range(4)] >>> b = [mx.nd.ones(shape, gpu) for gpu in gpus] - >>> kv.push(3, b) - >>> kv.pull(3, out=a) + >>> kv.push('3', b) + >>> kv.pull('3', out=a) >>> print a.asnumpy() [[ 4. 4. 4.] [ 4. 4. 4.]] @@ -157,10 +152,11 @@ def push(self, key, value, priority=0): [ 4. 4. 4.]] """ ckeys, cvals = _ctype_key_value(key, value) - check_call(_LIB.MXKVStorePush( + check_call(_LIB.MXKVStorePushEx( self.handle, mx_uint(len(ckeys)), ckeys, cvals, ctypes.c_int(priority))) + def pull(self, key, out=None, priority=0): """ Pulls a single value or a sequence of values from the store. @@ -190,21 +186,21 @@ def pull(self, key, out=None, priority=0): -------- >>> # pull a single key-value pair >>> a = mx.nd.zeros(shape) - >>> kv.pull(3, out=a) + >>> kv.pull('3', out=a) >>> print a.asnumpy() [[ 2. 2. 2.] [ 2. 2. 2.]] >>> # pull into multiple devices >>> b = [mx.nd.ones(shape, gpu) for gpu in gpus] - >>> kv.pull(3, out=b) + >>> kv.pull('3', out=b) >>> print b[1].asnumpy() [[ 2. 2. 2.] [ 2. 2. 2.]] >>> # pull a list of key-value pairs. >>> # On single device - >>> keys = [5, 7, 9] + >>> keys = ['5', '7', '9'] >>> b = [mx.nd.zeros(shape)]*len(keys) >>> kv.pull(keys, out=b) >>> print b[1].asnumpy() @@ -219,7 +215,7 @@ def pull(self, key, out=None, priority=0): """ assert(out is not None) ckeys, cvals = _ctype_key_value(key, out) - check_call(_LIB.MXKVStorePull( + check_call(_LIB.MXKVStorePullEx( self.handle, mx_uint(len(ckeys)), ckeys, cvals, ctypes.c_int(priority))) @@ -348,13 +344,13 @@ def _set_updater(self, updater): ... print "update on key: %d" % key ... stored += input * 2 >>> kv._set_updater(update) - >>> kv.pull(3, out=a) + >>> kv.pull('3', out=a) >>> print a.asnumpy() [[ 4. 4. 4.] [ 4. 4. 4.]] - >>> kv.push(3, mx.nd.ones(shape)) + >>> kv.push('3', mx.nd.ones(shape)) update on key: 3 - >>> kv.pull(3, out=a) + >>> kv.pull('3', out=a) >>> print a.asnumpy() [[ 6. 6. 6.] [ 6. 6. 6.]] diff --git a/python/mxnet/model.py b/python/mxnet/model.py index 189f301e91f7..a476d84efd92 100644 --- a/python/mxnet/model.py +++ b/python/mxnet/model.py @@ -80,34 +80,37 @@ def _initialize_kvstore(kvstore, param_arrays, arg_params, param_names, update_on_kvstore): """Initialize kvstore""" for idx, param_on_devs in enumerate(param_arrays): - kvstore.init(idx, arg_params[param_names[idx]]) + name = param_names[idx] + kvstore.init(name, arg_params[name]) if update_on_kvstore: - kvstore.pull(idx, param_on_devs, priority=-idx) + kvstore.pull(name, param_on_devs, priority=-idx) -def _update_params_on_kvstore(param_arrays, grad_arrays, kvstore): +def _update_params_on_kvstore(param_arrays, grad_arrays, kvstore, param_names): """Perform update of param_arrays from grad_arrays on kvstore.""" for index, pair in enumerate(zip(param_arrays, grad_arrays)): arg_list, grad_list = pair if grad_list[0] is None: continue + name = param_names[index] # push gradient, priority is negative index - kvstore.push(index, grad_list, priority=-index) + kvstore.push(name, grad_list, priority=-index) # pull back the weights - kvstore.pull(index, arg_list, priority=-index) + kvstore.pull(name, arg_list, priority=-index) def _update_params(param_arrays, grad_arrays, updater, num_device, - kvstore=None): + kvstore=None, param_names=None): """Perform update of param_arrays from grad_arrays not on kvstore.""" for index, pair in enumerate(zip(param_arrays, grad_arrays)): arg_list, grad_list = pair if grad_list[0] is None: continue if kvstore: + name = param_names[index] # push gradient, priority is negative index - kvstore.push(index, grad_list, priority=-index) + kvstore.push(name, grad_list, priority=-index) # pull back the sum gradients, to the same locations. - kvstore.pull(index, grad_list, priority=-index) + kvstore.pull(name, grad_list, priority=-index) for k, p in enumerate(zip(arg_list, grad_list)): # faked an index here, to make optimizer create diff # state for the same index but on diff devs, TODO(mli) @@ -245,13 +248,14 @@ def _train_multi_device(symbol, ctx, arg_names, param_names, aux_names, if update_on_kvstore: _update_params_on_kvstore(executor_manager.param_arrays, executor_manager.grad_arrays, - kvstore) + kvstore, executor_manager.param_names) else: _update_params(executor_manager.param_arrays, executor_manager.grad_arrays, updater=updater, num_device=len(ctx), - kvstore=kvstore) + kvstore=kvstore, + param_names=executor_manager.param_names) if monitor is not None: monitor.toc_print() diff --git a/python/mxnet/module/module.py b/python/mxnet/module/module.py index fef5c507d7e8..249122311274 100644 --- a/python/mxnet/module/module.py +++ b/python/mxnet/module/module.py @@ -572,13 +572,14 @@ def update(self): if self._update_on_kvstore: _update_params_on_kvstore(self._exec_group.param_arrays, self._exec_group.grad_arrays, - self._kvstore) + self._kvstore, self._exec_group.param_names) else: _update_params(self._exec_group.param_arrays, self._exec_group.grad_arrays, updater=self._updater, num_device=len(self._context), - kvstore=self._kvstore) + kvstore=self._kvstore, + param_names=self._exec_group.param_names) def get_outputs(self, merge_multi_context=True): """Gets outputs of the previous forward computation. diff --git a/src/c_api/c_api.cc b/src/c_api/c_api.cc index 9d60c8615027..bea6437b4c64 100644 --- a/src/c_api/c_api.cc +++ b/src/c_api/c_api.cc @@ -625,6 +625,21 @@ int MXKVStoreInit(KVStoreHandle handle, API_END(); } +int MXKVStoreInitEx(KVStoreHandle handle, + mx_uint num, + const char** keys, + NDArrayHandle* vals) { + API_BEGIN(); + std::vector v_keys(num); + std::vector v_vals(num); + for (mx_uint i = 0; i < num; ++i) { + v_keys[i] = keys[i]; + v_vals[i] = *static_cast(vals[i]); + } + static_cast(handle)->Init(v_keys, v_vals); + API_END(); +} + int MXKVStorePush(KVStoreHandle handle, mx_uint num, const int* keys, @@ -641,6 +656,22 @@ int MXKVStorePush(KVStoreHandle handle, API_END(); } +int MXKVStorePushEx(KVStoreHandle handle, + mx_uint num, + const char** keys, + NDArrayHandle* vals, + int priority) { + API_BEGIN(); + std::vector v_keys(num); + std::vector v_vals(num); + for (mx_uint i = 0; i < num; ++i) { + v_keys[i] = keys[i]; + v_vals[i] = *static_cast(vals[i]); + } + static_cast(handle)->Push(v_keys, v_vals, priority); + API_END(); +} + int MXKVStorePull(KVStoreHandle handle, mx_uint num, const int* keys, @@ -657,6 +688,22 @@ int MXKVStorePull(KVStoreHandle handle, API_END(); } +int MXKVStorePullEx(KVStoreHandle handle, + mx_uint num, + const char** keys, + NDArrayHandle* vals, + int priority) { + API_BEGIN(); + std::vector v_keys(num); + std::vector v_vals(num); + for (mx_uint i = 0; i < num; ++i) { + v_keys[i] = keys[i]; + v_vals[i] = static_cast(vals[i]); + } + static_cast(handle)->Pull(v_keys, v_vals, priority); + API_END(); +} + int MXKVStoreSetUpdater(KVStoreHandle handle, MXKVStoreUpdater updater, void* updater_handle) { diff --git a/src/kvstore/kvstore.cc b/src/kvstore/kvstore.cc index be5662e8a6db..78d4958096cc 100644 --- a/src/kvstore/kvstore.cc +++ b/src/kvstore/kvstore.cc @@ -7,7 +7,6 @@ #include #include #include "./kvstore_local.h" -// #include "./kvstore_device.h" #if MXNET_USE_DIST_KVSTORE #include "./kvstore_dist.h" #endif // MXNET_USE_DIST_KVSTORE diff --git a/src/kvstore/kvstore_local.h b/src/kvstore/kvstore_local.h index caa57a20d46e..dc5f7b786244 100644 --- a/src/kvstore/kvstore_local.h +++ b/src/kvstore/kvstore_local.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include "./comm.h" @@ -47,6 +48,20 @@ class KVStoreLocal : public KVStore { } } + void Init(const std::vector& str_keys, + const std::vector& values) override { + std::vector keys(str_keys.size()); + for (size_t i = 0; i < str_keys.size(); ++i) { + auto &str_key = str_keys[i]; + CHECK(str_key_dict_.find(str_key) == str_key_dict_.end()) + << "duplicate init of key " << str_key; + auto key = next_str_key_++; + str_key_dict_[str_key] = key; + keys[i] = key; + } + Init(keys, values); + } + void Push(const std::vector& keys, const std::vector& values, int priority) override { @@ -87,6 +102,22 @@ class KVStoreLocal : public KVStore { } } + void Push(const std::vector& str_keys, + const std::vector& values, + int priority) override { + std::vector keys(str_keys.size()); + LookupKeys(str_keys, &keys); + Push(keys, values, priority); + } + + void Pull(const std::vector& str_keys, + const std::vector& values, + int priority) override { + std::vector keys(str_keys.size()); + LookupKeys(str_keys, &keys); + Pull(keys, values, priority); + } + protected: /** * \brief group values on keys @@ -118,12 +149,27 @@ class KVStoreLocal : public KVStore { } } } + + void LookupKeys(const std::vector& str_keys, + std::vector *keys) { + for (size_t i = 0; i < str_keys.size(); ++i) { + auto &str_key = str_keys[i]; + CHECK(str_key_dict_.find(str_key) != str_key_dict_.end()) + << "key " << str_key << " doesn't exist. Did you init?"; + keys->at(i) = str_key_dict_[str_key]; + } + } + /// reducer and broadcaster Comm* comm_; /// pinned context Context pinned_ctx_; /// \brief buffer for storing local values std::unordered_map local_; + /// key mapping for string -> integer + std::unordered_map str_key_dict_; + /// the next available integer for string->int key mapping + int next_str_key_ = 0; }; } // namespace kvstore } // namespace mxnet diff --git a/tests/python/unittest/test_kvstore.py b/tests/python/unittest/test_kvstore.py index dd8149d4822e..87e5e0027241 100644 --- a/tests/python/unittest/test_kvstore.py +++ b/tests/python/unittest/test_kvstore.py @@ -4,6 +4,8 @@ shape = (4, 4) keys = [5, 7, 11] +str_keys = ['b', 'c', 'd'] + def init_kv(): """init kv """ kv = mx.kv.create() @@ -13,6 +15,14 @@ def init_kv(): kv.init(keys, [mx.nd.zeros(shape)] * len(keys)) return kv +def init_kv_with_str(): + """init kv """ + kv = mx.kv.create() + # single + kv.init('a', mx.nd.zeros(shape)) + # list + kv.init(str_keys, [mx.nd.zeros(shape)] * len(keys)) + return kv def check_diff_to_scalar(A, x): """ assert A == x""" @@ -20,59 +30,67 @@ def check_diff_to_scalar(A, x): def test_single_kv_pair(): """single key-value pair push & pull""" + def check_single_kv_pair(kv, key): + kv.push(key, mx.nd.ones(shape)) + val = mx.nd.empty(shape) + kv.pull(key, out = val) + check_diff_to_scalar(val, 1) - kv = init_kv() - kv.push(3, mx.nd.ones(shape)) - val = mx.nd.empty(shape) - kv.pull(3, out = val) - check_diff_to_scalar(val, 1) + check_single_kv_pair(init_kv(), 3) + check_single_kv_pair(init_kv_with_str(), 'a') def test_init(): """test init""" - kv = mx.kv.create() - kv.init(3, mx.nd.ones(shape)*4) - a = mx.nd.zeros(shape) - kv.pull(3, out=a) - check_diff_to_scalar(a, 4) + def check_init(kv, key): + kv.init(key, mx.nd.ones(shape)*4) + a = mx.nd.zeros(shape) + kv.pull(key, out=a) + check_diff_to_scalar(a, 4) + + check_init(mx.kv.create(), 3) + check_init(mx.kv.create(), 'a') def test_list_kv_pair(): """list key-value pair push & pull""" + def check_list_kv_pair(kv, key): + kv.push(key, [mx.nd.ones(shape)*4] * len(key)) + val = [mx.nd.empty(shape)] * len(key) + kv.pull(key, out = val) + for v in val: + check_diff_to_scalar(v, 4) - kv = init_kv() - - kv.push(keys, [mx.nd.ones(shape)*4] * len(keys)) - val = [mx.nd.empty(shape)] * len(keys) - kv.pull(keys, out = val) - for v in val: - check_diff_to_scalar(v, 4) + check_list_kv_pair(init_kv(), keys) + check_list_kv_pair(init_kv_with_str(), str_keys) def test_aggregator(): """aggregate value on muliple devices""" - kv = init_kv() + def check_aggregator(kv, key, key_list): + # devices + num_devs = 4 + devs = [mx.Context('cpu', i) for i in range(num_devs)] - # devices - num_devs = 4 - devs = [mx.Context('cpu', i) for i in range(num_devs)] + # single + vals = [mx.nd.ones(shape, d) for d in devs] - # single - vals = [mx.nd.ones(shape, d) for d in devs] + kv.push(key, vals) + kv.pull(key, out = vals) - kv.push(3, vals) - kv.pull(3, out = vals) + for v in vals: + check_diff_to_scalar(v, num_devs) - for v in vals: - check_diff_to_scalar(v, num_devs) + # list + vals = [[mx.nd.ones(shape, d)*2.0 for d in devs]] * len(key_list) + kv.push(key_list, vals) + kv.pull(key_list, out = vals) - # list - vals = [[mx.nd.ones(shape, d)*2.0 for d in devs]] * len(keys) - kv.push(keys, vals) - kv.pull(keys, out = vals) + for vv in vals: + for v in vv: + check_diff_to_scalar(v, num_devs * 2.0) - for vv in vals: - for v in vv: - check_diff_to_scalar(v, num_devs * 2.0) + check_aggregator(init_kv(), 3, keys) + check_aggregator(init_kv_with_str(), 'a', str_keys) def updater(key, recv, local): @@ -82,34 +100,41 @@ def updater(key, recv, local): def test_updater(dev = 'cpu'): """updater""" - kv = init_kv() - kv._set_updater(updater) + def check_updater(kv, key, key_list): + # devices + num_devs = 4 + devs = [mx.Context(dev, i) for i in range(num_devs)] - # devices - num_devs = 4 - devs = [mx.Context(dev, i) for i in range(num_devs)] + # single + vals = [mx.nd.ones(shape, d) for d in devs] - # single - vals = [mx.nd.ones(shape, d) for d in devs] + kv.push(key, vals) + kv.pull(key, out = vals) - kv.push(3, vals) - kv.pull(3, out = vals) + for v in vals: + check_diff_to_scalar(v, num_devs) - for v in vals: - check_diff_to_scalar(v, num_devs) + # list + vals = [[mx.nd.ones(shape, d) for d in devs]] * len(key_list) - # list - vals = [[mx.nd.ones(shape, d) for d in devs]] * len(keys) + num_push = 4 + for i in range(num_push): + kv.push(key_list, vals) + + kv.pull(key_list, out = vals) + + for vv in vals: + for v in vv: + check_diff_to_scalar(v, num_devs * num_push) - num_push = 4 - for i in range(num_push): - kv.push(keys, vals) + kv = init_kv() + kv._set_updater(updater) + check_updater(kv, 3, keys) - kv.pull(keys, out = vals) + str_kv = init_kv_with_str() + str_kv._set_updater(updater) + check_updater(str_kv, 'a', str_keys) - for vv in vals: - for v in vv: - check_diff_to_scalar(v, num_devs * num_push) def test_get_type(): kvtype = 'local_allreduce_cpu' From d6d61915aa36d6a2530e3a646bc4e0015bc62469 Mon Sep 17 00:00:00 2001 From: Jameson Date: Thu, 22 Jun 2017 17:32:09 -0700 Subject: [PATCH 109/834] allow extra params in module set params (#6753) * allow extra params in Module Loading * fix lint + one missing init_param * allow_extra_params -> allow_extra * added tests --- python/mxnet/module/base_module.py | 16 ++++-- python/mxnet/module/bucketing_module.py | 17 ++++-- python/mxnet/module/executor_group.py | 8 ++- python/mxnet/module/module.py | 22 +++++--- python/mxnet/module/python_module.py | 6 ++- python/mxnet/module/sequential_module.py | 8 ++- tests/python/unittest/common.py | 10 ++++ tests/python/unittest/test_module.py | 66 ++++++++++++++++++++++++ 8 files changed, 135 insertions(+), 18 deletions(-) diff --git a/python/mxnet/module/base_module.py b/python/mxnet/module/base_module.py index 820841087a9c..a5c4c709780a 100644 --- a/python/mxnet/module/base_module.py +++ b/python/mxnet/module/base_module.py @@ -590,7 +590,7 @@ def get_params(self): raise NotImplementedError() def init_params(self, initializer=Uniform(0.01), arg_params=None, aux_params=None, - allow_missing=False, force_init=False): + allow_missing=False, force_init=False, allow_extra=False): """Initializes the parameters and auxiliary states. Parameters @@ -608,6 +608,10 @@ def init_params(self, initializer=Uniform(0.01), arg_params=None, aux_params=Non called to fill those missing params. force_init : bool If ``True``, `force_init` will force re-initialize even if already initialized. + allow_extra : boolean, optional + Whether allow extra parameters that are not needed by symbol. + If this is True, no error will be thrown when arg_params or aux_params + contain extra parameters that is not needed by the executor. Examples -------- @@ -616,7 +620,8 @@ def init_params(self, initializer=Uniform(0.01), arg_params=None, aux_params=Non """ raise NotImplementedError() - def set_params(self, arg_params, aux_params, allow_missing=False, force_init=True): + def set_params(self, arg_params, aux_params, allow_missing=False, force_init=True, + allow_extra=False): """Assigns parameter and aux state values. Parameters @@ -630,6 +635,10 @@ def set_params(self, arg_params, aux_params, allow_missing=False, force_init=Tru called to fill those missing params. force_init : bool If ``True``, will force re-initialize even if already initialized. + allow_extra : boolean, optional + Whether allow extra parameters that are not needed by symbol. + If this is True, no error will be thrown when arg_params or aux_params + contain extra parameters that is not needed by the executor. Examples -------- @@ -638,7 +647,8 @@ def set_params(self, arg_params, aux_params, allow_missing=False, force_init=Tru >>> mod.set_params(arg_params=arg_params, aux_params=aux_params) """ self.init_params(initializer=None, arg_params=arg_params, aux_params=aux_params, - allow_missing=allow_missing, force_init=force_init) + allow_missing=allow_missing, force_init=force_init, + allow_extra=allow_extra) def save_params(self, fname): """Saves model parameters to file. diff --git a/python/mxnet/module/bucketing_module.py b/python/mxnet/module/bucketing_module.py index 11922ddafb56..7a1be96d093c 100644 --- a/python/mxnet/module/bucketing_module.py +++ b/python/mxnet/module/bucketing_module.py @@ -141,7 +141,8 @@ def get_params(self): self._params_dirty = False return params - def set_params(self, arg_params, aux_params, allow_missing=False, force_init=True): + def set_params(self, arg_params, aux_params, allow_missing=False, force_init=True, + allow_extra=False): """Assigns parameters and aux state values. Parameters @@ -155,6 +156,10 @@ def set_params(self, arg_params, aux_params, allow_missing=False, force_init=Tru called to fill those missing params. force_init : bool If true, will force re-initialize even if already initialized. + allow_extra : boolean, optional + Whether allow extra parameters that are not needed by symbol. + If this is True, no error will be thrown when arg_params or aux_params + contain extra parameters that is not needed by the executor. Examples -------- @@ -173,14 +178,14 @@ def set_params(self, arg_params, aux_params, allow_missing=False, force_init=Tru return self._curr_module.set_params(arg_params, aux_params, allow_missing=allow_missing, - force_init=force_init) + force_init=force_init, allow_extra=allow_extra) # because we didn't update self._arg_params, they are dirty now. self._params_dirty = True self.params_initialized = True def init_params(self, initializer=Uniform(0.01), arg_params=None, aux_params=None, - allow_missing=False, force_init=False): + allow_missing=False, force_init=False, allow_extra=False): """Initializes parameters. Parameters @@ -197,13 +202,17 @@ def init_params(self, initializer=Uniform(0.01), arg_params=None, aux_params=Non In this case, missing values will be filled with `initializer`. force_init : bool Defaults to ``False``. + allow_extra : boolean, optional + Whether allow extra parameters that are not needed by symbol. + If this is True, no error will be thrown when arg_params or aux_params + contain extra parameters that is not needed by the executor. """ if self.params_initialized and not force_init: return assert self.binded, 'call bind before initializing the parameters' self._curr_module.init_params(initializer=initializer, arg_params=arg_params, aux_params=aux_params, allow_missing=allow_missing, - force_init=force_init) + force_init=force_init, allow_extra=allow_extra) self._params_dirty = False self.params_initialized = True diff --git a/python/mxnet/module/executor_group.py b/python/mxnet/module/executor_group.py index 7e2e5e26bb35..169e81ee326e 100755 --- a/python/mxnet/module/executor_group.py +++ b/python/mxnet/module/executor_group.py @@ -330,7 +330,7 @@ def reshape(self, data_shapes, label_shapes): self._default_execs = [i for i in self.execs] self.bind_exec(data_shapes, label_shapes, reshape=True) - def set_params(self, arg_params, aux_params): + def set_params(self, arg_params, aux_params, allow_extra=False): """Assign, i.e. copy parameters to all the executors. Parameters @@ -339,9 +339,13 @@ def set_params(self, arg_params, aux_params): A dictionary of name to `NDArray` parameter mapping. aux_params : dict A dictionary of name to `NDArray` auxiliary variable mapping. + allow_extra : boolean, optional + Whether allow extra parameters that are not needed by symbol. + If this is True, no error will be thrown when arg_params or aux_params + contain extra parameters that is not needed by the executor. """ for exec_ in self.execs: - exec_.copy_params_from(arg_params, aux_params) + exec_.copy_params_from(arg_params, aux_params, allow_extra_params=allow_extra) def get_params(self, arg_params, aux_params): """ Copy data from each executor to `arg_params` and `aux_params`. diff --git a/python/mxnet/module/module.py b/python/mxnet/module/module.py index 249122311274..f5f3c2ade3d0 100644 --- a/python/mxnet/module/module.py +++ b/python/mxnet/module/module.py @@ -226,7 +226,7 @@ def get_params(self): return (self._arg_params, self._aux_params) def init_params(self, initializer=Uniform(0.01), arg_params=None, aux_params=None, - allow_missing=False, force_init=False): + allow_missing=False, force_init=False, allow_extra=False): """Initializes the parameters and auxiliary states. Parameters @@ -244,6 +244,10 @@ def init_params(self, initializer=Uniform(0.01), arg_params=None, aux_params=Non called to fill those missing params. force_init : bool If ``True``, will force re-initialize even if already initialized. + allow_extra : boolean, optional + Whether allow extra parameters that are not needed by symbol. + If this is True, no error will be thrown when arg_params or aux_params + contain extra parameters that is not needed by the executor. """ if self.params_initialized and not force_init: warnings.warn("Parameters already initialized and force_init=False. " @@ -281,9 +285,11 @@ def _impl(name, arr, cache): self._params_dirty = False # copy the initialized parameters to devices - self._exec_group.set_params(self._arg_params, self._aux_params) + self._exec_group.set_params(self._arg_params, self._aux_params, + allow_extra=allow_extra) - def set_params(self, arg_params, aux_params, allow_missing=False, force_init=True): + def set_params(self, arg_params, aux_params, allow_missing=False, force_init=True, + allow_extra=False): """Assigns parameter and aux state values. Parameters @@ -297,7 +303,10 @@ def set_params(self, arg_params, aux_params, allow_missing=False, force_init=Tru called to fill those missing params. force_init : bool If ``True``, will force re-initialize even if already initialized. - + allow_extra : boolean, optional + Whether allow extra parameters that are not needed by symbol. + If this is True, no error will be thrown when arg_params or aux_params + contain extra parameters that is not needed by the executor. Examples -------- >>> # An example of setting module parameters. @@ -306,7 +315,8 @@ def set_params(self, arg_params, aux_params, allow_missing=False, force_init=Tru """ if not allow_missing: self.init_params(initializer=None, arg_params=arg_params, aux_params=aux_params, - allow_missing=allow_missing, force_init=force_init) + allow_missing=allow_missing, force_init=force_init, + allow_extra=allow_extra) return if self.params_initialized and not force_init: @@ -314,7 +324,7 @@ def set_params(self, arg_params, aux_params, allow_missing=False, force_init=Tru "set_params call ignored.", stacklevel=2) return - self._exec_group.set_params(arg_params, aux_params) + self._exec_group.set_params(arg_params, aux_params, allow_extra=allow_extra) # because we didn't update self._arg_params, they are dirty now. self._params_dirty = True diff --git a/python/mxnet/module/python_module.py b/python/mxnet/module/python_module.py index f46ea280aaff..af7c76e646a1 100644 --- a/python/mxnet/module/python_module.py +++ b/python/mxnet/module/python_module.py @@ -88,7 +88,7 @@ def get_params(self): return (dict(), dict()) def init_params(self, initializer=Uniform(0.01), arg_params=None, aux_params=None, - allow_missing=False, force_init=False): + allow_missing=False, force_init=False, allow_extra=False): """Initializes the parameters and auxiliary states. By default this function does nothing. Subclass should override this method if contains parameters. @@ -107,6 +107,10 @@ def init_params(self, initializer=Uniform(0.01), arg_params=None, aux_params=Non called to fill those missing params. force_init : bool If ``True``, will force re-initialize even if already initialized. + allow_extra : boolean, optional + Whether allow extra parameters that are not needed by symbol. + If this is True, no error will be thrown when arg_params or aux_params + contain extra parameters that is not needed by the executor. """ pass diff --git a/python/mxnet/module/sequential_module.py b/python/mxnet/module/sequential_module.py index 21e30fb3b0ce..b2644c8e3864 100644 --- a/python/mxnet/module/sequential_module.py +++ b/python/mxnet/module/sequential_module.py @@ -154,7 +154,7 @@ def get_params(self): return (arg_params, aux_params) def init_params(self, initializer=Uniform(0.01), arg_params=None, aux_params=None, - allow_missing=False, force_init=False): + allow_missing=False, force_init=False, allow_extra=False): """Initializes parameters. Parameters @@ -171,6 +171,10 @@ def init_params(self, initializer=Uniform(0.01), arg_params=None, aux_params=Non In this case, missing values will be filled with `initializer`. force_init : bool Default ``False``. + allow_extra : boolean, optional + Whether allow extra parameters that are not needed by symbol. + If this is True, no error will be thrown when arg_params or aux_params + contain extra parameters that is not needed by the executor. """ if self.params_initialized and not force_init: return @@ -179,7 +183,7 @@ def init_params(self, initializer=Uniform(0.01), arg_params=None, aux_params=Non for module in self._modules: module.init_params(initializer=initializer, arg_params=arg_params, aux_params=aux_params, allow_missing=allow_missing, - force_init=force_init) + force_init=force_init, allow_extra=allow_extra) # make sure we do not have duplicated parameter names def _check_name(known_names, new_names, modules, i): diff --git a/tests/python/unittest/common.py b/tests/python/unittest/common.py index 29c489f0bf3c..38b4bd6b0227 100644 --- a/tests/python/unittest/common.py +++ b/tests/python/unittest/common.py @@ -5,3 +5,13 @@ import models import get_data + + +def assertRaises(expected_exception, func, *args, **kwargs): + try: + func(*args, **kwargs) + except expected_exception as e: + pass + else: + # Did not raise exception + assert False, "%s did not raise %s" % (func.__name__, expected_exception.__name__) diff --git a/tests/python/unittest/test_module.py b/tests/python/unittest/test_module.py index 9f3cff8e1265..8990aafe2838 100644 --- a/tests/python/unittest/test_module.py +++ b/tests/python/unittest/test_module.py @@ -3,6 +3,7 @@ import numpy as np from functools import reduce from mxnet.module.executor_group import DataParallelExecutorGroup +from common import assertRaises def test_module_dtype(): @@ -215,6 +216,70 @@ def create_bucketing_module(key): assert total_bytes_after == total_bytes_before + +def test_module_set_params(): + # data iter + mx.random.seed(11) + data = mx.nd.array([[0.05, .10]]); + label = mx.nd.array([[.01, 0.99]]); + train_data = mx.io.NDArrayIter(data, label, batch_size=1) + + # symbols + x = mx.symbol.Variable('data') + x = mx.symbol.FullyConnected(name='fc_0', data=x, num_hidden=2) + x = mx.symbol.Activation(name="act_0", data=x, act_type='sigmoid') + x = mx.symbol.FullyConnected(name='fc_1', data=x, num_hidden=2) + x = mx.symbol.Activation(name="act_1", data=x, act_type='sigmoid') + x = mx.symbol.LinearRegressionOutput(data=x, name='softmax', grad_scale=2) + + # create module + mod = mx.mod.Module(x, context=[mx.cpu()]); + mod.bind(train_data.provide_data, label_shapes=train_data.provide_label, + for_training=True) + + arg_params_correct = {'fc_0_weight': mx.nd.array([[.15, .20], [.25, .30]]), + 'fc_0_bias' : mx.nd.array([.35, .35]), + 'fc_1_weight': mx.nd.array([[.40, .45], [.50, .55]]), + 'fc_1_bias' : mx.nd.array([.60, .60])} + + arg_params_missing = {'fc_0_weight': mx.nd.array([[.15, .20], [.25, .30]]), + 'fc_0_bias' : mx.nd.array([.35, .35]), + 'fc_1_weight': mx.nd.array([[.40, .45], [.50, .55]])} + + arg_params_extra = {'fc_0_weight': mx.nd.array([[.15, .20], [.25, .30]]), + 'fc_0_bias' : mx.nd.array([.35, .35]), + 'fc_1_weight': mx.nd.array([[.40, .45], [.50, .55]]), + 'fc_1_bias' : mx.nd.array([.60, .60]), + 'fc_2_weight': mx.nd.array([.60, .60])} + + arg_params_missing_extra = {'fc_2_weight': mx.nd.array([.60, .60])} + + # test regular set_params + mod.set_params(force_init=True, arg_params=arg_params_correct, aux_params={}) + + # test allow missing + mod.set_params(force_init=True, arg_params=arg_params_missing, aux_params={}, allow_missing=True) + assertRaises(RuntimeError, mod.set_params, + force_init=True, arg_params=arg_params_missing, + aux_params={}, allow_missing=False) + + # test allow extra + mod.set_params(force_init=True, arg_params=arg_params_extra, aux_params={}, allow_missing=True, allow_extra=True) + assertRaises(ValueError, mod.set_params, + force_init=True, arg_params=arg_params_extra, + aux_params={}, allow_missing=True, allow_extra=False) + + # test allow missing + extra, + assertRaises(RuntimeError, mod.set_params, + force_init=True, arg_params=arg_params_missing_extra, + aux_params={}, allow_missing=False, allow_extra=False) + + # test allow missing + extra, this will throw a runtime error + assertRaises(ValueError, mod.set_params, + force_init=True, arg_params=arg_params_missing_extra, + aux_params={}, allow_missing=True, allow_extra=False) + + def test_monitor(): # data iter mx.random.seed(11) @@ -380,6 +445,7 @@ def test_shared_exec_group(exec_grp_shared, exec_grp_created, shared_arg_names=N test_module_input_grads() test_module_states() test_module_reshape() + test_module_set_params() test_save_load() test_module_layout() test_module_switch_bucket() From eb09f4402ab1cc0a6d6b8fd14e943ca1a65aae84 Mon Sep 17 00:00:00 2001 From: Hessel Tuinhof Date: Sat, 24 Jun 2017 01:43:11 +0200 Subject: [PATCH 110/834] [R] switch order of LRN and pooling layer (#6795) Original paper (section 3.5) performs local response normalization of relu. --- example/image-classification/symbol_alexnet.R | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/example/image-classification/symbol_alexnet.R b/example/image-classification/symbol_alexnet.R index 097444335451..b6698288cd42 100644 --- a/example/image-classification/symbol_alexnet.R +++ b/example/image-classification/symbol_alexnet.R @@ -5,13 +5,13 @@ get_symbol <- function(num_classes = 1000) { # stage 1 conv1 <- mx.symbol.Convolution(data = input_data, kernel = c(11, 11), stride = c(4, 4), num_filter = 96) relu1 <- mx.symbol.Activation(data = conv1, act_type = "relu") - pool1 <- mx.symbol.Pooling(data = relu1, pool_type = "max", kernel = c(3, 3), stride = c(2, 2)) - lrn1 <- mx.symbol.LRN(data = pool1, alpha = 0.0001, beta = 0.75, knorm = 2, nsize = 5) + lrn1 <- mx.symbol.LRN(data = relu1, alpha = 0.0001, beta = 0.75, knorm = 2, nsize = 5) + pool1 <- mx.symbol.Pooling(data = lrn1, kernel = c(3, 3), stride = c(2, 2), pool_type = "max") # stage 2 conv2 <- mx.symbol.Convolution(data = lrn1, kernel = c(5, 5), pad = c(2, 2), num_filter = 256) relu2 <- mx.symbol.Activation(data = conv2, act_type = "relu") - pool2 <- mx.symbol.Pooling(data = relu2, kernel = c(3, 3), stride = c(2, 2), pool_type = "max") - lrn2 <- mx.symbol.LRN(data = pool2, alpha = 0.0001, beta = 0.75, knorm = 2, nsize = 5) + lrn2 <- mx.symbol.LRN(data = relu2, alpha = 0.0001, beta = 0.75, knorm = 2, nsize = 5) + pool2 <- mx.symbol.Pooling(data = lrn2, kernel = c(3, 3), stride = c(2, 2), pool_type = "max") # stage 3 conv3 <- mx.symbol.Convolution(data = lrn2, kernel = c(3, 3), pad = c(1, 1), num_filter = 384) relu3 <- mx.symbol.Activation(data = conv3, act_type = "relu") From c160976d2ab8d31277fe2d24e6f6a13c80345e8b Mon Sep 17 00:00:00 2001 From: Przemyslaw Tredak Date: Fri, 23 Jun 2017 20:15:20 -0700 Subject: [PATCH 111/834] Removing unnecessary copies from backward pass of add and add_n (#6800) * Removing identity * Updating NNVM and modifying call to Gradient pass to work with lack of identity in backward_add Modifying add_n to not generate identity as well --- nnvm | 2 +- src/executor/graph_executor.cc | 2 +- src/operator/elemwise_op_common.h | 10 ++++++++++ src/operator/tensor/elemwise_binary_op_basic.cc | 2 +- src/operator/tensor/elemwise_sum.cc | 2 +- 5 files changed, 14 insertions(+), 4 deletions(-) diff --git a/nnvm b/nnvm index 7796ac76ccea..d73d6c5b37e7 160000 --- a/nnvm +++ b/nnvm @@ -1 +1 @@ -Subproject commit 7796ac76ccea1fba31afc32056c83f6da38b6c57 +Subproject commit d73d6c5b37e7376c1eb30a1ae5c7a42b1fbe22f5 diff --git a/src/executor/graph_executor.cc b/src/executor/graph_executor.cc index d60c5e46e52c..2be000112711 100644 --- a/src/executor/graph_executor.cc +++ b/src/executor/graph_executor.cc @@ -246,7 +246,7 @@ nnvm::Graph GraphExecutor::InitFullGraph(nnvm::Symbol symbol, nnvm::Graph g_grad = nnvm::pass::Gradient( g, symbol.outputs, xs, head_grad_entry_, AggregateGradient, need_mirror, nullptr, - zero_ops); + zero_ops, "_copy"); CHECK_EQ(g_grad.outputs.size(), xs.size()); for (const auto &e : g_grad.outputs) { g.outputs.push_back(e); diff --git a/src/operator/elemwise_op_common.h b/src/operator/elemwise_op_common.h index 9402ac0bf468..aa95d2d8696a 100644 --- a/src/operator/elemwise_op_common.h +++ b/src/operator/elemwise_op_common.h @@ -131,6 +131,16 @@ struct ElemwiseGradUseNone { } }; +struct CloneGradient { + const char *op_name; + std::vector operator()(const nnvm::NodePtr& n, + const std::vector& ograds) { + std::vector ret; + for (size_t i = 0; i < n->inputs.size(); ++i) + ret.emplace_back(ograds[0]); + return ret; + } +}; } // namespace op } // namespace mxnet diff --git a/src/operator/tensor/elemwise_binary_op_basic.cc b/src/operator/tensor/elemwise_binary_op_basic.cc index 1f363a114375..635f2a8692aa 100644 --- a/src/operator/tensor/elemwise_binary_op_basic.cc +++ b/src/operator/tensor/elemwise_binary_op_basic.cc @@ -12,7 +12,7 @@ MXNET_OPERATOR_REGISTER_BINARY(elemwise_add) .add_alias("_add").add_alias("_plus").add_alias("_Plus") .describe("Adds arguments element-wise.") .set_attr("FCompute", BinaryCompute) -.set_attr("FGradient", ElemwiseGradUseNone{"_backward_add"}); +.set_attr("FGradient", CloneGradient{"_backward_add"}); // specialized gradient add function to do add to optimization // this must differ from elemwise_add to prevent add to optimization in forward pass. diff --git a/src/operator/tensor/elemwise_sum.cc b/src/operator/tensor/elemwise_sum.cc index 06ec01e8ebd0..3c4bf719e18f 100644 --- a/src/operator/tensor/elemwise_sum.cc +++ b/src/operator/tensor/elemwise_sum.cc @@ -83,7 +83,7 @@ NNVM_REGISTER_OP(add_n) }) .set_attr("FInferShape", ElementWiseSumShape) .set_attr("FInferType", ElementWiseSumType) -.set_attr("FGradient", ElementWiseSumGrad) +.set_attr("FGradient", CloneGradient{"_backward_add_n"}) .add_argument("args", "NDArray-or-Symbol[]", "Positional input arguments"); From 5f2f12d8279efd920c531d7b03f25d992955000f Mon Sep 17 00:00:00 2001 From: Gu Wang Date: Mon, 26 Jun 2017 01:35:40 +0800 Subject: [PATCH 112/834] fix typos in the docstring of resnets (#6808) --- example/image-classification/symbols/resnet-v1-fp16.py | 4 ++-- example/image-classification/symbols/resnet-v1.py | 4 ++-- example/image-classification/symbols/resnet.py | 4 ++-- example/image-classification/symbols/resnet_fp16.py | 4 ++-- example/image-classification/symbols/resnext.py | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/example/image-classification/symbols/resnet-v1-fp16.py b/example/image-classification/symbols/resnet-v1-fp16.py index ae3f3c46fe7b..1f0e2fe28023 100755 --- a/example/image-classification/symbols/resnet-v1-fp16.py +++ b/example/image-classification/symbols/resnet-v1-fp16.py @@ -19,9 +19,9 @@ def residual_unit(data, num_filter, stride, dim_match, name, bottle_neck=True, b Number of output channels bnf : int Bottle neck channels factor with regard to num_filter - stride : tupe + stride : tuple Stride used in convolution - dim_match : Boolen + dim_match : Boolean True means channel number between input and output is the same, otherwise means differ name : str Base name of the operators diff --git a/example/image-classification/symbols/resnet-v1.py b/example/image-classification/symbols/resnet-v1.py index 9c515ba82cb9..2b898e5b8564 100755 --- a/example/image-classification/symbols/resnet-v1.py +++ b/example/image-classification/symbols/resnet-v1.py @@ -18,9 +18,9 @@ def residual_unit(data, num_filter, stride, dim_match, name, bottle_neck=True, b Number of output channels bnf : int Bottle neck channels factor with regard to num_filter - stride : tupe + stride : tuple Stride used in convolution - dim_match : Boolen + dim_match : Boolean True means channel number between input and output is the same, otherwise means differ name : str Base name of the operators diff --git a/example/image-classification/symbols/resnet.py b/example/image-classification/symbols/resnet.py index 9c121e8f9da7..a0c7002dcad9 100644 --- a/example/image-classification/symbols/resnet.py +++ b/example/image-classification/symbols/resnet.py @@ -18,9 +18,9 @@ def residual_unit(data, num_filter, stride, dim_match, name, bottle_neck=True, b Number of output channels bnf : int Bottle neck channels factor with regard to num_filter - stride : tupe + stride : tuple Stride used in convolution - dim_match : Boolen + dim_match : Boolean True means channel number between input and output is the same, otherwise means differ name : str Base name of the operators diff --git a/example/image-classification/symbols/resnet_fp16.py b/example/image-classification/symbols/resnet_fp16.py index b7759fc033fe..22d6d39dc36f 100755 --- a/example/image-classification/symbols/resnet_fp16.py +++ b/example/image-classification/symbols/resnet_fp16.py @@ -19,9 +19,9 @@ def residual_unit(data, num_filter, stride, dim_match, name, bottle_neck=True, b Number of output channels bnf : int Bottle neck channels factor with regard to num_filter - stride : tupe + stride : tuple Stride used in convolution - dim_match : Boolen + dim_match : Boolean True means channel number between input and output is the same, otherwise means differ name : str Base name of the operators diff --git a/example/image-classification/symbols/resnext.py b/example/image-classification/symbols/resnext.py index cc621602b87c..b6e7d7101fc3 100644 --- a/example/image-classification/symbols/resnext.py +++ b/example/image-classification/symbols/resnext.py @@ -17,9 +17,9 @@ def residual_unit(data, num_filter, stride, dim_match, name, bottle_neck=True, n Number of output channels bnf : int Bottle neck channels factor with regard to num_filter - stride : tupe + stride : tuple Stride used in convolution - dim_match : Boolen + dim_match : Boolean True means channel number between input and output is the same, otherwise means differ name : str Base name of the operators From 906c3585dd1f81bf4e966357e0e2db5cdbe4a315 Mon Sep 17 00:00:00 2001 From: "Qiang Kou (KK)" Date: Mon, 26 Jun 2017 11:13:09 -0700 Subject: [PATCH 113/834] [R] use ctx to be consistent. close #6818 (#6819) --- R-package/R/mlp.R | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/R-package/R/mlp.R b/R-package/R/mlp.R index ab3b56350d1e..33134ffbf48e 100644 --- a/R-package/R/mlp.R +++ b/R-package/R/mlp.R @@ -7,7 +7,7 @@ #' @param dropout a number in [0,1) containing the dropout ratio from the last hidden layer to the output layer. #' @param activation either a single string or a vector containing the names of the activation functions. #' @param out_activation a single string containing the name of the output activation function. -#' @param device whether train on cpu (default) or gpu. +#' @param ctx whether train on cpu (default) or gpu. #' @param eval_metric the evaluation metric/ #' @param ... other parameters passing to \code{mx.model.FeedForward.create}/ #' @@ -28,7 +28,7 @@ #' @export mx.mlp <- function(data, label, hidden_node = 1, out_node, dropout = NULL, activation = "tanh", out_activation = "softmax", - device=mx.ctx.default(), ...) { + ctx = mx.ctx.default(), ...) { m <- length(hidden_node) if (!is.null(dropout)) { @@ -64,6 +64,6 @@ mx.mlp <- function(data, label, hidden_node = 1, out_node, dropout = NULL, } else { stop("Not supported yet.") } - model <- mx.model.FeedForward.create(out, X=data, y=label, ctx=device, ...) + model <- mx.model.FeedForward.create(out, X=data, y=label, ctx = ctx, ...) return(model) } From 984aaa13b4d09f68900ce34944e4f2c1733dd7a6 Mon Sep 17 00:00:00 2001 From: Yao Wang Date: Mon, 26 Jun 2017 14:03:32 -0700 Subject: [PATCH 114/834] Change community to github for small screen (#6820) --- docs/_static/mxnet-theme/navbar.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_static/mxnet-theme/navbar.html b/docs/_static/mxnet-theme/navbar.html index e99cb179b1d8..1887f8cf520d 100644 --- a/docs/_static/mxnet-theme/navbar.html +++ b/docs/_static/mxnet-theme/navbar.html @@ -91,7 +91,7 @@

{% endfor %}
  • Architecture
    • -
  • Community
    • +
  • Github
    • From cc62aded9ce8bd078458553dc6bcb5ee16472f51 Mon Sep 17 00:00:00 2001 From: "Joshua Z. Zhang" Date: Mon, 26 Jun 2017 20:40:00 -0700 Subject: [PATCH 115/834] Better ssd (#6827) * adjust according to # ctx increase default cpu threads add ms coco dataset fix det_aug_default when label is invalid fix bracelets revert grad_scale and lr change lr-steps and end-epoch rescale by num_ctx only fix mxnet path remove hard-coded class names add coco utils for ground-truth fix os.path add new inception v3 ssd symbol rename fix rename add resnet 50 ssd symbol * rewrite symbol composing methods fix name add symbol to path modify lr-steps and end-epoch better names rename conv_act_layer names update demo,deploy, evalute remove handcrafted symbols add doc fix vgg16 fc->relu modify inceptionv3-ssd relax data_shape constraints fix resnet101 parameter tuning add back legacy models change default lr fix typo * add model link, update readme, wrap up fix path * fix lost commit train_net.py --- example/ssd/README.md | 35 +- example/ssd/config/config.py | 3 +- example/ssd/dataset/imdb.py | 32 +- example/ssd/dataset/mscoco.py | 115 +++++ example/ssd/dataset/names/mscoco.names | 80 ++++ example/ssd/dataset/names/pascal_voc.names | 20 + example/ssd/dataset/pascal_voc.py | 10 +- example/ssd/dataset/pycocotools/README.md | 2 + example/ssd/dataset/pycocotools/__init__.py | 1 + example/ssd/dataset/pycocotools/coco.py | 435 ++++++++++++++++++ example/ssd/demo.py | 66 ++- example/ssd/deploy.py | 25 +- example/ssd/evaluate.py | 23 +- example/ssd/evaluate/evaluate_net.py | 6 +- example/ssd/symbol/README.md | 49 ++ example/ssd/symbol/common.py | 100 +++- example/ssd/symbol/inceptionv3.py | 168 +++++++ ...g16_ssd_300.py => legacy_vgg16_ssd_300.py} | 24 +- ...g16_ssd_512.py => legacy_vgg16_ssd_512.py} | 22 +- example/ssd/symbol/resnet.py | 169 +++++++ example/ssd/symbol/symbol_builder.py | 166 +++++++ example/ssd/symbol/symbol_factory.py | 122 +++++ example/ssd/symbol/vgg16_reduced.py | 86 ++++ example/ssd/tools/prepare_coco.sh | 4 + example/ssd/tools/prepare_dataset.py | 30 ++ example/ssd/tools/visualize_net.py | 17 +- example/ssd/train.py | 14 +- example/ssd/train/train_net.py | Bin 9900 -> 9651 bytes src/io/image_det_aug_default.cc | 6 +- 29 files changed, 1723 insertions(+), 107 deletions(-) create mode 100644 example/ssd/dataset/mscoco.py create mode 100644 example/ssd/dataset/names/mscoco.names create mode 100644 example/ssd/dataset/names/pascal_voc.names create mode 100755 example/ssd/dataset/pycocotools/README.md create mode 100755 example/ssd/dataset/pycocotools/__init__.py create mode 100755 example/ssd/dataset/pycocotools/coco.py create mode 100644 example/ssd/symbol/README.md create mode 100644 example/ssd/symbol/inceptionv3.py rename example/ssd/symbol/{symbol_vgg16_ssd_300.py => legacy_vgg16_ssd_300.py} (90%) rename example/ssd/symbol/{symbol_vgg16_ssd_512.py => legacy_vgg16_ssd_512.py} (89%) create mode 100644 example/ssd/symbol/resnet.py create mode 100644 example/ssd/symbol/symbol_builder.py create mode 100644 example/ssd/symbol/symbol_factory.py create mode 100644 example/ssd/symbol/vgg16_reduced.py create mode 100644 example/ssd/tools/prepare_coco.sh diff --git a/example/ssd/README.md b/example/ssd/README.md index 8703a7cbbf73..5759fca611f2 100644 --- a/example/ssd/README.md +++ b/example/ssd/README.md @@ -17,6 +17,8 @@ remarkable traits of MXNet. Due to the permission issue, this example is maintained in this [repository](https://github.com/zhreshold/mxnet-ssd) separately. You can use the link regarding specific per example [issues](https://github.com/zhreshold/mxnet-ssd/issues). ### What's new +* Added multiple trained models. +* Added a much simpler way to compose network from mainstream classification networks (resnet, inception...) and [Guide](symbol/README.md). * Update to the latest version according to caffe version, with 5% mAP increase. * Use C++ record iterator based on back-end multi-thread engine to achieve huge speed up on multi-gpu environments. * Monitor validation mAP during training. @@ -30,11 +32,12 @@ Due to the permission issue, this example is maintained in this [repository](htt ![demo3](https://cloud.githubusercontent.com/assets/3307514/19171086/a9346842-8be0-11e6-8011-c17716b22ad3.png) ### mAP -| Model | Training data | Test data | mAP | -|:-----------------:|:----------------:|:---------:|:----:| -| [VGG16_reduced 300x300](https://github.com/zhreshold/mxnet-ssd/releases/download/v0.5-beta/vgg16_ssd_300_voc0712_trainval.zip) | VOC07+12 trainval| VOC07 test| 77.8| -| [VGG16_reduced 512x512](https://github.com/zhreshold/mxnet-ssd/releases/download/v0.5-beta/vgg16_ssd_512_voc0712_trainval.zip) | VOC07+12 trainval | VOC07 test| 79.9| -*More to be added* +| Model | Training data | Test data | mAP | Note | +|:-----------------:|:----------------:|:---------:|:----:|:-----| +| [VGG16_reduced 300x300](https://github.com/zhreshold/mxnet-ssd/releases/download/v0.5-beta/vgg16_ssd_300_voc0712_trainval.zip) | VOC07+12 trainval| VOC07 test| 77.8| fast | +| [VGG16_reduced 512x512](https://github.com/zhreshold/mxnet-ssd/releases/download/v0.5-beta/vgg16_ssd_512_voc0712_trainval.zip) | VOC07+12 trainval | VOC07 test| 79.9| slow | +| [Inception-v3 512x512](https://github.com/zhreshold/mxnet-ssd/releases/download/v0.6/inceptionv3_ssd_512_voc0712_trainval.zip) | VOC07+12 trainval| VOC07 test| 78.9 | fastest | +| [Resnet-50 512x512](https://github.com/zhreshold/mxnet-ssd/releases/download/v0.6/resnet50_ssd_512_voc0712_trainval.zip) | VOC07+12 trainval| VOC07 test| 78.9 | fast | ### Speed | Model | GPU | CUDNN | Batch-size | FPS* | @@ -65,13 +68,14 @@ Remember to enable CUDA if you want to be able to train, since CPU training is insanely slow. Using CUDNN is optional, but highly recommended. ### Try the demo -* Download the pretrained model: [`ssd_300_voc_0712.zip`](https://github.com/zhreshold/mxnet-ssd/releases/download/v0.5-beta/vgg16_ssd_300_voc0712_trainval.zip), and extract to `model/` directory. +* Download the pretrained model: [`ssd_resnet50_0712.zip`](https://github.com/zhreshold/mxnet-ssd/releases/download/v0.6/resnet50_ssd_512_voc0712_trainval.zip), and extract to `model/` directory. * Run ``` -# cd /path/to/mxnet/example/ssd -python demo.py +# cd /path/to/mxnet-ssd +python demo.py --gpu 0 # play with examples: python demo.py --epoch 0 --images ./data/demo/dog.jpg --thresh 0.5 +python demo.py --cpu --network resnet50 --data-shape 512 # wait for library to load for the first time ``` * Check `python demo.py --help` for more options. @@ -93,7 +97,7 @@ tar -xvf VOCtrainval_11-May-2012.tar tar -xvf VOCtrainval_06-Nov-2007.tar tar -xvf VOCtest_06-Nov-2007.tar ``` -* We are goint to use `trainval` set in VOC2007/2012 as a common strategy. +* We are going to use `trainval` set in VOC2007/2012 as a common strategy. The suggested directory structure is to store `VOC2007` and `VOC2012` directories in the same `VOCdevkit` folder. * Then link `VOCdevkit` folder to `data/VOCdevkit` by default: @@ -114,12 +118,12 @@ python tools/prepare_dataset.py --dataset pascal --year 2007 --set test --target # cd /path/to/mxnet/example/ssd python train.py ``` -* By default, this example will use `batch-size=32` and `learning_rate=0.004`. +* By default, this example will use `batch-size=32` and `learning_rate=0.002`. You might need to change the parameters a bit if you have different configurations. Check `python train.py --help` for more training options. For example, if you have 4 GPUs, use: ``` # note that a perfect training parameter set is yet to be discovered for multi-GPUs -python train.py --gpus 0,1,2,3 --batch-size 128 --lr 0.001 +python train.py --gpus 0,1,2,3 --batch-size 32 ``` ### Evalute trained model @@ -148,3 +152,12 @@ python convert_model.py deploy.prototxt name_of_pretrained_caffe_model.caffemode python demo.py --prefix ssd_converted --epoch 1 --deploy ``` There is no guarantee that conversion will always work, but at least it's good for now. + +### Legacy models +Since the new interface for composing network is introduced, the old models have inconsistent names for weights. +You can still load the previous model by rename the symbol to `legacy_xxx.py` +and call with `python train/demo.py --network legacy_xxx ` +For example: +``` +python demo.py --network 'legacy_vgg16_ssd_300.py' --prefix model/ssd_300 --epoch 0 +``` diff --git a/example/ssd/config/config.py b/example/ssd/config/config.py index 931ad16f14eb..278b770febe9 100644 --- a/example/ssd/config/config.py +++ b/example/ssd/config/config.py @@ -53,7 +53,7 @@ cfg.train.rand_mirror_prob = 0.5 cfg.train.shuffle = True cfg.train.seed = 233 -cfg.train.preprocess_threads = 6 +cfg.train.preprocess_threads = 48 cfg.train = config_as_dict(cfg.train) # convert to normal dict # validation @@ -64,4 +64,5 @@ cfg.valid.rand_mirror_prob = 0 cfg.valid.shuffle = False cfg.valid.seed = 0 +cfg.valid.preprocess_threads = 32 cfg.valid = config_as_dict(cfg.valid) # convert to normal dict diff --git a/example/ssd/dataset/imdb.py b/example/ssd/dataset/imdb.py index 95b082d594d9..279fe9cab4a1 100644 --- a/example/ssd/dataset/imdb.py +++ b/example/ssd/dataset/imdb.py @@ -14,7 +14,7 @@ def __init__(self, name): self.name = name self.classes = [] self.num_classes = 0 - self.image_set_index = [] + self.image_set_index = None self.num_images = 0 self.labels = None self.padding = 0 @@ -59,9 +59,22 @@ def save_imglist(self, fname=None, root=None, shuffle=False): fname : str saved filename """ + def progress_bar(count, total, suffix=''): + import sys + bar_len = 24 + filled_len = int(round(bar_len * count / float(total))) + + percents = round(100.0 * count / float(total), 1) + bar = '=' * filled_len + '-' * (bar_len - filled_len) + sys.stdout.write('[%s] %s%s ...%s\r' % (bar, percents, '%', suffix)) + sys.stdout.flush() + str_list = [] for index in range(self.num_images): + progress_bar(index, self.num_images) label = self.label_from_index(index) + if label.size < 1: + continue path = self.image_path_from_index(index) if root: path = osp.relpath(path, root) @@ -78,3 +91,20 @@ def save_imglist(self, fname=None, root=None, shuffle=False): f.write(line) else: raise RuntimeError("No image in imdb") + + def _load_class_names(self, filename, dirname): + """ + load class names from text file + + Parameters: + ---------- + filename: str + file stores class names + dirname: str + file directory + """ + full_path = osp.join(dirname, filename) + classes = [] + with open(full_path, 'r') as f: + classes = [l.strip() for l in f.readlines()] + return classes diff --git a/example/ssd/dataset/mscoco.py b/example/ssd/dataset/mscoco.py new file mode 100644 index 000000000000..b46b22745413 --- /dev/null +++ b/example/ssd/dataset/mscoco.py @@ -0,0 +1,115 @@ +import os +import numpy as np +from imdb import Imdb +from pycocotools.coco import COCO + + +class Coco(Imdb): + """ + Implementation of Imdb for MSCOCO dataset: https://http://mscoco.org + + Parameters: + ---------- + anno_file : str + annotation file for coco, a json file + image_dir : str + image directory for coco images + shuffle : bool + whether initially shuffle image list + + """ + def __init__(self, anno_file, image_dir, shuffle=True, names='mscoco.names'): + assert os.path.isfile(anno_file), "Invalid annotation file: " + anno_file + basename = os.path.splitext(os.path.basename(anno_file))[0] + super(Coco, self).__init__('coco_' + basename) + self.image_dir = image_dir + + self.classes = self._load_class_names(names, + os.path.join(os.path.dirname(__file__), 'names')) + + self.num_classes = len(self.classes) + self._load_all(anno_file, shuffle) + self.num_images = len(self.image_set_index) + + + def image_path_from_index(self, index): + """ + given image index, find out full path + + Parameters: + ---------- + index: int + index of a specific image + Returns: + ---------- + full path of this image + """ + assert self.image_set_index is not None, "Dataset not initialized" + name = self.image_set_index[index] + image_file = os.path.join(self.image_dir, 'images', name) + assert os.path.isfile(image_file), 'Path does not exist: {}'.format(image_file) + return image_file + + def label_from_index(self, index): + """ + given image index, return preprocessed ground-truth + + Parameters: + ---------- + index: int + index of a specific image + Returns: + ---------- + ground-truths of this image + """ + assert self.labels is not None, "Labels not processed" + return self.labels[index] + + def _load_all(self, anno_file, shuffle): + """ + initialize all entries given annotation json file + + Parameters: + ---------- + anno_file: str + annotation json file + shuffle: bool + whether to shuffle image list + """ + image_set_index = [] + labels = [] + coco = COCO(anno_file) + img_ids = coco.getImgIds() + for img_id in img_ids: + # filename + image_info = coco.loadImgs(img_id)[0] + filename = image_info["file_name"] + subdir = filename.split('_')[1] + height = image_info["height"] + width = image_info["width"] + # label + anno_ids = coco.getAnnIds(imgIds=img_id) + annos = coco.loadAnns(anno_ids) + label = [] + for anno in annos: + cat_id = int(anno["category_id"]) + bbox = anno["bbox"] + assert len(bbox) == 4 + xmin = float(bbox[0]) / width + ymin = float(bbox[1]) / height + xmax = xmin + float(bbox[2]) / width + ymax = ymin + float(bbox[3]) / height + label.append([cat_id, xmin, ymin, xmax, ymax, 0]) + if label: + labels.append(np.array(label)) + image_set_index.append(os.path.join(subdir, filename)) + + if shuffle: + import random + indices = range(len(image_set_index)) + random.shuffle(indices) + image_set_index = [image_set_index[i] for i in indices] + labels = [labels[i] for i in indices] + # store the results + self.image_set_index = image_set_index + self.labels = labels diff --git a/example/ssd/dataset/names/mscoco.names b/example/ssd/dataset/names/mscoco.names new file mode 100644 index 000000000000..ca76c80b5b2c --- /dev/null +++ b/example/ssd/dataset/names/mscoco.names @@ -0,0 +1,80 @@ +person +bicycle +car +motorbike +aeroplane +bus +train +truck +boat +traffic light +fire hydrant +stop sign +parking meter +bench +bird +cat +dog +horse +sheep +cow +elephant +bear +zebra +giraffe +backpack +umbrella +handbag +tie +suitcase +frisbee +skis +snowboard +sports ball +kite +baseball bat +baseball glove +skateboard +surfboard +tennis racket +bottle +wine glass +cup +fork +knife +spoon +bowl +banana +apple +sandwich +orange +broccoli +carrot +hot dog +pizza +donut +cake +chair +sofa +pottedplant +bed +diningtable +toilet +tvmonitor +laptop +mouse +remote +keyboard +cell phone +microwave +oven +toaster +sink +refrigerator +book +clock +vase +scissors +teddy bear +hair drier +toothbrush diff --git a/example/ssd/dataset/names/pascal_voc.names b/example/ssd/dataset/names/pascal_voc.names new file mode 100644 index 000000000000..8420ab35ede7 --- /dev/null +++ b/example/ssd/dataset/names/pascal_voc.names @@ -0,0 +1,20 @@ +aeroplane +bicycle +bird +boat +bottle +bus +car +cat +chair +cow +diningtable +dog +horse +motorbike +person +pottedplant +sheep +sofa +train +tvmonitor diff --git a/example/ssd/dataset/pascal_voc.py b/example/ssd/dataset/pascal_voc.py index 2c61be74e156..31e287e3cbac 100644 --- a/example/ssd/dataset/pascal_voc.py +++ b/example/ssd/dataset/pascal_voc.py @@ -24,7 +24,8 @@ class PascalVoc(Imdb): is_train : boolean if true, will load annotations """ - def __init__(self, image_set, year, devkit_path, shuffle=False, is_train=False): + def __init__(self, image_set, year, devkit_path, shuffle=False, is_train=False, + names='pascal_voc.names'): super(PascalVoc, self).__init__('voc_' + year + '_' + image_set) self.image_set = image_set self.year = year @@ -33,11 +34,8 @@ def __init__(self, image_set, year, devkit_path, shuffle=False, is_train=False): self.extension = '.jpg' self.is_train = is_train - self.classes = ['aeroplane', 'bicycle', 'bird', 'boat', - 'bottle', 'bus', 'car', 'cat', 'chair', - 'cow', 'diningtable', 'dog', 'horse', - 'motorbike', 'person', 'pottedplant', - 'sheep', 'sofa', 'train', 'tvmonitor'] + self.classes = self._load_class_names(names, + os.path.join(os.path.dirname(__file__), 'names')) self.config = {'use_difficult': True, 'comp_id': 'comp4',} diff --git a/example/ssd/dataset/pycocotools/README.md b/example/ssd/dataset/pycocotools/README.md new file mode 100755 index 000000000000..d358f53105da --- /dev/null +++ b/example/ssd/dataset/pycocotools/README.md @@ -0,0 +1,2 @@ +This is a modified version of https://github.com/pdollar/coco python API. +No `make` is required, but this will not support mask functions. diff --git a/example/ssd/dataset/pycocotools/__init__.py b/example/ssd/dataset/pycocotools/__init__.py new file mode 100755 index 000000000000..3f7d85bba884 --- /dev/null +++ b/example/ssd/dataset/pycocotools/__init__.py @@ -0,0 +1 @@ +__author__ = 'tylin' diff --git a/example/ssd/dataset/pycocotools/coco.py b/example/ssd/dataset/pycocotools/coco.py new file mode 100755 index 000000000000..a8939f64a38c --- /dev/null +++ b/example/ssd/dataset/pycocotools/coco.py @@ -0,0 +1,435 @@ +__author__ = 'tylin' +__version__ = '2.0' +# Interface for accessing the Microsoft COCO dataset. + +# Microsoft COCO is a large image dataset designed for object detection, +# segmentation, and caption generation. pycocotools is a Python API that +# assists in loading, parsing and visualizing the annotations in COCO. +# Please visit http://mscoco.org/ for more information on COCO, including +# for the data, paper, and tutorials. The exact format of the annotations +# is also described on the COCO website. For example usage of the pycocotools +# please see pycocotools_demo.ipynb. In addition to this API, please download both +# the COCO images and annotations in order to run the demo. + +# An alternative to using the API is to load the annotations directly +# into Python dictionary +# Using the API provides additional utility functions. Note that this API +# supports both *instance* and *caption* annotations. In the case of +# captions not all functions are defined (e.g. categories are undefined). + +# The following API functions are defined: +# COCO - COCO api class that loads COCO annotation file and prepare data structures. +# decodeMask - Decode binary mask M encoded via run-length encoding. +# encodeMask - Encode binary mask M using run-length encoding. +# getAnnIds - Get ann ids that satisfy given filter conditions. +# getCatIds - Get cat ids that satisfy given filter conditions. +# getImgIds - Get img ids that satisfy given filter conditions. +# loadAnns - Load anns with the specified ids. +# loadCats - Load cats with the specified ids. +# loadImgs - Load imgs with the specified ids. +# annToMask - Convert segmentation in an annotation to binary mask. +# showAnns - Display the specified annotations. +# loadRes - Load algorithm results and create API for accessing them. +# download - Download COCO images from mscoco.org server. +# Throughout the API "ann"=annotation, "cat"=category, and "img"=image. +# Help on each functions can be accessed by: "help COCO>function". + +# See also COCO>decodeMask, +# COCO>encodeMask, COCO>getAnnIds, COCO>getCatIds, +# COCO>getImgIds, COCO>loadAnns, COCO>loadCats, +# COCO>loadImgs, COCO>annToMask, COCO>showAnns + +# Microsoft COCO Toolbox. version 2.0 +# Data, paper, and tutorials available at: http://mscoco.org/ +# Code written by Piotr Dollar and Tsung-Yi Lin, 2014. +# Licensed under the Simplified BSD License [see bsd.txt] + +import json +import time +import matplotlib.pyplot as plt +from matplotlib.collections import PatchCollection +from matplotlib.patches import Polygon +import numpy as np +import copy +import itertools +# from . import mask as maskUtils +import os +from collections import defaultdict +import sys +PYTHON_VERSION = sys.version_info[0] +if PYTHON_VERSION == 2: + from urllib import urlretrieve +elif PYTHON_VERSION == 3: + from urllib.request import urlretrieve + +class COCO: + def __init__(self, annotation_file=None): + """ + Constructor of Microsoft COCO helper class for reading and visualizing annotations. + :param annotation_file (str): location of annotation file + :param image_folder (str): location to the folder that hosts images. + :return: + """ + # load dataset + self.dataset,self.anns,self.cats,self.imgs = dict(),dict(),dict(),dict() + self.imgToAnns, self.catToImgs = defaultdict(list), defaultdict(list) + if not annotation_file == None: + print('loading annotations into memory...') + tic = time.time() + dataset = json.load(open(annotation_file, 'r')) + assert type(dataset)==dict, 'annotation file format {} not supported'.format(type(dataset)) + print('Done (t={:0.2f}s)'.format(time.time()- tic)) + self.dataset = dataset + self.createIndex() + + def createIndex(self): + # create index + print('creating index...') + anns, cats, imgs = {}, {}, {} + imgToAnns,catToImgs = defaultdict(list),defaultdict(list) + if 'annotations' in self.dataset: + for ann in self.dataset['annotations']: + imgToAnns[ann['image_id']].append(ann) + anns[ann['id']] = ann + + if 'images' in self.dataset: + for img in self.dataset['images']: + imgs[img['id']] = img + + if 'categories' in self.dataset: + for cat in self.dataset['categories']: + cats[cat['id']] = cat + + if 'annotations' in self.dataset and 'categories' in self.dataset: + for ann in self.dataset['annotations']: + catToImgs[ann['category_id']].append(ann['image_id']) + + print('index created!') + + # create class members + self.anns = anns + self.imgToAnns = imgToAnns + self.catToImgs = catToImgs + self.imgs = imgs + self.cats = cats + + def info(self): + """ + Print information about the annotation file. + :return: + """ + for key, value in self.dataset['info'].items(): + print('{}: {}'.format(key, value)) + + def getAnnIds(self, imgIds=[], catIds=[], areaRng=[], iscrowd=None): + """ + Get ann ids that satisfy given filter conditions. default skips that filter + :param imgIds (int array) : get anns for given imgs + catIds (int array) : get anns for given cats + areaRng (float array) : get anns for given area range (e.g. [0 inf]) + iscrowd (boolean) : get anns for given crowd label (False or True) + :return: ids (int array) : integer array of ann ids + """ + imgIds = imgIds if type(imgIds) == list else [imgIds] + catIds = catIds if type(catIds) == list else [catIds] + + if len(imgIds) == len(catIds) == len(areaRng) == 0: + anns = self.dataset['annotations'] + else: + if not len(imgIds) == 0: + lists = [self.imgToAnns[imgId] for imgId in imgIds if imgId in self.imgToAnns] + anns = list(itertools.chain.from_iterable(lists)) + else: + anns = self.dataset['annotations'] + anns = anns if len(catIds) == 0 else [ann for ann in anns if ann['category_id'] in catIds] + anns = anns if len(areaRng) == 0 else [ann for ann in anns if ann['area'] > areaRng[0] and ann['area'] < areaRng[1]] + if not iscrowd == None: + ids = [ann['id'] for ann in anns if ann['iscrowd'] == iscrowd] + else: + ids = [ann['id'] for ann in anns] + return ids + + def getCatIds(self, catNms=[], supNms=[], catIds=[]): + """ + filtering parameters. default skips that filter. + :param catNms (str array) : get cats for given cat names + :param supNms (str array) : get cats for given supercategory names + :param catIds (int array) : get cats for given cat ids + :return: ids (int array) : integer array of cat ids + """ + catNms = catNms if type(catNms) == list else [catNms] + supNms = supNms if type(supNms) == list else [supNms] + catIds = catIds if type(catIds) == list else [catIds] + + if len(catNms) == len(supNms) == len(catIds) == 0: + cats = self.dataset['categories'] + else: + cats = self.dataset['categories'] + cats = cats if len(catNms) == 0 else [cat for cat in cats if cat['name'] in catNms] + cats = cats if len(supNms) == 0 else [cat for cat in cats if cat['supercategory'] in supNms] + cats = cats if len(catIds) == 0 else [cat for cat in cats if cat['id'] in catIds] + ids = [cat['id'] for cat in cats] + return ids + + def getImgIds(self, imgIds=[], catIds=[]): + ''' + Get img ids that satisfy given filter conditions. + :param imgIds (int array) : get imgs for given ids + :param catIds (int array) : get imgs with all given cats + :return: ids (int array) : integer array of img ids + ''' + imgIds = imgIds if type(imgIds) == list else [imgIds] + catIds = catIds if type(catIds) == list else [catIds] + + if len(imgIds) == len(catIds) == 0: + ids = self.imgs.keys() + else: + ids = set(imgIds) + for i, catId in enumerate(catIds): + if i == 0 and len(ids) == 0: + ids = set(self.catToImgs[catId]) + else: + ids &= set(self.catToImgs[catId]) + return list(ids) + + def loadAnns(self, ids=[]): + """ + Load anns with the specified ids. + :param ids (int array) : integer ids specifying anns + :return: anns (object array) : loaded ann objects + """ + if type(ids) == list: + return [self.anns[id] for id in ids] + elif type(ids) == int: + return [self.anns[ids]] + + def loadCats(self, ids=[]): + """ + Load cats with the specified ids. + :param ids (int array) : integer ids specifying cats + :return: cats (object array) : loaded cat objects + """ + if type(ids) == list: + return [self.cats[id] for id in ids] + elif type(ids) == int: + return [self.cats[ids]] + + def loadImgs(self, ids=[]): + """ + Load anns with the specified ids. + :param ids (int array) : integer ids specifying img + :return: imgs (object array) : loaded img objects + """ + if type(ids) == list: + return [self.imgs[id] for id in ids] + elif type(ids) == int: + return [self.imgs[ids]] + + def showAnns(self, anns): + """ + Display the specified annotations. + :param anns (array of object): annotations to display + :return: None + """ + if len(anns) == 0: + return 0 + if 'segmentation' in anns[0] or 'keypoints' in anns[0]: + datasetType = 'instances' + elif 'caption' in anns[0]: + datasetType = 'captions' + else: + raise Exception('datasetType not supported') + if datasetType == 'instances': + ax = plt.gca() + ax.set_autoscale_on(False) + polygons = [] + color = [] + for ann in anns: + c = (np.random.random((1, 3))*0.6+0.4).tolist()[0] + if 'segmentation' in ann: + if type(ann['segmentation']) == list: + # polygon + for seg in ann['segmentation']: + poly = np.array(seg).reshape((int(len(seg)/2), 2)) + polygons.append(Polygon(poly)) + color.append(c) + else: + # mask + t = self.imgs[ann['image_id']] + if type(ann['segmentation']['counts']) == list: + # rle = maskUtils.frPyObjects([ann['segmentation']], t['height'], t['width']) + raise NotImplementedError("maskUtils disabled!") + else: + rle = [ann['segmentation']] + # m = maskUtils.decode(rle) + raise NotImplementedError("maskUtils disabled!") + img = np.ones( (m.shape[0], m.shape[1], 3) ) + if ann['iscrowd'] == 1: + color_mask = np.array([2.0,166.0,101.0])/255 + if ann['iscrowd'] == 0: + color_mask = np.random.random((1, 3)).tolist()[0] + for i in range(3): + img[:,:,i] = color_mask[i] + ax.imshow(np.dstack( (img, m*0.5) )) + if 'keypoints' in ann and type(ann['keypoints']) == list: + # turn skeleton into zero-based index + sks = np.array(self.loadCats(ann['category_id'])[0]['skeleton'])-1 + kp = np.array(ann['keypoints']) + x = kp[0::3] + y = kp[1::3] + v = kp[2::3] + for sk in sks: + if np.all(v[sk]>0): + plt.plot(x[sk],y[sk], linewidth=3, color=c) + plt.plot(x[v>0], y[v>0],'o',markersize=8, markerfacecolor=c, markeredgecolor='k',markeredgewidth=2) + plt.plot(x[v>1], y[v>1],'o',markersize=8, markerfacecolor=c, markeredgecolor=c, markeredgewidth=2) + p = PatchCollection(polygons, facecolor=color, linewidths=0, alpha=0.4) + ax.add_collection(p) + p = PatchCollection(polygons, facecolor='none', edgecolors=color, linewidths=2) + ax.add_collection(p) + elif datasetType == 'captions': + for ann in anns: + print(ann['caption']) + + def loadRes(self, resFile): + """ + Load result file and return a result api object. + :param resFile (str) : file name of result file + :return: res (obj) : result api object + """ + res = COCO() + res.dataset['images'] = [img for img in self.dataset['images']] + + print('Loading and preparing results...') + tic = time.time() + if type(resFile) == str or type(resFile) == unicode: + anns = json.load(open(resFile)) + elif type(resFile) == np.ndarray: + anns = self.loadNumpyAnnotations(resFile) + else: + anns = resFile + assert type(anns) == list, 'results in not an array of objects' + annsImgIds = [ann['image_id'] for ann in anns] + assert set(annsImgIds) == (set(annsImgIds) & set(self.getImgIds())), \ + 'Results do not correspond to current coco set' + if 'caption' in anns[0]: + imgIds = set([img['id'] for img in res.dataset['images']]) & set([ann['image_id'] for ann in anns]) + res.dataset['images'] = [img for img in res.dataset['images'] if img['id'] in imgIds] + for id, ann in enumerate(anns): + ann['id'] = id+1 + elif 'bbox' in anns[0] and not anns[0]['bbox'] == []: + res.dataset['categories'] = copy.deepcopy(self.dataset['categories']) + for id, ann in enumerate(anns): + bb = ann['bbox'] + x1, x2, y1, y2 = [bb[0], bb[0]+bb[2], bb[1], bb[1]+bb[3]] + if not 'segmentation' in ann: + ann['segmentation'] = [[x1, y1, x1, y2, x2, y2, x2, y1]] + ann['area'] = bb[2]*bb[3] + ann['id'] = id+1 + ann['iscrowd'] = 0 + elif 'segmentation' in anns[0]: + res.dataset['categories'] = copy.deepcopy(self.dataset['categories']) + for id, ann in enumerate(anns): + # now only support compressed RLE format as segmentation results + # ann['area'] = maskUtils.area(ann['segmentation']) + raise NotImplementedError("maskUtils disabled!") + if not 'bbox' in ann: + # ann['bbox'] = maskUtils.toBbox(ann['segmentation']) + raise NotImplementedError("maskUtils disabled!") + ann['id'] = id+1 + ann['iscrowd'] = 0 + elif 'keypoints' in anns[0]: + res.dataset['categories'] = copy.deepcopy(self.dataset['categories']) + for id, ann in enumerate(anns): + s = ann['keypoints'] + x = s[0::3] + y = s[1::3] + x0,x1,y0,y1 = np.min(x), np.max(x), np.min(y), np.max(y) + ann['area'] = (x1-x0)*(y1-y0) + ann['id'] = id + 1 + ann['bbox'] = [x0,y0,x1-x0,y1-y0] + print('DONE (t={:0.2f}s)'.format(time.time()- tic)) + + res.dataset['annotations'] = anns + res.createIndex() + return res + + def download(self, tarDir = None, imgIds = [] ): + ''' + Download COCO images from mscoco.org server. + :param tarDir (str): COCO results directory name + imgIds (list): images to be downloaded + :return: + ''' + if tarDir is None: + print('Please specify target directory') + return -1 + if len(imgIds) == 0: + imgs = self.imgs.values() + else: + imgs = self.loadImgs(imgIds) + N = len(imgs) + if not os.path.exists(tarDir): + os.makedirs(tarDir) + for i, img in enumerate(imgs): + tic = time.time() + fname = os.path.join(tarDir, img['file_name']) + if not os.path.exists(fname): + urlretrieve(img['coco_url'], fname) + print('downloaded {}/{} images (t={:0.1f}s)'.format(i, N, time.time()- tic)) + + def loadNumpyAnnotations(self, data): + """ + Convert result data from a numpy array [Nx7] where each row contains {imageID,x1,y1,w,h,score,class} + :param data (numpy.ndarray) + :return: annotations (python nested list) + """ + print('Converting ndarray to lists...') + assert(type(data) == np.ndarray) + print(data.shape) + assert(data.shape[1] == 7) + N = data.shape[0] + ann = [] + for i in range(N): + if i % 1000000 == 0: + print('{}/{}'.format(i,N)) + ann += [{ + 'image_id' : int(data[i, 0]), + 'bbox' : [ data[i, 1], data[i, 2], data[i, 3], data[i, 4] ], + 'score' : data[i, 5], + 'category_id': int(data[i, 6]), + }] + return ann + + def annToRLE(self, ann): + """ + Convert annotation which can be polygons, uncompressed RLE to RLE. + :return: binary mask (numpy 2D array) + """ + t = self.imgs[ann['image_id']] + h, w = t['height'], t['width'] + segm = ann['segmentation'] + if type(segm) == list: + # polygon -- a single object might consist of multiple parts + # we merge all parts into one mask rle code + # rles = maskUtils.frPyObjects(segm, h, w) + # rle = maskUtils.merge(rles) + raise NotImplementedError("maskUtils disabled!") + elif type(segm['counts']) == list: + # uncompressed RLE + # rle = maskUtils.frPyObjects(segm, h, w) + raise NotImplementedError("maskUtils disabled!") + else: + # rle + rle = ann['segmentation'] + return rle + + def annToMask(self, ann): + """ + Convert annotation which can be polygons, uncompressed RLE, or RLE to binary mask. + :return: binary mask (numpy 2D array) + """ + rle = self.annToRLE(ann) + # m = maskUtils.decode(rle) + raise NotImplementedError("maskUtils disabled!") + return m diff --git a/example/ssd/demo.py b/example/ssd/demo.py index ededbdb59b88..bda4606543d7 100644 --- a/example/ssd/demo.py +++ b/example/ssd/demo.py @@ -2,18 +2,12 @@ import tools.find_mxnet import mxnet as mx import os -import importlib import sys from detect.detector import Detector +from symbol.symbol_factory import get_symbol -CLASSES = ('aeroplane', 'bicycle', 'bird', 'boat', - 'bottle', 'bus', 'car', 'cat', 'chair', - 'cow', 'diningtable', 'dog', 'horse', - 'motorbike', 'person', 'pottedplant', - 'sheep', 'sofa', 'train', 'tvmonitor') - -def get_detector(net, prefix, epoch, data_shape, mean_pixels, ctx, - nms_thresh=0.5, force_nms=True): +def get_detector(net, prefix, epoch, data_shape, mean_pixels, ctx, num_class, + nms_thresh=0.5, force_nms=True, nms_topk=400): """ wrapper for initialize a detector @@ -31,23 +25,25 @@ def get_detector(net, prefix, epoch, data_shape, mean_pixels, ctx, mean pixel values (R, G, B) ctx : mx.ctx running context, mx.cpu() or mx.gpu(?) + num_class : int + number of classes + nms_thresh : float + non-maximum suppression threshold force_nms : bool force suppress different categories """ - sys.path.append(os.path.join(os.getcwd(), 'symbol')) if net is not None: - net = importlib.import_module("symbol_" + net) \ - .get_symbol(len(CLASSES), nms_thresh, force_nms) - detector = Detector(net, prefix + "_" + str(data_shape), epoch, \ - data_shape, mean_pixels, ctx=ctx) + net = get_symbol(net, data_shape, num_classes=num_class, nms_thresh=nms_thresh, + force_nms=force_nms, nms_topk=nms_topk) + detector = Detector(net, prefix, epoch, data_shape, mean_pixels, ctx=ctx) return detector def parse_args(): parser = argparse.ArgumentParser(description='Single-shot detection network demo') - parser.add_argument('--network', dest='network', type=str, default='vgg16_ssd_300', - choices=['vgg16_ssd_300', 'vgg16_ssd_512'], help='which network to use') + parser.add_argument('--network', dest='network', type=str, default='resnet50', + help='which network to use') parser.add_argument('--images', dest='images', type=str, default='./data/demo/dog.jpg', - help='run demo with images, use comma(without space) to seperate multiple images') + help='run demo with images, use comma to seperate multiple images') parser.add_argument('--dir', dest='dir', nargs='?', help='demo image directory, optional', type=str) parser.add_argument('--ext', dest='extension', help='image extension, optional', @@ -55,12 +51,13 @@ def parse_args(): parser.add_argument('--epoch', dest='epoch', help='epoch of trained model', default=0, type=int) parser.add_argument('--prefix', dest='prefix', help='trained model prefix', - default=os.path.join(os.getcwd(), 'model', 'ssd'), type=str) + default=os.path.join(os.getcwd(), 'model', 'ssd_'), + type=str) parser.add_argument('--cpu', dest='cpu', help='(override GPU) use CPU to detect', action='store_true', default=False) parser.add_argument('--gpu', dest='gpu_id', type=int, default=0, help='GPU device id to detect with') - parser.add_argument('--data-shape', dest='data_shape', type=int, default=300, + parser.add_argument('--data-shape', dest='data_shape', type=int, default=512, help='set image shape') parser.add_argument('--mean-r', dest='mean_r', type=float, default=123, help='red mean value') @@ -78,9 +75,29 @@ def parse_args(): help='show detection time') parser.add_argument('--deploy', dest='deploy_net', action='store_true', default=False, help='Load network from json file, rather than from symbol') + parser.add_argument('--class-names', dest='class_names', type=str, + default='aeroplane, bicycle, bird, boat, bottle, bus, \ + car, cat, chair, cow, diningtable, dog, horse, motorbike, \ + person, pottedplant, sheep, sofa, train, tvmonitor', + help='string of comma separated names, or text filename') args = parser.parse_args() return args +def parse_class_names(class_names): + """ parse # classes and class_names if applicable """ + if len(class_names) > 0: + if os.path.isfile(class_names): + # try to open it to read class names + with open(class_names, 'r') as f: + class_names = [l.strip() for l in f.readlines()] + else: + class_names = [c.strip() for c in class_names.split(',')] + for name in class_names: + assert len(name) > 0 + else: + raise RuntimeError("No valid class_name provided...") + return class_names + if __name__ == '__main__': args = parse_args() if args.cpu: @@ -93,10 +110,15 @@ def parse_args(): assert len(image_list) > 0, "No valid image specified to detect" network = None if args.deploy_net else args.network - detector = get_detector(network, args.prefix, args.epoch, + class_names = parse_class_names(args.class_names) + if args.prefix.endswith('_'): + prefix = args.prefix + args.network + '_' + str(args.data_shape) + else: + prefix = args.prefix + detector = get_detector(network, prefix, args.epoch, args.data_shape, (args.mean_r, args.mean_g, args.mean_b), - ctx, args.nms_thresh, args.force_nms) + ctx, len(class_names), args.nms_thresh, args.force_nms) # run detection detector.detect_and_visualize(image_list, args.dir, args.extension, - CLASSES, args.thresh, args.show_timer) + class_names, args.thresh, args.show_timer) diff --git a/example/ssd/deploy.py b/example/ssd/deploy.py index 264314a59f70..aa70cac548c4 100644 --- a/example/ssd/deploy.py +++ b/example/ssd/deploy.py @@ -5,32 +5,41 @@ import os import importlib import sys +from symbol.symbol_factory import get_symbol def parse_args(): parser = argparse.ArgumentParser(description='Convert a trained model to deploy model') - parser.add_argument('--network', dest='network', type=str, default='vgg16_ssd_300', - choices=['vgg16_ssd_300', 'vgg16_ssd_512'], help='which network to use') + parser.add_argument('--network', dest='network', type=str, default='vgg16_reduced', + help='which network to use') parser.add_argument('--epoch', dest='epoch', help='epoch of trained model', default=0, type=int) parser.add_argument('--prefix', dest='prefix', help='trained model prefix', - default=os.path.join(os.getcwd(), 'model', 'ssd_300'), type=str) + default=os.path.join(os.getcwd(), 'model', 'ssd_'), type=str) + parser.add_argument('--data-shape', dest='data_shape', type=int, default=300, + help='data shape') parser.add_argument('--num-class', dest='num_classes', help='number of classes', default=20, type=int) parser.add_argument('--nms', dest='nms_thresh', type=float, default=0.5, help='non-maximum suppression threshold, default 0.5') parser.add_argument('--force', dest='force_nms', type=bool, default=True, help='force non-maximum suppression on different class') + parser.add_argument('--topk', dest='nms_topk', type=int, default=400, + help='apply nms only to top k detections based on scores.') args = parser.parse_args() return args if __name__ == '__main__': args = parse_args() - sys.path.append(os.path.join(os.getcwd(), 'symbol')) - net = importlib.import_module("symbol_" + args.network) \ - .get_symbol(args.num_classes, args.nms_thresh, args.force_nms) - _, arg_params, aux_params = mx.model.load_checkpoint(args.prefix, args.epoch) + net = get_symbol(args.network, args.data_shape, + num_classes=args.num_classes, nms_thresh=args.nms_thresh, + force_suppress=args.force_nms, nms_topk=args.nms_topk) + if args.prefix.endswith('_'): + prefix = args.prefix + args.network + '_' + str(args.data_shape) + else: + prefix = args.prefix + _, arg_params, aux_params = mx.model.load_checkpoint(prefix, args.epoch) # new name - tmp = args.prefix.rsplit('/', 1) + tmp = prefix.rsplit('/', 1) save_prefix = '/deploy_'.join(tmp) mx.model.save_checkpoint(save_prefix, args.epoch, net, arg_params, aux_params) print("Saved model: {}-{:04d}.param".format(save_prefix, args.epoch)) diff --git a/example/ssd/evaluate.py b/example/ssd/evaluate.py index a38a7f6e6a6e..65e0b30b985c 100644 --- a/example/ssd/evaluate.py +++ b/example/ssd/evaluate.py @@ -5,30 +5,27 @@ import sys from evaluate.evaluate_net import evaluate_net -CLASSES = ('aeroplane', 'bicycle', 'bird', 'boat', - 'bottle', 'bus', 'car', 'cat', 'chair', - 'cow', 'diningtable', 'dog', 'horse', - 'motorbike', 'person', 'pottedplant', - 'sheep', 'sofa', 'train', 'tvmonitor') - def parse_args(): parser = argparse.ArgumentParser(description='Evaluate a network') parser.add_argument('--rec-path', dest='rec_path', help='which record file to use', default=os.path.join(os.getcwd(), 'data', 'val.rec'), type=str) parser.add_argument('--list-path', dest='list_path', help='which list file to use', default="", type=str) - parser.add_argument('--network', dest='network', type=str, default='vgg16_ssd_300', - choices=['vgg16_ssd_300', 'vgg16_ssd_512'], help='which network to use') + parser.add_argument('--network', dest='network', type=str, default='vgg16_reduced', + help='which network to use') parser.add_argument('--batch-size', dest='batch_size', type=int, default=32, help='evaluation batch size') parser.add_argument('--num-class', dest='num_class', type=int, default=20, help='number of classes') - parser.add_argument('--class-names', dest='class_names', type=str, default=",".join(CLASSES), + parser.add_argument('--class-names', dest='class_names', type=str, + default='aeroplane, bicycle, bird, boat, bottle, bus, \ + car, cat, chair, cow, diningtable, dog, horse, motorbike, \ + person, pottedplant, sheep, sofa, train, tvmonitor', help='string of comma separated names, or text filename') parser.add_argument('--epoch', dest='epoch', help='epoch of pretrained model', default=0, type=int) parser.add_argument('--prefix', dest='prefix', help='load model prefix', - default=os.path.join(os.getcwd(), 'model', 'ssd'), type=str) + default=os.path.join(os.getcwd(), 'model', 'ssd_'), type=str) parser.add_argument('--gpus', dest='gpu_id', help='GPU devices to evaluate with', default='0', type=str) parser.add_argument('--cpu', dest='cpu', help='use cpu to evaluate, this can be slow', @@ -79,9 +76,13 @@ def parse_args(): class_names = None network = None if args.deploy_net else args.network + if args.prefix.endswith('_'): + prefix = args.prefix + args.network + else: + prefix = args.prefix evaluate_net(network, args.rec_path, num_class, (args.mean_r, args.mean_g, args.mean_b), args.data_shape, - args.prefix, args.epoch, ctx, batch_size=args.batch_size, + prefix, args.epoch, ctx, batch_size=args.batch_size, path_imglist=args.list_path, nms_thresh=args.nms_thresh, force_nms=args.force_nms, ovp_thresh=args.overlap_thresh, use_difficult=args.use_difficult, class_names=class_names, diff --git a/example/ssd/evaluate/evaluate_net.py b/example/ssd/evaluate/evaluate_net.py index 8d86f8eefd56..4c629f892bd6 100644 --- a/example/ssd/evaluate/evaluate_net.py +++ b/example/ssd/evaluate/evaluate_net.py @@ -7,6 +7,7 @@ from config.config import cfg from evaluate.eval_metric import MApMetric, VOC07MApMetric import logging +from symbol.symbol_factory import get_symbol def evaluate_net(net, path_imgrec, num_classes, mean_pixels, data_shape, model_prefix, epoch, ctx=mx.cpu(), batch_size=1, @@ -71,9 +72,8 @@ class names in string, must correspond to num_classes if set if net is None: net = load_net else: - sys.path.append(os.path.join(cfg.ROOT_DIR, 'symbol')) - net = importlib.import_module("symbol_" + net) \ - .get_symbol(num_classes, nms_thresh, force_nms) + net = get_symbol(net, data_shape[1], num_classes=num_classes, + nms_thresh=nms_thresh, force_suppress=force_nms) if not 'label' in net.list_arguments(): label = mx.sym.Variable(name='label') net = mx.sym.Group([net, label]) diff --git a/example/ssd/symbol/README.md b/example/ssd/symbol/README.md new file mode 100644 index 000000000000..8fee31985a0d --- /dev/null +++ b/example/ssd/symbol/README.md @@ -0,0 +1,49 @@ +## How to compose SSD network on top of mainstream classification networks + +1. Have the base network ready in this directory as `name.py`, such as `inceptionv3.py`. +2. Add configuration to `symbol_factory.py`, an example would be: +``` +if network == 'vgg16_reduced': + if data_shape >= 448: + from_layers = ['relu4_3', 'relu7', '', '', '', '', ''] + num_filters = [512, -1, 512, 256, 256, 256, 256] + strides = [-1, -1, 2, 2, 2, 2, 1] + pads = [-1, -1, 1, 1, 1, 1, 1] + sizes = [[.07, .1025], [.15,.2121], [.3, .3674], [.45, .5196], [.6, .6708], \ + [.75, .8216], [.9, .9721]] + ratios = [[1,2,.5], [1,2,.5,3,1./3], [1,2,.5,3,1./3], [1,2,.5,3,1./3], \ + [1,2,.5,3,1./3], [1,2,.5], [1,2,.5]] + normalizations = [20, -1, -1, -1, -1, -1, -1] + steps = [] if data_shape != 512 else [x / 512.0 for x in + [8, 16, 32, 64, 128, 256, 512]] + else: + from_layers = ['relu4_3', 'relu7', '', '', '', ''] + num_filters = [512, -1, 512, 256, 256, 256] + strides = [-1, -1, 2, 2, 1, 1] + pads = [-1, -1, 1, 1, 0, 0] + sizes = [[.1, .141], [.2,.272], [.37, .447], [.54, .619], [.71, .79], [.88, .961]] + ratios = [[1,2,.5], [1,2,.5,3,1./3], [1,2,.5,3,1./3], [1,2,.5,3,1./3], \ + [1,2,.5], [1,2,.5]] + normalizations = [20, -1, -1, -1, -1, -1] + steps = [] if data_shape != 300 else [x / 300.0 for x in [8, 16, 32, 64, 100, 300]] + return locals() +elif network == 'inceptionv3': + from_layers = ['ch_concat_mixed_7_chconcat', 'ch_concat_mixed_10_chconcat', '', '', '', ''] + num_filters = [-1, -1, 512, 256, 256, 128] + strides = [-1, -1, 2, 2, 2, 2] + pads = [-1, -1, 1, 1, 1, 1] + sizes = [[.1, .141], [.2,.272], [.37, .447], [.54, .619], [.71, .79], [.88, .961]] + ratios = [[1,2,.5], [1,2,.5,3,1./3], [1,2,.5,3,1./3], [1,2,.5,3,1./3], \ + [1,2,.5], [1,2,.5]] + normalizations = -1 + steps = [] + return locals() +``` +Here `from_layers` indicate the feature layer you would like to extract from the base network. +`''` indicate that we want add extra new layers on top of the last feature layer, +and the number of filters must be specified in `num_filters`. Similarly, `strides` and `pads` +are required to compose these new layers. `sizes` and `ratios` are the parameters controlling +the anchor generation algorithm. `normalizations` is used to normalize and rescale feature if +not `-1`. `steps`: optional, used to calculate the anchor sliding steps. + +3. Train or test with arguments `--network name --data-shape xxx --pretrained pretrained_model` diff --git a/example/ssd/symbol/common.py b/example/ssd/symbol/common.py index 12ea71826e22..474d3ea944b9 100644 --- a/example/ssd/symbol/common.py +++ b/example/ssd/symbol/common.py @@ -6,6 +6,42 @@ def conv_act_layer(from_layer, name, num_filter, kernel=(1,1), pad=(0,0), \ """ wrapper for a small Convolution group + Parameters: + ---------- + from_layer : mx.symbol + continue on which layer + name : str + base name of the new layers + num_filter : int + how many filters to use in Convolution layer + kernel : tuple (int, int) + kernel size (h, w) + pad : tuple (int, int) + padding size (h, w) + stride : tuple (int, int) + stride size (h, w) + act_type : str + activation type, can be relu... + use_batchnorm : bool + whether to use batch normalization + + Returns: + ---------- + (conv, relu) mx.Symbols + """ + conv = mx.symbol.Convolution(data=from_layer, kernel=kernel, pad=pad, \ + stride=stride, num_filter=num_filter, name="{}_conv".format(name)) + if use_batchnorm: + conv = mx.symbol.BatchNorm(data=conv, name="{}_bn".format(name)) + relu = mx.symbol.Activation(data=conv, act_type=act_type, \ + name="{}_{}".format(name, act_type)) + return relu + +def legacy_conv_act_layer(from_layer, name, num_filter, kernel=(1,1), pad=(0,0), \ + stride=(1,1), act_type="relu", use_batchnorm=False): + """ + wrapper for a small Convolution group + Parameters: ---------- from_layer : mx.symbol @@ -40,9 +76,66 @@ def conv_act_layer(from_layer, name, num_filter, kernel=(1,1), pad=(0,0), \ relu = mx.symbol.BatchNorm(data=relu, name="bn{}".format(name)) return conv, relu +def multi_layer_feature(body, from_layers, num_filters, strides, pads, min_filter=128): + """Wrapper function to extract features from base network, attaching extra + layers and SSD specific layers + + Parameters + ---------- + from_layers : list of str + feature extraction layers, use '' for add extra layers + For example: + from_layers = ['relu4_3', 'fc7', '', '', '', ''] + which means extract feature from relu4_3 and fc7, adding 4 extra layers + on top of fc7 + num_filters : list of int + number of filters for extra layers, you can use -1 for extracted features, + however, if normalization and scale is applied, the number of filter for + that layer must be provided. + For example: + num_filters = [512, -1, 512, 256, 256, 256] + strides : list of int + strides for the 3x3 convolution appended, -1 can be used for extracted + feature layers + pads : list of int + paddings for the 3x3 convolution, -1 can be used for extracted layers + min_filter : int + minimum number of filters used in 1x1 convolution + + Returns + ------- + list of mx.Symbols + + """ + # arguments check + assert len(from_layers) > 0 + assert isinstance(from_layers[0], str) and len(from_layers[0].strip()) > 0 + assert len(from_layers) == len(num_filters) == len(strides) == len(pads) + + internals = body.get_internals() + layers = [] + for k, params in enumerate(zip(from_layers, num_filters, strides, pads)): + from_layer, num_filter, s, p = params + if from_layer.strip(): + # extract from base network + layer = internals[from_layer.strip() + '_output'] + layers.append(layer) + else: + # attach from last feature layer + assert len(layers) > 0 + assert num_filter > 0 + layer = layers[-1] + num_1x1 = max(min_filter, num_filter // 2) + conv_1x1 = conv_act_layer(layer, 'multi_feat_%d_conv_1x1' % (k), + num_1x1, kernel=(1, 1), pad=(0, 0), stride=(1, 1), act_type='relu') + conv_3x3 = conv_act_layer(conv_1x1, 'multi_feat_%d_conv_3x3' % (k), + num_filter, kernel=(3, 3), pad=(p, p), stride=(s, s), act_type='relu') + layers.append(conv_3x3) + return layers + def multibox_layer(from_layers, num_classes, sizes=[.2, .95], ratios=[1], normalization=-1, num_channels=[], - clip=True, interm_layer=0, steps=[]): + clip=False, interm_layer=0, steps=[]): """ the basic aggregation module for SSD detection. Takes in multiple layers, generate multiple object detection targets by customized layers @@ -106,7 +199,7 @@ def multibox_layer(from_layers, num_classes, sizes=[.2, .95], normalization = [normalization] * len(from_layers) assert len(normalization) == len(from_layers) - assert sum(x > 0 for x in normalization) == len(num_channels), \ + assert sum(x > 0 for x in normalization) <= len(num_channels), \ "must provide number of channels for each normalized layer" if steps: @@ -125,7 +218,8 @@ def multibox_layer(from_layers, num_classes, sizes=[.2, .95], mode="channel", name="{}_norm".format(from_name)) scale = mx.symbol.Variable(name="{}_scale".format(from_name), shape=(1, num_channels.pop(0), 1, 1), - init=mx.init.Constant(normalization[k])) + init=mx.init.Constant(normalization[k]), + attr={'__wd_mult__': '0.1'}) from_layer = mx.symbol.broadcast_mul(lhs=scale, rhs=from_layer) if interm_layer > 0: from_layer = mx.symbol.Convolution(data=from_layer, kernel=(3,3), \ diff --git a/example/ssd/symbol/inceptionv3.py b/example/ssd/symbol/inceptionv3.py new file mode 100644 index 000000000000..1c38ae6d57c9 --- /dev/null +++ b/example/ssd/symbol/inceptionv3.py @@ -0,0 +1,168 @@ +""" +Inception V3, suitable for images with around 299 x 299 + +Reference: + +Szegedy, Christian, et al. "Rethinking the Inception Architecture for Computer Vision." arXiv preprint arXiv:1512.00567 (2015). +""" +import mxnet as mx + +def Conv(data, num_filter, kernel=(1, 1), stride=(1, 1), pad=(0, 0), name=None, suffix=''): + conv = mx.sym.Convolution(data=data, num_filter=num_filter, kernel=kernel, stride=stride, pad=pad, no_bias=True, name='%s%s_conv2d' %(name, suffix)) + bn = mx.sym.BatchNorm(data=conv, name='%s%s_batchnorm' %(name, suffix), fix_gamma=True) + act = mx.sym.Activation(data=bn, act_type='relu', name='%s%s_relu' %(name, suffix)) + return act + + +def Inception7A(data, + num_1x1, + num_3x3_red, num_3x3_1, num_3x3_2, + num_5x5_red, num_5x5, + pool, proj, + name): + tower_1x1 = Conv(data, num_1x1, name=('%s_conv' % name)) + tower_5x5 = Conv(data, num_5x5_red, name=('%s_tower' % name), suffix='_conv') + tower_5x5 = Conv(tower_5x5, num_5x5, kernel=(5, 5), pad=(2, 2), name=('%s_tower' % name), suffix='_conv_1') + tower_3x3 = Conv(data, num_3x3_red, name=('%s_tower_1' % name), suffix='_conv') + tower_3x3 = Conv(tower_3x3, num_3x3_1, kernel=(3, 3), pad=(1, 1), name=('%s_tower_1' % name), suffix='_conv_1') + tower_3x3 = Conv(tower_3x3, num_3x3_2, kernel=(3, 3), pad=(1, 1), name=('%s_tower_1' % name), suffix='_conv_2') + pooling = mx.sym.Pooling(data=data, kernel=(3, 3), stride=(1, 1), pad=(1, 1), pool_type=pool, name=('%s_pool_%s_pool' % (pool, name))) + cproj = Conv(pooling, proj, name=('%s_tower_2' % name), suffix='_conv') + concat = mx.sym.Concat(*[tower_1x1, tower_5x5, tower_3x3, cproj], name='ch_concat_%s_chconcat' % name) + return concat + +# First Downsample +def Inception7B(data, + num_3x3, + num_d3x3_red, num_d3x3_1, num_d3x3_2, + pool, + name): + tower_3x3 = Conv(data, num_3x3, kernel=(3, 3), pad=(0, 0), stride=(2, 2), name=('%s_conv' % name)) + tower_d3x3 = Conv(data, num_d3x3_red, name=('%s_tower' % name), suffix='_conv') + tower_d3x3 = Conv(tower_d3x3, num_d3x3_1, kernel=(3, 3), pad=(1, 1), stride=(1, 1), name=('%s_tower' % name), suffix='_conv_1') + tower_d3x3 = Conv(tower_d3x3, num_d3x3_2, kernel=(3, 3), pad=(0, 0), stride=(2, 2), name=('%s_tower' % name), suffix='_conv_2') + pooling = mx.symbol.Pooling(data=data, kernel=(3, 3), stride=(2, 2), pad=(0,0), pool_type="max", name=('max_pool_%s_pool' % name)) + concat = mx.sym.Concat(*[tower_3x3, tower_d3x3, pooling], name='ch_concat_%s_chconcat' % name) + return concat + +def Inception7C(data, + num_1x1, + num_d7_red, num_d7_1, num_d7_2, + num_q7_red, num_q7_1, num_q7_2, num_q7_3, num_q7_4, + pool, proj, + name): + tower_1x1 = Conv(data=data, num_filter=num_1x1, kernel=(1, 1), name=('%s_conv' % name)) + tower_d7 = Conv(data=data, num_filter=num_d7_red, name=('%s_tower' % name), suffix='_conv') + tower_d7 = Conv(data=tower_d7, num_filter=num_d7_1, kernel=(1, 7), pad=(0, 3), name=('%s_tower' % name), suffix='_conv_1') + tower_d7 = Conv(data=tower_d7, num_filter=num_d7_2, kernel=(7, 1), pad=(3, 0), name=('%s_tower' % name), suffix='_conv_2') + tower_q7 = Conv(data=data, num_filter=num_q7_red, name=('%s_tower_1' % name), suffix='_conv') + tower_q7 = Conv(data=tower_q7, num_filter=num_q7_1, kernel=(7, 1), pad=(3, 0), name=('%s_tower_1' % name), suffix='_conv_1') + tower_q7 = Conv(data=tower_q7, num_filter=num_q7_2, kernel=(1, 7), pad=(0, 3), name=('%s_tower_1' % name), suffix='_conv_2') + tower_q7 = Conv(data=tower_q7, num_filter=num_q7_3, kernel=(7, 1), pad=(3, 0), name=('%s_tower_1' % name), suffix='_conv_3') + tower_q7 = Conv(data=tower_q7, num_filter=num_q7_4, kernel=(1, 7), pad=(0, 3), name=('%s_tower_1' % name), suffix='_conv_4') + pooling = mx.sym.Pooling(data=data, kernel=(3, 3), stride=(1, 1), pad=(1, 1), pool_type=pool, name=('%s_pool_%s_pool' % (pool, name))) + cproj = Conv(data=pooling, num_filter=proj, kernel=(1, 1), name=('%s_tower_2' % name), suffix='_conv') + # concat + concat = mx.sym.Concat(*[tower_1x1, tower_d7, tower_q7, cproj], name='ch_concat_%s_chconcat' % name) + return concat + +def Inception7D(data, + num_3x3_red, num_3x3, + num_d7_3x3_red, num_d7_1, num_d7_2, num_d7_3x3, + pool, + name): + tower_3x3 = Conv(data=data, num_filter=num_3x3_red, name=('%s_tower' % name), suffix='_conv') + tower_3x3 = Conv(data=tower_3x3, num_filter=num_3x3, kernel=(3, 3), pad=(0,0), stride=(2, 2), name=('%s_tower' % name), suffix='_conv_1') + tower_d7_3x3 = Conv(data=data, num_filter=num_d7_3x3_red, name=('%s_tower_1' % name), suffix='_conv') + tower_d7_3x3 = Conv(data=tower_d7_3x3, num_filter=num_d7_1, kernel=(1, 7), pad=(0, 3), name=('%s_tower_1' % name), suffix='_conv_1') + tower_d7_3x3 = Conv(data=tower_d7_3x3, num_filter=num_d7_2, kernel=(7, 1), pad=(3, 0), name=('%s_tower_1' % name), suffix='_conv_2') + tower_d7_3x3 = Conv(data=tower_d7_3x3, num_filter=num_d7_3x3, kernel=(3, 3), stride=(2, 2), name=('%s_tower_1' % name), suffix='_conv_3') + pooling = mx.sym.Pooling(data=data, kernel=(3, 3), stride=(2, 2), pool_type=pool, name=('%s_pool_%s_pool' % (pool, name))) + # concat + concat = mx.sym.Concat(*[tower_3x3, tower_d7_3x3, pooling], name='ch_concat_%s_chconcat' % name) + return concat + +def Inception7E(data, + num_1x1, + num_d3_red, num_d3_1, num_d3_2, + num_3x3_d3_red, num_3x3, num_3x3_d3_1, num_3x3_d3_2, + pool, proj, + name): + tower_1x1 = Conv(data=data, num_filter=num_1x1, kernel=(1, 1), name=('%s_conv' % name)) + tower_d3 = Conv(data=data, num_filter=num_d3_red, name=('%s_tower' % name), suffix='_conv') + tower_d3_a = Conv(data=tower_d3, num_filter=num_d3_1, kernel=(1, 3), pad=(0, 1), name=('%s_tower' % name), suffix='_mixed_conv') + tower_d3_b = Conv(data=tower_d3, num_filter=num_d3_2, kernel=(3, 1), pad=(1, 0), name=('%s_tower' % name), suffix='_mixed_conv_1') + tower_3x3_d3 = Conv(data=data, num_filter=num_3x3_d3_red, name=('%s_tower_1' % name), suffix='_conv') + tower_3x3_d3 = Conv(data=tower_3x3_d3, num_filter=num_3x3, kernel=(3, 3), pad=(1, 1), name=('%s_tower_1' % name), suffix='_conv_1') + tower_3x3_d3_a = Conv(data=tower_3x3_d3, num_filter=num_3x3_d3_1, kernel=(1, 3), pad=(0, 1), name=('%s_tower_1' % name), suffix='_mixed_conv') + tower_3x3_d3_b = Conv(data=tower_3x3_d3, num_filter=num_3x3_d3_2, kernel=(3, 1), pad=(1, 0), name=('%s_tower_1' % name), suffix='_mixed_conv_1') + pooling = mx.sym.Pooling(data=data, kernel=(3, 3), stride=(1, 1), pad=(1, 1), pool_type=pool, name=('%s_pool_%s_pool' % (pool, name))) + cproj = Conv(data=pooling, num_filter=proj, kernel=(1, 1), name=('%s_tower_2' % name), suffix='_conv') + # concat + concat = mx.sym.Concat(*[tower_1x1, tower_d3_a, tower_d3_b, tower_3x3_d3_a, tower_3x3_d3_b, cproj], name='ch_concat_%s_chconcat' % name) + return concat + +# In[49]: + +def get_symbol(num_classes=1000, **kwargs): + data = mx.symbol.Variable(name="data") + # stage 1 + conv = Conv(data, 32, kernel=(3, 3), stride=(2, 2), name="conv") + conv_1 = Conv(conv, 32, kernel=(3, 3), name="conv_1") + conv_2 = Conv(conv_1, 64, kernel=(3, 3), pad=(1, 1), name="conv_2") + pool = mx.sym.Pooling(data=conv_2, kernel=(3, 3), stride=(2, 2), pool_type="max", name="pool") + # stage 2 + conv_3 = Conv(pool, 80, kernel=(1, 1), name="conv_3") + conv_4 = Conv(conv_3, 192, kernel=(3, 3), name="conv_4") + pool1 = mx.sym.Pooling(data=conv_4, kernel=(3, 3), stride=(2, 2), pool_type="max", name="pool1") + # stage 3 + in3a = Inception7A(pool1, 64, + 64, 96, 96, + 48, 64, + "avg", 32, "mixed") + in3b = Inception7A(in3a, 64, + 64, 96, 96, + 48, 64, + "avg", 64, "mixed_1") + in3c = Inception7A(in3b, 64, + 64, 96, 96, + 48, 64, + "avg", 64, "mixed_2") + in3d = Inception7B(in3c, 384, + 64, 96, 96, + "max", "mixed_3") + # stage 4 + in4a = Inception7C(in3d, 192, + 128, 128, 192, + 128, 128, 128, 128, 192, + "avg", 192, "mixed_4") + in4b = Inception7C(in4a, 192, + 160, 160, 192, + 160, 160, 160, 160, 192, + "avg", 192, "mixed_5") + in4c = Inception7C(in4b, 192, + 160, 160, 192, + 160, 160, 160, 160, 192, + "avg", 192, "mixed_6") + in4d = Inception7C(in4c, 192, + 192, 192, 192, + 192, 192, 192, 192, 192, + "avg", 192, "mixed_7") + in4e = Inception7D(in4d, 192, 320, + 192, 192, 192, 192, + "max", "mixed_8") + # stage 5 + in5a = Inception7E(in4e, 320, + 384, 384, 384, + 448, 384, 384, 384, + "avg", 192, "mixed_9") + in5b = Inception7E(in5a, 320, + 384, 384, 384, + 448, 384, 384, 384, + "max", 192, "mixed_10") + # pool + pool = mx.sym.Pooling(data=in5b, kernel=(8, 8), stride=(1, 1), pool_type="avg", name="global_pool") + flatten = mx.sym.Flatten(data=pool, name="flatten") + fc1 = mx.symbol.FullyConnected(data=flatten, num_hidden=num_classes, name='fc1') + softmax = mx.symbol.SoftmaxOutput(data=fc1, name='softmax') + return softmax diff --git a/example/ssd/symbol/symbol_vgg16_ssd_300.py b/example/ssd/symbol/legacy_vgg16_ssd_300.py similarity index 90% rename from example/ssd/symbol/symbol_vgg16_ssd_300.py rename to example/ssd/symbol/legacy_vgg16_ssd_300.py index e40674661300..257fdd6acb7a 100644 --- a/example/ssd/symbol/symbol_vgg16_ssd_300.py +++ b/example/ssd/symbol/legacy_vgg16_ssd_300.py @@ -1,8 +1,9 @@ import mxnet as mx -from common import conv_act_layer +from common import legacy_conv_act_layer from common import multibox_layer -def get_symbol_train(num_classes=20, nms_thresh=0.5, force_suppress=False, nms_topk=400): +def get_symbol_train(num_classes=20, nms_thresh=0.5, force_suppress=False, + nms_topk=400, **kwargs): """ Single-shot multi-box detection with VGG 16 layers ConvNet This is a modified version, with fc6/fc7 layers replaced by conv layers @@ -96,21 +97,21 @@ def get_symbol_train(num_classes=20, nms_thresh=0.5, force_suppress=False, nms_t # drop7 = mx.symbol.Dropout(data=relu7, p=0.5, name="drop7") ### ssd extra layers ### - conv8_1, relu8_1 = conv_act_layer(relu7, "8_1", 256, kernel=(1,1), pad=(0,0), \ + conv8_1, relu8_1 = legacy_conv_act_layer(relu7, "8_1", 256, kernel=(1,1), pad=(0,0), \ stride=(1,1), act_type="relu", use_batchnorm=False) - conv8_2, relu8_2 = conv_act_layer(relu8_1, "8_2", 512, kernel=(3,3), pad=(1,1), \ + conv8_2, relu8_2 = legacy_conv_act_layer(relu8_1, "8_2", 512, kernel=(3,3), pad=(1,1), \ stride=(2,2), act_type="relu", use_batchnorm=False) - conv9_1, relu9_1 = conv_act_layer(relu8_2, "9_1", 128, kernel=(1,1), pad=(0,0), \ + conv9_1, relu9_1 = legacy_conv_act_layer(relu8_2, "9_1", 128, kernel=(1,1), pad=(0,0), \ stride=(1,1), act_type="relu", use_batchnorm=False) - conv9_2, relu9_2 = conv_act_layer(relu9_1, "9_2", 256, kernel=(3,3), pad=(1,1), \ + conv9_2, relu9_2 = legacy_conv_act_layer(relu9_1, "9_2", 256, kernel=(3,3), pad=(1,1), \ stride=(2,2), act_type="relu", use_batchnorm=False) - conv10_1, relu10_1 = conv_act_layer(relu9_2, "10_1", 128, kernel=(1,1), pad=(0,0), \ + conv10_1, relu10_1 = legacy_conv_act_layer(relu9_2, "10_1", 128, kernel=(1,1), pad=(0,0), \ stride=(1,1), act_type="relu", use_batchnorm=False) - conv10_2, relu10_2 = conv_act_layer(relu10_1, "10_2", 256, kernel=(3,3), pad=(0,0), \ + conv10_2, relu10_2 = legacy_conv_act_layer(relu10_1, "10_2", 256, kernel=(3,3), pad=(0,0), \ stride=(1,1), act_type="relu", use_batchnorm=False) - conv11_1, relu11_1 = conv_act_layer(relu10_2, "11_1", 128, kernel=(1,1), pad=(0,0), \ + conv11_1, relu11_1 = legacy_conv_act_layer(relu10_2, "11_1", 128, kernel=(1,1), pad=(0,0), \ stride=(1,1), act_type="relu", use_batchnorm=False) - conv11_2, relu11_2 = conv_act_layer(relu11_1, "11_2", 256, kernel=(3,3), pad=(0,0), \ + conv11_2, relu11_2 = legacy_conv_act_layer(relu11_1, "11_2", 256, kernel=(3,3), pad=(0,0), \ stride=(1,1), act_type="relu", use_batchnorm=False) # specific parameters for VGG16 network @@ -154,7 +155,8 @@ def get_symbol_train(num_classes=20, nms_thresh=0.5, force_suppress=False, nms_t out = mx.symbol.Group([cls_prob, loc_loss, cls_label, det]) return out -def get_symbol(num_classes=20, nms_thresh=0.5, force_suppress=False, nms_topk=400): +def get_symbol(num_classes=20, nms_thresh=0.5, force_suppress=False, + nms_topk=400, **kwargs): """ Single-shot multi-box detection with VGG 16 layers ConvNet This is a modified version, with fc6/fc7 layers replaced by conv layers diff --git a/example/ssd/symbol/symbol_vgg16_ssd_512.py b/example/ssd/symbol/legacy_vgg16_ssd_512.py similarity index 89% rename from example/ssd/symbol/symbol_vgg16_ssd_512.py rename to example/ssd/symbol/legacy_vgg16_ssd_512.py index 1cc243b0406b..15424f93e643 100644 --- a/example/ssd/symbol/symbol_vgg16_ssd_512.py +++ b/example/ssd/symbol/legacy_vgg16_ssd_512.py @@ -1,5 +1,5 @@ import mxnet as mx -from common import conv_act_layer +from common import legacy_conv_act_layer from common import multibox_layer def get_symbol_train(num_classes=20, nms_thresh=0.5, force_suppress=False, nms_topk=400): @@ -96,25 +96,25 @@ def get_symbol_train(num_classes=20, nms_thresh=0.5, force_suppress=False, nms_t # drop7 = mx.symbol.Dropout(data=relu7, p=0.5, name="drop7") ### ssd extra layers ### - conv8_1, relu8_1 = conv_act_layer(relu7, "8_1", 256, kernel=(1,1), pad=(0,0), \ + conv8_1, relu8_1 = legacy_conv_act_layer(relu7, "8_1", 256, kernel=(1,1), pad=(0,0), \ stride=(1,1), act_type="relu", use_batchnorm=False) - conv8_2, relu8_2 = conv_act_layer(relu8_1, "8_2", 512, kernel=(3,3), pad=(1,1), \ + conv8_2, relu8_2 = legacy_conv_act_layer(relu8_1, "8_2", 512, kernel=(3,3), pad=(1,1), \ stride=(2,2), act_type="relu", use_batchnorm=False) - conv9_1, relu9_1 = conv_act_layer(relu8_2, "9_1", 128, kernel=(1,1), pad=(0,0), \ + conv9_1, relu9_1 = legacy_conv_act_layer(relu8_2, "9_1", 128, kernel=(1,1), pad=(0,0), \ stride=(1,1), act_type="relu", use_batchnorm=False) - conv9_2, relu9_2 = conv_act_layer(relu9_1, "9_2", 256, kernel=(3,3), pad=(1,1), \ + conv9_2, relu9_2 = legacy_conv_act_layer(relu9_1, "9_2", 256, kernel=(3,3), pad=(1,1), \ stride=(2,2), act_type="relu", use_batchnorm=False) - conv10_1, relu10_1 = conv_act_layer(relu9_2, "10_1", 128, kernel=(1,1), pad=(0,0), \ + conv10_1, relu10_1 = legacy_conv_act_layer(relu9_2, "10_1", 128, kernel=(1,1), pad=(0,0), \ stride=(1,1), act_type="relu", use_batchnorm=False) - conv10_2, relu10_2 = conv_act_layer(relu10_1, "10_2", 256, kernel=(3,3), pad=(1,1), \ + conv10_2, relu10_2 = legacy_conv_act_layer(relu10_1, "10_2", 256, kernel=(3,3), pad=(1,1), \ stride=(2,2), act_type="relu", use_batchnorm=False) - conv11_1, relu11_1 = conv_act_layer(relu10_2, "11_1", 128, kernel=(1,1), pad=(0,0), \ + conv11_1, relu11_1 = legacy_conv_act_layer(relu10_2, "11_1", 128, kernel=(1,1), pad=(0,0), \ stride=(1,1), act_type="relu", use_batchnorm=False) - conv11_2, relu11_2 = conv_act_layer(relu11_1, "11_2", 256, kernel=(3,3), pad=(1,1), \ + conv11_2, relu11_2 = legacy_conv_act_layer(relu11_1, "11_2", 256, kernel=(3,3), pad=(1,1), \ stride=(2,2), act_type="relu", use_batchnorm=False) - conv12_1, relu12_1 = conv_act_layer(relu11_2, "12_1", 128, kernel=(1,1), pad=(0,0), \ + conv12_1, relu12_1 = legacy_conv_act_layer(relu11_2, "12_1", 128, kernel=(1,1), pad=(0,0), \ stride=(1,1), act_type="relu", use_batchnorm=False) - conv12_2, relu12_2 = conv_act_layer(relu12_1, "12_2", 256, kernel=(4,4), pad=(1,1), \ + conv12_2, relu12_2 = legacy_conv_act_layer(relu12_1, "12_2", 256, kernel=(4,4), pad=(1,1), \ stride=(1,1), act_type="relu", use_batchnorm=False) # specific parameters for VGG16 network diff --git a/example/ssd/symbol/resnet.py b/example/ssd/symbol/resnet.py new file mode 100644 index 000000000000..9c121e8f9da7 --- /dev/null +++ b/example/ssd/symbol/resnet.py @@ -0,0 +1,169 @@ +''' +Adapted from https://github.com/tornadomeet/ResNet/blob/master/symbol_resnet.py +Original author Wei Wu + +Implemented the following paper: + +Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun. "Identity Mappings in Deep Residual Networks" +''' +import mxnet as mx + +def residual_unit(data, num_filter, stride, dim_match, name, bottle_neck=True, bn_mom=0.9, workspace=256, memonger=False): + """Return ResNet Unit symbol for building ResNet + Parameters + ---------- + data : str + Input data + num_filter : int + Number of output channels + bnf : int + Bottle neck channels factor with regard to num_filter + stride : tupe + Stride used in convolution + dim_match : Boolen + True means channel number between input and output is the same, otherwise means differ + name : str + Base name of the operators + workspace : int + Workspace used in convolution operator + """ + if bottle_neck: + # the same as https://github.com/facebook/fb.resnet.torch#notes, a bit difference with origin paper + bn1 = mx.sym.BatchNorm(data=data, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn1') + act1 = mx.sym.Activation(data=bn1, act_type='relu', name=name + '_relu1') + conv1 = mx.sym.Convolution(data=act1, num_filter=int(num_filter*0.25), kernel=(1,1), stride=(1,1), pad=(0,0), + no_bias=True, workspace=workspace, name=name + '_conv1') + bn2 = mx.sym.BatchNorm(data=conv1, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn2') + act2 = mx.sym.Activation(data=bn2, act_type='relu', name=name + '_relu2') + conv2 = mx.sym.Convolution(data=act2, num_filter=int(num_filter*0.25), kernel=(3,3), stride=stride, pad=(1,1), + no_bias=True, workspace=workspace, name=name + '_conv2') + bn3 = mx.sym.BatchNorm(data=conv2, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn3') + act3 = mx.sym.Activation(data=bn3, act_type='relu', name=name + '_relu3') + conv3 = mx.sym.Convolution(data=act3, num_filter=num_filter, kernel=(1,1), stride=(1,1), pad=(0,0), no_bias=True, + workspace=workspace, name=name + '_conv3') + if dim_match: + shortcut = data + else: + shortcut = mx.sym.Convolution(data=act1, num_filter=num_filter, kernel=(1,1), stride=stride, no_bias=True, + workspace=workspace, name=name+'_sc') + if memonger: + shortcut._set_attr(mirror_stage='True') + return conv3 + shortcut + else: + bn1 = mx.sym.BatchNorm(data=data, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_bn1') + act1 = mx.sym.Activation(data=bn1, act_type='relu', name=name + '_relu1') + conv1 = mx.sym.Convolution(data=act1, num_filter=num_filter, kernel=(3,3), stride=stride, pad=(1,1), + no_bias=True, workspace=workspace, name=name + '_conv1') + bn2 = mx.sym.BatchNorm(data=conv1, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_bn2') + act2 = mx.sym.Activation(data=bn2, act_type='relu', name=name + '_relu2') + conv2 = mx.sym.Convolution(data=act2, num_filter=num_filter, kernel=(3,3), stride=(1,1), pad=(1,1), + no_bias=True, workspace=workspace, name=name + '_conv2') + if dim_match: + shortcut = data + else: + shortcut = mx.sym.Convolution(data=act1, num_filter=num_filter, kernel=(1,1), stride=stride, no_bias=True, + workspace=workspace, name=name+'_sc') + if memonger: + shortcut._set_attr(mirror_stage='True') + return conv2 + shortcut + +def resnet(units, num_stages, filter_list, num_classes, image_shape, bottle_neck=True, bn_mom=0.9, workspace=256, memonger=False): + """Return ResNet symbol of + Parameters + ---------- + units : list + Number of units in each stage + num_stages : int + Number of stage + filter_list : list + Channel size of each stage + num_classes : int + Ouput size of symbol + dataset : str + Dataset type, only cifar10 and imagenet supports + workspace : int + Workspace used in convolution operator + """ + num_unit = len(units) + assert(num_unit == num_stages) + data = mx.sym.Variable(name='data') + data = mx.sym.identity(data=data, name='id') + data = mx.sym.BatchNorm(data=data, fix_gamma=True, eps=2e-5, momentum=bn_mom, name='bn_data') + (nchannel, height, width) = image_shape + if height <= 32: # such as cifar10 + body = mx.sym.Convolution(data=data, num_filter=filter_list[0], kernel=(3, 3), stride=(1,1), pad=(1, 1), + no_bias=True, name="conv0", workspace=workspace) + else: # often expected to be 224 such as imagenet + body = mx.sym.Convolution(data=data, num_filter=filter_list[0], kernel=(7, 7), stride=(2,2), pad=(3, 3), + no_bias=True, name="conv0", workspace=workspace) + body = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn0') + body = mx.sym.Activation(data=body, act_type='relu', name='relu0') + body = mx.symbol.Pooling(data=body, kernel=(3, 3), stride=(2,2), pad=(1,1), pool_type='max') + + for i in range(num_stages): + body = residual_unit(body, filter_list[i+1], (1 if i==0 else 2, 1 if i==0 else 2), False, + name='stage%d_unit%d' % (i + 1, 1), bottle_neck=bottle_neck, workspace=workspace, + memonger=memonger) + for j in range(units[i]-1): + body = residual_unit(body, filter_list[i+1], (1,1), True, name='stage%d_unit%d' % (i + 1, j + 2), + bottle_neck=bottle_neck, workspace=workspace, memonger=memonger) + bn1 = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn1') + relu1 = mx.sym.Activation(data=bn1, act_type='relu', name='relu1') + # Although kernel is not used here when global_pool=True, we should put one + pool1 = mx.symbol.Pooling(data=relu1, global_pool=True, kernel=(7, 7), pool_type='avg', name='pool1') + flat = mx.symbol.Flatten(data=pool1) + fc1 = mx.symbol.FullyConnected(data=flat, num_hidden=num_classes, name='fc1') + return mx.symbol.SoftmaxOutput(data=fc1, name='softmax') + +def get_symbol(num_classes, num_layers, image_shape, conv_workspace=256, **kwargs): + """ + Adapted from https://github.com/tornadomeet/ResNet/blob/master/train_resnet.py + Original author Wei Wu + """ + image_shape = [int(l) for l in image_shape.split(',')] + (nchannel, height, width) = image_shape + if height <= 28: + num_stages = 3 + if (num_layers-2) % 9 == 0 and num_layers >= 164: + per_unit = [(num_layers-2)//9] + filter_list = [16, 64, 128, 256] + bottle_neck = True + elif (num_layers-2) % 6 == 0 and num_layers < 164: + per_unit = [(num_layers-2)//6] + filter_list = [16, 16, 32, 64] + bottle_neck = False + else: + raise ValueError("no experiments done on num_layers {}, you can do it yourself".format(num_layers)) + units = per_unit * num_stages + else: + if num_layers >= 50: + filter_list = [64, 256, 512, 1024, 2048] + bottle_neck = True + else: + filter_list = [64, 64, 128, 256, 512] + bottle_neck = False + num_stages = 4 + if num_layers == 18: + units = [2, 2, 2, 2] + elif num_layers == 34: + units = [3, 4, 6, 3] + elif num_layers == 50: + units = [3, 4, 6, 3] + elif num_layers == 101: + units = [3, 4, 23, 3] + elif num_layers == 152: + units = [3, 8, 36, 3] + elif num_layers == 200: + units = [3, 24, 36, 3] + elif num_layers == 269: + units = [3, 30, 48, 8] + else: + raise ValueError("no experiments done on num_layers {}, you can do it yourself".format(num_layers)) + + return resnet(units = units, + num_stages = num_stages, + filter_list = filter_list, + num_classes = num_classes, + image_shape = image_shape, + bottle_neck = bottle_neck, + workspace = conv_workspace) diff --git a/example/ssd/symbol/symbol_builder.py b/example/ssd/symbol/symbol_builder.py new file mode 100644 index 000000000000..48c9d1d53f34 --- /dev/null +++ b/example/ssd/symbol/symbol_builder.py @@ -0,0 +1,166 @@ +import mxnet as mx +from common import multi_layer_feature, multibox_layer + + +def import_module(module_name): + """Helper function to import module""" + import sys, os + import importlib + sys.path.append(os.path.dirname(__file__)) + return importlib.import_module(module_name) + +def get_symbol_train(network, num_classes, from_layers, num_filters, strides, pads, + sizes, ratios, normalizations=-1, steps=[], min_filter=128, + nms_thresh=0.5, force_suppress=False, nms_topk=400, **kwargs): + """Build network symbol for training SSD + + Parameters + ---------- + network : str + base network symbol name + num_classes : int + number of object classes not including background + from_layers : list of str + feature extraction layers, use '' for add extra layers + For example: + from_layers = ['relu4_3', 'fc7', '', '', '', ''] + which means extract feature from relu4_3 and fc7, adding 4 extra layers + on top of fc7 + num_filters : list of int + number of filters for extra layers, you can use -1 for extracted features, + however, if normalization and scale is applied, the number of filter for + that layer must be provided. + For example: + num_filters = [512, -1, 512, 256, 256, 256] + strides : list of int + strides for the 3x3 convolution appended, -1 can be used for extracted + feature layers + pads : list of int + paddings for the 3x3 convolution, -1 can be used for extracted layers + sizes : list or list of list + [min_size, max_size] for all layers or [[], [], []...] for specific layers + ratios : list or list of list + [ratio1, ratio2...] for all layers or [[], [], ...] for specific layers + normalizations : int or list of int + use normalizations value for all layers or [...] for specific layers, + -1 indicate no normalizations and scales + steps : list + specify steps for each MultiBoxPrior layer, leave empty, it will calculate + according to layer dimensions + min_filter : int + minimum number of filters used in 1x1 convolution + nms_thresh : float + non-maximum suppression threshold + force_suppress : boolean + whether suppress different class objects + nms_topk : int + apply NMS to top K detections + + Returns + ------- + mx.Symbol + + """ + label = mx.sym.Variable('label') + body = import_module(network).get_symbol(num_classes, **kwargs) + layers = multi_layer_feature(body, from_layers, num_filters, strides, pads, + min_filter=min_filter) + + loc_preds, cls_preds, anchor_boxes = multibox_layer(layers, \ + num_classes, sizes=sizes, ratios=ratios, normalization=normalizations, \ + num_channels=num_filters, clip=False, interm_layer=0, steps=steps) + + tmp = mx.contrib.symbol.MultiBoxTarget( + *[anchor_boxes, label, cls_preds], overlap_threshold=.5, \ + ignore_label=-1, negative_mining_ratio=3, minimum_negative_samples=0, \ + negative_mining_thresh=.5, variances=(0.1, 0.1, 0.2, 0.2), + name="multibox_target") + loc_target = tmp[0] + loc_target_mask = tmp[1] + cls_target = tmp[2] + + cls_prob = mx.symbol.SoftmaxOutput(data=cls_preds, label=cls_target, \ + ignore_label=-1, use_ignore=True, grad_scale=1., multi_output=True, \ + normalization='valid', name="cls_prob") + loc_loss_ = mx.symbol.smooth_l1(name="loc_loss_", \ + data=loc_target_mask * (loc_preds - loc_target), scalar=1.0) + loc_loss = mx.symbol.MakeLoss(loc_loss_, grad_scale=1., \ + normalization='valid', name="loc_loss") + + # monitoring training status + cls_label = mx.symbol.MakeLoss(data=cls_target, grad_scale=0, name="cls_label") + det = mx.contrib.symbol.MultiBoxDetection(*[cls_prob, loc_preds, anchor_boxes], \ + name="detection", nms_threshold=nms_thresh, force_suppress=force_suppress, + variances=(0.1, 0.1, 0.2, 0.2), nms_topk=nms_topk) + det = mx.symbol.MakeLoss(data=det, grad_scale=0, name="det_out") + + # group output + out = mx.symbol.Group([cls_prob, loc_loss, cls_label, det]) + return out + +def get_symbol(network, num_classes, from_layers, num_filters, sizes, ratios, + strides, pads, normalizations=-1, steps=[], min_filter=128, + nms_thresh=0.5, force_suppress=False, nms_topk=400, **kwargs): + """Build network for testing SSD + + Parameters + ---------- + network : str + base network symbol name + num_classes : int + number of object classes not including background + from_layers : list of str + feature extraction layers, use '' for add extra layers + For example: + from_layers = ['relu4_3', 'fc7', '', '', '', ''] + which means extract feature from relu4_3 and fc7, adding 4 extra layers + on top of fc7 + num_filters : list of int + number of filters for extra layers, you can use -1 for extracted features, + however, if normalization and scale is applied, the number of filter for + that layer must be provided. + For example: + num_filters = [512, -1, 512, 256, 256, 256] + strides : list of int + strides for the 3x3 convolution appended, -1 can be used for extracted + feature layers + pads : list of int + paddings for the 3x3 convolution, -1 can be used for extracted layers + sizes : list or list of list + [min_size, max_size] for all layers or [[], [], []...] for specific layers + ratios : list or list of list + [ratio1, ratio2...] for all layers or [[], [], ...] for specific layers + normalizations : int or list of int + use normalizations value for all layers or [...] for specific layers, + -1 indicate no normalizations and scales + steps : list + specify steps for each MultiBoxPrior layer, leave empty, it will calculate + according to layer dimensions + min_filter : int + minimum number of filters used in 1x1 convolution + nms_thresh : float + non-maximum suppression threshold + force_suppress : boolean + whether suppress different class objects + nms_topk : int + apply NMS to top K detections + + Returns + ------- + mx.Symbol + + """ + body = import_module(network).get_symbol(num_classes, **kwargs) + layers = multi_layer_feature(body, from_layers, num_filters, strides, pads, + min_filter=min_filter) + + loc_preds, cls_preds, anchor_boxes = multibox_layer(layers, \ + num_classes, sizes=sizes, ratios=ratios, normalization=normalizations, \ + num_channels=num_filters, clip=False, interm_layer=0, steps=steps) + + cls_prob = mx.symbol.SoftmaxActivation(data=cls_preds, mode='channel', \ + name='cls_prob') + out = mx.contrib.symbol.MultiBoxDetection(*[cls_prob, loc_preds, anchor_boxes], \ + name="detection", nms_threshold=nms_thresh, force_suppress=force_suppress, + variances=(0.1, 0.1, 0.2, 0.2), nms_topk=nms_topk) + return out diff --git a/example/ssd/symbol/symbol_factory.py b/example/ssd/symbol/symbol_factory.py new file mode 100644 index 000000000000..ef156f194c88 --- /dev/null +++ b/example/ssd/symbol/symbol_factory.py @@ -0,0 +1,122 @@ +"""Presets for various network configurations""" +import logging +import symbol_builder + +def get_config(network, data_shape, **kwargs): + """Configuration factory for various networks + + Parameters + ---------- + network : str + base network name, such as vgg_reduced, inceptionv3, resnet... + data_shape : int + input data dimension + kwargs : dict + extra arguments + """ + if network == 'vgg16_reduced': + if data_shape >= 448: + from_layers = ['relu4_3', 'relu7', '', '', '', '', ''] + num_filters = [512, -1, 512, 256, 256, 256, 256] + strides = [-1, -1, 2, 2, 2, 2, 1] + pads = [-1, -1, 1, 1, 1, 1, 1] + sizes = [[.07, .1025], [.15,.2121], [.3, .3674], [.45, .5196], [.6, .6708], \ + [.75, .8216], [.9, .9721]] + ratios = [[1,2,.5], [1,2,.5,3,1./3], [1,2,.5,3,1./3], [1,2,.5,3,1./3], \ + [1,2,.5,3,1./3], [1,2,.5], [1,2,.5]] + normalizations = [20, -1, -1, -1, -1, -1, -1] + steps = [] if data_shape != 512 else [x / 512.0 for x in + [8, 16, 32, 64, 128, 256, 512]] + else: + from_layers = ['relu4_3', 'relu7', '', '', '', ''] + num_filters = [512, -1, 512, 256, 256, 256] + strides = [-1, -1, 2, 2, 1, 1] + pads = [-1, -1, 1, 1, 0, 0] + sizes = [[.1, .141], [.2,.272], [.37, .447], [.54, .619], [.71, .79], [.88, .961]] + ratios = [[1,2,.5], [1,2,.5,3,1./3], [1,2,.5,3,1./3], [1,2,.5,3,1./3], \ + [1,2,.5], [1,2,.5]] + normalizations = [20, -1, -1, -1, -1, -1] + steps = [] if data_shape != 300 else [x / 300.0 for x in [8, 16, 32, 64, 100, 300]] + if not (data_shape == 300 or data_shape == 512): + logging.warn('data_shape %d was not tested, use with caucious.' % data_shape) + return locals() + elif network == 'inceptionv3': + from_layers = ['ch_concat_mixed_7_chconcat', 'ch_concat_mixed_10_chconcat', '', '', '', ''] + num_filters = [-1, -1, 512, 256, 256, 128] + strides = [-1, -1, 2, 2, 2, 2] + pads = [-1, -1, 1, 1, 1, 1] + sizes = [[.1, .141], [.2,.272], [.37, .447], [.54, .619], [.71, .79], [.88, .961]] + ratios = [[1,2,.5], [1,2,.5,3,1./3], [1,2,.5,3,1./3], [1,2,.5,3,1./3], \ + [1,2,.5], [1,2,.5]] + normalizations = -1 + steps = [] + return locals() + elif network == 'resnet50': + num_layers = 50 + image_shape = '3,224,224' # resnet require it as shape check + network = 'resnet' + from_layers = ['_plus12', '_plus15', '', '', '', ''] + num_filters = [-1, -1, 512, 256, 256, 128] + strides = [-1, -1, 2, 2, 2, 2] + pads = [-1, -1, 1, 1, 1, 1] + sizes = [[.1, .141], [.2,.272], [.37, .447], [.54, .619], [.71, .79], [.88, .961]] + ratios = [[1,2,.5], [1,2,.5,3,1./3], [1,2,.5,3,1./3], [1,2,.5,3,1./3], \ + [1,2,.5], [1,2,.5]] + normalizations = -1 + steps = [] + return locals() + elif network == 'resnet101': + num_layers = 101 + image_shape = '3,224,224' + network = 'resnet' + from_layers = ['_plus12', '_plus15', '', '', '', ''] + num_filters = [-1, -1, 512, 256, 256, 128] + strides = [-1, -1, 2, 2, 2, 2] + pads = [-1, -1, 1, 1, 1, 1] + sizes = [[.1, .141], [.2,.272], [.37, .447], [.54, .619], [.71, .79], [.88, .961]] + ratios = [[1,2,.5], [1,2,.5,3,1./3], [1,2,.5,3,1./3], [1,2,.5,3,1./3], \ + [1,2,.5], [1,2,.5]] + normalizations = -1 + steps = [] + return locals() + else: + msg = 'No configuration found for %s with data_shape %d' % (network, data_shape) + raise NotImplementedError(msg) + +def get_symbol_train(network, data_shape, **kwargs): + """Wrapper for get symbol for train + + Parameters + ---------- + network : str + name for the base network symbol + data_shape : int + input shape + kwargs : dict + see symbol_builder.get_symbol_train for more details + """ + if network.startswith('legacy'): + logging.warn('Using legacy model.') + return symbol_builder.import_module(network).get_symbol_train(**kwargs) + config = get_config(network, data_shape, **kwargs).copy() + config.update(kwargs) + return symbol_builder.get_symbol_train(**config) + +def get_symbol(network, data_shape, **kwargs): + """Wrapper for get symbol for test + + Parameters + ---------- + network : str + name for the base network symbol + data_shape : int + input shape + kwargs : dict + see symbol_builder.get_symbol for more details + """ + if network.startswith('legacy'): + logging.warn('Using legacy model.') + return symbol_builder.import_module(network).get_symbol(**kwargs) + config = get_config(network, data_shape, **kwargs).copy() + config.update(kwargs) + return symbol_builder.get_symbol(**config) diff --git a/example/ssd/symbol/vgg16_reduced.py b/example/ssd/symbol/vgg16_reduced.py new file mode 100644 index 000000000000..c6a55b117cf8 --- /dev/null +++ b/example/ssd/symbol/vgg16_reduced.py @@ -0,0 +1,86 @@ +import mxnet as mx + +def get_symbol(num_classes=1000, **kwargs): + """ + VGG 16 layers network + This is a modified version, with fc6/fc7 layers replaced by conv layers + And the network is slightly smaller than original VGG 16 network + """ + data = mx.symbol.Variable(name="data") + label = mx.symbol.Variable(name="label") + + # group 1 + conv1_1 = mx.symbol.Convolution( + data=data, kernel=(3, 3), pad=(1, 1), num_filter=64, name="conv1_1") + relu1_1 = mx.symbol.Activation(data=conv1_1, act_type="relu", name="relu1_1") + conv1_2 = mx.symbol.Convolution( + data=relu1_1, kernel=(3, 3), pad=(1, 1), num_filter=64, name="conv1_2") + relu1_2 = mx.symbol.Activation(data=conv1_2, act_type="relu", name="relu1_2") + pool1 = mx.symbol.Pooling( + data=relu1_2, pool_type="max", kernel=(2, 2), stride=(2, 2), name="pool1") + # group 2 + conv2_1 = mx.symbol.Convolution( + data=pool1, kernel=(3, 3), pad=(1, 1), num_filter=128, name="conv2_1") + relu2_1 = mx.symbol.Activation(data=conv2_1, act_type="relu", name="relu2_1") + conv2_2 = mx.symbol.Convolution( + data=relu2_1, kernel=(3, 3), pad=(1, 1), num_filter=128, name="conv2_2") + relu2_2 = mx.symbol.Activation(data=conv2_2, act_type="relu", name="relu2_2") + pool2 = mx.symbol.Pooling( + data=relu2_2, pool_type="max", kernel=(2, 2), stride=(2, 2), name="pool2") + # group 3 + conv3_1 = mx.symbol.Convolution( + data=pool2, kernel=(3, 3), pad=(1, 1), num_filter=256, name="conv3_1") + relu3_1 = mx.symbol.Activation(data=conv3_1, act_type="relu", name="relu3_1") + conv3_2 = mx.symbol.Convolution( + data=relu3_1, kernel=(3, 3), pad=(1, 1), num_filter=256, name="conv3_2") + relu3_2 = mx.symbol.Activation(data=conv3_2, act_type="relu", name="relu3_2") + conv3_3 = mx.symbol.Convolution( + data=relu3_2, kernel=(3, 3), pad=(1, 1), num_filter=256, name="conv3_3") + relu3_3 = mx.symbol.Activation(data=conv3_3, act_type="relu", name="relu3_3") + pool3 = mx.symbol.Pooling( + data=relu3_3, pool_type="max", kernel=(2, 2), stride=(2, 2), \ + pooling_convention="full", name="pool3") + # group 4 + conv4_1 = mx.symbol.Convolution( + data=pool3, kernel=(3, 3), pad=(1, 1), num_filter=512, name="conv4_1") + relu4_1 = mx.symbol.Activation(data=conv4_1, act_type="relu", name="relu4_1") + conv4_2 = mx.symbol.Convolution( + data=relu4_1, kernel=(3, 3), pad=(1, 1), num_filter=512, name="conv4_2") + relu4_2 = mx.symbol.Activation(data=conv4_2, act_type="relu", name="relu4_2") + conv4_3 = mx.symbol.Convolution( + data=relu4_2, kernel=(3, 3), pad=(1, 1), num_filter=512, name="conv4_3") + relu4_3 = mx.symbol.Activation(data=conv4_3, act_type="relu", name="relu4_3") + pool4 = mx.symbol.Pooling( + data=relu4_3, pool_type="max", kernel=(2, 2), stride=(2, 2), name="pool4") + # group 5 + conv5_1 = mx.symbol.Convolution( + data=pool4, kernel=(3, 3), pad=(1, 1), num_filter=512, name="conv5_1") + relu5_1 = mx.symbol.Activation(data=conv5_1, act_type="relu", name="relu5_1") + conv5_2 = mx.symbol.Convolution( + data=relu5_1, kernel=(3, 3), pad=(1, 1), num_filter=512, name="conv5_2") + relu5_2 = mx.symbol.Activation(data=conv5_2, act_type="relu", name="relu5_2") + conv5_3 = mx.symbol.Convolution( + data=relu5_2, kernel=(3, 3), pad=(1, 1), num_filter=512, name="conv5_3") + relu5_3 = mx.symbol.Activation(data=conv5_3, act_type="relu", name="relu5_3") + pool5 = mx.symbol.Pooling( + data=relu5_3, pool_type="max", kernel=(3, 3), stride=(1, 1), + pad=(1,1), name="pool5") + # group 6 + conv6 = mx.symbol.Convolution( + data=pool5, kernel=(3, 3), pad=(6, 6), dilate=(6, 6), + num_filter=1024, name="fc6") + relu6 = mx.symbol.Activation(data=conv6, act_type="relu", name="relu6") + # drop6 = mx.symbol.Dropout(data=relu6, p=0.5, name="drop6") + # group 7 + conv7 = mx.symbol.Convolution( + data=relu6, kernel=(1, 1), pad=(0, 0), num_filter=1024, name="fc7") + relu7 = mx.symbol.Activation(data=conv7, act_type="relu", name="relu7") + # drop7 = mx.symbol.Dropout(data=relu7, p=0.5, name="drop7") + + gpool = mx.symbol.Pooling(data=relu7, pool_type='avg', kernel=(7, 7), + global_pool=True, name='global_pool') + conv8 = mx.symbol.Convolution(data=gpool, num_filter=num_classes, kernel=(1, 1), + name='fc8') + flat = mx.symbol.Flatten(data=conv8) + softmax = mx.symbol.SoftmaxOutput(data=flat, name='softmax') + return softmax diff --git a/example/ssd/tools/prepare_coco.sh b/example/ssd/tools/prepare_coco.sh new file mode 100644 index 000000000000..1b11f9384946 --- /dev/null +++ b/example/ssd/tools/prepare_coco.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +python $DIR/prepare_dataset.py --dataset coco --set train2014,valminusminival2014 --target $DIR/../data/train.lst --root $DIR/../data/coco +python $DIR/prepare_dataset.py --dataset coco --set minival2014 --target $DIR/../data/val.lst --shuffle False --root $DIR/../data/coco diff --git a/example/ssd/tools/prepare_dataset.py b/example/ssd/tools/prepare_dataset.py index 7bd696840766..12b090c27841 100644 --- a/example/ssd/tools/prepare_dataset.py +++ b/example/ssd/tools/prepare_dataset.py @@ -5,6 +5,7 @@ curr_path = os.path.abspath(os.path.dirname(__file__)) sys.path.append(os.path.join(curr_path, '..')) from dataset.pascal_voc import PascalVoc +from dataset.mscoco import Coco from dataset.concat_db import ConcatDB def load_pascal(image_set, year, devkit_path, shuffle=False): @@ -46,6 +47,30 @@ def load_pascal(image_set, year, devkit_path, shuffle=False): else: return imdbs[0] +def load_coco(image_set, dirname, shuffle=False): + """ + wrapper function for loading ms coco dataset + + Parameters: + ---------- + image_set : str + train2014, val2014, valminusminival2014, minival2014 + dirname: str + root dir for coco + shuffle: boolean + initial shuffle + """ + anno_files = ['instances_' + y.strip() + '.json' for y in image_set.split(',')] + assert anno_files, "No image set specified" + imdbs = [] + for af in anno_files: + af_path = os.path.join(dirname, 'annotations', af) + imdbs.append(Coco(af_path, dirname, shuffle=shuffle)) + if len(imdbs) > 1: + return ConcatDB(imdbs, shuffle) + else: + return imdbs[0] + def parse_args(): parser = argparse.ArgumentParser(description='Prepare lists for dataset') parser.add_argument('--dataset', dest='dataset', help='dataset to use', @@ -69,6 +94,11 @@ def parse_args(): args = parse_args() if args.dataset == 'pascal': db = load_pascal(args.set, args.year, args.root_path, args.shuffle) + print("saving list to disk...") + db.save_imglist(args.target, root=args.root_path) + elif args.dataset == 'coco': + db = load_coco(args.set, args.root_path, args.shuffle) + print("saving list to disk...") db.save_imglist(args.target, root=args.root_path) else: raise NotImplementedError("No implementation for dataset: " + args.dataset) diff --git a/example/ssd/tools/visualize_net.py b/example/ssd/tools/visualize_net.py index e619c230bb90..f72d6a6678fa 100644 --- a/example/ssd/tools/visualize_net.py +++ b/example/ssd/tools/visualize_net.py @@ -1,13 +1,14 @@ from __future__ import print_function import find_mxnet import mxnet as mx -import importlib import argparse -import sys +import sys, os +sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'symbol')) +import symbol_factory + parser = argparse.ArgumentParser(description='network visualization') -parser.add_argument('--network', type=str, default='vgg16_ssd_300', - choices = ['vgg16_ssd_300', 'vgg16_ssd_512'], +parser.add_argument('--network', type=str, default='vgg16_reduced', help = 'the cnn to use') parser.add_argument('--num-classes', type=int, default=20, help='the number of classes') @@ -16,13 +17,11 @@ parser.add_argument('--train', action='store_true', default=False, help='show train net') args = parser.parse_args() -sys.path.append('../symbol') - if not args.train: - net = importlib.import_module("symbol_" + args.network).get_symbol(args.num_classes) + net = symbol_factory.get_symbol(args.network, args.data_shape, num_classes=args.num_classes) a = mx.viz.plot_network(net, shape={"data":(1,3,args.data_shape,args.data_shape)}, \ node_attrs={"shape":'rect', "fixedsize":'false'}) - a.render("ssd_" + args.network) + a.render("ssd_" + args.network + '_' + str(args.data_shape)) else: - net = importlib.import_module("symbol_" + args.network).get_symbol_train(args.num_classes) + net = symbol_factory.get_symbol_train(args.network, args.data_shape, num_classes=args.num_classes) print(net.tojson()) diff --git a/example/ssd/train.py b/example/ssd/train.py index fcd5fb95e8f1..62ad2c4e0a78 100644 --- a/example/ssd/train.py +++ b/example/ssd/train.py @@ -15,8 +15,8 @@ def parse_args(): default=os.path.join(os.getcwd(), 'data', 'val.rec'), type=str) parser.add_argument('--val-list', dest='val_list', help='validation list to use', default="", type=str) - parser.add_argument('--network', dest='network', type=str, default='vgg16_ssd_300', - choices=['vgg16_ssd_300', 'vgg16_ssd_512'], help='which network to use') + parser.add_argument('--network', dest='network', type=str, default='vgg16_reduced', + help='which network to use') parser.add_argument('--batch-size', dest='batch_size', type=int, default=32, help='training batch size') parser.add_argument('--resume', dest='resume', type=int, default=-1, @@ -41,7 +41,7 @@ def parse_args(): help='set image shape') parser.add_argument('--label-width', dest='label_width', type=int, default=350, help='force padding label width to sync across train and validation') - parser.add_argument('--lr', dest='learning_rate', type=float, default=0.004, + parser.add_argument('--lr', dest='learning_rate', type=float, default=0.002, help='learning rate') parser.add_argument('--momentum', dest='momentum', type=float, default=0.9, help='momentum') @@ -53,7 +53,7 @@ def parse_args(): help='green mean value') parser.add_argument('--mean-b', dest='mean_b', type=float, default=104, help='blue mean value') - parser.add_argument('--lr-steps', dest='lr_refactor_step', type=str, default='150, 200', + parser.add_argument('--lr-steps', dest='lr_refactor_step', type=str, default='80, 160', help='refactor learning rate at specified epochs') parser.add_argument('--lr-factor', dest='lr_refactor_ratio', type=str, default=0.1, help='ratio to refactor learning rate') @@ -92,9 +92,9 @@ def parse_class_names(args): num_class = args.num_class if len(args.class_names) > 0: if os.path.isfile(args.class_names): - # try to open it to read class names - with open(args.class_names, 'r') as f: - class_names = [l.strip() for l in f.readlines()] + # try to open it to read class names + with open(args.class_names, 'r') as f: + class_names = [l.strip() for l in f.readlines()] else: class_names = [c.strip() for c in args.class_names.split(',')] assert len(class_names) == num_class, str(len(class_names)) diff --git a/example/ssd/train/train_net.py b/example/ssd/train/train_net.py index 3f73ee880ef5786e2ca6773310e4c2ee3993b08e..54cae05a4421bf0587e445174ace680f90aa73f5 100644 GIT binary patch delta 304 zcmZXPO-chn5QP~-Km~Uu;I2S~32BmyxDv(-h+ZH_(~dRHK>txy17_s~+zKATh0qtu zNrEFvj98oZz4xGsdcR56ALH!2rr>~A?mpPrx9Hkb5@-dji-80lQS?tEWLlp_S&=0g zsNuA&!3O~rjC^?rJ3}ZPdB`m(xD6gB>2-f|H1sGiO)ulP_xy$BzvX!!D|j$sG&d&V zvba;=JEtpan32mJR@;l-v6hCAoAMVG)PXA0+{M^eT>i0XeR6Sd26M3J^Gcp3&_A=~ RD_8(pMkr@?N-vI9 zPtMOPGmB5kOe|K9Rj^e63F;N(7ig%bB}0WZvFa#K%}mcI!J-EwQ4f^Crp+9?MNnbv zI&fHI9uJj3=t@b=0lPn1T{A7&0wx~|cRoTMBn|QM<`O1D*3CEBzw&OrC7Q`pUt9_F zequ?6USdH(YF>&)K9rl4pP8qToR+Q^m0?`a)0JRiXYJyCJs? obj.left && obj.bottom > obj.top) { + objects_.push_back(obj); + } } } From 9e66154b1edda0d8b1d64b9b1fd95e6fed8b18cc Mon Sep 17 00:00:00 2001 From: Yao Wang Date: Mon, 26 Jun 2017 22:35:16 -0700 Subject: [PATCH 116/834] Add mxnet issue and roadmap link (#6825) --- docs/how_to/index.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/how_to/index.md b/docs/how_to/index.md index d7e42eaa2094..1e96a66e230a 100644 --- a/docs/how_to/index.md +++ b/docs/how_to/index.md @@ -47,3 +47,12 @@ and full working examples, visit the [tutorials section](../tutorials/index.md). * [How do I set MXNet's environmental variables?](http://mxnet.io/how_to/env_var.html) * [How do I use MXNet as a front end for Torch?](http://mxnet.io/how_to/torch.html) + +## Questions about Using MXNet +If you are not sure of how to use MXNet for something, or have questions about applying it to a particular kind of problem, please post a question at [Stackoverflow](http://stackoverflow.com/) with tag - ```mxnet```. You can view StackOverflow questions about mxnet [here](http://stackoverflow.com/questions/tagged/mxnet). + +## Issue Tracker +We track bugs and new feature requests in the MXNet Github repo in the issues folder: [mxnet/issues](https://github.com/dmlc/mxnet/issues). + +## Roadmap +MXNet is evolving fast. To see what's next and what we are working on internally, go to the [MXNet Roadmap](https://github.com/dmlc/mxnet/labels/Roadmap). From 5acc2c90d026015ad02b66839c19f72bdce35dfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=A2=81=E5=BE=B7=E6=BE=8E?= Date: Tue, 27 Jun 2017 22:50:29 +0800 Subject: [PATCH 117/834] [Scala] support str key type in kvstore (#6829) --- .../main/scala/ml/dmlc/mxnet/KVStore.scala | 30 ++++---- .../main/scala/ml/dmlc/mxnet/LibInfo.scala | 14 ++++ .../src/main/scala/ml/dmlc/mxnet/Model.scala | 22 ++++-- .../scala/ml/dmlc/mxnet/module/Module.scala | 4 +- .../scala/ml/dmlc/mxnet/KVStoreSuite.scala | 40 +++++++--- .../main/native/ml_dmlc_mxnet_native_c_api.cc | 77 ++++++++++++++++++- 6 files changed, 152 insertions(+), 35 deletions(-) diff --git a/scala-package/core/src/main/scala/ml/dmlc/mxnet/KVStore.scala b/scala-package/core/src/main/scala/ml/dmlc/mxnet/KVStore.scala index 32e0acec1572..94dd25497a9e 100644 --- a/scala-package/core/src/main/scala/ml/dmlc/mxnet/KVStore.scala +++ b/scala-package/core/src/main/scala/ml/dmlc/mxnet/KVStore.scala @@ -83,13 +83,13 @@ class KVStore(private[mxnet] val handle: KVStoreHandle) { * @param keys The keys. * @param values The values. */ - def init(keys: Array[Int], values: Array[NDArray]): Unit = { + def init(keys: Array[String], values: Array[NDArray]): Unit = { require(keys.length == values.length, "len(keys) != len(values)") val valuePtrs = values.map(_.handle) - checkCall(_LIB.mxKVStoreInit(handle, keys.length, keys, valuePtrs)) + checkCall(_LIB.mxKVStoreInitEx(handle, keys.length, keys, valuePtrs)) } - def init(key: Int, value: NDArray): Unit = { + def init(key: String, value: NDArray): Unit = { init(Array(key), Array(value)) } @@ -107,24 +107,24 @@ class KVStore(private[mxnet] val handle: KVStoreHandle) { * The higher the priority, the faster this action is likely * to be executed before other push actions. */ - def push(keys: Array[Int], values: Array[NDArray], priority: Int): Unit = { + def push(keys: Array[String], values: Array[NDArray], priority: Int): Unit = { require(keys.length == values.length, "len(keys) != len(values)") val valuePtrs = values.map(_.handle) - checkCall(_LIB.mxKVStorePush(handle, keys.length, keys, valuePtrs, priority)) + checkCall(_LIB.mxKVStorePushEx(handle, keys.length, keys, valuePtrs, priority)) } - def push(keys: Array[Int], values: Array[NDArray]): Unit = push(keys, values, 0) + def push(keys: Array[String], values: Array[NDArray]): Unit = push(keys, values, 0) - def push(key: Int, value: NDArray, priority: Int = 0): Unit = { + def push(key: String, value: NDArray, priority: Int = 0): Unit = { push(Array(key), Array(value), priority) } - def push(key: Int, values: Array[NDArray], priority: Int): Unit = { + def push(key: String, values: Array[NDArray], priority: Int): Unit = { val keys = Array.fill(values.length)(key) push(keys, values, priority) } - def push(key: Int, values: Array[NDArray]): Unit = { + def push(key: String, values: Array[NDArray]): Unit = { push(key, values, 0) } @@ -143,24 +143,24 @@ class KVStore(private[mxnet] val handle: KVStoreHandle) { * The higher the priority, the faster this action is likely * to be executed before other push actions. */ - def pull(keys: Array[Int], outs: Array[NDArray], priority: Int): Unit = { + def pull(keys: Array[String], outs: Array[NDArray], priority: Int): Unit = { require(keys.length == outs.length, "len(keys) != len(outs)") val outPtrs = outs.map(_.handle) - checkCall(_LIB.mxKVStorePull(handle, keys.length, keys, outPtrs, priority)) + checkCall(_LIB.mxKVStorePullEx(handle, keys.length, keys, outPtrs, priority)) } - def pull(keys: Array[Int], outs: Array[NDArray]): Unit = pull(keys, outs, 0) + def pull(keys: Array[String], outs: Array[NDArray]): Unit = pull(keys, outs, 0) - def pull(key: Int, out: NDArray, priority: Int = 0): Unit = { + def pull(key: String, out: NDArray, priority: Int = 0): Unit = { pull(Array(key), Array(out), priority) } - def pull(key: Int, outs: Array[NDArray], priority: Int): Unit = { + def pull(key: String, outs: Array[NDArray], priority: Int): Unit = { val keys = Array.fill(outs.length)(key) pull(keys, outs, priority) } - def pull(key: Int, outs: Array[NDArray]): Unit = { + def pull(key: String, outs: Array[NDArray]): Unit = { pull(key, outs, 0) } diff --git a/scala-package/core/src/main/scala/ml/dmlc/mxnet/LibInfo.scala b/scala-package/core/src/main/scala/ml/dmlc/mxnet/LibInfo.scala index 97ba81528d30..a943e314055f 100644 --- a/scala-package/core/src/main/scala/ml/dmlc/mxnet/LibInfo.scala +++ b/scala-package/core/src/main/scala/ml/dmlc/mxnet/LibInfo.scala @@ -117,16 +117,30 @@ private[mxnet] class LibInfo { len: MXUint, keys: Array[Int], values: Array[NDArrayHandle]): Int + @native def mxKVStoreInitEx(handle: KVStoreHandle, + len: MXUint, + keys: Array[String], + values: Array[NDArrayHandle]): Int @native def mxKVStorePush(handle: KVStoreHandle, len: MXUint, keys: Array[Int], values: Array[NDArrayHandle], priority: Int): Int + @native def mxKVStorePushEx(handle: KVStoreHandle, + len: MXUint, + keys: Array[String], + values: Array[NDArrayHandle], + priority: Int): Int @native def mxKVStorePull(handle: KVStoreHandle, len: MXUint, keys: Array[Int], outs: Array[NDArrayHandle], priority: Int): Int + @native def mxKVStorePullEx(handle: KVStoreHandle, + len: MXUint, + keys: Array[String], + outs: Array[NDArrayHandle], + priority: Int): Int @native def mxKVStoreSetUpdater(handle: KVStoreHandle, updaterFunc: MXKVStoreUpdater): Int @native def mxKVStoreIsWorkerNode(isWorker: RefInt): Int @native def mxKVStoreGetType(handle: KVStoreHandle, kvType: RefString): Int diff --git a/scala-package/core/src/main/scala/ml/dmlc/mxnet/Model.scala b/scala-package/core/src/main/scala/ml/dmlc/mxnet/Model.scala index 69fe68255274..81ff1cfb3c7d 100644 --- a/scala-package/core/src/main/scala/ml/dmlc/mxnet/Model.scala +++ b/scala-package/core/src/main/scala/ml/dmlc/mxnet/Model.scala @@ -163,9 +163,10 @@ object Model { require(paramArrays.length == paramNames.length) for (idx <- 0 until paramArrays.length) { val paramOnDevs = paramArrays(idx) - kvStore.init(idx, argParams(paramNames(idx))) + val name = paramNames(idx) + kvStore.init(name, argParams(paramNames(idx))) if (updateOnKVStore) { - kvStore.pull(idx, paramOnDevs, -idx) + kvStore.pull(name, paramOnDevs, -idx) } } } @@ -173,13 +174,15 @@ object Model { // Perform update of param_arrays from grad_arrays on kvstore private[mxnet] def updateParamsOnKVStore(paramArrays: IndexedSeq[Array[NDArray]], gradArrays: IndexedSeq[Array[NDArray]], - kvStore: Option[KVStore]): Unit = { + kvStore: Option[KVStore], + paramNames: IndexedSeq[String]): Unit = { (paramArrays zip gradArrays).zipWithIndex.foreach { case ((argList, gradList), index) => if (gradList != null) { + val name = paramNames(index) // push gradient, priority is negative index - kvStore.foreach(_.push(index, gradList, -index)) + kvStore.foreach(_.push(name, gradList, -index)) // pull back the weights - kvStore.foreach(_.pull(index, argList, -index)) + kvStore.foreach(_.pull(name, argList, -index)) } } } @@ -189,14 +192,16 @@ object Model { gradArrays: IndexedSeq[Array[NDArray]], updater: MXKVStoreUpdater, numDevice: Int, + paramNames: IndexedSeq[String], kvStore: Option[KVStore] = None) { (paramArrays zip gradArrays).zipWithIndex.foreach { case ((argList, gradList), index) => if (gradList != null) { kvStore.foreach(kv => { + val name = paramNames(index) // push gradient, priority is negative index - kv.push(index, gradList, -index) + kv.push(name, gradList, -index) // pull back the sum gradients, to the same locations. - kv.pull(index, gradList, -index) + kv.pull(name, gradList, -index) }) (argList zip gradList).zipWithIndex.foreach { case ((w: NDArray, g: NDArray), k: Int) => // faked an index here, to make optimizer create diff @@ -295,11 +300,12 @@ object Model { if (updateOnKVStore) { updateParamsOnKVStore(executorManager.paramArrays, executorManager.gradArrays, - kvStore) + kvStore, executorManager.paramNames) } else { updateParams(executorManager.paramArrays, executorManager.gradArrays, updaterLocal, ctx.length, + executorManager.paramNames, kvStore) } monitor.foreach(_.tocPrint()) diff --git a/scala-package/core/src/main/scala/ml/dmlc/mxnet/module/Module.scala b/scala-package/core/src/main/scala/ml/dmlc/mxnet/module/Module.scala index f0b8da0ecfcb..2b1d743ea648 100644 --- a/scala-package/core/src/main/scala/ml/dmlc/mxnet/module/Module.scala +++ b/scala-package/core/src/main/scala/ml/dmlc/mxnet/module/Module.scala @@ -365,11 +365,11 @@ class Module(symbolVar: Symbol, paramsDirty = true if (updateOnKVStore) { Model.updateParamsOnKVStore(execGroup.paramArrays, - execGroup.gradArrays, kvstore) + execGroup.gradArrays, kvstore, execGroup.paramNames) } else { require(updater != None) Model.updateParams(execGroup.paramArrays, - execGroup.gradArrays, updater.orNull, contexts.length, kvstore) + execGroup.gradArrays, updater.orNull, contexts.length, execGroup.paramNames, kvstore) } } diff --git a/scala-package/core/src/test/scala/ml/dmlc/mxnet/KVStoreSuite.scala b/scala-package/core/src/test/scala/ml/dmlc/mxnet/KVStoreSuite.scala index f024e8d16e41..8df6d18e2a33 100644 --- a/scala-package/core/src/test/scala/ml/dmlc/mxnet/KVStoreSuite.scala +++ b/scala-package/core/src/test/scala/ml/dmlc/mxnet/KVStoreSuite.scala @@ -25,8 +25,8 @@ class KVStoreSuite extends FunSuite with BeforeAndAfterAll { val shape = Shape(2, 1) val ndArray = NDArray.zeros(shape) - kv.init(3, NDArray.ones(shape)) - kv.pull(3, ndArray) + kv.init("3", NDArray.ones(shape)) + kv.pull("3", ndArray) assert(ndArray.toArray === Array(1f, 1f)) } @@ -35,12 +35,34 @@ class KVStoreSuite extends FunSuite with BeforeAndAfterAll { val shape = Shape(2, 1) val ndArray = NDArray.zeros(shape) - kv.init(3, NDArray.ones(shape)) - kv.push(3, NDArray.ones(shape) * 4) - kv.pull(3, ndArray) + kv.init("3", NDArray.ones(shape)) + kv.push("3", NDArray.ones(shape) * 4) + kv.pull("3", ndArray) assert(ndArray.toArray === Array(4f, 4f)) } + test("test aggregate") { + val shape = Shape(4, 4) + val keys = Array("b", "c", "d") + val kv = KVStore.create() + kv.init("a", NDArray.zeros(shape)) + kv.init(keys, Array.fill(keys.length)(NDArray.zeros(shape))) + val numDevs = 4 + val devs = (0 until numDevs).map(Context.cpu(_)) + val vals = devs.map(d => NDArray.ones(shape, d)).toArray + kv.push("a", vals) + kv.pull("a", outs = vals) + assert(vals.map(v => v.toArray.map(x => x - numDevs).sum).sum == 0f) + + val valss = keys.map { k => + val tmpVals = devs.map(d => NDArray.ones(shape, d) * 2f).toArray + kv.push(k, tmpVals) + kv.pull(k, outs = tmpVals) + tmpVals + }.flatten + assert(valss.map(v => v.toArray.map(x => x - numDevs * 2f).sum).sum == 0f) + } + test("updater runs when push") { val kv = KVStore.create() val updater = new MXKVStoreUpdater { @@ -57,12 +79,12 @@ class KVStoreSuite extends FunSuite with BeforeAndAfterAll { val shape = Shape(2, 1) val ndArray = NDArray.zeros(shape) - kv.init(3, NDArray.ones(shape) * 4) - kv.pull(3, ndArray) + kv.init("3", NDArray.ones(shape) * 4) + kv.pull("3", ndArray) assert(ndArray.toArray === Array(4f, 4f)) - kv.push(3, NDArray.ones(shape)) - kv.pull(3, ndArray) + kv.push("3", NDArray.ones(shape)) + kv.pull("3", ndArray) assert(ndArray.toArray === Array(6f, 6f)) } diff --git a/scala-package/native/src/main/native/ml_dmlc_mxnet_native_c_api.cc b/scala-package/native/src/main/native/ml_dmlc_mxnet_native_c_api.cc index 65bf2b77579b..07fd07596ee7 100644 --- a/scala-package/native/src/main/native/ml_dmlc_mxnet_native_c_api.cc +++ b/scala-package/native/src/main/native/ml_dmlc_mxnet_native_c_api.cc @@ -654,6 +654,30 @@ JNIEXPORT jint JNICALL Java_ml_dmlc_mxnet_LibInfo_mxKVStoreInit return ret; } +JNIEXPORT jint JNICALL Java_ml_dmlc_mxnet_LibInfo_mxKVStoreInitEx + (JNIEnv *env, jobject obj, jlong kvStorePtr, jint len, jobjectArray keys, jlongArray values) { + const char **keyArray = new const char *[len]; + for (int i = 0; i < len; i++) { + jstring jkey = reinterpret_cast(env->GetObjectArrayElement(keys, i)); + const char *key = env->GetStringUTFChars(jkey, 0); + keyArray[i] = key; + env->DeleteLocalRef(jkey); + } + jlong *valueArray = env->GetLongArrayElements(values, NULL); + int ret = MXKVStoreInitEx(reinterpret_cast(kvStorePtr), + static_cast(len), + keyArray, + reinterpret_cast(valueArray)); + env->ReleaseLongArrayElements(values, valueArray, 0); + for (int i = 0; i < len; i++) { + jstring jkey = reinterpret_cast(env->GetObjectArrayElement(keys, i)); + env->ReleaseStringUTFChars(jkey, keyArray[i]); + env->DeleteLocalRef(jkey); + } + delete[] keyArray; + return ret; +} + JNIEXPORT jint JNICALL Java_ml_dmlc_mxnet_LibInfo_mxKVStorePush (JNIEnv *env, jobject obj, jlong kvStorePtr, jint len, jintArray keys, jlongArray values, jint priority) { @@ -664,11 +688,36 @@ JNIEXPORT jint JNICALL Java_ml_dmlc_mxnet_LibInfo_mxKVStorePush static_cast(keyArray), reinterpret_cast(valueArray), priority); - env->ReleaseIntArrayElements(keys, keyArray, 0); env->ReleaseLongArrayElements(values, valueArray, 0); return ret; } +JNIEXPORT jint JNICALL Java_ml_dmlc_mxnet_LibInfo_mxKVStorePushEx + (JNIEnv *env, jobject obj, jlong kvStorePtr, jint len, jobjectArray keys, + jlongArray values, jint priority) { + const char **keyArray = new const char *[len]; + for (int i = 0; i < len; i++) { + jstring jkey = reinterpret_cast(env->GetObjectArrayElement(keys, i)); + const char *key = env->GetStringUTFChars(jkey, 0); + keyArray[i] = key; + env->DeleteLocalRef(jkey); + } + jlong *valueArray = env->GetLongArrayElements(values, NULL); + int ret = MXKVStorePushEx(reinterpret_cast(kvStorePtr), + static_cast(len), + keyArray, + reinterpret_cast(valueArray), + priority); + env->ReleaseLongArrayElements(values, valueArray, 0); + for (int i = 0; i < len; i++) { + jstring jkey = reinterpret_cast(env->GetObjectArrayElement(keys, i)); + env->ReleaseStringUTFChars(jkey, keyArray[i]); + env->DeleteLocalRef(jkey); + } + delete[] keyArray; + return ret; +} + JNIEXPORT jint JNICALL Java_ml_dmlc_mxnet_LibInfo_mxKVStorePull (JNIEnv *env, jobject obj, jlong kvStorePtr, jint len, jintArray keys, jlongArray outs, jint priority) { @@ -684,6 +733,32 @@ JNIEXPORT jint JNICALL Java_ml_dmlc_mxnet_LibInfo_mxKVStorePull return ret; } +JNIEXPORT jint JNICALL Java_ml_dmlc_mxnet_LibInfo_mxKVStorePullEx + (JNIEnv *env, jobject obj, jlong kvStorePtr, jint len, jobjectArray keys, + jlongArray outs, jint priority) { + const char **keyArray = new const char *[len]; + for (int i = 0; i < len; i++) { + jstring jkey = reinterpret_cast(env->GetObjectArrayElement(keys, i)); + const char *key = env->GetStringUTFChars(jkey, 0); + keyArray[i] = key; + env->DeleteLocalRef(jkey); + } + jlong *outArray = env->GetLongArrayElements(outs, NULL); + int ret = MXKVStorePullEx(reinterpret_cast(kvStorePtr), + static_cast(len), + keyArray, + reinterpret_cast(outArray), + priority); + env->ReleaseLongArrayElements(outs, outArray, 0); + for (int i = 0; i < len; i++) { + jstring jkey = reinterpret_cast(env->GetObjectArrayElement(keys, i)); + env->ReleaseStringUTFChars(jkey, keyArray[i]); + env->DeleteLocalRef(jkey); + } + delete[] keyArray; + return ret; +} + JNIEXPORT jint JNICALL Java_ml_dmlc_mxnet_LibInfo_mxKVStoreGetType (JNIEnv *env, jobject obj, jlong kvStorePtr, jobject kvType) { const char *type; From 459efce26648e93100c880d5e820f785ce86eb09 Mon Sep 17 00:00:00 2001 From: vafl Date: Tue, 27 Jun 2017 19:13:51 +0200 Subject: [PATCH 118/834] fix numerical issue in softrelu (#6834) --- src/operator/mshadow_op.h | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/operator/mshadow_op.h b/src/operator/mshadow_op.h index eac2a2c50d24..245df1bea616 100644 --- a/src/operator/mshadow_op.h +++ b/src/operator/mshadow_op.h @@ -136,13 +136,20 @@ struct tanh_grad { struct softrelu { template MSHADOW_XINLINE static DType Map(DType a) { - return DType(log1pf(expf(a))); + // Avoid overflow of exp for large inputs. + // Thresholds 20.0 is chosen such that softrelu(a) = a + // for a > 20 using floating precision. + if (a > DType(20.0)) { + return a; + } else { + return DType(log1pf(expf(a))); + } } }; struct softrelu_grad { template MSHADOW_XINLINE static DType Map(DType a) { - return DType(DType(1.0f) - expf(-a)); + return -DType(expm1f(-a)); } }; From ff968225167a9f10e255600044028280f5696469 Mon Sep 17 00:00:00 2001 From: Yao Wang Date: Tue, 27 Jun 2017 10:15:12 -0700 Subject: [PATCH 119/834] Module forward reshape resolving conflicts (#6805) * Add module reshape * Module forward reshape * Small fix * Pass dtype * Resolve conflict * More fix * Fix lint --- python/mxnet/module/base_module.py | 25 +++-- python/mxnet/module/module.py | 33 ++++++- tests/python/unittest/test_module.py | 131 +++++++++++++++++++++++++-- 3 files changed, 172 insertions(+), 17 deletions(-) diff --git a/python/mxnet/module/base_module.py b/python/mxnet/module/base_module.py index a5c4c709780a..cb6cfccb2759 100644 --- a/python/mxnet/module/base_module.py +++ b/python/mxnet/module/base_module.py @@ -751,7 +751,11 @@ def prepare(self, data_batch): pass def forward(self, data_batch, is_train=None): - """Forward computation. + """Forward computation. It supports data batches with different shapes, such as + different batch sizes or different image sizes. + If reshaping of data batch relates to modification of symbol or module, such as + changing image layout ordering or switching from training to predicting, module + rebinding is required. Parameters ---------- @@ -762,16 +766,25 @@ def forward(self, data_batch, is_train=None): Examples -------- - >>> # An example of forward computation. + >>> import mxnet as mx >>> from collections import namedtuple >>> Batch = namedtuple('Batch', ['data']) - >>> mod.bind(data_shapes=[('data', (1, 10, 10))]) + >>> data = mx.sym.Variable('data') + >>> out = data * 2 + >>> mod = mx.mod.Module(symbol=out, label_names=None) + >>> mod.bind(data_shapes=[('data', (1, 10))]) >>> mod.init_params() - >>> data1 = [mx.nd.ones([1, 10, 10])] + >>> data1 = [mx.nd.ones((1, 10))] >>> mod.forward(Batch(data1)) >>> print mod.get_outputs()[0].asnumpy() - [[ 0.09999977 0.10000153 0.10000716 0.10000195 0.09999853 0.09999743 - 0.10000272 0.10000113 0.09999088 0.09999888]] + [[ 2. 2. 2. 2. 2. 2. 2. 2. 2. 2.]] + >>> # Forward with data batch of different shape + >>> data2 = [mx.nd.ones((3, 5))] + >>> mod.forward(Batch(data2)) + >>> print mod.get_outputs()[0].asnumpy() + [[ 2. 2. 2. 2. 2.] + [ 2. 2. 2. 2. 2.] + [ 2. 2. 2. 2. 2.]] """ raise NotImplementedError() diff --git a/python/mxnet/module/module.py b/python/mxnet/module/module.py index f5f3c2ade3d0..1b5ecbc3c301 100644 --- a/python/mxnet/module/module.py +++ b/python/mxnet/module/module.py @@ -15,6 +15,7 @@ from ..model import _create_kvstore, _initialize_kvstore, _update_params, _update_params_on_kvstore from ..model import load_checkpoint from ..initializer import Uniform, InitDesc +from ..io import DataDesc from .base_module import BaseModule, _check_input_names, _parse_data_desc @@ -535,7 +536,11 @@ def borrow_optimizer(self, shared_module): self.optimizer_initialized = True def forward(self, data_batch, is_train=None): - """Forward computation. + """Forward computation. It supports data batches with different shapes, such as + different batch sizes or different image sizes. + If reshaping of data batch relates to modification of symbol or module, such as + changing image layout ordering or switching from training to predicting, module + rebinding is required. See Also ---------- @@ -549,6 +554,32 @@ def forward(self, data_batch, is_train=None): Default is ``None``, which means ``is_train`` takes the value of ``self.for_training``. """ assert self.binded and self.params_initialized + + # If start to inference, force rebind module. + if self._label_shapes and not data_batch.label: + raise RuntimeError("If you are trying to do inference, rebind module " + "with 'force_rebind=True' and 'for_training=False'") + + curr_data_shapes = (i.shape for i in self._data_shapes) + new_data_shapes = (i.shape for i in data_batch.data) + + if curr_data_shapes != new_data_shapes: + if hasattr(data_batch, "provide_data") and data_batch.provide_data: + new_dshape = data_batch.provide_data + else: + new_dshape = [DataDesc(i.name, shape, i.dtype, i.layout) \ + for i, shape in zip(self._data_shapes, new_data_shapes)] + + if hasattr(data_batch, "provide_label") and data_batch.provide_label: + new_lshape = data_batch.provide_label + elif hasattr(data_batch, "label") and data_batch.label: + new_lshape = [DataDesc(i.name, j.shape, i.dtype, i.layout) \ + for i, j in zip(self._label_shapes, data_batch.label)] + else: + new_lshape = None + + self.reshape(new_dshape, new_lshape) + self._exec_group.forward(data_batch, is_train) def backward(self, out_grads=None): diff --git a/tests/python/unittest/test_module.py b/tests/python/unittest/test_module.py index 8990aafe2838..766995dd2ac9 100644 --- a/tests/python/unittest/test_module.py +++ b/tests/python/unittest/test_module.py @@ -4,6 +4,7 @@ from functools import reduce from mxnet.module.executor_group import DataParallelExecutorGroup from common import assertRaises +from collections import namedtuple def test_module_dtype(): @@ -440,14 +441,124 @@ def test_shared_exec_group(exec_grp_shared, exec_grp_created, shared_arg_names=N shared_arg_names=shared_arg_names, extra_args=extra_args) +def test_forward_reshape(): + num_class=10 + data1 = mx.sym.Variable('data1') + data2 = mx.sym.Variable('data2') + conv1 = mx.sym.Convolution(data=data1, kernel=(2, 2), num_filter=2, stride=(2, 2)) + conv2 = mx.sym.Convolution(data=data2, kernel=(3, 3), num_filter=3, stride=(1, 1)) + pooling1 = mx.sym.Pooling(data=conv1, kernel=(2, 2), stride=(1, 1), pool_type="avg") + pooling2 = mx.sym.Pooling(data=conv2, kernel=(2, 2), stride=(1, 1), pool_type="max") + flatten1 = mx.sym.flatten(data=pooling1) + flatten2 = mx.sym.flatten(data=pooling2) + sum = mx.sym.sum(data=flatten1, axis=1) + mx.sym.sum(data=flatten2, axis=1) + fc = mx.sym.FullyConnected(data=sum, num_hidden=num_class) + sym = mx.sym.SoftmaxOutput(data=fc, name='softmax') + + dshape1 = (10, 3, 64, 64) + dshape2 = (10, 3, 32, 32) + lshape = (10,) + + mod = mx.mod.Module(symbol=sym, data_names=['data1', 'data2'], + label_names=['softmax_label']) + mod.bind(data_shapes=[('data1', dshape1), ('data2', dshape2)], + label_shapes=[('softmax_label', lshape)]) + mod.init_params() + mod.init_optimizer(optimizer_params={'learning_rate': 0.01}) + + # Train with original data shapes + data_batch = mx.io.DataBatch(data=[mx.nd.random_uniform(0, 9, dshape1), + mx.nd.random_uniform(5, 15, dshape2)], + label=[mx.nd.ones(lshape)]) + mod.forward(data_batch) + assert mod.get_outputs()[0].shape == tuple([lshape[0], num_class]) + mod.backward() + mod.update() + + # Train with different batch size + dshape1 = (3, 3, 64, 64) + dshape2 = (3, 3, 32, 32) + lshape = (3,) + data_batch = mx.io.DataBatch(data=[mx.nd.random_uniform(0, 9, dshape1), + mx.nd.random_uniform(5, 15, dshape2)], + label=[mx.nd.ones(lshape)]) + mod.forward(data_batch) + assert mod.get_outputs()[0].shape == tuple([lshape[0], num_class]) + mod.backward() + mod.update() + + dshape1 = (20, 3, 64, 64) + dshape2 = (20, 3, 32, 32) + lshape = (20,) + data_batch = mx.io.DataBatch(data=[mx.nd.random_uniform(3, 5, dshape1), + mx.nd.random_uniform(10, 25, dshape2)], + label=[mx.nd.ones(lshape)]) + mod.forward(data_batch) + assert mod.get_outputs()[0].shape == tuple([lshape[0], num_class]) + mod.backward() + mod.update() + + #Train with both different batch size and data shapes + dshape1 = (20, 3, 120, 120) + dshape2 = (20, 3, 32, 64) + lshape = (20,) + data_batch = mx.io.DataBatch(data=[mx.nd.random_uniform(0, 9, dshape1), + mx.nd.random_uniform(5, 15, dshape2)], + label=[mx.nd.ones(lshape)]) + mod.forward(data_batch) + assert mod.get_outputs()[0].shape == tuple([lshape[0], num_class]) + mod.backward() + mod.update() + + dshape1 = (5, 3, 28, 40) + dshape2 = (5, 3, 24, 16) + lshape = (5,) + data_batch = mx.io.DataBatch(data=[mx.nd.random_uniform(0, 9, dshape1), + mx.nd.random_uniform(15, 25, dshape2)], + label=[mx.nd.ones(lshape)]) + mod.forward(data_batch) + assert mod.get_outputs()[0].shape == tuple([lshape[0], num_class]) + mod.backward() + mod.update() + + #Test score + dataset_shape1 = (30, 3, 30, 30) + dataset_shape2 = (30, 3, 20, 40) + labelset_shape = (30,) + + eval_dataiter = mx.io.NDArrayIter(data=[mx.nd.random_uniform(0, 9, dataset_shape1), + mx.nd.random_uniform(15, 25, dataset_shape2)], + label=[mx.nd.ones(labelset_shape)], + batch_size=5) + assert len(mod.score(eval_data=eval_dataiter, eval_metric='acc')) == 1 + + #Test prediction + dshape1 = (1, 3, 30, 30) + dshape2 = (1, 3, 20, 40) + dataset_shape1 = (10, 3, 30, 30) + dataset_shape2 = (10, 3, 20, 40) + + pred_dataiter = mx.io.NDArrayIter(data=[mx.nd.random_uniform(0, 9, dataset_shape1), + mx.nd.random_uniform(15, 25, dataset_shape2)]) + mod.bind(data_shapes=[('data1', dshape1), ('data2', dshape2)], + for_training=False, force_rebind=True) + assert mod.predict(pred_dataiter).shape == tuple([10, num_class]) + + #Test forward with other data batch API + Batch = namedtuple('Batch', ['data']) + data = mx.sym.Variable('data') + out = data * 2 + mod = mx.mod.Module(symbol=out, label_names=None) + mod.bind(data_shapes=[('data', (1, 10))]) + mod.init_params() + data1 = [mx.nd.ones((1, 10))] + mod.forward(Batch(data1)) + assert mod.get_outputs()[0].shape == (1, 10) + data2 = [mx.nd.ones((3, 5))] + mod.forward(Batch(data2)) + assert mod.get_outputs()[0].shape == (3, 5) + + if __name__ == '__main__': - test_module_dtype() - test_module_input_grads() - test_module_states() - test_module_reshape() - test_module_set_params() - test_save_load() - test_module_layout() - test_module_switch_bucket() - test_monitor() - test_executor_group() + import nose + nose.runmodule() From 17620fe513742014d2cebd4594bc309105808567 Mon Sep 17 00:00:00 2001 From: Pepe Mandioca Date: Tue, 27 Jun 2017 14:51:32 -0300 Subject: [PATCH 120/834] Fixed broken SpatialTransformerNetwork example (#5798) * re-added option to include a SpatialTransformerNetwork layer in the lenet/mnist example, and a cmd argument in train_mnist to activate it * Update lenet.py --- example/image-classification/symbols/lenet.py | 18 +++++++++++++++++- example/image-classification/train_mnist.py | 5 ++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/example/image-classification/symbols/lenet.py b/example/image-classification/symbols/lenet.py index f6cfd6893afe..6df02991e4ad 100644 --- a/example/image-classification/symbols/lenet.py +++ b/example/image-classification/symbols/lenet.py @@ -5,9 +5,25 @@ """ import mxnet as mx +def get_loc(data, attr={'lr_mult':'0.01'}): + """ + the localisation network in lenet-stn, it will increase acc about more than 1%, + when num-epoch >=15 + """ + loc = mx.symbol.Convolution(data=data, num_filter=30, kernel=(5, 5), stride=(2,2)) + loc = mx.symbol.Activation(data = loc, act_type='relu') + loc = mx.symbol.Pooling(data=loc, kernel=(2, 2), stride=(2, 2), pool_type='max') + loc = mx.symbol.Convolution(data=loc, num_filter=60, kernel=(3, 3), stride=(1,1), pad=(1, 1)) + loc = mx.symbol.Activation(data = loc, act_type='relu') + loc = mx.symbol.Pooling(data=loc, global_pool=True, kernel=(2, 2), pool_type='avg') + loc = mx.symbol.Flatten(data=loc) + loc = mx.symbol.FullyConnected(data=loc, num_hidden=6, name="stn_loc", attr=attr) + return loc + + def get_symbol(num_classes=10, add_stn=False, **kwargs): data = mx.symbol.Variable('data') - if(add_stn): + if add_stn: data = mx.sym.SpatialTransformer(data=data, loc=get_loc(data), target_shape = (28,28), transform_type="affine", sampler_type="bilinear") # first conv diff --git a/example/image-classification/train_mnist.py b/example/image-classification/train_mnist.py index 61162e637520..31ecbfb463c4 100644 --- a/example/image-classification/train_mnist.py +++ b/example/image-classification/train_mnist.py @@ -53,6 +53,9 @@ def get_mnist_iter(args, kv): help='the number of classes') parser.add_argument('--num-examples', type=int, default=60000, help='the number of training examples') + + parser.add_argument('--add_stn', action="store_true", default=False, help='Add Spatial Transformer Network Layer (lenet only)') + fit.add_fit_args(parser) parser.set_defaults( # network @@ -63,7 +66,7 @@ def get_mnist_iter(args, kv): disp_batches = 100, num_epochs = 20, lr = .05, - lr_step_epochs = '10', + lr_step_epochs = '10' ) args = parser.parse_args() From 49cd2e66941937791940efd50ec1e00db09b035e Mon Sep 17 00:00:00 2001 From: Przemyslaw Tredak Date: Wed, 28 Jun 2017 09:32:44 -0700 Subject: [PATCH 121/834] Don't use __CUDACC_VER__ (#6844) --- src/operator/tensor/sort_op-inl.cuh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/operator/tensor/sort_op-inl.cuh b/src/operator/tensor/sort_op-inl.cuh index 10ba61f5431d..725f9b8c5f9e 100644 --- a/src/operator/tensor/sort_op-inl.cuh +++ b/src/operator/tensor/sort_op-inl.cuh @@ -7,8 +7,8 @@ #define MXNET_OPERATOR_TENSOR_SORT_OP_INL_CUH_ #include #include -#if defined(_MSC_VER) && __CUDACC_VER__ != 80044 -// Many CUDA compilers other than V8.0.44 crash on Windows +#if defined(_MSC_VER) && __CUDACC_VER_MAJOR__ == 8 && __CUDACC_VER_BUILD__ != 44 +// Many CUDA 8 compilers other than V8.0.44 crash on Windows #pragma warning("Potential crash on CUDA compiler detected. Switching sorting from CUB to Thrust") #define SORT_WITH_THRUST #else From cb69e15cd0ac063b1199248c6ff70669059cf837 Mon Sep 17 00:00:00 2001 From: Viacheslav Kovalevskyi Date: Wed, 28 Jun 2017 09:33:41 -0700 Subject: [PATCH 122/834] Documentation and tutorials are not 2 separated links. (#6845) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a06c8f0682ff..a929234dd682 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ What's New Contents -------- -* [Documentation and Tutorials](http://mxnet.io/) +* [Documentation](http://mxnet.io/) and [Tutorials](http://mxnet.io/tutorials/) * [Design Notes](http://mxnet.io/architecture/index.html) * [Code Examples](https://github.com/dmlc/mxnet/tree/master/example) * [Installation](http://mxnet.io/get_started/install.html) From 4b1dd223954535b689f0d7e9e66d2a83d5d64d1a Mon Sep 17 00:00:00 2001 From: Viacheslav Kovalevskyi Date: Wed, 28 Jun 2017 15:35:06 -0700 Subject: [PATCH 123/834] lstm_bucketing now shows more information if the file is not present.wq (#6862) --- example/rnn/lstm_bucketing.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/example/rnn/lstm_bucketing.py b/example/rnn/lstm_bucketing.py index 4bc934a01ad0..6c4371b2fd4e 100644 --- a/example/rnn/lstm_bucketing.py +++ b/example/rnn/lstm_bucketing.py @@ -1,6 +1,7 @@ import numpy as np import mxnet as mx import argparse +import os parser = argparse.ArgumentParser(description="Train RNN on Penn Tree Bank", formatter_class=argparse.ArgumentDefaultsHelpFormatter) @@ -32,6 +33,8 @@ def tokenize_text(fname, vocab=None, invalid_label=-1, start_label=0): + if not os.path.isfile(fname): + raise IOError("Please use get_ptb_data.sh to download requied file (data/ptb.train.txt)") lines = open(fname).readlines() lines = [filter(None, i.split(' ')) for i in lines] sentences, vocab = mx.rnn.encode_sentences(lines, vocab=vocab, invalid_label=invalid_label, From 8d3aa29a18d109772d5f81b877d88a1d727c2a2d Mon Sep 17 00:00:00 2001 From: CNevd Date: Thu, 29 Jun 2017 06:56:20 +0800 Subject: [PATCH 124/834] add Adamax & Nadam (#6784) * add Adamax & Nadam * change abs to NDabs * update nadam * Update optimizer.py --- python/mxnet/optimizer.py | 121 +++++++++++++++++++++++++++++++++++++- 1 file changed, 119 insertions(+), 2 deletions(-) diff --git a/python/mxnet/optimizer.py b/python/mxnet/optimizer.py index 3d318003715a..57fadf44335e 100644 --- a/python/mxnet/optimizer.py +++ b/python/mxnet/optimizer.py @@ -4,7 +4,7 @@ import logging import warnings import numpy -from .ndarray import NDArray, zeros, clip, sqrt, sign, array +from .ndarray import (NDArray, zeros, clip, sqrt, sign, array, maximum, abs as NDabs) from .ndarray import (sgd_update, sgd_mom_update, adam_update, rmsprop_update, rmspropalex_update, mp_sgd_update, mp_sgd_mom_update) from .random import normal @@ -777,7 +777,124 @@ def update(self, index, weight, grad, state): # update weight weight[:] = (sign(dn) * self.lamda1 - dn) / \ - ((self.beta + sqrt(n)) / lr + wd) * (NDArray.abs(dn) > self.lamda1) + ((self.beta + sqrt(n)) / lr + wd) * (NDabs(dn) > self.lamda1) + +@register +class Adamax(Optimizer): + """The AdaMax optimizer. + + It is a variant of Adam based on the infinity norm + available at http://arxiv.org/abs/1412.6980 Section 7. + + This optimizer accepts the following parameters in addition to those accepted + by :class:`.Optimizer`. + + Parameters + ---------- + beta1 : float, optional + Exponential decay rate for the first moment estimates. + beta2 : float, optional + Exponential decay rate for the second moment estimates. + """ + def __init__(self, learning_rate=0.002, beta1=0.9, beta2=0.999, **kwargs): + super(Adamax, self).__init__(learning_rate=learning_rate, **kwargs) + self.beta1 = beta1 + self.beta2 = beta2 + + def create_state(self, index, weight): + return (zeros(weight.shape, weight.context, dtype=weight.dtype), # mean + zeros(weight.shape, weight.context, dtype=weight.dtype)) # variance + + def update(self, index, weight, grad, state): + assert(isinstance(weight, NDArray)) + assert(isinstance(grad, NDArray)) + lr = self._get_lr(index) + wd = self._get_wd(index) + self._update_count(index) + + t = self._index_update_count[index] + lr /= (1. - self.beta1**t) + + # preprocess grad + grad = grad * self.rescale_grad + wd * weight + if self.clip_gradient is not None: + grad = clip(grad, -self.clip_gradient, self.clip_gradient) + + # update m_t and u_t + m_t, u_t = state + m_t[:] = self.beta1 * m_t + (1. - self.beta1) * grad + u_t[:] = maximum(self.beta2 * u_t, NDabs(grad)) + + # update weight + weight[:] -= lr * m_t / u_t + +@register +class Nadam(Optimizer): + """The Nesterov Adam optimizer. + + Much like Adam is essentially RMSprop with momentum, + Nadam is Adam RMSprop with Nesterov momentum available + at http://cs229.stanford.edu/proj2015/054_report.pdf. + + This optimizer accepts the following parameters in addition to those accepted + by :class:`.Optimizer`. + + Parameters + ---------- + beta1 : float, optional + Exponential decay rate for the first moment estimates. + beta2 : float, optional + Exponential decay rate for the second moment estimates. + epsilon : float, optional + Small value to avoid division by 0. + schedule_decay : float, optional + Exponential decay rate for the momentum schedule + """ + def __init__(self, learning_rate=0.001, beta1=0.9, beta2=0.999, epsilon=1e-8, + schedule_decay=0.004, **kwargs): + super(Nadam, self).__init__(learning_rate=learning_rate, **kwargs) + self.beta1 = beta1 + self.beta2 = beta2 + self.epsilon = epsilon + self.schedule_decay = schedule_decay + self.m_schedule = 1. + + def create_state(self, index, weight): + return (zeros(weight.shape, weight.context, dtype=weight.dtype), # mean + zeros(weight.shape, weight.context, dtype=weight.dtype)) # variance + + def update(self, index, weight, grad, state): + assert(isinstance(weight, NDArray)) + assert(isinstance(grad, NDArray)) + lr = self._get_lr(index) + wd = self._get_wd(index) + self._update_count(index) + + t = self._index_update_count[index] + + # preprocess grad + grad *= self.rescale_grad + wd * weight + if self.clip_gradient is not None: + grad = clip(grad, -self.clip_gradient, self.clip_gradient) + + # warming momentum schedule + momentum_t = self.beta1 * (1. - 0.5 * (pow(0.96, t * self.schedule_decay))) + momentum_t_1 = self.beta1 * (1. - 0.5 * (pow(0.96, (t + 1) * self.schedule_decay))) + self.m_schedule = self.m_schedule * momentum_t + m_schedule_next = self.m_schedule * momentum_t_1 + + # update m_t and v_t + m_t, v_t = state + m_t[:] = self.beta1 * m_t + (1. - self.beta1) * grad + v_t[:] = self.beta2 * v_t + (1. - self.beta2) * grad * grad + + grad_prime = grad / (1. - self.m_schedule) + m_t_prime = m_t / (1. - m_schedule_next) + v_t_prime = v_t / (1. - pow(self.beta2, t)) + m_t_bar = (1. - momentum_t) * grad_prime + momentum_t_1 * m_t_prime + + # update weight + weight[:] -= lr * m_t_bar / (sqrt(v_t_prime) + self.epsilon) @register class Test(Optimizer): From 78b12084d14b267ea73c180a3aee54422d3a7fb0 Mon Sep 17 00:00:00 2001 From: "Qiang Kou (KK)" Date: Wed, 28 Jun 2017 16:00:37 -0700 Subject: [PATCH 125/834] [R] instruction to build GPU-enabled pkg on Windows. close #3902 (#6841) * [R] instruction to build GPU-enabled pkg on Windows. close #3902 * [R][Doc] add link into install.md --- docs/get_started/install.md | 2 +- docs/get_started/windows_setup.md | 69 +++++++++++++++++++++++++------ 2 files changed, 57 insertions(+), 14 deletions(-) diff --git a/docs/get_started/install.md b/docs/get_started/install.md index 766730cde089..3fdb0e83d674 100644 --- a/docs/get_started/install.md +++ b/docs/get_started/install.md @@ -880,7 +880,7 @@ install.packages("mxnet")
    -Will be available soon. +You can also follow the installation instructions [in this guide](./windows_setup.md) to build MXNet from source.
    diff --git a/docs/get_started/windows_setup.md b/docs/get_started/windows_setup.md index 9025add50cd6..b40ed3e34aae 100755 --- a/docs/get_started/windows_setup.md +++ b/docs/get_started/windows_setup.md @@ -53,7 +53,7 @@ Next, we install ```graphviz``` library that we use for visualizing network grap We have installed MXNet core library. Next, we will install MXNet interface package for programming language of your choice: - [Python](#install-the-mxnet-package-for-python) -- [R](#install-the-mxnet-package-for-r) +- [R](#install-mxnet-for-r) - [Julia](#install-the-mxnet-package-for-julia) - [Scala](#install-the-mxnet-package-for-scala) @@ -108,6 +108,7 @@ Run the following commands to install the MXNet dependencies and build the MXNet ```r Rscript -e "install.packages('devtools', repo = 'https://cran.rstudio.com')" ``` + ```bash cd R-package Rscript -e "library(devtools); library(methods); options(repos=c(CRAN='https://cran.rstudio.com')); install_deps(dependencies = TRUE)" @@ -125,7 +126,7 @@ These commands create the MXNet R package as a tar.gz file that you can install ### Installing MXNet on a Computer with a GPU Processor -To install MXNet on a computer with a GPU processor, you need the following: +To install MXNet R package on a computer with a GPU processor, you need the following: * Microsoft Visual Studio 2013 @@ -137,20 +138,62 @@ To install MXNet on a computer with a GPU processor, you need the following: To install the required dependencies and install MXNet for R: -1. If [Microsoft Visual Studio 2013](https://www.visualstudio.com/downloads/) is not already installed, download and install it. You can download and install the free community edition. -2. Install the [CUDA Toolkit](https://developer.nvidia.com/cuda-toolkit). The CUDA Toolkit depends on Visual Studio. To check whether your GPU is compatible with the CUDA Toolkit and for information on installing it, see NVidia's [CUDA Installation Guide](http://docs.nvidia.com/cuda/cuda-installation-guide-microsoft-windows/). -3. Download the MXNet package as a .zip file from the [MXNet Github repository](https://github.com/dmlc/mxnet/) and unpack it. You will be editing the ```"/mxnet/R-package"``` folder. -4. Download the most recent GPU-enabled MXNet package from the [Releases](https://github.com/dmlc/mxnet/releases) tab. Unzip this file and navigate to the ```/nocudnn``` folder. -**Note:** You will copy some of these extracted files into MXNet's R-package folder. We are now working two folders, ```R-package/``` and ```nocudnn/```. -5. Download and install [CuDNN V3](https://developer.nvidia.com/cudnn). To get access to the download link, register as an NVIDIA community user. Unpack the .zip file. You will see three folders: ```/bin```, ```/include```, and ```/lib```. Copy these folders into ```nocudnn/3rdparty/cudnn/```, replacing the folders that are already there. You can also unpack the .zip file directly into the nocudnn/ folder. +1. Install the [CUDA Toolkit](https://developer.nvidia.com/cuda-toolkit). The CUDA Toolkit depends on Visual Studio. To check whether your GPU is compatible with the CUDA Toolkit and for information on installing it, see NVidia's [CUDA Installation Guide](http://docs.nvidia.com/cuda/cuda-installation-guide-microsoft-windows/). +3. Clone the MXNet github repo. + +```sh +git clone --recursive https://github.com/dmlc/mxnet +``` + +The `--recursive` is to clone all the submodules used by MXNet. You will be editing the ```"/mxnet/R-package"``` folder. +4. Download prebuilt GPU-enabled MXNet libraries for Windows from https://github.com/yajiedesign/mxnet/releases. You will need `mxnet_x64_vc14_gpu.7z` and `prebuildbase_win10_x64_vc14.7z`. +5. Download and install [CuDNN](https://developer.nvidia.com/cudnn). 6. Create a folder called ```R-package/inst/libs/x64```. MXNet supports only 64-bit operating systems, so you need the x64 folder. 7. Copy the following shared libraries (.dll files) into the ```R-package/inst/libs/x64``` folder: - * nocudnn/lib/libmxnet.dll. - * The *.dll files in all four subfolders of the nocudnn/3rdparty/ directory. The cudnn and openblas .dll files are in the /bin folders. -You should now have 11 .dll files in the R-package/inst/libs/x64 folder. -8. Copy the ```nocudnn/include/``` folder into ```R-package/inst/```. You should now have a folder called ```R-package/inst/include/``` with three subfolders. +``` +cublas64_80.dll +cudart64_80.dll +cudnn64_5.dll +curand64_80.dll +libgcc_s_seh-1.dll +libgfortran-3.dll +libmxnet.dll +libmxnet.lib +libopenblas.dll +libquadmath-0.dll +nvrtc64_80.dll +``` +These dlls can be found in `prebuildbase_win10_x64_vc14/3rdparty/cudart`, `prebuildbase_win10_x64_vc14/3rdparty/openblas/bin`, `mxnet_x64_vc14_gpu/build`, `mxnet_x64_vc14_gpu/lib` and the `cuDNN` downloaded from NVIDIA. +8. Copy the header files from `dmlc`, `mxnet` and `nnvm` into `./R-package/inst/include`. It should look like: + +``` +./R-package/inst +└── include + ├── dmlc + ├── mxnet + └── nnvm +``` 9. Make sure that R is added to your ```PATH``` in the environment variables. Running the ```where R``` command at the command prompt should return the location. -10. Run ```R CMD INSTALL --no-multiarch R-package```. +10. Now open the Windows CMD and change the directory to the `mxnet` folder. Then use the following commands +to build R package: + +```bat +echo import(Rcpp) > R-package\NAMESPACE +echo import(methods) >> R-package\NAMESPACE +Rscript -e "install.packages('devtools', repos = 'https://cloud.r-project.org')" +cd R-package +Rscript -e "library(devtools); library(methods); options(repos=c(CRAN='https://cloud.r-project.org')); install_deps(dependencies = TRUE)" +cd .. + +R CMD INSTALL --no-multiarch R-package + +Rscript -e "require(mxnet); mxnet:::mxnet.export('R-package')" +rm R-package/NAMESPACE +Rscript -e "require(devtools); install_version('roxygen2', version = '5.0.1', repos = 'https://cloud.r-project.org/', quiet = TRUE)" +Rscript -e "require(roxygen2); roxygen2::roxygenise('R-package')" + +R CMD INSTALL --build --no-multiarch R-package +``` **Note:** To maximize its portability, the MXNet library is built with the Rcpp end. Computers running Windows need [MSVC](https://en.wikipedia.org/wiki/Visual_C%2B%2B) (Microsoft Visual C++) to handle CUDA toolchain compatibilities. From e906f883642423479600abd9cc1c101af91ca908 Mon Sep 17 00:00:00 2001 From: "Qiang Kou (KK)" Date: Wed, 28 Jun 2017 16:01:23 -0700 Subject: [PATCH 126/834] [R] add R test into jenkins; add rpkgtest into makefile (#6733) * simpify the makefile;add R test into jenkins;some basic test for R * fix GPU test * fix GPU test for R pkg --- Jenkinsfile | 28 +++++ Makefile | 13 ++- R-package/tests/testthat/get_data.R | 39 +++++++ R-package/tests/testthat/test_io.R | 86 +++++++++++++++ R-package/tests/testthat/test_model.R | 119 +++++++++++---------- R-package/tests/testthat/test_ndarray.R | 1 + R-package/tests/testthat/test_symbol.R | 48 ++++++++- tests/ci_build/Dockerfile.cpu | 2 + tests/ci_build/Dockerfile.gpu | 2 + tests/ci_build/install/ubuntu_install_r.sh | 10 ++ 10 files changed, 287 insertions(+), 61 deletions(-) create mode 100644 R-package/tests/testthat/get_data.R create mode 100644 R-package/tests/testthat/test_io.R create mode 100755 tests/ci_build/install/ubuntu_install_r.sh diff --git a/Jenkinsfile b/Jenkinsfile index f2beae0d4a31..cd6ab3166b79 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -271,6 +271,34 @@ stage('Unit Test') { } } }, + 'R: CPU': { + node('linux') { + ws('workspace/ut-r-cpu') { + init_git() + unpack_lib('cpu') + timeout(time: max_time, unit: 'MINUTES') { + sh "${docker_run} cpu mkdir -p /workspace/ut-r-cpu/site-library" + sh "${docker_run} cpu make rpkg USE_BLAS=openblas R_LIBS=/workspace/ut-r-cpu/site-library" + sh "${docker_run} cpu R CMD INSTALL --library=/workspace/ut-r-cpu/site-library mxnet_current_r.tar.gz" + sh "${docker_run} cpu make rpkgtest R_LIBS=/workspace/ut-r-cpu/site-library" + } + } + } + }, + 'R: GPU': { + node('GPU' && 'linux') { + ws('workspace/ut-r-gpu') { + init_git() + unpack_lib('gpu') + timeout(time: max_time, unit: 'MINUTES') { + sh "${docker_run} gpu mkdir -p /workspace/ut-r-gpu/site-library" + sh "${docker_run} gpu make rpkg USE_BLAS=openblas R_LIBS=/workspace/ut-r-gpu/site-library" + sh "${docker_run} gpu R CMD INSTALL --library=/workspace/ut-r-gpu/site-library mxnet_current_r.tar.gz" + sh "${docker_run} gpu make rpkgtest R_LIBS=/workspace/ut-r-gpu/site-library" + } + } + } + }, 'Python2/3: CPU Win':{ node('windows') { ws('workspace/ut-python-cpu') { diff --git a/Makefile b/Makefile index 69527f42d248..749c761f062f 100644 --- a/Makefile +++ b/Makefile @@ -383,17 +383,24 @@ rpkg: cp -rf include/* R-package/inst/include cp -rf dmlc-core/include/* R-package/inst/include/ cp -rf nnvm/include/* R-package/inst/include + Rscript -e "if(!require(devtools)){install.packages('devtools', repo = 'https://cloud.r-project.org/')}" + Rscript -e "library(devtools); library(methods); options(repos=c(CRAN='https://cloud.r-project.org/')); install_deps(pkg='R-package', dependencies = TRUE)" echo "import(Rcpp)" > R-package/NAMESPACE echo "import(methods)" >> R-package/NAMESPACE R CMD INSTALL R-package - Rscript -e "require(mxnet); mxnet:::mxnet.export(\"R-package\")" + Rscript -e "require(mxnet); mxnet:::mxnet.export('R-package')" rm -rf R-package/NAMESPACE - Rscript -e "require(devtools); install_version(\"roxygen2\", version = \"5.0.1\", repos = \"https://cloud.r-project.org/\", quiet = TRUE)" - Rscript -e "require(roxygen2); roxygen2::roxygenise(\"R-package\")" + Rscript -e "if (!require('roxygen2')||packageVersion('roxygen2')!= '5.0.1'){\ + devtools::install_version('roxygen2',version='5.0.1',\ + repo='https://cloud.r-project.org/',quiet=TRUE)}" + Rscript -e "require(roxygen2); roxygen2::roxygenise('R-package')" R CMD build --no-build-vignettes R-package rm -rf mxnet_current_r.tar.gz mv mxnet_*.tar.gz mxnet_current_r.tar.gz +rpkgtest: + Rscript -e "require(testthat);res<-test_dir('R-package/tests/testthat');if(!testthat:::all_passed(res)){stop('Test failures', call. = FALSE)}" + scalapkg: (cd $(ROOTDIR)/scala-package; \ mvn clean package -P$(SCALA_PKG_PROFILE) -Dcxx="$(CXX)" \ diff --git a/R-package/tests/testthat/get_data.R b/R-package/tests/testthat/get_data.R new file mode 100644 index 000000000000..021cbfeafb5f --- /dev/null +++ b/R-package/tests/testthat/get_data.R @@ -0,0 +1,39 @@ + +GetMNIST_ubyte <- function() { + if (!dir.exists("data")) { + dir.create("data/") + } + if (!file.exists('data/train-images-idx3-ubyte') | + !file.exists('data/train-labels-idx1-ubyte') | + !file.exists('data/t10k-images-idx3-ubyte') | + !file.exists('data/t10k-labels-idx1-ubyte')) { + download.file('http://data.mxnet.io/mxnet/data/mnist.zip', destfile = 'data/mnist.zip') + unzip('data/mnist.zip', exdir = 'data/') + } +} + +GetMNIST_csv <- function() { + if (!dir.exists("data")) { + dir.create("data/") + } + if (!file.exists('data/train.csv') | + !file.exists('data/test.csv')) { + download.file('https://s3-us-west-2.amazonaws.com/apache-mxnet/R/data/mnist_csv.zip', + destfile = 'data/mnist_csv.zip') + unzip('data/mnist_csv.zip', exdir = 'data/') + } +} + +GetCifar10 <- function() { + if (!dir.exists("data")) { + dir.create("data/") + } + if (!file.exists('data/cifar/train.rec') | + !file.exists('data/cifar/test.rec') | + !file.exists('data/cifar/train.lst') | + !file.exists('data/cifar/test.lst')) { + download.file('http://data.mxnet.io/mxnet/data/cifar10.zip', + destfile = 'data/cifar10.zip') + unzip('data/cifar10.zip', exdir = 'data/') + } +} diff --git a/R-package/tests/testthat/test_io.R b/R-package/tests/testthat/test_io.R new file mode 100644 index 000000000000..d619856cbb99 --- /dev/null +++ b/R-package/tests/testthat/test_io.R @@ -0,0 +1,86 @@ +require(mxnet) + +context("io") + +source("get_data.R") + +test_that("MNISTIter", { + GetMNIST_ubyte() + batch.size <- 100 + train_dataiter <- mx.io.MNISTIter( + image = "data/train-images-idx3-ubyte", + label = "data/train-labels-idx1-ubyte", + data.shape = c(784), + batch.size = batch.size, + shuffle = TRUE, + flat = TRUE, + silent = 0, + seed = 10 + ) + train_dataiter$reset() + batch_count = 0 + while (train_dataiter$iter.next()) { + batch_count = batch_count + 1 + } + nbatch = 60000 / batch.size + expect_equal(batch_count, nbatch) + train_dataiter$reset() + train_dataiter$iter.next() + label_0 <- as.array(train_dataiter$value()$label) + train_dataiter$iter.next() + train_dataiter$iter.next() + train_dataiter$iter.next() + train_dataiter$iter.next() + train_dataiter$reset() + train_dataiter$iter.next() + label_1 <- as.array(train_dataiter$value()$label) + expect_equal(label_0, label_1) +}) + +test_that("Cifar10Rec", { + GetCifar10() + dataiter <- mx.io.ImageRecordIter( + path.imgrec = "./data/cifar/train.rec", + path.imglist = "./data/cifar/train.lst", + mean.img = "./data/cifar/cifar10_mean.bin", + batch.size = 100, + data.shape = c(28, 28, 3), + rand.crop = TRUE, + rand.mirror = TRUE + ) + labelcount = rep(0, 10) + dataiter$reset() + while (dataiter$iter.next()) { + label = as.array(dataiter$value()$label) + for (i in label) { + labelcount[i + 1] = labelcount[i + 1] + 1 + } + } + + expect_equal(labelcount, rep(5000, 10)) +}) + +test_that("mx.io.arrayiter", { + X <- matrix(c(1:10000), 100, 100) + y <- c(1:100) + dataiter <- mx.io.arrayiter(X, y, batch.size = 20, shuffle = FALSE) + dataiter$reset() + batch_count = 0 + while (dataiter$iter.next()) { + batch_count = batch_count + 1 + } + expect_equal(batch_count, 100 / 20) + + y <- round(y / 10) + dataiter <- mx.io.arrayiter(X, y, batch.size = 30, shuffle = FALSE) + labelcount <- rep(0, 11) + dataiter$reset() + while (dataiter$iter.next()) { + label <- as.array(dataiter$value()$label) + for (i in label) { + labelcount[i + 1] = labelcount[i + 1] + 1 + } + } + + expect_equal(labelcount, c(5, 9, 11, 9, 11, 9, 11, 13, 22, 14, 6)) +}) diff --git a/R-package/tests/testthat/test_model.R b/R-package/tests/testthat/test_model.R index 93784a622bbb..9c85afac956a 100644 --- a/R-package/tests/testthat/test_model.R +++ b/R-package/tests/testthat/test_model.R @@ -1,62 +1,67 @@ require(mxnet) +source("get_data.R") + context("models") -# test_that("basic symbol operation", { +test_that("basic symbol operation", { # # Network configuration -# batch.size <- 100 -# data <- mx.symbol.Variable("data") -# fc1 <- mx.symbol.FullyConnected(data, name="fc1", num_hidden=128) -# act1 <- mx.symbol.Activation(fc1, name="relu1", act_type="relu") -# fc2 <- mx.symbol.FullyConnected(act1, name = "fc2", num_hidden = 64) -# act2 <- mx.symbol.Activation(fc2, name="relu2", act_type="relu") -# fc3 <- mx.symbol.FullyConnected(act2, name="fc3", num_hidden=10) -# softmax <- mx.symbol.Softmax(fc3, name = "sm") -# -# dtrain = mx.io.MNISTIter( -# image="data/train-images-idx3-ubyte", -# label="data/train-labels-idx1-ubyte", -# data.shape=c(784), -# batch.size=batch.size, -# shuffle=TRUE, -# flat=TRUE, -# silent=0, -# seed=10) -# -# dtest = mx.io.MNISTIter( -# image="data/t10k-images-idx3-ubyte", -# label="data/t10k-labels-idx1-ubyte", -# data.shape=c(784), -# batch.size=batch.size, -# shuffle=FALSE, -# flat=TRUE, -# silent=0) -# -# mx.set.seed(0) -# devices = lapply(1:2, function(i) { -# mx.cpu(i) -# }) -# -# # create the model -# model <- mx.model.FeedForward.create(softmax, X=dtrain, eval.data=dtest, -# ctx=devices, num.round=1, -# learning.rate=0.1, momentum=0.9, -# initializer=mx.init.uniform(0.07), -# epoch.end.callback=mx.callback.save.checkpoint("chkpt"), -# batch.end.callback=mx.callback.log.train.metric(100)) -# -# # do prediction -# pred <- predict(model, dtest) -# label <- mx.io.extract(dtest, "label") -# dataX <- mx.io.extract(dtest, "data") -# # Predict with R's array -# pred2 <- predict(model, X=dataX) -# -# accuracy <- function(label, pred) { -# ypred = max.col(t(as.array(pred))) -# return(sum((as.array(label) + 1) == ypred) / length(label)) -# } -# -# print(paste0("Finish prediction... accuracy=", accuracy(label, pred))) -# print(paste0("Finish prediction... accuracy2=", accuracy(label, pred2))) -# }) + GetMNIST_ubyte() + batch.size <- 100 + data <- mx.symbol.Variable("data") + fc1 <- mx.symbol.FullyConnected(data, name="fc1", num_hidden=128) + act1 <- mx.symbol.Activation(fc1, name="relu1", act_type="relu") + fc2 <- mx.symbol.FullyConnected(act1, name = "fc2", num_hidden = 64) + act2 <- mx.symbol.Activation(fc2, name="relu2", act_type="relu") + fc3 <- mx.symbol.FullyConnected(act2, name="fc3", num_hidden=10) + softmax <- mx.symbol.Softmax(fc3, name = "sm") + + dtrain = mx.io.MNISTIter( + image="data/train-images-idx3-ubyte", + label="data/train-labels-idx1-ubyte", + data.shape=c(784), + batch.size=batch.size, + shuffle=TRUE, + flat=TRUE, + silent=0, + seed=10) + + dtest = mx.io.MNISTIter( + image="data/t10k-images-idx3-ubyte", + label="data/t10k-labels-idx1-ubyte", + data.shape=c(784), + batch.size=batch.size, + shuffle=FALSE, + flat=TRUE, + silent=0) + + mx.set.seed(0) + devices = lapply(1:2, function(i) { + mx.cpu(i) + }) + + # create the model + model <- mx.model.FeedForward.create(softmax, X=dtrain, eval.data=dtest, + ctx=devices, num.round=1, + learning.rate=0.1, momentum=0.9, + initializer=mx.init.uniform(0.07), + epoch.end.callback=mx.callback.save.checkpoint("chkpt"), + batch.end.callback=mx.callback.log.train.metric(100)) + + # do prediction + pred <- predict(model, dtest) + label <- mx.io.extract(dtest, "label") + dataX <- mx.io.extract(dtest, "data") + # Predict with R's array + pred2 <- predict(model, X=dataX) + + accuracy <- function(label, pred) { + ypred = max.col(t(as.array(pred))) + return(sum((as.array(label) + 1) == ypred) / length(label)) + } + + expect_equal(accuracy(label, pred), accuracy(label, pred2)) + + file.remove("chkpt-0001.params") + file.remove("chkpt-symbol.json") +}) diff --git a/R-package/tests/testthat/test_ndarray.R b/R-package/tests/testthat/test_ndarray.R index 0be603eb5c2a..a6b85daa2516 100644 --- a/R-package/tests/testthat/test_ndarray.R +++ b/R-package/tests/testthat/test_ndarray.R @@ -46,4 +46,5 @@ test_that("ndarray ones, zeros, save and load", { mat2 = mx.nd.load('temp.mat') expect_true(is.mx.ndarray(mat2[[1]])) expect_equal(as.array(mat), as.array(mat2[[1]])) + file.remove('temp.mat') }) diff --git a/R-package/tests/testthat/test_symbol.R b/R-package/tests/testthat/test_symbol.R index 7e733e8690c4..be38762d85bb 100644 --- a/R-package/tests/testthat/test_symbol.R +++ b/R-package/tests/testthat/test_symbol.R @@ -8,6 +8,7 @@ test_that("basic symbol operation", { net1 = mx.symbol.FullyConnected(data=net1, name='fc2', num_hidden=100) expect_equal(arguments(net1), c('data', 'fc1_weight', 'fc1_bias', 'fc2_weight', 'fc2_bias')) + expect_equal(outputs(net1), 'fc2_output') net2 = mx.symbol.FullyConnected(name='fc3', num_hidden=10) net2 = mx.symbol.Activation(data=net2, act_type='relu') @@ -16,6 +17,52 @@ test_that("basic symbol operation", { composed = mx.apply(net2, fc3_data=net1, name='composed') expect_equal(arguments(composed), c('data', 'fc1_weight', 'fc1_bias', 'fc2_weight', 'fc2_bias', 'fc3_weight', 'fc3_bias', 'fc4_weight', 'fc4_bias')) + expect_equal(outputs(composed), 'composed_output') + + multi_out = mx.symbol.Group(c(composed, net1)) + expect_equal(outputs(multi_out), c('composed_output', 'fc2_output')) +}) + +test_that("symbol internal", { + data = mx.symbol.Variable('data') + oldfc = mx.symbol.FullyConnected(data=data, name='fc1', num_hidden=10) + net1 = mx.symbol.FullyConnected(data=oldfc, name='fc2', num_hidden=100) + + expect_equal(arguments(net1), c("data", "fc1_weight", "fc1_bias", "fc2_weight", "fc2_bias")) + + internal = net1$get.internals() + fc1 = internal[[match("fc1_output", internal$outputs)]] + + expect_equal(arguments(fc1), arguments(oldfc)) +}) + +test_that("symbol infer type", { + num_hidden = 128 + num_dim = 64 + num_sample = 10 + + data = mx.symbol.Variable('data') + prev = mx.symbol.Variable('prevstate') + x2h = mx.symbol.FullyConnected(data=data, name='x2h', num_hidden=num_hidden) + h2h = mx.symbol.FullyConnected(data=prev, name='h2h', num_hidden=num_hidden) + + out = mx.symbol.Activation(data=mx.symbol.elemwise_add(x2h, h2h), name='out', act_type='relu') + + # shape inference will fail because information is not available for h2h + ret = mx.symbol.infer.shape(out, data = c(num_dim, num_sample)) + + expect_equal(ret, NULL) +}) + +test_that("symbol save/load", { + data <- mx.symbol.Variable("data") + fc1 <- mx.symbol.FullyConnected(data, num_hidden=1) + lro <- mx.symbol.LinearRegressionOutput(fc1) + mx.symbol.save(lro, "tmp_r_sym.json") + data2 = mx.symbol.load("tmp_r_sym.json") + + expect_equal(data2$as.json(), lro$as.json()) + file.remove("tmp_r_sym.json") }) test_that("symbol attributes access", { @@ -31,4 +78,3 @@ test_that("symbol attributes access", { expect_equal(y$attributes$`__shape__`, str) }) - diff --git a/tests/ci_build/Dockerfile.cpu b/tests/ci_build/Dockerfile.cpu index aabda4e99ce6..c9ba57c6ad46 100644 --- a/tests/ci_build/Dockerfile.cpu +++ b/tests/ci_build/Dockerfile.cpu @@ -6,3 +6,5 @@ COPY install/ubuntu_install_python.sh /install/ RUN /install/ubuntu_install_python.sh COPY install/ubuntu_install_scala.sh /install/ RUN /install/ubuntu_install_scala.sh +COPY install/ubuntu_install_r.sh /install/ +RUN /install/ubuntu_install_r.sh diff --git a/tests/ci_build/Dockerfile.gpu b/tests/ci_build/Dockerfile.gpu index 46d36312de7c..cd9986ec01a2 100644 --- a/tests/ci_build/Dockerfile.gpu +++ b/tests/ci_build/Dockerfile.gpu @@ -6,3 +6,5 @@ COPY install/ubuntu_install_python.sh /install/ RUN /install/ubuntu_install_python.sh COPY install/ubuntu_install_scala.sh /install/ RUN /install/ubuntu_install_scala.sh +COPY install/ubuntu_install_r.sh /install/ +RUN /install/ubuntu_install_r.sh diff --git a/tests/ci_build/install/ubuntu_install_r.sh b/tests/ci_build/install/ubuntu_install_r.sh new file mode 100755 index 000000000000..10851a6d0276 --- /dev/null +++ b/tests/ci_build/install/ubuntu_install_r.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# install libraries for mxnet's r package on ubuntu + +echo "deb http://cran.rstudio.com/bin/linux/ubuntu trusty/" >> /etc/apt/sources.list +gpg --keyserver keyserver.ubuntu.com --recv-key E084DAB9 +gpg -a --export E084DAB9 | apt-key add - + +apt-get update +apt-get install -y r-base r-base-dev libxml2-dev libssl-dev + From 52abdabd919d0c69b962c1926bfb7b9c98652268 Mon Sep 17 00:00:00 2001 From: "Qiang Kou (KK)" Date: Wed, 28 Jun 2017 16:07:33 -0700 Subject: [PATCH 127/834] [R] R >= 3.3.0 (#6840) --- R-package/DESCRIPTION | 8 ++++---- R-package/R/executor.R | 2 +- R-package/R/initializer.R | 16 ++++++++-------- R-package/R/model.R | 18 ++++++------------ R-package/R/util.R | 17 ----------------- 5 files changed, 19 insertions(+), 42 deletions(-) diff --git a/R-package/DESCRIPTION b/R-package/DESCRIPTION index c879c737ca0c..fb57b4b7e31f 100644 --- a/R-package/DESCRIPTION +++ b/R-package/DESCRIPTION @@ -1,10 +1,10 @@ Package: mxnet Type: Package -Title: MXNet +Title: MXNet: A Flexible and Efficient Machine Learning Library for Heterogeneous Distributed Systems Version: 0.10.1 -Date: 2017-06-02 +Date: 2017-06-27 Author: Tianqi Chen, Qiang Kou, Tong He -Maintainer: Qiang Kou +Maintainer: Qiang Kou Repository: DMLC Description: MXNet is a deep learning framework designed for both efficiency and flexibility. It allows you to mix the flavours of deep learning programs @@ -28,7 +28,7 @@ Suggests: rmarkdown, imager Depends: - R (>= 3.2.0) + R (>= 3.3.0) LinkingTo: Rcpp VignetteBuilder: knitr RoxygenNote: 5.0.1 diff --git a/R-package/R/executor.R b/R-package/R/executor.R index f5d773b5c471..d33aeefc32ae 100644 --- a/R-package/R/executor.R +++ b/R-package/R/executor.R @@ -16,7 +16,7 @@ mx.simple.bind <- function(symbol, ctx, grad.req = "null", ...) { mx.nd.zeros(shape, ctx) }, simplify = FALSE, USE.NAMES = TRUE) grad.reqs <- lapply(names(slist$arg.shapes), function(nm) { - if (!mx.util.str.endswith(nm, "label") && !mx.util.str.endswith(nm, "data")) { + if (!endsWith(nm, "label") && !endsWith(nm, "data")) { grad.req } else { "null" diff --git a/R-package/R/initializer.R b/R-package/R/initializer.R index ab2b151be465..7a1ffb2b182a 100644 --- a/R-package/R/initializer.R +++ b/R-package/R/initializer.R @@ -4,11 +4,11 @@ #' @param shape the shape of the array to be generated. #' mx.init.internal.default <- function(name, shape, ctx, allow.unknown=FALSE) { - if (mx.util.str.endswith(name, "bias")) return (mx.nd.zeros(shape, ctx)) - if (mx.util.str.endswith(name, "gamma")) return (mx.nd.ones(shape, ctx)) - if (mx.util.str.endswith(name, "beta")) return (mx.nd.zeros(shape, ctx)) - if (mx.util.str.endswith(name, "moving_mean")) return (mx.nd.zeros(shape, ctx)) - if (mx.util.str.endswith(name, "moving_var")) return (mx.nd.ones(shape, ctx)) + if (endsWith(name, "bias")) return (mx.nd.zeros(shape, ctx)) + if (endsWith(name, "gamma")) return (mx.nd.ones(shape, ctx)) + if (endsWith(name, "beta")) return (mx.nd.zeros(shape, ctx)) + if (endsWith(name, "moving_mean")) return (mx.nd.zeros(shape, ctx)) + if (endsWith(name, "moving_var")) return (mx.nd.ones(shape, ctx)) if (allow.unknown) return(NULL) stop(paste("Unkown initialization pattern for ", name)) } @@ -20,7 +20,7 @@ mx.init.internal.default <- function(name, shape, ctx, allow.unknown=FALSE) { #' @export mx.init.uniform <- function(scale) { function(name, shape, ctx, allow.unknown=FALSE) { - if (!mx.util.str.endswith(name, "weight")) { + if (!endsWith(name, "weight")) { return (mx.init.internal.default(name, shape, ctx, allow.unknown)) } return (mx.runif(shape, -scale, scale, ctx)) @@ -34,7 +34,7 @@ mx.init.uniform <- function(scale) { #' @export mx.init.normal <- function(sd) { function(name, shape, ctx, allow.unknown=FALSE) { - if (!mx.util.str.endswith(name, "weight")) { + if (!endsWith(name, "weight")) { return (mx.init.internal.default(name, shape, ctx, allow.unknown)) } return (mx.rnorm(shape, 0, sd, ctx)) @@ -55,7 +55,7 @@ mx.init.normal <- function(sd) { mx.init.Xavier <- function(rnd_type = "uniform", factor_type = "avg", magnitude = 3){ function(name, shape, ctx, allow.unknown = FALSE){ - if (!mx.util.str.endswith(name, "weight")) { + if (!endsWith(name, "weight")) { return (mx.init.internal.default(name, shape, ctx, allow.unknown)) } diff --git a/R-package/R/model.R b/R-package/R/model.R index ccdd93635dc2..069c564599b5 100644 --- a/R-package/R/model.R +++ b/R-package/R/model.R @@ -17,14 +17,14 @@ mx.model.check.arguments <- function(symbol) { data <- NULL label <- NULL for (nm in arguments(symbol)) { - if (mx.util.str.endswith(nm, "data")) { + if (endsWith(nm, "data")) { if (!is.null(data)) { stop("Multiple fields contains suffix data") } else { data <- nm } } - if (mx.util.str.endswith(nm, "label")) { + if (endsWith(nm, "label")) { if (!is.null(label)) { stop("Multiple fields contains suffix label") } else { @@ -106,10 +106,7 @@ mx.model.train <- function(symbol, ctx, input.shape, output.shape, sliceinfo <- mx.model.slice.shape(input.shape, ndevice) sliceinfo2 <- mx.model.slice.shape(output.shape, ndevice) arg_names <- arguments(symbol) - tmp <- unlist(lapply(arg_names, function(a) { - mxnet:::mx.util.str.endswith(a, "label") - })) - label_name <- arg_names[tmp] + label_name <- arg_names[endsWith(arg_names, "label")] train.execs <- lapply(1:ndevice, function(i) { arg_lst <- list(symbol = symbol, ctx = ctx[[i]], grad.req = "write", data=sliceinfo[[i]]$shape) @@ -271,10 +268,7 @@ mx.model.train <- function(symbol, ctx, input.shape, output.shape, mx.model.init.params <- function(symbol, input.shape, output.shape, initializer, ctx) { if (!is.MXSymbol(symbol)) stop("symbol need to be MXSymbol") arg_names <- arguments(symbol) - tmp <- unlist(lapply(arg_names, function(a) { - mxnet:::mx.util.str.endswith(a, "label") - })) - label_name <- arg_names[tmp] + label_name <- arg_names[endsWith(arg_names, "label")] arg_lst <- list(symbol = symbol, data=input.shape) arg_lst[[label_name]] = output.shape @@ -535,10 +529,10 @@ mx.model.load <- function(prefix, iteration) { save.dict <- mx.nd.load(sprintf("%s-%04d.params", prefix, iteration)) names <- names(save.dict) arg.index <- as.integer(mx.util.filter.null(lapply(1:length(names), function(i) { - if (mx.util.str.startswith(names[[i]], "arg:")) i else NULL + if (startsWith(names[[i]], "arg:")) i else NULL }))) aux.index <- as.integer(mx.util.filter.null(lapply(1:length(names), function(i) { - if (mx.util.str.startswith(names[[i]], "aux:")) i else NULL + if (startsWith(names[[i]], "aux:")) i else NULL }))) if (length(arg.index) != 0) { diff --git a/R-package/R/util.R b/R-package/R/util.R index 9eaddf260a83..2b292d1ad786 100644 --- a/R-package/R/util.R +++ b/R-package/R/util.R @@ -1,20 +1,3 @@ -# Internal function to check if name end with suffix -mx.util.str.endswith <- function(name, suffix) { - slen <- nchar(suffix) - nlen <- nchar(name) - if (slen > nlen) return (FALSE) - nsuf <- substr(name, nlen - slen + 1, nlen) - return (nsuf == suffix) -} - -mx.util.str.startswith <- function(name, prefix) { - slen <- nchar(prefix) - nlen <- nchar(name) - if (slen > nlen) return (FALSE) - npre <- substr(name, 1, slen) - return (npre == prefix) -} - # filter out null, keep the names mx.util.filter.null <- function(lst) { lst[!sapply(lst, is.null)] From b9e02cf1a1d48bd3515afaee1195495a3d0946c7 Mon Sep 17 00:00:00 2001 From: Jean Kossaifi Date: Wed, 28 Jun 2017 16:08:01 -0700 Subject: [PATCH 128/834] Updated installation instruction (#6861) Do not use sudo needlessly. Use pip rather than python setup.py. --- CONTRIBUTORS.md | 1 + docs/get_started/install.md | 31 ++++++++++++++++++++++++------- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index f4cc011059dc..2ee54f07c85e 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -144,3 +144,4 @@ List of Contributors * [Yi Li](https://github.com/liyi14/) * [Guodong Zhang](https://github.com/gd-zhang/) * [Xizhou Zhu](https://github.com/einsiedler0408/) +* [Jean Kossaifi](https://github.com/JeanKossaifi/) diff --git a/docs/get_started/install.md b/docs/get_started/install.md index 3fdb0e83d674..fc4dcf093036 100644 --- a/docs/get_started/install.md +++ b/docs/get_started/install.md @@ -241,13 +241,16 @@ $ make -j $(nproc) USE_OPENCV=1 USE_BLAS=openblas $ sudo apt-get install -y python-dev python-setuptools python-numpy ``` -**Step 2** Build the MXNet Python binding. +**Step 2** Install the MXNet Python binding. ```bash $ cd python -$ sudo python setup.py install +$ pip install --upgrade pip +$ pip install -e . ``` +Note that the `-e` flag is optional. It is equivalent to `--editable` and means that if you edit the source files, these changes will be reflected in the package installed. + **Step 3** Install [Graphviz](http://www.graphviz.org/). (Optional, needed for graph visualization using `mxnet.viz` package). ```bash sudo apt-get install graphviz @@ -453,7 +456,7 @@ $ make -j $(nproc) USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/
    -**Build the MXNet Python binding** +**Install the MXNet Python binding** **Step 1** Install prerequisites - python setup tools and numpy. @@ -461,13 +464,16 @@ $ make -j $(nproc) USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/ $ sudo apt-get install -y python-dev python-setuptools python-numpy ``` -**Step 2** Build the MXNet Python binding. +**Step 2** Install the MXNet Python binding. ```bash $ cd python -$ sudo python setup.py install +$ pip install --upgrade pip +$ pip install -e . ``` +Note that the `-e` flag is optional. It is equivalent to `--editable` and means that if you edit the source files, these changes will be reflected in the package installed. + **Step 3** Install [Graphviz](http://www.graphviz.org/). (Optional, needed for graph visualization using `mxnet.viz` package). ```bash sudo apt-get install graphviz @@ -987,9 +993,12 @@ To install python bindings run the following commands in the MXNet directory: ```bash cd python - sudo python setup.py install + pip install --upgrade pip + pip install -e . ``` +Note that the `-e` flag is optional. It is equivalent to `--editable` and means that if you edit the source files, these changes will be reflected in the package installed. + You are now ready to run MXNet on your Raspberry Pi device. You can get started by following the tutorial on [Real-time Object Detection with MXNet On The Raspberry Pi](http://mxnet.io/tutorials/embedded/wine_detector.html). *Note - Because the complete MXNet library takes up a significant amount of the Raspberry Pi's limited RAM, when loading training data or large models into memory, you might have to turn off the GUI and terminate running processes to free RAM.* @@ -1066,7 +1075,15 @@ To install python bindings run the following commands in the MXNet directory: ```bash cd python - sudo python setup.py install + pip install --upgrade pip + pip install -e . +``` + +Note that the `-e` flag is optional. It is equivalent to `--editable` and means that if you edit the source files, these changes will be reflected in the package installed. + +Add the mxnet folder to the path: + +```bash cd .. export MXNET_HOME=$(pwd) echo "export PYTHONPATH=$MXNET_HOME/python:$PYTHONPATH" >> ~/.bashrc From 49018a2bff9152519cd24659659fffdd3c875058 Mon Sep 17 00:00:00 2001 From: "Qiang Kou (KK)" Date: Wed, 28 Jun 2017 16:08:55 -0700 Subject: [PATCH 129/834] [R] doc update for regression and classification (#6826) * [R] softmax output is zero-indexed. close #3086 * [R] add doc for linear regression with multiple outputs. close #2138 --- .../vignettes/fiveMinutesNeuralNetwork.Rmd | 53 +++++++++++++++++-- 1 file changed, 49 insertions(+), 4 deletions(-) diff --git a/R-package/vignettes/fiveMinutesNeuralNetwork.Rmd b/R-package/vignettes/fiveMinutesNeuralNetwork.Rmd index c1b707fadc51..5cb9aafb8088 100644 --- a/R-package/vignettes/fiveMinutesNeuralNetwork.Rmd +++ b/R-package/vignettes/fiveMinutesNeuralNetwork.Rmd @@ -5,9 +5,10 @@ This is the first tutorial for new users of the R package `mxnet`. You will lear We will show you how to do classification and regression tasks respectively. The data we use comes from the package `mlbench`. -Preface -------- +## Preface + This tutorial is written in Rmarkdown. + - You can directly view the hosted version of the tutorial from [MXNet R Document](http://mxnet.readthedocs.io/en/latest/packages/r/fiveMinutesNeuralNetwork.html) - You can find the download the Rmarkdown source from [here](https://github.com/dmlc/mxnet/blob/master/R-package/vignettes/fiveMinutesNeuralNetwork.Rmd) @@ -29,7 +30,14 @@ test.x <- data.matrix(Sonar[-train.ind, 1:60]) test.y <- Sonar[-train.ind, 61] ``` -Next we are going to use a multi-layer perceptron as our classifier. In `mxnet`, we have a function called `mx.mlp` so that users can build a general multi-layer neural network to do classification or regression. +Next we are going to use a multi-layer perceptron (MLP) as our classifier. +In `mxnet`, we have a function called `mx.mlp` so that users can build a general multi-layer neural network to do classification (`out_activation="softmax"`) or regression (`out_activation="rmse"`). +Note for the `softmax` activation, the output is zero-indexed not one-indexed. In the data we use: + +```{r} +table(train.y) +table(test.y) +``` There are several parameters we have to feed to `mx.mlp`: @@ -38,7 +46,7 @@ There are several parameters we have to feed to `mx.mlp`: - Number of nodes in the output layer. - Type of the activation. - Type of the output loss. -- The device to train (GPU or CPU). +- The device to train `mx.gpu()` for GPU or `mx.cpu()` for CPU. - Other parameters for `mx.model.FeedForward.create`. The following code piece is showing a possible usage of `mx.mlp`: @@ -130,6 +138,43 @@ model <- mx.model.FeedForward.create(lro, X=train.x, y=train.y, learning.rate=2e-6, momentum=0.9, eval.metric=demo.metric.mae) ``` +In the previous example, our target is to predict the last column ("medv") in the dataset. +It is also possible to build a regression model with multiple outputs. +This time we use the last two columns as the targets: + +```{r} +train.x <- data.matrix(BostonHousing[train.ind, -(13:14)]) +train.y <- BostonHousing[train.ind, c(13:14)] +test.x <- data.matrix(BostonHousing[-train.ind, -(13:14)]) +test.y <- BostonHousing[-train.ind, c(13:14)] +``` + +and build a similar network symbol: + +```{r} +data <- mx.symbol.Variable("data") +fc2 <- mx.symbol.FullyConnected(data, num_hidden=2) +lro2 <- mx.symbol.LinearRegressionOutput(fc2) +``` + +We use `mx.io.arrayiter` to build an iter for our training set and train the model using `mx.model.FeedForward.create`: + +```{r} +mx.set.seed(0) +train_iter = mx.io.arrayiter(data = t(train.x), label = t(train.y)) + +model <- mx.model.FeedForward.create(lro2, X=train_iter, + ctx=mx.cpu(), num.round=50, array.batch.size=20, + learning.rate=2e-6, momentum=0.9) +``` + +After training, we can see that the dimension of the prediction is the same with our target. + +```{r} +preds <- t(predict(model, test.x)) +dim(preds) +dim(test.y) +``` Congratulations! Now you have learnt the basic for using `mxnet`. Please check the other tutorials for advanced features. From 09e08f59ffb092bd1a28bfc27531e04ccaa56386 Mon Sep 17 00:00:00 2001 From: "Qiang Kou (KK)" Date: Wed, 28 Jun 2017 16:11:07 -0700 Subject: [PATCH 130/834] [R] use AWS S3 to host all pre-built binary packages. close #6543 (#6722) --- R-package/README.md | 9 +++++---- docs/get_started/install.md | 27 ++++++++++++++++++++------- docs/get_started/osx_setup.md | 7 ++++--- docs/get_started/windows_setup.md | 19 ++++++++++++++++--- 4 files changed, 45 insertions(+), 17 deletions(-) diff --git a/R-package/README.md b/R-package/README.md index 75d4da5da450..e150f17bb732 100644 --- a/R-package/README.md +++ b/R-package/README.md @@ -20,12 +20,13 @@ Resources Installation ------------ -For Windows/Mac users, we provide a pre-built binary package using CPU. -You can install a weekly updated package directly from the R console: +We provide pre-built binary packages for Windows/OSX users. +You can install the CPU package directly from the R console: ```r -install.packages("drat", repos="https://cran.rstudio.com") -drat:::addRepo("dmlc") +cran <- getOption("repos") +cran["dmlc"] <- "https://s3-us-west-2.amazonaws.com/apache-mxnet/R/CRAN/" +options(repos = cran) install.packages("mxnet") ``` diff --git a/docs/get_started/install.md b/docs/get_started/install.md index fc4dcf093036..b9420bc0d926 100644 --- a/docs/get_started/install.md +++ b/docs/get_started/install.md @@ -712,8 +712,9 @@ You could also run distributed deeplearning with *MXNet* on AWS using [Cloudform The CPU version of MXNet R package can be installed in R like other packages ```r -install.packages("drat") -drat::addRepo("dmlc") +cran <- getOption("repos") +cran["dmlc"] <- "https://s3-us-west-2.amazonaws.com/apache-mxnet/R/CRAN/" +options(repos = cran) install.packages("mxnet") ``` @@ -876,23 +877,35 @@ The CPU version of MXNet R package can be installed in R like other packages ```r -install.packages("drat") -drat::addRepo("dmlc") +cran <- getOption("repos") +cran["dmlc"] <- "https://s3-us-west-2.amazonaws.com/apache-mxnet/R/CRAN/" +options(repos = cran) install.packages("mxnet") ``` - + +
    -You can also follow the installation instructions [in this guide](./windows_setup.md) to build MXNet from source. +The GPU version of MXNet R package can be installed in R like other packages + + +```r +cran <- getOption("repos") +cran["dmlc"] <- "https://s3-us-west-2.amazonaws.com/apache-mxnet/R/CRAN/GPU" +options(repos = cran) +install.packages("mxnet") +``` + +Alternatively, You can also follow the installation instructions [in this guide](./windows_setup.md) to build MXNet from source.
    - +
    diff --git a/docs/get_started/osx_setup.md b/docs/get_started/osx_setup.md index 1d58ddf36731..2b2858f9d058 100644 --- a/docs/get_started/osx_setup.md +++ b/docs/get_started/osx_setup.md @@ -117,9 +117,10 @@ You have 2 options: For OS X (Mac) users, MXNet provides a prebuilt binary package for CPUs. The prebuilt package is updated weekly. You can install the package directly in the R console using the following commands: ```r - install.packages("drat", repos="https://cran.rstudio.com") - drat:::addRepo("dmlc") - install.packages("mxnet") + cran <- getOption("repos") + cran["dmlc"] <- "https://s3-us-west-2.amazonaws.com/apache-mxnet/R/CRAN/" + options(repos = cran) + install.packages("mxnet") ``` ### Building MXNet from Source Code diff --git a/docs/get_started/windows_setup.md b/docs/get_started/windows_setup.md index b40ed3e34aae..47e0c0814b13 100755 --- a/docs/get_started/windows_setup.md +++ b/docs/get_started/windows_setup.md @@ -93,11 +93,24 @@ To install MXNet on a computer with a CPU processor, choose from two options: * Build the library from source code #### Building MXNet with the Prebuilt Binary Package -For Windows users, MXNet provides a prebuilt binary package for CPUs. The prebuilt package is updated weekly. You can install the package directly in the R console using the following commands: +For Windows users, MXNet provides prebuilt binary packages. +You can install the package directly in the R console. + +For CPU-only package: + +```r + cran <- getOption("repos") + cran["dmlc"] <- "https://s3-us-west-2.amazonaws.com/apache-mxnet/R/CRAN/" + options(repos = cran) + install.packages("mxnet") +``` + +For GPU-enabled package: ```r - install.packages("drat", repos="https://cran.rstudio.com") - drat:::addRepo("dmlc") + cran <- getOption("repos") + cran["dmlc"] <- "https://s3-us-west-2.amazonaws.com/apache-mxnet/R/CRAN/GPU" + options(repos = cran) install.packages("mxnet") ``` From 8c81ee48c197dd66276fa8d4008cbad0dcd2c8fb Mon Sep 17 00:00:00 2001 From: Arik Poznanski Date: Thu, 29 Jun 2017 02:11:52 +0300 Subject: [PATCH 131/834] * extended caffe to mxnet converter and improved converter test (#6822) - added support for networks which uses batch normalization without a scale layer following the batch norm, i.e. gamma is fixed to 1 - extended naming convention used when implementing batch normalization in caffe - added support for old caffe versions where dilation didn't exist. This is needed to convert models which depends on old caffe - added support for deconvolution layer - added support for older version of caffe where kernel_size, pad and stride parameters were not iterable - fixed crash happening when a bottom layer doesn't exist in the internal top_to_layers dictionary, this can happen if the name of the input is not 'data' - added ignore-by-design support for converting 'Crop' layers - fixed batch norm layer comparison to take into account the rescaling factor - added careful condition in tester to swap (RGB,BGR) input channels only if they are of size 3 or 4, which is the same check the conversion does - allow comparing layers of models with no mean file - added support for comparing the parameters of deconvolution layers --- tools/caffe_converter/caffe_proto_utils.py | 38 +++++++++++++++------ tools/caffe_converter/compare_layers.py | 39 ++++++++++++++++------ tools/caffe_converter/convert_model.py | 29 +++++++++++++--- tools/caffe_converter/convert_symbol.py | 15 +++++---- 4 files changed, 90 insertions(+), 31 deletions(-) diff --git a/tools/caffe_converter/caffe_proto_utils.py b/tools/caffe_converter/caffe_proto_utils.py index 65b2030fc2c5..4404f39b8698 100644 --- a/tools/caffe_converter/caffe_proto_utils.py +++ b/tools/caffe_converter/caffe_proto_utils.py @@ -26,6 +26,9 @@ def process_network_proto(caffe_root, deploy_proto): class LayerRecord(object): + """ + A record which describe basic layer parameters + """ def __init__(self, layer_def): @@ -35,15 +38,24 @@ def __init__(self, layer_def): # keep filter, stride and pad if layer_def.type == 'Convolution': - self.filter = list(layer_def.convolution_param.kernel_size) + if LayerRecord._is_iterable(layer_def.convolution_param.kernel_size): + self.filter = list(layer_def.convolution_param.kernel_size) + else: + self.filter = list([layer_def.convolution_param.kernel_size]) if len(self.filter) == 1: self.filter *= 2 - self.pad = list(layer_def.convolution_param.pad) + if LayerRecord._is_iterable(layer_def.convolution_param.pad): + self.pad = list(layer_def.convolution_param.pad) + else: + self.pad = list([layer_def.convolution_param.pad]) if len(self.pad) == 0: self.pad = [0, 0] elif len(self.pad) == 1: self.pad *= 2 - self.stride = list(layer_def.convolution_param.stride) + if LayerRecord._is_iterable(layer_def.convolution_param.stride): + self.stride = list(layer_def.convolution_param.stride) + else: + self.stride = list([layer_def.convolution_param.stride]) if len(self.stride) == 0: self.stride = [1, 1] elif len(self.stride) == 1: @@ -81,6 +93,9 @@ def __init__(self, layer_def): # list of child layers self.children = [] + @staticmethod + def _is_iterable(obj): + return hasattr(obj, '__iter__') def read_network_dag(processed_deploy_prototxt): """ @@ -123,16 +138,17 @@ def read_network_dag(processed_deploy_prototxt): top_to_layers[top].append(layer.name) # find parents and children of all layers - for child_layer_name in layer_name_to_record.keys(): + for child_layer_name in layer_name_to_record.keys(): # pylint: disable=too-many-nested-blocks child_layer_def = layer_name_to_record[child_layer_name] for bottom in child_layer_def.bottoms: - for parent_layer_name in top_to_layers[bottom]: - if parent_layer_name in layer_name_to_record: - parent_layer_def = layer_name_to_record[parent_layer_name] - if parent_layer_def not in child_layer_def.parents: - child_layer_def.parents.append(parent_layer_def) - if child_layer_def not in parent_layer_def.children: - parent_layer_def.children.append(child_layer_def) + if bottom in top_to_layers: + for parent_layer_name in top_to_layers[bottom]: + if parent_layer_name in layer_name_to_record: + parent_layer_def = layer_name_to_record[parent_layer_name] + if parent_layer_def not in child_layer_def.parents: + child_layer_def.parents.append(parent_layer_def) + if child_layer_def not in parent_layer_def.children: + parent_layer_def.children.append(child_layer_def) # update filter, strid, pad for maxout "structures" for layer_name in layer_name_to_record.keys(): diff --git a/tools/caffe_converter/compare_layers.py b/tools/caffe_converter/compare_layers.py index 097d86215515..bb4451d1b5fc 100644 --- a/tools/caffe_converter/compare_layers.py +++ b/tools/caffe_converter/compare_layers.py @@ -79,6 +79,8 @@ def convert_and_compare_caffe_to_mxnet(image_url, gpu, caffe_prototxt_path, caff if isinstance(caffe_mean, str): caffe_mean = read_caffe_mean(caffe_mean) + elif caffe_mean is None: + pass elif len(caffe_mean) == 3: # swap channels from Caffe BGR to RGB caffe_mean = caffe_mean[::-1] @@ -188,7 +190,8 @@ def _process_layer_parameters(layer): normalized_layer_name = re.sub('[-/]', '_', layer.name) # handle weight and bias of convolution and fully-connected layers - if layer.name in caffe_net.params and layer.type in ['Convolution', 'InnerProduct']: + if layer.name in caffe_net.params and layer.type in ['Convolution', 'InnerProduct', + 'Deconvolution']: has_bias = len(caffe_net.params[layer.name]) > 1 @@ -199,8 +202,10 @@ def _process_layer_parameters(layer): if layer.type == 'Convolution' and compare_layers_from_nets.is_first_convolution: compare_layers_from_nets.is_first_convolution = False - # swapping BGR of caffe into RGB in mxnet - mx_beta = mx_beta[:, ::-1, :, :] + # if RGB or RGBA + if mx_beta.shape[1] == 3 or mx_beta.shape[1] == 4: + # Swapping BGR of caffe into RGB in mxnet + mx_beta[:, [0, 2], :, :] = mx_beta[:, [2, 0], :, :] caf_beta = caffe_net.params[layer.name][0].data _compare_blob(caf_beta, mx_beta, layer.name, mx_name_weight, 'weight', '') @@ -213,7 +218,13 @@ def _process_layer_parameters(layer): elif layer.name in caffe_net.params and layer.type == 'Scale': - bn_name = normalized_layer_name.replace('scale', 'bn') + if 'scale' in normalized_layer_name: + bn_name = normalized_layer_name.replace('scale', 'bn') + elif 'sc' in normalized_layer_name: + bn_name = normalized_layer_name.replace('sc', 'bn') + else: + assert False, 'Unknown name convention for bn/scale' + beta_name = '{}_beta'.format(bn_name) gamma_name = '{}_gamma'.format(bn_name) @@ -230,17 +241,19 @@ def _process_layer_parameters(layer): mean_name = '{}_moving_mean'.format(normalized_layer_name) var_name = '{}_moving_var'.format(normalized_layer_name) + caf_rescale_factor = caffe_net.params[layer.name][2].data + mx_mean = aux_params[mean_name].asnumpy() - caf_mean = caffe_net.params[layer.name][0].data + caf_mean = caffe_net.params[layer.name][0].data / caf_rescale_factor _compare_blob(caf_mean, mx_mean, layer.name, mean_name, 'mean', '') mx_var = aux_params[var_name].asnumpy() - caf_var = caffe_net.params[layer.name][1].data + caf_var = caffe_net.params[layer.name][1].data / caf_rescale_factor _compare_blob(caf_var, mx_var, layer.name, var_name, 'var', 'expect 1e-04 change due to cudnn eps') elif layer.type in ['Input', 'Pooling', 'ReLU', 'Eltwise', 'Softmax', 'LRN', 'Concat', - 'Dropout']: + 'Dropout', 'Crop']: # no parameters to check for these layers pass @@ -262,8 +275,11 @@ def _process_layer_output(caffe_blob_name): # data should change from BGR to RGB if caffe_blob_name == 'data': - # swapping BGR of caffe into RGB in mxnet - caf_blob = caf_blob[:, ::-1, :, :] + + # if RGB or RGBA + if caf_blob.shape[1] == 3 or caf_blob.shape[1] == 4: + # Swapping BGR of caffe into RGB in mxnet + caf_blob[:, [0, 2], :, :] = caf_blob[:, [2, 0], :, :] mx_name = 'data' else: @@ -271,7 +287,10 @@ def _process_layer_output(caffe_blob_name): last_layer_name = top_to_layers[caffe_blob_name][-1] normalized_last_layer_name = re.sub('[-/]', '_', last_layer_name) mx_name = '{}_output'.format(normalized_last_layer_name) - mx_name = mx_name.replace('scale', 'bn') + if 'scale' in mx_name: + mx_name = mx_name.replace('scale', 'bn') + elif 'sc' in mx_name: + mx_name = mx_name.replace('sc', 'bn') if mx_name not in exe.output_dict: logging.error('mxnet blob %s is missing, time to extend the compare tool..', mx_name) diff --git a/tools/caffe_converter/convert_model.py b/tools/caffe_converter/convert_model.py index 03641aa80d51..2d8c9941ddea 100644 --- a/tools/caffe_converter/convert_model.py +++ b/tools/caffe_converter/convert_model.py @@ -48,8 +48,9 @@ def convert_model(prototxt_fname, caffemodel_fname, output_prefix=None): layers_proto = caffe_parser.get_layers(caffe_parser.read_prototxt(prototxt_fname)) for layer_name, layer_type, layer_blobs in layer_iter: - if layer_type == 'Convolution' or layer_type == 'InnerProduct' \ - or layer_type == 4 or layer_type == 14 or layer_type == 'PReLU': + if layer_type == 'Convolution' or layer_type == 'InnerProduct' \ + or layer_type == 4 or layer_type == 14 or layer_type == 'PReLU' \ + or layer_type == 'Deconvolution' or layer_type == 39: if layer_type == 'PReLU': assert (len(layer_blobs) == 1) wmat = layer_blobs[0].data @@ -108,7 +109,13 @@ def convert_model(prototxt_fname, caffemodel_fname, output_prefix=None): first_conv = False elif layer_type == 'Scale': - bn_name = layer_name.replace('scale', 'bn') + if 'scale' in layer_name: + bn_name = layer_name.replace('scale', 'bn') + elif 'sc' in layer_name: + bn_name = layer_name.replace('sc', 'bn') + else: + assert False, 'Unknown name convention for bn/scale' + gamma = np.array(layer_blobs[0].data) beta = np.array(layer_blobs[1].data) # beta = np.expand_dims(beta, 1) @@ -154,9 +161,23 @@ def convert_model(prototxt_fname, caffemodel_fname, output_prefix=None): assert mean.flags['C_CONTIGUOUS'] is True print('converting batchnorm layer, mean shape = {}, var shape = {}'.format( mean.shape, var.shape)) + + fix_gamma = layers_proto[bn_index+1].type != 'Scale' + if fix_gamma: + gamma_name = '{}_gamma'.format(bn_name) + gamma = np.array(np.ones(arg_shape_dic[gamma_name])) + beta_name = '{}_beta'.format(bn_name) + beta = np.array(np.zeros(arg_shape_dic[beta_name])) + arg_params[beta_name] = mx.nd.zeros(beta.shape) + arg_params[gamma_name] = mx.nd.zeros(gamma.shape) + arg_params[beta_name][:] = beta + arg_params[gamma_name][:] = gamma + assert gamma.flags['C_CONTIGUOUS'] is True + assert beta.flags['C_CONTIGUOUS'] is True + else: - assert len(layer_blobs) == 0 print('\tskipping layer {} of type {}'.format(layer_name, layer_type)) + assert len(layer_blobs) == 0 if output_prefix is not None: model = mx.mod.Module(symbol=sym, label_names=['prob_label', ]) diff --git a/tools/caffe_converter/convert_symbol.py b/tools/caffe_converter/convert_symbol.py index e4fb743494c3..fad89c41e83c 100644 --- a/tools/caffe_converter/convert_symbol.py +++ b/tools/caffe_converter/convert_symbol.py @@ -69,10 +69,11 @@ def _convert_conv_param(param): param_string += ", stride=(%d,%d)" % (stride, stride) dilate = 1 - if isinstance(param.dilation, int): - dilate = param.dilation - else: - dilate = 1 if len(param.dilation) == 0 else param.dilation[0] + if hasattr(param, 'dilation'): + if isinstance(param.dilation, int): + dilate = param.dilation + else: + dilate = 1 if len(param.dilation) == 0 else param.dilation[0] param_string += ", no_bias=%s" % (not param.bias_term) @@ -189,8 +190,10 @@ def _parse_proto(prototxt_fname): epsilon = param.eps if (epsilon <= 1e-05): epsilon = 1e-04 - param_string = 'use_global_stats=%s, fix_gamma=False, eps=%f' % ( - param.use_global_stats, epsilon) + # if next layer is scale, don't fix gamma + fix_gamma = layers[i+1].type != 'Scale' + param_string = 'use_global_stats=%s, fix_gamma=%s, eps=%f' % ( + param.use_global_stats, fix_gamma, epsilon) need_flatten[name] = need_flatten[mapping[layer.bottom[0]]] if layer.type == 'Scale': assert layers[i-1].type == 'BatchNorm' From aec0460b036136f68edd17fa939a210e4eac2278 Mon Sep 17 00:00:00 2001 From: Mu Li Date: Thu, 29 Jun 2017 12:14:44 -0700 Subject: [PATCH 132/834] update mklml path in jenkins docker (#6876) --- tests/ci_build/Dockerfile.mklml_gpu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci_build/Dockerfile.mklml_gpu b/tests/ci_build/Dockerfile.mklml_gpu index eb488267c0db..1c29ca3248ec 100644 --- a/tests/ci_build/Dockerfile.mklml_gpu +++ b/tests/ci_build/Dockerfile.mklml_gpu @@ -9,7 +9,7 @@ RUN /install/ubuntu_install_python.sh COPY install/ubuntu_install_scala.sh /install/ RUN /install/ubuntu_install_scala.sh -RUN wget --no-check-certificate -O /tmp/mklml.tgz https://github.com/dmlc/web-data/raw/master/mxnet/mklml-release/mklml_lnx_2017.0.2.20170209.tgz +RUN wget --no-check-certificate -O /tmp/mklml.tgz https://github.com/01org/mkl-dnn/releases/download/v0.7/mklml_lnx_2018.0.20170425.tgz RUN tar -zxvf /tmp/mklml.tgz && cp -rf mklml_*/* /usr/local/ && rm -rf mklml_* ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib From bfb79bd2d7adf2fdde8266fe62565f6b3ff6ec40 Mon Sep 17 00:00:00 2001 From: "Qiang Kou (KK)" Date: Thu, 29 Jun 2017 19:17:54 -0700 Subject: [PATCH 133/834] [R] fix concat for symbol and NDArray. close #6650 (#6696) --- Jenkinsfile | 2 + R-package/R/symbol.R | 40 ++++++++- R-package/src/export.cc | 2 +- R-package/src/ndarray.cc | 10 ++- R-package/src/symbol.cc | 8 ++ R-package/src/symbol.h | 6 ++ R-package/tests/testthat/test_ndarray.R | 106 +++++++++++++++++++----- R-package/tests/testthat/test_symbol.R | 57 ++++++++++--- 8 files changed, 190 insertions(+), 41 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index cd6ab3166b79..881fef529f11 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -277,6 +277,7 @@ stage('Unit Test') { init_git() unpack_lib('cpu') timeout(time: max_time, unit: 'MINUTES') { + sh "${docker_run} cpu rm -rf .Renviron" sh "${docker_run} cpu mkdir -p /workspace/ut-r-cpu/site-library" sh "${docker_run} cpu make rpkg USE_BLAS=openblas R_LIBS=/workspace/ut-r-cpu/site-library" sh "${docker_run} cpu R CMD INSTALL --library=/workspace/ut-r-cpu/site-library mxnet_current_r.tar.gz" @@ -291,6 +292,7 @@ stage('Unit Test') { init_git() unpack_lib('gpu') timeout(time: max_time, unit: 'MINUTES') { + sh "${docker_run} cpu rm -rf .Renviron" sh "${docker_run} gpu mkdir -p /workspace/ut-r-gpu/site-library" sh "${docker_run} gpu make rpkg USE_BLAS=openblas R_LIBS=/workspace/ut-r-gpu/site-library" sh "${docker_run} gpu R CMD INSTALL --library=/workspace/ut-r-gpu/site-library mxnet_current_r.tar.gz" diff --git a/R-package/R/symbol.R b/R-package/R/symbol.R index 091a6468c1a1..f222c55ff74e 100644 --- a/R-package/R/symbol.R +++ b/R-package/R/symbol.R @@ -32,14 +32,32 @@ mx.symbol.Group <- function(...) { #' @return out The result mx.symbol #' #' @export -mx.symbol.Concat <- function(data, num.args, dim = NULL, name = NULL) { +mx.symbol.concat <- function(data, num.args, dim = NULL, name = NULL) { data[['num.args']] <- num.args if(!is.null(dim)) data[['dim']] <- dim if(!is.null(name)) data[['name']] <- name - mx.varg.symbol.Concat(data) + mx.varg.symbol.concat(data) +} + +#' Perform an feature concat on channel dim (dim 1) over all the inputs. +#' +#' @param data list, required +#' List of tensors to concatenate +#' @param num.args int, required +#' Number of inputs to be concated. +#' @param dim int, optional, default='1' +#' the dimension to be concated. +#' @param name string, optional +#' Name of the resulting symbol. +#' @return out The result mx.symbol +#' +#' @export +mx.symbol.Concat <- function(data, num.args, dim = NULL, name = NULL) { + warning("mx.symbol.Concat is deprecated. Use mx.symbol.concat instead.") + mx.symbol.concat(data, num.args, dim, name) } #' Save an mx.symbol object @@ -123,6 +141,24 @@ mx.apply <- function(x, ...) { x$apply(list(...)) } +#' Get a symbol that contains all the internals +#' @param x The input symbol +#' +#' @export +internals <- function(x) { + if (!is.MXSymbol(x)) stop("only for MXSymbol type") + x$get.internals() +} + +#' Gets a new grouped symbol whose output contains inputs to output nodes of the original symbol. +#' @param x The input symbol +#' +#' @export +children <- function(x) { + if (!is.MXSymbol(x)) stop("only for MXSymbol type") + x$get.children() +} + #' Get the outputs of a symbol. #' @param x The input symbol #' diff --git a/R-package/src/export.cc b/R-package/src/export.cc index ed8f4bc0c7fe..0e77c1c3b58e 100644 --- a/R-package/src/export.cc +++ b/R-package/src/export.cc @@ -93,7 +93,7 @@ void Exporter::Export(const std::string& path) { std::string fname = Rcpp::as(func_names[i]); // skip internal functions if (fname.find("internal.") != std::string::npos) continue; - if (fname == "mx.varg.symbol.Concat") continue; + if (fname == "mx.varg.symbol.Concat" || fname == "mx.varg.symbol.concat") continue; Rcpp::List func_info(scope->get_function(fname)); std::string docstr = Rcpp::as(func_info[2]); if (docstr.find("@export") == std::string::npos) continue; diff --git a/R-package/src/ndarray.cc b/R-package/src/ndarray.cc index c2bfe0c945a6..d270a303fcbf 100644 --- a/R-package/src/ndarray.cc +++ b/R-package/src/ndarray.cc @@ -398,10 +398,16 @@ SEXP NDArrayFunction::operator() (SEXP* args) { std::vector param_vals; std::vector out_args; - for (mx_uint i = 0; i < arg_names_.size() - 1; ++i) { if (arg_nd_array_[i]) { - nd_args.push_back(NDArray(args[i])->handle); + if (TYPEOF(args[i]) == 22) { + nd_args.push_back(NDArray(args[i])->handle); + } else if (TYPEOF(args[i]) == 19) { + Rcpp::List data_lst = Rcpp::as(args[i]); + for (size_t k = 0; k < data_lst.size(); k++) { + nd_args.push_back(NDArray((SEXP)data_lst[k])->handle); + } + } } else { if (args[i] != R_NilValue) { param_keys.push_back(arg_names_[i].c_str()); diff --git a/R-package/src/symbol.cc b/R-package/src/symbol.cc index aa901ff64d9a..b5d6eca5fbdd 100644 --- a/R-package/src/symbol.cc +++ b/R-package/src/symbol.cc @@ -134,6 +134,12 @@ Symbol::RObjectType Symbol::GetInternals() const { return Symbol::RObject(out); } +Symbol::RObjectType Symbol::GetChildren() const { + SymbolHandle out; + MX_CALL(MXSymbolGetChildren(handle_, &out)); + return Symbol::RObject(out); +} + Symbol::RObjectType Symbol::GetOutput(mx_uint index) const { SymbolHandle out; MX_CALL(MXSymbolGetOutput(handle_, index - 1, &out)); @@ -344,6 +350,8 @@ void Symbol::InitRcppModule() { "List the auxiliary state names of the symbol") .method("get.internals", &Symbol::GetInternals, "Get a symbol that contains all the internals") + .method("get.children", &Symbol::GetChildren, + "Get a symbol that contains all the children") .method("get.output", &Symbol::GetOutput, "Get index-th output symbol of current one") .method("[[", &Symbol::GetOutput, diff --git a/R-package/src/symbol.h b/R-package/src/symbol.h index a361bdf2426d..4bfdebf2b959 100644 --- a/R-package/src/symbol.h +++ b/R-package/src/symbol.h @@ -69,6 +69,12 @@ class Symbol { * \return The internal of the symbol. */ RObjectType GetInternals() const; + /*! + * \brief Gets a new grouped symbol whose output contains + * inputs to output nodes of the original symbol. + * \return The children of the symbol. + */ + RObjectType GetChildren() const; /*! * \brief Get index-th outputs of the symbol. * \param symbol The symbol diff --git a/R-package/tests/testthat/test_ndarray.R b/R-package/tests/testthat/test_ndarray.R index a6b85daa2516..e574ea74dbd6 100644 --- a/R-package/tests/testthat/test_ndarray.R +++ b/R-package/tests/testthat/test_ndarray.R @@ -6,34 +6,39 @@ test_that("element-wise calculation for vector", { x = 1:10 mat = mx.nd.array(as.array(x), mx.cpu(0)) expect_equal(x, as.array(mat)) - expect_equal(x+1, as.array(mat+1)) - expect_equal(x-10, as.array(mat-10)) - expect_equal(x*20, as.array(mat*20)) - expect_equal(x/3, as.array(mat/3), tolerance = 1e-5) - expect_equal(-1-x, as.array(-1-mat)) - expect_equal(-5/x, as.array(-5/mat), tolerance = 1e-5) - expect_equal(x+x, as.array(mat+mat)) - expect_equal(x/x, as.array(mat/mat)) - expect_equal(x*x, as.array(mat*mat)) - expect_equal(x-x, as.array(mat-mat)) - expect_equal(as.array(1-mat), as.array(1-mat)) + expect_equal(x + 1, as.array(mat + 1)) + expect_equal(x - 10, as.array(mat - 10)) + expect_equal(x * 20, as.array(mat * 20)) + expect_equal(x / 3, as.array(mat / 3), tolerance = 1e-5) + expect_equal(-1 - x, as.array(-1 - mat)) + expect_equal(-5 / x, as.array(-5 / mat), tolerance = 1e-5) + expect_equal(x + x, as.array(mat + mat)) + expect_equal(x / x, as.array(mat / mat)) + expect_equal(x * x, as.array(mat * mat)) + expect_equal(x - x, as.array(mat - mat)) + expect_equal(as.array(1 - mat), as.array(1 - mat)) + + x <- runif(10,-10, 10) + nd = mx.nd.array(as.array(x)) + expect_equal(sqrt(abs(x)), as.array(mx.nd.sqrt(mx.nd.abs(nd))), tolerance = 1e-6) + expect_equal(x ^ 2, as.array(mx.nd.square(nd)), tolerance = 1e-6) }) -test_that("element-wise calculation for matrix", { +test_that("element-wise calculation for matrix", { x = matrix(1:4, 2, 2) mat = mx.nd.array(as.array(x), mx.cpu(0)) expect_equal(x, as.array(mat)) - expect_equal(x+1, as.array(mat+1)) - expect_equal(x-10, as.array(mat-10)) - expect_equal(x*20, as.array(mat*20)) - expect_equal(x/3, as.array(mat/3), tolerance = 1e-5) - expect_equal(-1-x, as.array(-1-mat)) - expect_equal(-5/x, as.array(-5/mat), tolerance = 1e-5) - expect_equal(x+x, as.array(mat+mat)) - expect_equal(x/x, as.array(mat/mat)) - expect_equal(x*x, as.array(mat*mat)) - expect_equal(x-x, as.array(mat-mat)) - expect_equal(as.array(1-mat), as.array(1-mat)) + expect_equal(x + 1, as.array(mat + 1)) + expect_equal(x - 10, as.array(mat - 10)) + expect_equal(x * 20, as.array(mat * 20)) + expect_equal(x / 3, as.array(mat / 3), tolerance = 1e-5) + expect_equal(-1 - x, as.array(-1 - mat)) + expect_equal(-5 / x, as.array(-5 / mat), tolerance = 1e-5) + expect_equal(x + x, as.array(mat + mat)) + expect_equal(x / x, as.array(mat / mat)) + expect_equal(x * x, as.array(mat * mat)) + expect_equal(x - x, as.array(mat - mat)) + expect_equal(as.array(1 - mat), as.array(1 - mat)) }) test_that("ndarray ones, zeros, save and load", { @@ -48,3 +53,58 @@ test_that("ndarray ones, zeros, save and load", { expect_equal(as.array(mat), as.array(mat2[[1]])) file.remove('temp.mat') }) + +test_that("ndarray concatenate", { + shapes <- matrix(c(2, 3, 4, 2, 2, 2, 4, 2, 2, 1, 4, 2), nrow = 3, byrow = TRUE) + array_r <- apply(shapes, 2, function(s) { runif(s, -10, 10) }) + array_nd <- apply(array_r, 1, function(s) { mx.nd.array(matrix(s, nrow = 1)) }) + array_nd_concat <- mx.nd.concat(data = array_nd, num_args = 3, dim = 1) + expect_equal(array_r, as.matrix(array_nd_concat), tolerance = 1e-6) + + x1 <- mx.nd.array(c(1:24)) + x2 <- mx.nd.array(c(25:48)) + x3 <- mx.nd.concat(data = c(x1, x2), num_args = 2, dim = 0) + expect_equal(c(1:48), as.array(x3)) + expect_equal(dim(x3), 48) + + x1 <- array(1:24, dim = c(4, 3, 2)) + x2 <- array(25:48, dim = c(4, 3, 2)) + x3 <- c(1:4, 25:28, 5:8, 29:32, 9:12, 33:36, 13:16, 37:40, 17:20, 41:44, 21:24, 45:48) + y1 <- mx.nd.array(x1) + y2 <- mx.nd.array(x2) + y3 <- mx.nd.concat(data = c(y1, y2), num_args = 2, dim = 2) + expect_equal(dim(y3), c(8, 3, 2)) + expect_equal(as.array(y3), array(x3, dim = c(8, 3, 2))) +}) + +test_that("ndarray clip", { + nd <- mx.nd.array(runif(10,-10, 10)) + nd2 <- mx.nd.clip(nd,-2, 3) + arr <- as.array(nd2) + expect_equal(arr >= -2 | arr <= 3, rep(TRUE, length(arr))) +}) + +test_that("ndarray dot", { + a <- matrix(runif(12), nrow = 3) + b <- matrix(runif(20), nrow = 4) + c <- a %*% b + + A <- mx.nd.array(t(a)) + B <- mx.nd.array(t(b)) + C <- mx.nd.dot(A, B) + + expect_equal(c, t(as.matrix(C)), tolerance = 1e-6) +}) + +test_that("ndarray crop", { + x <- mx.nd.ones(c(2, 3, 4)) + y <- mx.nd.crop(x, begin = c(0, 0, 0), end = c(2, 1, 3)) + expect_equal(array(1, dim = c(2, 1, 3)), as.array(y)) + + z <- mx.nd.zeros(c(2, 1, 3)) + x <- mxnet:::mx.nd.internal.crop.assign(x, z, begin = c(0, 0, 0), end = c(2, 1, 3)) + arr_x <- array(1, dim = dim(x)) + arr_x[c(1:2), 1 , c(1:3)] <- 0 + + expect_equal(as.array(x), arr_x) +}) \ No newline at end of file diff --git a/R-package/tests/testthat/test_symbol.R b/R-package/tests/testthat/test_symbol.R index be38762d85bb..656d146cd87c 100644 --- a/R-package/tests/testthat/test_symbol.R +++ b/R-package/tests/testthat/test_symbol.R @@ -4,17 +4,17 @@ context("symbol") test_that("basic symbol operation", { data = mx.symbol.Variable('data') - net1 = mx.symbol.FullyConnected(data=data, name='fc1', num_hidden=10) - net1 = mx.symbol.FullyConnected(data=net1, name='fc2', num_hidden=100) + net1 = mx.symbol.FullyConnected(data = data, name = 'fc1', num_hidden = 10) + net1 = mx.symbol.FullyConnected(data = net1, name = 'fc2', num_hidden = 100) expect_equal(arguments(net1), c('data', 'fc1_weight', 'fc1_bias', 'fc2_weight', 'fc2_bias')) expect_equal(outputs(net1), 'fc2_output') - net2 = mx.symbol.FullyConnected(name='fc3', num_hidden=10) - net2 = mx.symbol.Activation(data=net2, act_type='relu') - net2 = mx.symbol.FullyConnected(data=net2, name='fc4', num_hidden=20) + net2 = mx.symbol.FullyConnected(name = 'fc3', num_hidden = 10) + net2 = mx.symbol.Activation(data = net2, act_type = 'relu') + net2 = mx.symbol.FullyConnected(data = net2, name = 'fc4', num_hidden = 20) - composed = mx.apply(net2, fc3_data=net1, name='composed') + composed = mx.apply(net2, fc3_data = net1, name = 'composed') expect_equal(arguments(composed), c('data', 'fc1_weight', 'fc1_bias', 'fc2_weight', 'fc2_bias', 'fc3_weight', 'fc3_bias', 'fc4_weight', 'fc4_bias')) expect_equal(outputs(composed), 'composed_output') @@ -25,17 +25,35 @@ test_that("basic symbol operation", { test_that("symbol internal", { data = mx.symbol.Variable('data') - oldfc = mx.symbol.FullyConnected(data=data, name='fc1', num_hidden=10) - net1 = mx.symbol.FullyConnected(data=oldfc, name='fc2', num_hidden=100) + oldfc = mx.symbol.FullyConnected(data = data, name = 'fc1', num_hidden = 10) + net1 = mx.symbol.FullyConnected(data = oldfc, name = 'fc2', num_hidden = 100) expect_equal(arguments(net1), c("data", "fc1_weight", "fc1_bias", "fc2_weight", "fc2_bias")) - internal = net1$get.internals() + internal = internals(net1) fc1 = internal[[match("fc1_output", internal$outputs)]] expect_equal(arguments(fc1), arguments(oldfc)) }) +test_that("symbol children", { + data = mx.symbol.Variable('data') + oldfc = mx.symbol.FullyConnected(data = data, + name = 'fc1', + num_hidden = 10) + net1 = mx.symbol.FullyConnected(data = oldfc, name = 'fc2', num_hidden = 100) + + expect_equal(outputs(children(net1)), c('fc1_output', 'fc2_weight', 'fc2_bias')) + expect_equal(outputs(children(children(net1))), c('data', 'fc1_weight', 'fc1_bias')) + + net2 = net1$get.children() + expect_equal(net2[[match('fc2_weight', net2$outputs)]]$arguments, 'fc2_weight') + + data = mx.symbol.Variable('data') + sliced = mx.symbol.SliceChannel(data, num_outputs = 3, name = 'slice') + expect_equal(outputs(children(sliced)), 'data') +}) + test_that("symbol infer type", { num_hidden = 128 num_dim = 64 @@ -43,10 +61,10 @@ test_that("symbol infer type", { data = mx.symbol.Variable('data') prev = mx.symbol.Variable('prevstate') - x2h = mx.symbol.FullyConnected(data=data, name='x2h', num_hidden=num_hidden) - h2h = mx.symbol.FullyConnected(data=prev, name='h2h', num_hidden=num_hidden) + x2h = mx.symbol.FullyConnected(data = data, name = 'x2h', num_hidden = num_hidden) + h2h = mx.symbol.FullyConnected(data = prev, name = 'h2h', num_hidden = num_hidden) - out = mx.symbol.Activation(data=mx.symbol.elemwise_add(x2h, h2h), name='out', act_type='relu') + out = mx.symbol.Activation(data = mx.symbol.elemwise_add(x2h, h2h), name = 'out', act_type = 'relu') # shape inference will fail because information is not available for h2h ret = mx.symbol.infer.shape(out, data = c(num_dim, num_sample)) @@ -56,7 +74,7 @@ test_that("symbol infer type", { test_that("symbol save/load", { data <- mx.symbol.Variable("data") - fc1 <- mx.symbol.FullyConnected(data, num_hidden=1) + fc1 <- mx.symbol.FullyConnected(data, num_hidden = 1) lro <- mx.symbol.LinearRegressionOutput(fc1) mx.symbol.save(lro, "tmp_r_sym.json") data2 = mx.symbol.load("tmp_r_sym.json") @@ -78,3 +96,16 @@ test_that("symbol attributes access", { expect_equal(y$attributes$`__shape__`, str) }) +test_that("symbol concat", { + s1 <- mx.symbol.Variable("data1") + s2 <- mx.symbol.Variable("data2") + s3 <- mx.symbol.concat(data = c(s1, s2), num.args = 2, name = "concat") + expect_equal(outputs(s3), "concat_output") + expect_equal(outputs(children(s3)), c("data1", "data2")) + expect_equal(arguments(s3), c("data1", "data2")) + + s4 <- mx.symbol.Concat(data = c(s1, s2), num.args = 2, name = "concat") + expect_equal(outputs(s3), outputs(s4)) + expect_equal(outputs(children(s3)), outputs(children(s4))) + expect_equal(arguments(s3), arguments(s4)) +}) From a4fe4b4d2feb45434ad960df7060a104b6186709 Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Thu, 29 Jun 2017 21:55:44 -0700 Subject: [PATCH 134/834] fix warnings (#6881) --- src/operator/mshadow_op.h | 37 +++++++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/src/operator/mshadow_op.h b/src/operator/mshadow_op.h index 245df1bea616..e2860819ca5c 100644 --- a/src/operator/mshadow_op.h +++ b/src/operator/mshadow_op.h @@ -24,6 +24,8 @@ __constant__ const float PI = 3.14159265358979323846; const float PI = 3.14159265358979323846; using std::isnan; #endif +using std::enable_if; +using std::is_unsigned; /*! \brief identity Operation */ struct identity { @@ -445,8 +447,15 @@ struct abs { /*! \brief used for generate element of sign */ struct sign { template - MSHADOW_XINLINE static DType Map(DType a) { - if (a < 0.0f) return DType(-1.0f); + MSHADOW_XINLINE static typename enable_if::value, DType>::type + Map(DType a) { + if (a < 0.0f) return DType(-DType(1.0f)); + if (a > 0.0f) return DType(1.0f); + return DType(0.0f); + } + template + MSHADOW_XINLINE static typename enable_if::value, DType>::type + Map(DType a) { if (a > 0.0f) return DType(1.0f); return DType(0.0f); } @@ -678,7 +687,8 @@ struct rdiv_grad { struct mod { template - MSHADOW_XINLINE static DType Map(DType a, DType b) { + MSHADOW_XINLINE static typename enable_if::value, DType>::type + Map(DType a, DType b) { if (b == DType(0)) { return DType(0); } else if (b < DType(0)) { @@ -699,6 +709,15 @@ struct mod { } } } + template + MSHADOW_XINLINE static typename enable_if::value, DType>::type + Map(DType a, DType b) { + if (b == DType(0)) { + return DType(0); + } else { + return DType(::fmod(static_cast(a), static_cast(b))); + } + } }; #ifdef __CUDACC__ template<> @@ -783,7 +802,8 @@ MSHADOW_XINLINE mshadow::half::half2_t mod_rgrad::Map struct rmod { template - MSHADOW_XINLINE static DType Map(DType a, DType b) { + MSHADOW_XINLINE static typename enable_if::value, DType>::type + Map(DType a, DType b) { if (a == DType(0)) { return DType(0); } else if (a < DType(0)) { @@ -804,6 +824,15 @@ struct rmod { } } } + template + MSHADOW_XINLINE static typename enable_if::value, DType>::type + Map(DType a, DType b) { + if (a == DType(0)) { + return DType(0); + } else { + return DType(::fmod(static_cast(b), static_cast(a))); + } + } }; #ifdef __CUDACC__ template<> From a94d0e2448f767f7deed01168e61b096a0b3598e Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Thu, 29 Jun 2017 22:06:13 -0700 Subject: [PATCH 135/834] fix pinned mem USE_CUDA=1 on host with no device (#6864) * fix pinned mem USE_CUDA=1 on host with no device * fix * fix * fix --- src/storage/storage.cc | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/src/storage/storage.cc b/src/storage/storage.cc index 997f033b927b..a7cdbf667021 100644 --- a/src/storage/storage.cc +++ b/src/storage/storage.cc @@ -27,21 +27,25 @@ class StorageImpl : public Storage { private: static constexpr size_t kMaxNumberOfDevices = Context::kMaxDevType + 1; static constexpr size_t kMaxNumberOfDeviceIDs = Context::kMaxDevID + 1; +#if MXNET_USE_CUDA + static int num_gpu_device; +#endif // MXNET_USE_CUDA static void ActivateDevice(Context ctx) { switch (ctx.dev_type) { case Context::kCPU: break; case Context::kGPU: case Context::kCPUPinned: { - int gpu_num = 0; #if MXNET_USE_CUDA - CUDA_CALL(cudaGetDeviceCount(&gpu_num)); -#endif // MXNET_USE_CUDA - if (gpu_num > 0) { -#if MXNET_USE_CUDA - CUDA_CALL(cudaSetDevice(ctx.dev_id)); -#endif // MXNET_USE_CUDA + num_gpu_device = 0; + cudaError_t e = cudaGetDeviceCount(&num_gpu_device); + if (e != cudaSuccess) { + num_gpu_device = 0; } + if (num_gpu_device > 0) { + CUDA_CALL(cudaSetDevice(ctx.dev_id)); + } +#endif // MXNET_USE_CUDA break; } default: @@ -52,6 +56,9 @@ class StorageImpl : public Storage { std::array, kMaxNumberOfDevices> storage_managers_; }; // struct Storage::Impl +#if MXNET_USE_CUDA +int StorageImpl::num_gpu_device = 0; +#endif // MXNET_USE_CUDA Storage::Handle StorageImpl::Alloc(size_t size, Context ctx) { // space already recycled, ignore request @@ -69,7 +76,11 @@ Storage::Handle StorageImpl::Alloc(size_t size, Context ctx) { } case Context::kCPUPinned: { #if MXNET_USE_CUDA - ptr = new storage::NaiveStorageManager(); + if (num_gpu_device > 0) { + ptr = new storage::NaiveStorageManager(); + } else { + ptr = new storage::NaiveStorageManager(); + } #else ptr = new storage::NaiveStorageManager(); #endif // MXNET_USE_CUDA From 4a2bae2a6a45a96da8bab05304265b801076d4a6 Mon Sep 17 00:00:00 2001 From: wac81 Date: Fri, 30 Jun 2017 13:11:32 +0800 Subject: [PATCH 136/834] create inception-v4 (#6847) * Create symbol_inception-v4.py * Update and rename symbol_inception-v4.py to inception-v4.py * Update inception-v4.py * Update inception-v4.py --- .../symbols/inception-v4.py | 191 ++++++++++++++++++ 1 file changed, 191 insertions(+) create mode 100644 example/image-classification/symbols/inception-v4.py diff --git a/example/image-classification/symbols/inception-v4.py b/example/image-classification/symbols/inception-v4.py new file mode 100644 index 000000000000..be81e30ccd31 --- /dev/null +++ b/example/image-classification/symbols/inception-v4.py @@ -0,0 +1,191 @@ +# -*- coding:utf-8 -*- +__author__ = 'zhangshuai' +modified_date = '16/7/5' +__modify__ = 'anchengwu' +modified_date = '17/2/22' + +''' +Inception v4 , suittable for image with around 299 x 299 + +Reference: + Inception-v4, Inception-ResNet and the Impact of Residual Connections on Learning + Christian Szegedy, Sergey Ioffe, Vincent Vanhoucke + arXiv.1602.07261 +''' +import find_mxnet +import mxnet as mx + +def Conv(data, num_filter, kernel=(1, 1), stride=(1, 1), pad=(0, 0), name=None, suffix=''): + conv = mx.symbol.Convolution(data=data, num_filter=num_filter, kernel=kernel, stride=stride, pad=pad, no_bias=True, name='%s%s_conv2d' %(name, suffix)) + bn = mx.symbol.BatchNorm(data=conv, name='%s%s_batchnorm' %(name, suffix), fix_gamma=True) + act = mx.symbol.Activation(data=bn, act_type='relu', name='%s%s_relu' %(name, suffix)) + + return act + + +def Inception_stem(data, name= None): + c = Conv(data, 32, kernel=(3, 3), stride=(2, 2), name='%s_conv1_3*3' %name) + c = Conv(c, 32, kernel=(3, 3), name='%s_conv2_3*3' %name) + c = Conv(c, 64, kernel=(3, 3), pad=(1, 1), name='%s_conv3_3*3' %name) + + p1 = mx.symbol.Pooling(c, kernel=(3, 3), stride=(2, 2), pool_type='max', name='%s_maxpool_1' %name) + c2 = Conv(c, 96, kernel=(3, 3), stride=(2, 2), name='%s_conv4_3*3' %name) + concat = mx.symbol.Concat(*[p1, c2], name='%s_concat_1' %name) + + c1 = Conv(concat, 64, kernel=(1, 1), pad=(0, 0), name='%s_conv5_1*1' %name) + c1 = Conv(c1, 96, kernel=(3, 3), name='%s_conv6_3*3' %name) + + c2 = Conv(concat, 64, kernel=(1, 1), pad=(0, 0), name='%s_conv7_1*1' %name) + c2 = Conv(c2, 64, kernel=(7, 1), pad=(3, 0), name='%s_conv8_7*1' %name) + c2 = Conv(c2, 64, kernel=(1, 7), pad=(0, 3), name='%s_conv9_1*7' %name) + c2 = Conv(c2, 96, kernel=(3, 3), pad=(0, 0), name='%s_conv10_3*3' %name) + + concat = mx.symbol.Concat(*[c1, c2], name='%s_concat_2' %name) + + c1 = Conv(concat, 192, kernel=(3, 3), stride=(2, 2), name='%s_conv11_3*3' %name) + p1 = mx.symbol.Pooling(concat, kernel=(3, 3), stride=(2, 2), pool_type='max', name='%s_maxpool_2' %name) + + concat = mx.symbol.Concat(*[c1, p1], name='%s_concat_3' %name) + + return concat + + +def InceptionA(input, name=None): + p1 = mx.symbol.Pooling(input, kernel=(3, 3), pad=(1, 1), pool_type='avg', name='%s_avgpool_1' %name) + c1 = Conv(p1, 96, kernel=(1, 1), pad=(0, 0), name='%s_conv1_1*1' %name) + + c2 = Conv(input, 96, kernel=(1, 1), pad=(0, 0), name='%s_conv2_1*1' %name) + + c3 = Conv(input, 64, kernel=(1, 1), pad=(0, 0), name='%s_conv3_1*1' %name) + c3 = Conv(c3, 96, kernel=(3, 3), pad=(1, 1), name='%s_conv4_3*3' %name) + + c4 = Conv(input, 64, kernel=(1, 1), pad=(0, 0), name='%s_conv5_1*1' % name) + c4 = Conv(c4, 96, kernel=(3, 3), pad=(1, 1), name='%s_conv6_3*3' % name) + c4 = Conv(c4, 96, kernel=(3, 3), pad=(1, 1), name='%s_conv7_3*3' %name) + + concat = mx.symbol.Concat(*[c1, c2, c3, c4], name='%s_concat_1' %name) + + return concat + + +def ReductionA(input, name=None): + p1 = mx.symbol.Pooling(input, kernel=(3, 3), stride=(2, 2), pool_type='max', name='%s_maxpool_1' %name) + + c2 = Conv(input, 384, kernel=(3, 3), stride=(2, 2), name='%s_conv1_3*3' %name) + + c3 = Conv(input, 192, kernel=(1, 1), pad=(0, 0), name='%s_conv2_1*1' %name) + c3 = Conv(c3, 224, kernel=(3, 3), pad=(1, 1), name='%s_conv3_3*3' %name) + c3 = Conv(c3, 256, kernel=(3, 3), stride=(2, 2), pad=(0, 0), name='%s_conv4_3*3' %name) + + concat = mx.symbol.Concat(*[p1, c2, c3], name='%s_concat_1' %name) + + return concat + +def InceptionB(input, name=None): + p1 = mx.symbol.Pooling(input, kernel=(3, 3), pad=(1, 1), pool_type='avg', name='%s_avgpool_1' %name) + c1 = Conv(p1, 128, kernel=(1, 1), pad=(0, 0), name='%s_conv1_1*1' %name) + + c2 = Conv(input, 384, kernel=(1, 1), pad=(0, 0), name='%s_conv2_1*1' %name) + + c3 = Conv(input, 192, kernel=(1, 1), pad=(0, 0), name='%s_conv3_1*1' %name) + c3 = Conv(c3, 224, kernel=(1, 7), pad=(0, 3), name='%s_conv4_1*7' %name) + #paper wrong + c3 = Conv(c3, 256, kernel=(7, 1), pad=(3, 0), name='%s_conv5_1*7' %name) + + c4 = Conv(input, 192, kernel=(1, 1), pad=(0, 0), name='%s_conv6_1*1' %name) + c4 = Conv(c4, 192, kernel=(1, 7), pad=(0, 3), name='%s_conv7_1*7' %name) + c4 = Conv(c4, 224, kernel=(7, 1), pad=(3, 0), name='%s_conv8_7*1' %name) + c4 = Conv(c4, 224, kernel=(1, 7), pad=(0, 3), name='%s_conv9_1*7' %name) + c4 = Conv(c4, 256, kernel=(7, 1), pad=(3, 0), name='%s_conv10_7*1' %name) + + concat = mx.sym.Concat(*[c1, c2, c3, c4], name='%s_concat_1' %name) + + return concat + +def ReductionB(input,name=None): + p1 = mx.symbol.Pooling(input, kernel=(3, 3), stride=(2, 2), pool_type='max', name='%s_maxpool_1' %name) + + c2 = Conv(input, 192, kernel=(1, 1), pad=(0, 0), name='%s_conv1_1*1' %name) + c2 = Conv(c2, 192, kernel=(3, 3), stride=(2, 2), name='%s_conv2_3*3' %name) + + c3 = Conv(input, 256, kernel=(1, 1), pad=(0, 0), name='%s_conv3_1*1' %name) + c3 = Conv(c3, 256, kernel=(1, 7), pad=(0, 3), name='%s_conv4_1*7' %name) + c3 = Conv(c3, 320, kernel=(7, 1), pad=(3, 0), name='%s_conv5_7*1' %name) + c3 = Conv(c3, 320, kernel=(3, 3), stride=(2, 2), name='%s_conv6_3*3' %name) + + concat = mx.symbol.Concat(*[p1, c2, c3], name='%s_concat_1' %name) + + return concat + + +def InceptionC(input, name=None): + p1 = mx.symbol.Pooling(input, kernel=(3, 3), pad=(1, 1), pool_type='avg', name='%s_avgpool_1' %name) + c1 = Conv(p1, 256, kernel=(1, 1), pad=(0, 0), name='%s_conv1_1*1' %name) + + c2 = Conv(input, 256, kernel=(1, 1), pad=(0, 0), name='%s_conv2_1*1' %name) + + c3 = Conv(input, 384, kernel=(1, 1), pad=(0, 0), name='%s_conv3_1*1' %name) + c3_1 = Conv(c3, 256, kernel=(1, 3), pad=(0, 1), name='%s_conv4_3*1' %name) + c3_2 = Conv(c3, 256, kernel=(3, 1), pad=(1, 0), name='%s_conv5_1*3' %name) + + c4 = Conv(input, 384, kernel=(1, 1), pad=(0, 0), name='%s_conv6_1*1' %name) + c4 = Conv(c4, 448, kernel=(1, 3), pad=(0, 1), name='%s_conv7_1*3' %name) + c4 = Conv(c4, 512, kernel=(3, 1), pad=(1, 0), name='%s_conv8_3*1' %name) + c4_1 = Conv(c4, 256, kernel=(3, 1), pad=(1, 0), name='%s_conv9_1*3' %name) + c4_2 = Conv(c4, 256, kernel=(1, 3), pad=(0, 1), name='%s_conv10_3*1' %name) + + concat = mx.symbol.Concat(*[c1, c2, c3_1, c3_2, c4_1, c4_2], name='%s_concat' %name) + + return concat + + +def get_symbol(num_classes=1000, **kwargs): + data = mx.symbol.Variable(name="data") + x = Inception_stem(data, name='in_stem') + + #4 * InceptionA + # x = InceptionA(x, name='in1A') + # x = InceptionA(x, name='in2A') + # x = InceptionA(x, name='in3A') + # x = InceptionA(x, name='in4A') + + for i in range(4): + x = InceptionA(x, name='in%dA' %(i+1)) + + #Reduction A + x = ReductionA(x, name='re1A') + + #7 * InceptionB + # x = InceptionB(x, name='in1B') + # x = InceptionB(x, name='in2B') + # x = InceptionB(x, name='in3B') + # x = InceptionB(x, name='in4B') + # x = InceptionB(x, name='in5B') + # x = InceptionB(x, name='in6B') + # x = InceptionB(x, name='in7B') + + for i in range(7): + x = InceptionB(x, name='in%dB' %(i+1)) + + #ReductionB + x = ReductionB(x, name='re1B') + + #3 * InceptionC + # x = InceptionC(x, name='in1C') + # x = InceptionC(x, name='in2C') + # x = InceptionC(x, name='in3C') + + for i in range(3): + x = InceptionC(x, name='in%dC' %(i+1)) + + #Average Pooling + x = mx.symbol.Pooling(x, kernel=(8, 8), pad=(1, 1), pool_type='avg', name='global_avgpool') + + #Dropout + x = mx.symbol.Dropout(x, p=0.2) + + flatten = mx.symbol.Flatten(x, name='flatten') + fc1 = mx.symbol.FullyConnected(flatten, num_hidden=num_classes, name='fc1') + softmax = mx.symbol.SoftmaxOutput(fc1, name='softmax') + + return softmax From d79488a17c94318b2bdacefe42d93cca514a3d92 Mon Sep 17 00:00:00 2001 From: lordofgod <1290401724@qq.com> Date: Fri, 30 Jun 2017 13:12:11 +0800 Subject: [PATCH 137/834] Update deconvolution-inl.h. Change the condition control statement in InferPad. Merge differences between v0.8 and v0.9. (#6019) * Update deconvolution-inl.h. Change the condition control statement in InferPad. Merge differences between v0.8 and v0.9. There seems to be a bug at the condition control statement in InferPad. Original version(v0.9) only use target_shape.ndim to decide the calculation of pad. But we find even the target_shape.ndim is not zero, values of target_shape can be zero. When we load v0.8 model with v0.9, this may cause error calculation of pad. * Try to fix lint * Update deconvolution-inl.h --- src/operator/deconvolution-inl.h | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/operator/deconvolution-inl.h b/src/operator/deconvolution-inl.h index 4edeb6979222..b15777c69517 100644 --- a/src/operator/deconvolution-inl.h +++ b/src/operator/deconvolution-inl.h @@ -92,17 +92,23 @@ struct DeconvolutionParam : public dmlc::Parameter { template void InferPad(TShape input, index_t (&o_pad)[ndim], index_t (&o_adj)[ndim] ) const { + // Modified by Li.bs + // Use tag to control the calculation of pad + bool bCal = false; if (target_shape.ndim() != 0) { + for (int i = 0; i < target_shape.ndim(); i++) { + if (target_shape[i] != 0) bCal = true; + } + } + + if (bCal) { size_t input_ndim = input.ndim(); for (unsigned int i = 0; i < ndim; i++) { // input.ndim() can be larger than ndim, in case that the complete input // shape was passed and not only the ndim last ones o_pad[i] = stride[i] * (input[(input_ndim - ndim) + i] - 1) + DilatedKernelSize(i); - - CHECK_GE(o_pad[i], target_shape[i]) - << "too big target shape"; - + CHECK_GE(o_pad[i], target_shape[i]) << "too big target shape"; o_pad[i] -= target_shape[i]; o_adj[i] = o_pad[i] % 2; o_pad[i] = (o_pad[i] + 1) / 2; From 9f1df688c5bd7fe984ba03afab31c7bface41ac2 Mon Sep 17 00:00:00 2001 From: Viacheslav Kovalevskyi Date: Fri, 30 Jun 2017 20:48:20 -0700 Subject: [PATCH 138/834] Fix 404 links on rnn page (#6900) * Fix for the broken links in the rnn example. * Fix for the broken links in the rnn example. --- docs/tutorials/nlp/rnn.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/docs/tutorials/nlp/rnn.md b/docs/tutorials/nlp/rnn.md index 0382b2cc23c5..e2d2265ecedf 100644 --- a/docs/tutorials/nlp/rnn.md +++ b/docs/tutorials/nlp/rnn.md @@ -3,12 +3,8 @@ This folder contains RNN examples using a low-level symbol interface. You can ge ## Python -- [lstm.py](lstm.py). Functions for building an LSTM Network -- [gru.py](gru.py). Functions for building a GRU Network -- [lstm_bucketing.py](lstm_bucketing.py). A PennTreeBank language model using LSTM -- [gru_bucketing.py](gru_bucketing.py). A PennTreeBank language model using GRU -- [char-rnn.ipynb](char-rnn.ipynb). A notebook that demonstrates how to train a character LSTM by using ```lstm.py``` - +- [https://github.com/dmlc/mxnet/blob/master/example/rnn/lstm_bucketing.py](lstm_bucketing.py). A PennTreeBank language model using LSTM +- [https://github.com/dmlc/mxnet/blob/master/example/rnn/cudnn_lstm_bucketing.py](cudnn_lstm_bucketing.py). A PennTreeBank language model using LSTM and CUDNN Performance Note: From 04617df09e49fecea850f269fdcc23fea81d7c64 Mon Sep 17 00:00:00 2001 From: Viacheslav Kovalevskyi Date: Fri, 30 Jun 2017 20:49:39 -0700 Subject: [PATCH 139/834] Link to install XCode from the App Store has been added. (#6901) --- docs/get_started/install.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/get_started/install.md b/docs/get_started/install.md index b9420bc0d926..898aa0899a1d 100644 --- a/docs/get_started/install.md +++ b/docs/get_started/install.md @@ -498,7 +498,7 @@ The following installation instructions have been tested on OSX Sierra and El Ca **Prerequisites** -If not already installed, [download and install Xcode](https://developer.apple.com/xcode/) for macOS. [Xcode](https://en.wikipedia.org/wiki/Xcode) is an integrated development environment for macOS containing a suite of software development tools like C/C++ compilers, BLAS library and more. +If not already installed, [download and install Xcode](https://developer.apple.com/xcode/) (or [insall it from the App Store](https://itunes.apple.com/us/app/xcode/id497799835)) for macOS. [Xcode](https://en.wikipedia.org/wiki/Xcode) is an integrated development environment for macOS containing a suite of software development tools like C/C++ compilers, BLAS library and more.

    From d79e0e0c1a706daa8dcafdb617e02a76ce7c5288 Mon Sep 17 00:00:00 2001 From: Eric Junyuan Xie Date: Fri, 30 Jun 2017 20:50:11 -0700 Subject: [PATCH 140/834] Add inplace identity (#6896) * add inplace identity * fix --- nnvm | 2 +- src/operator/tensor/elemwise_unary_op.cc | 34 +++++++++++++++++++++++- src/operator/tensor/matrix_op.cc | 18 ++++++++++--- 3 files changed, 49 insertions(+), 5 deletions(-) diff --git a/nnvm b/nnvm index d73d6c5b37e7..217d3d5adefe 160000 --- a/nnvm +++ b/nnvm @@ -1 +1 @@ -Subproject commit d73d6c5b37e7376c1eb30a1ae5c7a42b1fbe22f5 +Subproject commit 217d3d5adefe9b2bd9e3e3fe4fa0695f3a47c93f diff --git a/src/operator/tensor/elemwise_unary_op.cc b/src/operator/tensor/elemwise_unary_op.cc index 4af087749c2b..16591c9dd9d3 100644 --- a/src/operator/tensor/elemwise_unary_op.cc +++ b/src/operator/tensor/elemwise_unary_op.cc @@ -48,6 +48,10 @@ MXNET_OPERATOR_REGISTER_BINARY(_backward_sigmoid) MXNET_OPERATOR_REGISTER_UNARY(_copy) .MXNET_DESCRIBE("Returns a copy of the input.") .add_alias("identity") +.set_attr("FInplaceIdentity", + [](const NodeAttrs& attrs){ + return std::vector{true}; + }) .set_attr("FCompute", IdentityCompute) .set_attr("FGradient", ElemwiseGradUseNone{"_copy"}); @@ -59,6 +63,10 @@ NNVM_REGISTER_OP(_backward_copy) [](const NodeAttrs& attrs){ return std::vector >{{0, 0}}; }) +.set_attr("FInplaceIdentity", + [](const NodeAttrs& attrs){ + return std::vector{true}; + }) .set_attr("FCompute", IdentityCompute); MXNET_OPERATOR_REGISTER_UNARY(BlockGrad) @@ -89,6 +97,10 @@ Example:: [ 1. 1.] )code" ADD_FILELINE) +.set_attr("FInplaceIdentity", + [](const NodeAttrs& attrs){ + return std::vector{true}; + }) .set_attr("FCompute", IdentityCompute) .set_attr("FGradient", MakeZeroGradNodes); @@ -100,6 +112,10 @@ MXNET_OPERATOR_REGISTER_UNARY(make_loss) [](const NodeAttrs& attrs) { return std::vector{"loss"}; }) +.set_attr("FInplaceIdentity", + [](const NodeAttrs& attrs){ + return std::vector{true}; + }) .set_attr("FCompute", IdentityCompute) .set_attr("FGradient", [](const nnvm::NodePtr& n, const std::vector& ograds) { @@ -121,6 +137,10 @@ NNVM_REGISTER_OP(_identity_with_attr_like_rhs) "FInplaceOption", [](const NodeAttrs& attrs) { return std::vector >{{0, 0}}; }) +.set_attr("FInplaceIdentity", + [](const NodeAttrs& attrs){ + return std::vector{true}; + }) .set_attr("FIgnoreInputs", [](const NodeAttrs& attrs) { return std::vector(1, 1); }) .set_attr("FCompute", IdentityCompute) @@ -160,6 +180,10 @@ Example:: [](const NodeAttrs& attrs){ return std::vector >{{0, 0}}; }) +.set_attr("FInplaceIdentity", + [](const NodeAttrs& attrs){ + return std::vector{true}; + }) .set_attr("FCompute", CastCompute) .set_attr("FGradient", ElemwiseGradUseNone{"_backward_cast"}) .add_argument("data", "NDArray-or-Symbol", "The input.") @@ -167,6 +191,14 @@ Example:: NNVM_REGISTER_OP(_backward_cast) .set_attr("TIsBackward", true) +.set_attr("FInplaceOption", + [](const NodeAttrs& attrs){ + return std::vector >{{0, 0}}; + }) +.set_attr("FInplaceIdentity", + [](const NodeAttrs& attrs){ + return std::vector{true}; + }) .set_attr("FCompute", CastCompute); // negative @@ -262,7 +294,7 @@ Example:: MXNET_OPERATOR_REGISTER_UNARY(trunc) .describe(R"code(Return the element-wise truncated value of the input. -The truncated value of the scalar x is the nearest integer i which is closer to +The truncated value of the scalar x is the nearest integer i which is closer to zero than x is. In short, the fractional part of the signed number x is discarded. Example:: diff --git a/src/operator/tensor/matrix_op.cc b/src/operator/tensor/matrix_op.cc index f3d69733a814..6a51d46db25c 100644 --- a/src/operator/tensor/matrix_op.cc +++ b/src/operator/tensor/matrix_op.cc @@ -95,7 +95,11 @@ If the argument `reverse` is set to 1, then the special values are inferred from .set_attr("FInplaceOption", [](const NodeAttrs& attrs) { return std::vector >{{0, 0}}; -}) + }) +.set_attr("FInplaceIdentity", + [](const NodeAttrs& attrs){ + return std::vector{true}; + }) .add_argument("data", "NDArray-or-Symbol", "Input data to reshape.") .add_arguments(ReshapeParam::__FIELDS__()); @@ -133,8 +137,12 @@ Example:: .set_attr("FCompute", IdentityCompute) .set_attr("FInplaceOption", [](const NodeAttrs& attrs) { - return std::vector >{{0, 0}}; -}) + return std::vector >{{0, 0}}; + }) +.set_attr("FInplaceIdentity", + [](const NodeAttrs& attrs){ + return std::vector{true}; + }) .add_argument("data", "NDArray-or-Symbol", "Input array."); NNVM_REGISTER_OP(transpose) @@ -211,6 +219,10 @@ will return a new array with shape ``(2,1,3,4)``. [](const NodeAttrs& attrs){ return std::vector >{{0, 0}}; }) +.set_attr("FInplaceIdentity", + [](const NodeAttrs& attrs){ + return std::vector{true}; + }) .set_attr("FGradient", ElemwiseGradUseNone{"_backward_copy"}) .set_attr("FCompute", IdentityCompute) .add_argument("data", "NDArray-or-Symbol", "Source input") From bdcb45f60966744c4e834ad5debaa2a5c8b20ee7 Mon Sep 17 00:00:00 2001 From: Yao Wang Date: Fri, 30 Jun 2017 21:04:23 -0700 Subject: [PATCH 141/834] Change mxnet.io home page and api subtitle color (#6899) * Change mxnet.io home page and api subtitle color * Update index.html --- docs/_static/mxnet-theme/index.html | 68 ++++++++++-------- docs/_static/mxnet.css | 107 +++++++++++++++++++++++++--- 2 files changed, 136 insertions(+), 39 deletions(-) diff --git a/docs/_static/mxnet-theme/index.html b/docs/_static/mxnet-theme/index.html index a6dafbca25fa..76a6e1381fca 100644 --- a/docs/_static/mxnet-theme/index.html +++ b/docs/_static/mxnet-theme/index.html @@ -9,13 +9,6 @@ - - -
    @@ -24,35 +17,48 @@
    -
    -

    Flexible

    -

    Supports both imperative and symbolic programming

    -
    -
    -

    Portable

    -

    Runs on CPUs or GPUs, servers, desktops, or mobile phones

    -
    -
    -

    Multiple Languages

    -

    Supports C++, Python, R, Scala, Julia, Perl, Matlab and Javascript - All with the same amazing performance

    -
    -
    -

    Auto-Differentiation

    -

    Calculates the gradients automatically for training a model

    -
    -
    -

    Distributed on Cloud

    -

    Supports distributed training on multiple CPU/GPU machines, including AWS, - GCE, Azure, and Yarn clusters

    -
    -
    -

    Performance

    -

    Optimized C++ backend engine parallelizes both I/O and computation

    +
    +

    MXNet 0.10.0 Released

    +

    We're excited to announce the release of MXNet 0.10.0! Check out the release notes for latest updates.

    + Learn More +
    +
    +

    MXNet Joining Apache

    +

    We’re excited to announce that MXNet has been accepted to the Apache Incubator. +

    + Learn More +
    +
    +

    MXNet in AWS re:Invent 2016

    +

    Learn how to use MXNet to build neural network models for recommendation systems. +

    + Watch Video
    +
    +
    +
    +
    +

    Examples

    +

    Explore projects from simple demos to state-of-the-art research

    +
    + MXNet Examples +
    +
    +
    +

    Model Zoo

    +

    Off the shelf pre-trained models

    +
    + Model Zoo +
    +
    +
    +
    +
    +
    diff --git a/docs/_static/mxnet.css b/docs/_static/mxnet.css index b0eff89003d6..b5228bbf5aec 100644 --- a/docs/_static/mxnet.css +++ b/docs/_static/mxnet.css @@ -501,7 +501,6 @@ li.dropdown-submenu ul.dropdown-menu a { #why_mxnet_btn, #install_btn, #release_btn { border: 1.8px solid #FFFFFF; - border-radius: 2px; color: #FFFFFF; display: inline-block; font-size: 18px; @@ -565,27 +564,115 @@ li.dropdown-submenu ul.dropdown-menu a { .section-tout { padding:3em 0 3em; border-bottom:1px solid rgba(0,0,0,.05); - background-color:#eaf1f1 + background-color:#fff +} + +.section-tout .container { + height: 200px; +} + +.section-tout .row { + height: 100%; } .section-tout .row div { - height: 140px; + height: 100%; + padding-left: 50px; } -.section-tout .fa{ - margin-right:.5em +.section-tout .row a { + position: absolute; + bottom: 20px; } .section-tout h3{ font-size:20px; - color: #0079b2; + color: #444; } .section-tout p { - margin-bottom:2em + margin-bottom:2em; +} + +@media (max-width: 1199px) { + .section-tout .container { + height: auto; + } + + .section-tout .row a { + position: inherit; + } + + .section-tout .row div { + margin-bottom: 20px; + padding-left: 20px; + } +} + +.section-util { + background-color: #eaf1f1; + padding:3em 0 3em; + border-bottom:1px solid rgba(0,0,0,.05); + text-align: center; +} + +.section-util p { + color: #999; + position: absolute; + width: 50%; + margin: auto; + left: 0; + right: 0; +} + +.section-util .util-btn { + position: absolute; + margin: auto; + left: 0; + right: 0; + padding-top: 10px; + margin-top: 100px; +} + +.util-btn a { + display: inline-block; + border: 1.8px solid #0079b2; + border-radius: 30px; + width: 200px; + height: 50px; + -webkit-transition: .2s; + transition: .2s; + padding: 10px 30px; } -.section-inst{ +.util-btn a:hover { + background-color: #0079b2; + color: #FFFFFF; + opacity: 0.9; + text-decoration: none; +} + +.section-util .container { + height: 230px; +} + +@media (max-width: 1199px) { + .section-util .container { + height: auto + } + + .section-util .row div { + margin-bottom: 200px; + } +} + +@media (max-width: 767px) { + .section-util .row div { + margin-bottom: 250px; + } +} + +.section-inst { padding:3em 0 3em; border-bottom:1px solid rgba(0,0,0,.05); @@ -878,6 +965,10 @@ code { background-color: #f5f5f5; } +dt code { + color: #555; +} + dl.last.docutils dt{ background-color: transparent; border-bottom: none; From b760eb18e6b7b9f9a7f04b615d33ef9851226a11 Mon Sep 17 00:00:00 2001 From: Jongju Shin Date: Sat, 1 Jul 2017 13:30:35 +0900 Subject: [PATCH 142/834] add allow_extra parameter to module (#6903) * add allow_extra parameter to module * pass allow_extra to _curr_module --- example/rcnn/rcnn/core/module.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/example/rcnn/rcnn/core/module.py b/example/rcnn/rcnn/core/module.py index c823cb926e8d..bf28f8ee56f1 100644 --- a/example/rcnn/rcnn/core/module.py +++ b/example/rcnn/rcnn/core/module.py @@ -80,13 +80,13 @@ def get_params(self): return self._curr_module.get_params() def init_params(self, initializer=Uniform(0.01), arg_params=None, aux_params=None, - allow_missing=False, force_init=False): + allow_missing=False, force_init=False, allow_extra=False): if self.params_initialized and not force_init: return assert self.binded, 'call bind before initializing the parameters' self._curr_module.init_params(initializer=initializer, arg_params=arg_params, aux_params=aux_params, allow_missing=allow_missing, - force_init=force_init) + force_init=force_init, allow_extra=allow_extra) self.params_initialized = True def bind(self, data_shapes, label_shapes=None, for_training=True, From c8289d1a53579bd186b5a8180c13650abec1628f Mon Sep 17 00:00:00 2001 From: Sebastian Bodenstein Date: Sat, 1 Jul 2017 20:57:48 +0200 Subject: [PATCH 143/834] added docs for output of ctc loss layer (#6906) --- src/operator/contrib/ctc_loss.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/operator/contrib/ctc_loss.cc b/src/operator/contrib/ctc_loss.cc index 13d280044dee..918af0e27bbc 100644 --- a/src/operator/contrib/ctc_loss.cc +++ b/src/operator/contrib/ctc_loss.cc @@ -72,6 +72,8 @@ applies a softmax to each vector, which then becomes a vector of probabilities over the alphabet. Note that the 0th element of this vector is reserved for the special blank character. +``out`` is a list of CTC loss values, one per example in the batch. + See *Connectionist Temporal Classification: Labelling Unsegmented Sequence Data with Recurrent Neural Networks*, A. Graves *et al*. for more information. From 2e9943b20f772acfe87ed38f22731c892deb0035 Mon Sep 17 00:00:00 2001 From: "Qiang Kou (KK)" Date: Mon, 3 Jul 2017 08:33:24 -0700 Subject: [PATCH 144/834] [R] fix the operations between MXSymbol and scalar. close #4994 (#6758) --- R-package/R/symbol.R | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/R-package/R/symbol.R b/R-package/R/symbol.R index f222c55ff74e..541cce456f9f 100644 --- a/R-package/R/symbol.R +++ b/R-package/R/symbol.R @@ -176,34 +176,61 @@ init.symbol.methods <- function() { setMethod("+", signature(e1 = "Rcpp_MXSymbol", e2 = "numeric"), function(e1, e2) { mx.varg.symbol.internal.PlusScalar(list(e1, scalar = e2)) }) + setMethod("+", signature(e1 = "numeric", e2 = "Rcpp_MXSymbol"), function(e1, e2) { + mx.varg.symbol.internal.PlusScalar(list(e2, scalar = e1)) + }) setMethod("-", signature(e1 = "Rcpp_MXSymbol", e2 = "Rcpp_MXSymbol"), function(e1, e2) { mx.varg.symbol.internal.Minus(list(e1, e2)) }) setMethod("-", signature(e1 = "Rcpp_MXSymbol", e2 = "numeric"), function(e1, e2) { mx.varg.symbol.internal.MinusScalar(list(e1, scalar = e2)) }) + setMethod("-", signature(e1 = "numeric", e2 = "Rcpp_MXSymbol"), function(e1, e2) { + mx.varg.symbol.internal.rminus_scalar(list(e2, scalar = e1)) + }) setMethod("*", signature(e1 = "Rcpp_MXSymbol", e2 = "Rcpp_MXSymbol"), function(e1, e2) { mx.varg.symbol.internal.Mul(list(e1, e2)) }) setMethod("*", signature(e1 = "Rcpp_MXSymbol", e2 = "numeric"), function(e1, e2) { mx.varg.symbol.internal.MulScalar(list(e1, scalar = e2)) }) + setMethod("*", signature(e1 = "numeric", e2 = "Rcpp_MXSymbol"), function(e1, e2) { + mx.varg.symbol.internal.MulScalar(list(e2, scalar = e1)) + }) setMethod("/", signature(e1 = "Rcpp_MXSymbol", e2 = "Rcpp_MXSymbol"), function(e1, e2) { mx.varg.symbol.internal.Div(list(e1, e2)) }) setMethod("/", signature(e1 = "Rcpp_MXSymbol", e2 = "numeric"), function(e1, e2) { mx.varg.symbol.internal.DivScalar(list(e1, scalar = e2)) }) + setMethod("/", signature(e1 = "numeric", e2 = "Rcpp_MXSymbol"), function(e1, e2) { + mx.varg.symbol.internal.rdiv_scalar(list(e2, scalar = e1)) + }) setMethod("%%", signature(e1 = "Rcpp_MXSymbol", e2 = "Rcpp_MXSymbol"), function(e1, e2) { mx.varg.symbol.internal.Mod(list(e1, e2)) }) setMethod("%%", signature(e1 = "Rcpp_MXSymbol", e2 = "numeric"), function(e1, e2) { mx.varg.symbol.internal.ModScalar(list(e1, scalar = e2)) }) + setMethod("%%", signature(e1 = "numeric", e2 = "Rcpp_MXSymbol"), function(e1, e2) { + mx.varg.symbol.internal.RModScalar(list(e2, scalar = e1)) + }) setMethod("%/%", signature(e1 = "Rcpp_MXSymbol", e2 = "Rcpp_MXSymbol"), function(e1, e2) { mx.varg.symbol.internal.Mod(list(e1, e2)) }) setMethod("%/%", signature(e1 = "Rcpp_MXSymbol", e2 = "numeric"), function(e1, e2) { mx.varg.symbol.internal.ModScalar(list(e1, scalar = e2)) }) + setMethod("%/%", signature(e1 = "numeric", e2 = "Rcpp_MXSymbol"), function(e1, e2) { + mx.varg.symbol.internal.RModScalar(list(e2, scalar = e1)) + }) + setMethod("^", signature(e1 = "Rcpp_MXSymbol", e2 = "Rcpp_MXSymbol"), function(e1, e2) { + mx.varg.symbol.internal.power(list(e1, e2)) + }) + setMethod("^", signature(e1 = "Rcpp_MXSymbol", e2 = "numeric"), function(e1, e2) { + mx.varg.symbol.internal.power_scalar(list(e1, scalar = e2)) + }) + setMethod("^", signature(e1 = "numeric", e2 = "Rcpp_MXSymbol"), function(e1, e2) { + mx.varg.symbol.internal.rpower_scalar(list(e2, scalar = e1)) + }) } From 00b0f95211fe281b0a38ed774fa9c1551dc3dc9f Mon Sep 17 00:00:00 2001 From: Haozhi Qi Date: Tue, 4 Jul 2017 01:45:48 +0800 Subject: [PATCH 145/834] Fix a bug in deformable convolution operator (#6911) * Fix a bug in deformable convolution * Fix indentation * Fix lint error --- src/operator/contrib/deformable_convolution-inl.h | 6 ++++-- tests/python/unittest/test_operator.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/operator/contrib/deformable_convolution-inl.h b/src/operator/contrib/deformable_convolution-inl.h index da979e707aee..d8397cfb715d 100644 --- a/src/operator/contrib/deformable_convolution-inl.h +++ b/src/operator/contrib/deformable_convolution-inl.h @@ -218,9 +218,11 @@ class DeformableConvolutionOp : public Operator { req[conv::kData]); // gradient w.r.t. weight, dWeight should accumulate across the batch and group - im2col(s, in_data[conv::kData].dptr() + n*input_dim_, in_data[conv::kData].shape_, + deformable_im2col(s, in_data[conv::kData].dptr() + n*input_dim_, + in_data[conv::kOffset].dptr() + n*input_offset_dim_, in_data[conv::kData].shape_, col_buffer.shape_, param_.kernel, param_.pad, param_.stride, param_.dilate, - col_buffer.dptr()); + param_.num_deformable_group, col_buffer.dptr()); + for (index_t g = 0; g < group_; ++g) { if (0 == n) { ASSIGN_DISPATCH(dweight_3d[g], req[conv::kWeight], diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py index 8129a41ee751..7eca5650786a 100644 --- a/tests/python/unittest/test_operator.py +++ b/tests/python/unittest/test_operator.py @@ -3248,7 +3248,7 @@ def test_deformable_convolution(): for num_channel_data, num_deformable_group in itertools.product([4, 8], [1, 2]): for input_height, input_width in itertools.product([5, 6], [5, 6]): for dilate in [(1, 1), (2, 2)]: - for grad_nodes in [['im_data'], ['offset_data']]: + for grad_nodes in [['im_data'], ['offset_data'], ['weight']]: output_height = input_height output_width = input_width im_data = np.random.rand(num_batch, num_channel_data, input_height, input_width) From 998378a64f890920150ff13087d7c1acfa372530 Mon Sep 17 00:00:00 2001 From: Kenta Kubo Date: Tue, 4 Jul 2017 02:46:40 +0900 Subject: [PATCH 146/834] Fix Python 3 compatibilities (#6908) --- tools/im2rec.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tools/im2rec.py b/tools/im2rec.py index 380ad1e43162..a05bfb04621b 100644 --- a/tools/im2rec.py +++ b/tools/im2rec.py @@ -12,6 +12,9 @@ import time import traceback +if sys.version_info[0] == 3: + xrange = range + try: import multiprocessing except ImportError: @@ -89,7 +92,7 @@ def read_list(path_in): continue try: item = [int(line[0])] + [line[-1]] + [float(i) for i in line[1:-1]] - except Exception, e: + except Exception as e: print('Parsing lst met error for %s, detail: %s' %(line, e)) continue yield item @@ -108,7 +111,7 @@ def image_encode(args, i, item, q_out): img = fin.read() s = mx.recordio.pack(header, img) q_out.put((i, s, item)) - except Exception, e: + except Exception as e: traceback.print_exc() print('pack_img error:', item[1], e) q_out.put((i, None, item)) @@ -142,7 +145,7 @@ def image_encode(args, i, item, q_out): try: s = mx.recordio.pack_img(header, img, quality=args.quality, img_fmt=args.encoding) q_out.put((i, s, item)) - except Exception, e: + except Exception as e: traceback.print_exc() print('pack_img error on file: %s' % fullpath, e) q_out.put((i, None, item)) @@ -279,8 +282,11 @@ def parse_args(): write_process.join() else: print('multiprocessing not available, fall back to single threaded encoding') - import Queue - q_out = Queue.Queue() + try: + import Queue as queue + except ImportError: + import queue + q_out = queue.Queue() fname = os.path.basename(fname) fname_rec = os.path.splitext(fname)[0] + '.rec' fname_idx = os.path.splitext(fname)[0] + '.idx' From 5fb083212f2337963462295e782610df5e2345d4 Mon Sep 17 00:00:00 2001 From: Han Shen Date: Tue, 4 Jul 2017 02:37:28 +0800 Subject: [PATCH 147/834] fix label bug which harms voc accuracy (#6849) * fix label bug which harms voc accuracy * change padding rois sampling from neg rois only --- example/rcnn/rcnn/io/rcnn.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/example/rcnn/rcnn/io/rcnn.py b/example/rcnn/rcnn/io/rcnn.py index aad1a4617c0e..807447c61baa 100644 --- a/example/rcnn/rcnn/io/rcnn.py +++ b/example/rcnn/rcnn/io/rcnn.py @@ -146,12 +146,13 @@ def sample_rois(rois, fg_rois_per_image, rois_per_image, num_classes, # indexes selected keep_indexes = np.append(fg_indexes, bg_indexes) - + neg_idx = np.where(overlaps < config.TRAIN.FG_THRESH)[0] + neg_rois = rois[neg_idx] # pad more to ensure a fixed minibatch size while keep_indexes.shape[0] < rois_per_image: - gap = np.minimum(len(rois), rois_per_image - keep_indexes.shape[0]) - gap_indexes = npr.choice(range(len(rois)), size=gap, replace=False) - keep_indexes = np.append(keep_indexes, gap_indexes) + gap = np.minimum(len(neg_rois), rois_per_image - keep_indexes.shape[0]) + gap_indexes = npr.choice(range(len(neg_rois)), size=gap, replace=False) + keep_indexes = np.append(keep_indexes, neg_idx[gap_indexes]) # select labels labels = labels[keep_indexes] From e9cc791e8dd41089de2353aeda775d0a14da4759 Mon Sep 17 00:00:00 2001 From: "Qiang Kou (KK)" Date: Tue, 4 Jul 2017 10:55:35 -0700 Subject: [PATCH 148/834] [R] To use the latest R in the installation script for ubuntu. (#6897) --- docs/get_started/windows_setup.md | 6 +++--- setup-utils/install-mxnet-ubuntu-r.sh | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/get_started/windows_setup.md b/docs/get_started/windows_setup.md index 47e0c0814b13..86104c6be5f3 100755 --- a/docs/get_started/windows_setup.md +++ b/docs/get_started/windows_setup.md @@ -92,7 +92,7 @@ To install MXNet on a computer with a CPU processor, choose from two options: * Use the prebuilt binary package * Build the library from source code -#### Building MXNet with the Prebuilt Binary Package +#### Installing MXNet with the Prebuilt Binary Package For Windows users, MXNet provides prebuilt binary packages. You can install the package directly in the R console. @@ -119,12 +119,12 @@ For GPU-enabled package: Run the following commands to install the MXNet dependencies and build the MXNet R package. ```r - Rscript -e "install.packages('devtools', repo = 'https://cran.rstudio.com')" + Rscript -e "install.packages('devtools', repo = 'https://cloud.r-project.org/')" ``` ```bash cd R-package - Rscript -e "library(devtools); library(methods); options(repos=c(CRAN='https://cran.rstudio.com')); install_deps(dependencies = TRUE)" + Rscript -e "library(devtools); library(methods); options(repos=c(CRAN='https://cloud.r-project.org/')); install_deps(dependencies = TRUE)" cd .. make rpkg ``` diff --git a/setup-utils/install-mxnet-ubuntu-r.sh b/setup-utils/install-mxnet-ubuntu-r.sh index 5f1b04daef05..8f4c07d0325d 100644 --- a/setup-utils/install-mxnet-ubuntu-r.sh +++ b/setup-utils/install-mxnet-ubuntu-r.sh @@ -1,9 +1,6 @@ #!/usr/bin/env bash ###################################################################### # This script installs MXNet for R along with all required dependencies on a Ubuntu Machine. -# We recommend to install Microsoft RServer together with Intel MKL library for optimal performance -# More information can be found here: -# https://blogs.technet.microsoft.com/machinelearning/2016/09/15/building-deep-neural-networks-in-the-cloud-with-azure-gpu-vms-mxnet-and-microsoft-r-server/ # Tested on Ubuntu 14.04+ distro. ###################################################################### set -e @@ -22,6 +19,9 @@ is_rscript_installed=$(which Rscript | wc -l) if [ "$is_rscript_installed" = "0" ]; then read -p "Seems like Rscript is not installed. Install Rscript? [Y/n]" if [ x"$REPLY" = x"" -o x"$REPLY" = x"y" -o x"$REPLY" = x"Y" ]; then + sudo add-apt-repository -y "deb http://cran.rstudio.com/bin/linux/ubuntu `lsb_release -cs`/" + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E084DAB9 + sudo apt-get -qq update sudo apt-get install -y r-base-core fi fi From 2c7f1d85e0734fd0fa53638ae0b83069e35aea6a Mon Sep 17 00:00:00 2001 From: Liangfu Chen Date: Wed, 5 Jul 2017 03:20:09 +0800 Subject: [PATCH 149/834] convert to long int for direct comparison (#6869) * convert to long int for direct comparison ```self._data_shapes[0].shape``` is ```int``` based shape, and ```data_batch.data``` is ```long int``` base shape result. The change converts both shape variables into ```long int``` for direct comparison. * Update module.py --- python/mxnet/module/module.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/mxnet/module/module.py b/python/mxnet/module/module.py index 1b5ecbc3c301..75201292010c 100644 --- a/python/mxnet/module/module.py +++ b/python/mxnet/module/module.py @@ -560,8 +560,8 @@ def forward(self, data_batch, is_train=None): raise RuntimeError("If you are trying to do inference, rebind module " "with 'force_rebind=True' and 'for_training=False'") - curr_data_shapes = (i.shape for i in self._data_shapes) - new_data_shapes = (i.shape for i in data_batch.data) + curr_data_shapes = tuple(i.shape for i in self._data_shapes) + new_data_shapes = tuple(i.shape for i in data_batch.data) if curr_data_shapes != new_data_shapes: if hasattr(data_batch, "provide_data") and data_batch.provide_data: From a21cab7c0868675b47a16ddf76d75b8f64dcdd6f Mon Sep 17 00:00:00 2001 From: "Qiang Kou (KK)" Date: Tue, 4 Jul 2017 19:35:52 -0700 Subject: [PATCH 150/834] [R] add "fixed.param". close #3906 (#6922) --- R-package/R/executor.R | 6 ++-- R-package/R/model.R | 20 ++++++------ R-package/tests/testthat/test_model.R | 45 ++++++++++++++++++++++++++- 3 files changed, 57 insertions(+), 14 deletions(-) diff --git a/R-package/R/executor.R b/R-package/R/executor.R index d33aeefc32ae..571708268a7f 100644 --- a/R-package/R/executor.R +++ b/R-package/R/executor.R @@ -2,7 +2,7 @@ #' with information from input shapes. #' #' @export -mx.simple.bind <- function(symbol, ctx, grad.req = "null", ...) { +mx.simple.bind <- function(symbol, ctx, grad.req = "null", fixed.param = NULL, ...) { if (!is.MXSymbol(symbol)) stop("symbol need to be MXSymbol") slist <- symbol$infer.shape(list(...)) @@ -16,7 +16,9 @@ mx.simple.bind <- function(symbol, ctx, grad.req = "null", ...) { mx.nd.zeros(shape, ctx) }, simplify = FALSE, USE.NAMES = TRUE) grad.reqs <- lapply(names(slist$arg.shapes), function(nm) { - if (!endsWith(nm, "label") && !endsWith(nm, "data")) { + if (nm %in% fixed.param) { + "null" + } else if (!endsWith(nm, "label") && !endsWith(nm, "data")) { grad.req } else { "null" diff --git a/R-package/R/model.R b/R-package/R/model.R index 069c564599b5..0f6af3dd61cb 100644 --- a/R-package/R/model.R +++ b/R-package/R/model.R @@ -94,12 +94,9 @@ mx.model.create.kvstore <- function(kvstore, arg.params, ndevice, verbose=TRUE) mx.model.train <- function(symbol, ctx, input.shape, output.shape, arg.params, aux.params, begin.round, end.round, optimizer, - train.data, eval.data, - metric, - epoch.end.callback, - batch.end.callback, - kvstore, - verbose=TRUE) { + train.data, eval.data, metric, + epoch.end.callback, batch.end.callback, + kvstore, fixed.param = NULL, verbose = TRUE) { ndevice <- length(ctx) if(verbose) message(paste0("Start training with ", ndevice, " devices")) # create the executors @@ -109,8 +106,9 @@ mx.model.train <- function(symbol, ctx, input.shape, output.shape, label_name <- arg_names[endsWith(arg_names, "label")] train.execs <- lapply(1:ndevice, function(i) { arg_lst <- list(symbol = symbol, ctx = ctx[[i]], grad.req = "write", - data=sliceinfo[[i]]$shape) + data = sliceinfo[[i]]$shape) arg_lst[[label_name]] = sliceinfo2[[i]]$shape + arg_lst[["fixed.param"]] = fixed.param do.call(mx.simple.bind, arg_lst) }) # set the parameters into executors @@ -406,9 +404,8 @@ function(symbol, X, y=NULL, ctx=NULL, begin.round=1, eval.data=NULL, eval.metric=NULL, epoch.end.callback=NULL, batch.end.callback=NULL, array.batch.size=128, array.layout="auto", - kvstore="local", - verbose=TRUE, - arg.params=NULL, aux.params=NULL, + kvstore = "local", verbose = TRUE, + arg.params = NULL, aux.params = NULL, fixed.param = NULL, ...) { if (is.array(X) || is.matrix(X)) { if (array.layout == "auto") { @@ -463,7 +460,8 @@ function(symbol, X, y=NULL, ctx=NULL, begin.round=1, metric=eval.metric, epoch.end.callback=epoch.end.callback, batch.end.callback=batch.end.callback, - kvstore=kvstore, + kvstore=kvstore, + fixed.param = fixed.param, verbose=verbose) return (model) } diff --git a/R-package/tests/testthat/test_model.R b/R-package/tests/testthat/test_model.R index 9c85afac956a..9b804e16537b 100644 --- a/R-package/tests/testthat/test_model.R +++ b/R-package/tests/testthat/test_model.R @@ -4,7 +4,7 @@ source("get_data.R") context("models") -test_that("basic symbol operation", { +test_that("MNIST", { # # Network configuration GetMNIST_ubyte() batch.size <- 100 @@ -65,3 +65,46 @@ test_that("basic symbol operation", { file.remove("chkpt-0001.params") file.remove("chkpt-symbol.json") }) + +test_that("Regression", { + data(BostonHousing, package = "mlbench") + train.ind <- seq(1, 506, 3) + train.x <- data.matrix(BostonHousing[train.ind,-14]) + train.y <- BostonHousing[train.ind, 14] + test.x <- data.matrix(BostonHousing[-train.ind,-14]) + test.y <- BostonHousing[-train.ind, 14] + data <- mx.symbol.Variable("data") + fc1 <- mx.symbol.FullyConnected(data, num_hidden = 1) + lro <- mx.symbol.LinearRegressionOutput(fc1) + + demo.metric.mae <- mx.metric.custom("mae", function(label, pred) { + res <- mean(abs(label - pred)) + return(res) + }) + mx.set.seed(0) + model <- mx.model.FeedForward.create(lro, X = train.x, y = train.y, + ctx = mx.cpu(), num.round = 50, + array.batch.size = 20, + learning.rate = 2e-6, + momentum = 0.9, + eval.metric = demo.metric.mae) + +}) + +test_that("Classification", { + data(Sonar, package = "mlbench") + Sonar[, 61] <- as.numeric(Sonar[, 61]) - 1 + train.ind <- c(1:50, 100:150) + train.x <- data.matrix(Sonar[train.ind, 1:60]) + train.y <- Sonar[train.ind, 61] + test.x <- data.matrix(Sonar[-train.ind, 1:60]) + test.y <- Sonar[-train.ind, 61] + mx.set.seed(0) + model <- mx.mlp(train.x, train.y, hidden_node = 10, + out_node = 2, out_activation = "softmax", + num.round = 20, array.batch.size = 15, + learning.rate = 0.07, + momentum = 0.9, + eval.metric = mx.metric.accuracy) +}) + From de5b0fe4e584958600551ac151aa6c03755face8 Mon Sep 17 00:00:00 2001 From: Xu Dong Date: Wed, 5 Jul 2017 10:40:37 +0800 Subject: [PATCH 151/834] Add ConvRNN Cell, ConvLSTM Cell (#6832) * Add ConvLSTM cell * Fix lint * Fix typo * Add activation parameters to ConvLSTM and ConvGRU * Add leaky relu to activation options * Change defaut params * Remove h2h_pad * Fix python3 compatibility bug * Fix wrong padding * Add base class for Conv RNN --- python/mxnet/rnn/rnn_cell.py | 346 ++++++++++++++++++++++++++++++ tests/python/unittest/test_rnn.py | 47 ++++ 2 files changed, 393 insertions(+) diff --git a/python/mxnet/rnn/rnn_cell.py b/python/mxnet/rnn/rnn_cell.py index d0505f87ac40..320f78120c5c 100644 --- a/python/mxnet/rnn/rnn_cell.py +++ b/python/mxnet/rnn/rnn_cell.py @@ -6,6 +6,7 @@ from __future__ import print_function import warnings +import functools from .. import symbol, init, ndarray from ..base import string_types, numeric_types @@ -1068,3 +1069,348 @@ def unroll(self, length, inputs, begin_state=None, layout='NTC', merge_outputs=N states = [l_states, r_states] return outputs, states + + +class BaseConvRNNCell(BaseRNNCell): + """Abstract base class for Convolutional RNN cells + + Parameters + ---------- + input_shape : tuple of int + Shape of input in single timestep. + num_hidden : int + Number of units in output symbol. + h2h_kernel : tuple of int + Kernel of Convolution operator in state-to-state transitions. + h2h_dilate : tuple of int + Dilation of Convolution operator in state-to-state transitions. + i2h_kernel : tuple of int + Kernel of Convolution operator in input-to-state transitions. + i2h_stride : tuple of int + Stride of Convolution operator in input-to-state transitions. + i2h_pad : tuple of int + Pad of Convolution operator in input-to-state transitions. + i2h_dilate : tuple of int + Dilation of Convolution operator in input-to-state transitions. + activation : str or Symbol, + Type of activation function. + prefix : str, default '' + Prefix for name of layers (and name of weight if params is None). + params : RNNParams, default None + Container for weight sharing between cells. Created if None. + conv_layout : str, , default 'NCHW' + Layout of ConvolutionOp + """ + def __init__(self, input_shape, num_hidden, + h2h_kernel, h2h_dilate, + i2h_kernel, i2h_stride, + i2h_pad, i2h_dilate, + activation, + prefix='', params=None, conv_layout='NCHW'): + super(BaseConvRNNCell, self).__init__(prefix=prefix, params=params) + # Convolution setting + self._h2h_kernel = h2h_kernel + assert (self._h2h_kernel[0] % 2 == 1) and (self._h2h_kernel[1] % 2 == 1), \ + "Only support odd number, get h2h_kernel= %s" % str(h2h_kernel) + self._h2h_pad = (h2h_dilate[0] * (h2h_kernel[0] - 1) // 2, + h2h_dilate[1] * (h2h_kernel[1] - 1) // 2) + self._h2h_dilate = h2h_dilate + self._i2h_kernel = i2h_kernel + self._i2h_stride = i2h_stride + self._i2h_pad = i2h_pad + self._i2h_dilate = i2h_dilate + + self._num_hidden = num_hidden + self._input_shape = input_shape + self._conv_layout = conv_layout + self._activation = activation + + # Infer state shape + data = symbol.Variable('data') + self._state_shape = symbol.Convolution(data=data, + num_filter=self._num_hidden, + kernel=self._i2h_kernel, + stride=self._i2h_stride, + pad=self._i2h_pad, + dilate=self._i2h_dilate, + layout=conv_layout) + self._state_shape = self._state_shape.infer_shape(data=input_shape)[1][0] + self._state_shape = (0, ) + self._state_shape[1:] + + @property + def state_info(self): + return [{'shape': self._state_shape, '__layout__': self._conv_layout}, + {'shape': self._state_shape, '__layout__': self._conv_layout}] + + def __call__(self, inputs, states): + raise NotImplementedError("BaseConvRNNCell is abstract class for convolutional RNN") + +class ConvRNNCell(BaseConvRNNCell): + """Convolutional RNN cells + + Parameters + ---------- + input_shape : tuple of int + Shape of input in single timestep. + num_hidden : int + Number of units in output symbol. + h2h_kernel : tuple of int, default (3, 3) + Kernel of Convolution operator in state-to-state transitions. + h2h_dilate : tuple of int, default (1, 1) + Dilation of Convolution operator in state-to-state transitions. + i2h_kernel : tuple of int, default (3, 3) + Kernel of Convolution operator in input-to-state transitions. + i2h_stride : tuple of int, default (1, 1) + Stride of Convolution operator in input-to-state transitions. + i2h_pad : tuple of int, default (1, 1) + Pad of Convolution operator in input-to-state transitions. + i2h_dilate : tuple of int, default (1, 1) + Dilation of Convolution operator in input-to-state transitions. + activation : str or Symbol, + default functools.partial(symbol.LeakyReLU, act_type='leaky', slope=0.2) + Type of activation function. + prefix : str, default 'ConvRNN_' + Prefix for name of layers (and name of weight if params is None). + params : RNNParams, default None + Container for weight sharing between cells. Created if None. + conv_layout : str, , default 'NCHW' + Layout of ConvolutionOp + """ + def __init__(self, input_shape, num_hidden, + h2h_kernel=(3, 3), h2h_dilate=(1, 1), + i2h_kernel=(3, 3), i2h_stride=(1, 1), + i2h_pad=(1, 1), i2h_dilate=(1, 1), + activation=functools.partial(symbol.LeakyReLU, act_type='leaky', slope=0.2), + prefix='ConvRNN_', params=None, conv_layout='NCHW'): + super(ConvRNNCell, self).__init__(input_shape=input_shape, num_hidden=num_hidden, + h2h_kernel=h2h_kernel, h2h_dilate=h2h_dilate, + i2h_kernel=i2h_kernel, i2h_stride=i2h_stride, + i2h_pad=i2h_pad, i2h_dilate=i2h_dilate, + activation=activation, prefix=prefix, + params=params, conv_layout=conv_layout) + # Get params + self._iW = self.params.get('i2h_weight') + self._hW = self.params.get('h2h_weight') + self._iB = self.params.get('i2h_bias') + self._hB = self.params.get('h2h_bias') + + @property + def _gate_names(self): + return ('',) + + def __call__(self, inputs, states): + self._counter += 1 + name = '%st%d_'%(self._prefix, self._counter) + i2h = symbol.Convolution(name='%si2h'%name, + data=inputs, + num_filter=self._num_hidden, + kernel=self._i2h_kernel, + stride=self._i2h_stride, + pad=self._i2h_pad, + dilate=self._i2h_dilate, + weight=self._iW, + bias=self._iB,) + h2h = symbol.Convolution(name='%sh2h'%name, + data=states[0], + num_filter=self._num_hidden, + kernel=self._h2h_kernel, + dilate=self._h2h_dilate, + pad=self._h2h_pad, + stride=(1, 1), + weight=self._hW, + bias=self._hB) + output = self._get_activation(i2h + h2h, self._activation, + name='%sout'%name) + return output, [output] + + +class ConvLSTMCell(BaseConvRNNCell): + """Convolutional LSTM network cell. + + Reference: + Xingjian et al. NIPS2015 + + Parameters + ---------- + input_shape : tuple of int + Shape of input in single timestep. + num_hidden : int + Number of units in output symbol. + h2h_kernel : tuple of int, default (3, 3) + Kernel of Convolution operator in state-to-state transitions. + h2h_dilate : tuple of int, default (1, 1) + Dilation of Convolution operator in state-to-state transitions. + i2h_kernel : tuple of int, default (3, 3) + Kernel of Convolution operator in input-to-state transitions. + i2h_stride : tuple of int, default (1, 1) + Stride of Convolution operator in input-to-state transitions. + i2h_pad : tuple of int, default (1, 1) + Pad of Convolution operator in input-to-state transitions. + i2h_dilate : tuple of int, default (1, 1) + Dilation of Convolution operator in input-to-state transitions. + activation : str or Symbol + default functools.partial(symbol.LeakyReLU, act_type='leaky', slope=0.2) + Type of activation function. + prefix : str, default 'ConvLSTM_' + Prefix for name of layers (and name of weight if params is None). + params : RNNParams, default None + Container for weight sharing between cells. Created if None. + forget_bias : bias added to forget gate, default 1.0. + Jozefowicz et al. 2015 recommends setting this to 1.0 + conv_layout : str, , default 'NCHW' + Layout of ConvolutionOp + """ + def __init__(self, input_shape, num_hidden, + h2h_kernel=(3, 3), h2h_dilate=(1, 1), + i2h_kernel=(3, 3), i2h_stride=(1, 1), + i2h_pad=(1, 1), i2h_dilate=(1, 1), + activation=functools.partial(symbol.LeakyReLU, act_type='leaky', slope=0.2), + prefix='ConvLSTM_', params=None, forget_bias=1.0, + conv_layout='NCHW'): + super(ConvLSTMCell, self).__init__(input_shape=input_shape, num_hidden=num_hidden, + h2h_kernel=h2h_kernel, h2h_dilate=h2h_dilate, + i2h_kernel=i2h_kernel, i2h_stride=i2h_stride, + i2h_pad=i2h_pad, i2h_dilate=i2h_dilate, + activation=activation, prefix=prefix, + params=params, conv_layout=conv_layout) + + # Get params + self._iW = self.params.get('i2h_weight') + self._hW = self.params.get('h2h_weight') + # we add the forget_bias to i2h_bias, this adds the bias to the forget gate activation + self._iB = self.params.get('i2h_bias', init=init.LSTMBias(forget_bias=forget_bias)) + self._hB = self.params.get('h2h_bias') + + @property + def _gate_names(self): + return ['_i', '_f', '_c', '_o'] + + def __call__(self, inputs, states): + self._counter += 1 + name = '%st%d_'%(self._prefix, self._counter) + i2h = symbol.Convolution(name='%si2h'%name, + data=inputs, + num_filter=self._num_hidden*4, + kernel=self._i2h_kernel, + stride=self._i2h_stride, + pad=self._i2h_pad, + dilate=self._i2h_dilate, + weight=self._iW, + bias=self._iB,) + h2h = symbol.Convolution(name='%sh2h'%name, + data=states[0], + num_filter=self._num_hidden*4, + kernel=self._h2h_kernel, + dilate=self._h2h_dilate, + pad=self._h2h_pad, + stride=(1, 1), + weight=self._hW, + bias=self._hB) + + gates = i2h + h2h + slice_gates = symbol.SliceChannel(gates, num_outputs=4, axis=self._conv_layout.find('C'), + name="%sslice"%name) + in_gate = symbol.Activation(slice_gates[0], act_type="sigmoid", + name='%si'%name) + forget_gate = symbol.Activation(slice_gates[1], act_type="sigmoid", + name='%sf'%name) + in_transform = self._get_activation(slice_gates[2], self._activation, + name='%sc'%name) + out_gate = symbol.Activation(slice_gates[3], act_type="sigmoid", + name='%so'%name) + next_c = symbol._internal._plus(forget_gate * states[1], in_gate * in_transform, + name='%sstate'%name) + next_h = symbol._internal._mul(out_gate, self._get_activation(next_c, self._activation), + name='%sout'%name) + + return next_h, [next_h, next_c] + +class ConvGRUCell(BaseConvRNNCell): + """Convolutional Gated Rectified Unit (GRU) network cell. + + Parameters + ---------- + input_shape : tuple of int + Shape of input in single timestep. + num_hidden : int + Number of units in output symbol. + h2h_kernel : tuple of int, default (3, 3) + Kernel of Convolution operator in state-to-state transitions. + h2h_dilate : tuple of int, default (1, 1) + Dilation of Convolution operator in state-to-state transitions. + i2h_kernel : tuple of int, default (3, 3) + Kernel of Convolution operator in input-to-state transitions. + i2h_stride : tuple of int, default (1, 1) + Stride of Convolution operator in input-to-state transitions. + i2h_pad : tuple of int, default (1, 1) + Pad of Convolution operator in input-to-state transitions. + i2h_dilate : tuple of int, default (1, 1) + Dilation of Convolution operator in input-to-state transitions. + activation : str or Symbol, + default functools.partial(symbol.LeakyReLU, act_type='leaky', slope=0.2) + Type of activation function. + prefix : str, default 'ConvGRU_' + Prefix for name of layers (and name of weight if params is None). + params : RNNParams, default None + Container for weight sharing between cells. Created if None. + conv_layout : str, , default 'NCHW' + Layout of ConvolutionOp + """ + def __init__(self, input_shape, num_hidden, + h2h_kernel=(3, 3), h2h_dilate=(1, 1), + i2h_kernel=(3, 3), i2h_stride=(1, 1), + i2h_pad=(1, 1), i2h_dilate=(1, 1), + activation=functools.partial(symbol.LeakyReLU, act_type='leaky', slope=0.2), + prefix='ConvGRU_', params=None, conv_layout='NCHW'): + super(ConvGRUCell, self).__init__(input_shape=input_shape, num_hidden=num_hidden, + h2h_kernel=h2h_kernel, h2h_dilate=h2h_dilate, + i2h_kernel=i2h_kernel, i2h_stride=i2h_stride, + i2h_pad=i2h_pad, i2h_dilate=i2h_dilate, + activation=activation, prefix=prefix, + params=params, conv_layout=conv_layout) + # Get params + self._iW = self.params.get('i2h_weight') + self._hW = self.params.get('h2h_weight') + self._iB = self.params.get('i2h_bias') + self._hB = self.params.get('h2h_bias') + + @property + def _gate_names(self): + return ['_r', '_z', '_o'] + + def __call__(self, inputs, states): + self._counter += 1 + seq_idx = self._counter + name = '%st%d_' % (self._prefix, seq_idx) + i2h = symbol.Convolution(name='%s_i2h'%name, data=inputs, + num_filter=self._num_hidden * 3, + kernel=self._i2h_kernel, + stride=self._i2h_stride, + pad=self._i2h_pad, + dilate=self._i2h_dilate, + weight=self._iW, + bias=self._iB,) + h2h = symbol.Convolution(name='%s_h2h'%name, data=states[0], + num_filter=self._num_hidden * 3, + kernel=self._h2h_kernel, + dilate=self._h2h_dilate, + pad=self._h2h_pad, + stride=(1, 1), + weight=self._hW, + bias=self._hB) + + i2h_r, i2h_z, i2h = symbol.SliceChannel(i2h, num_outputs=3, name="%s_i2h_slice" % name) + h2h_r, h2h_z, h2h = symbol.SliceChannel(h2h, num_outputs=3, name="%s_h2h_slice" % name) + + reset_gate = symbol.Activation(i2h_r + h2h_r, act_type="sigmoid", + name="%s_r_act" % name) + update_gate = symbol.Activation(i2h_z + h2h_z, act_type="sigmoid", + name="%s_z_act" % name) + + next_h_tmp = self._get_activation(i2h + reset_gate * h2h, self._activation, + name="%s_h_act" % name) + + next_h = symbol._internal._plus((1. - update_gate) * next_h_tmp, update_gate * states[0], + name='%sout' % name) + + return next_h, [next_h] diff --git a/tests/python/unittest/test_rnn.py b/tests/python/unittest/test_rnn.py index 419104d57dd2..6df8452d0a0d 100644 --- a/tests/python/unittest/test_rnn.py +++ b/tests/python/unittest/test_rnn.py @@ -175,6 +175,50 @@ def test_unfuse(): args, outs, auxs = outputs.infer_shape(rnn_t0_data=(10,50), rnn_t1_data=(10,50), rnn_t2_data=(10,50)) assert outs == [(10, 200), (10, 200), (10, 200)] +def test_convrnn(): + cell = mx.rnn.ConvRNNCell(input_shape = (1, 3, 16, 10), num_hidden=10, + h2h_kernel=(3, 3), h2h_dilate=(1, 1), + i2h_kernel=(3, 3), i2h_stride=(1, 1), + i2h_pad=(1, 1), i2h_dilate=(1, 1), + prefix='rnn_') + inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)] + outputs, _ = cell.unroll(3, inputs) + outputs = mx.sym.Group(outputs) + assert sorted(cell.params._params.keys()) == ['rnn_h2h_bias', 'rnn_h2h_weight', 'rnn_i2h_bias', 'rnn_i2h_weight'] + assert outputs.list_outputs() == ['rnn_t0_out_output', 'rnn_t1_out_output', 'rnn_t2_out_output'] + + args, outs, auxs = outputs.infer_shape(rnn_t0_data=(1, 3, 16, 10), rnn_t1_data=(1, 3, 16, 10), rnn_t2_data=(1, 3, 16, 10)) + assert outs == [(1, 10, 16, 10), (1, 10, 16, 10), (1, 10, 16, 10)] + +def test_convlstm(): + cell = mx.rnn.ConvLSTMCell(input_shape = (1, 3, 16, 10), num_hidden=10, + h2h_kernel=(3, 3), h2h_dilate=(1, 1), + i2h_kernel=(3, 3), i2h_stride=(1, 1), + i2h_pad=(1, 1), i2h_dilate=(1, 1), + prefix='rnn_', forget_bias=1.0) + inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)] + outputs, _ = cell.unroll(3, inputs) + outputs = mx.sym.Group(outputs) + assert sorted(cell.params._params.keys()) == ['rnn_h2h_bias', 'rnn_h2h_weight', 'rnn_i2h_bias', 'rnn_i2h_weight'] + assert outputs.list_outputs() == ['rnn_t0_out_output', 'rnn_t1_out_output', 'rnn_t2_out_output'] + + args, outs, auxs = outputs.infer_shape(rnn_t0_data=(1, 3, 16, 10), rnn_t1_data=(1, 3, 16, 10), rnn_t2_data=(1, 3, 16, 10)) + assert outs == [(1, 10, 16, 10), (1, 10, 16, 10), (1, 10, 16, 10)] + +def test_convgru(): + cell = mx.rnn.ConvGRUCell(input_shape = (1, 3, 16, 10), num_hidden=10, + h2h_kernel=(3, 3), h2h_dilate=(1, 1), + i2h_kernel=(3, 3), i2h_stride=(1, 1), + i2h_pad=(1, 1), i2h_dilate=(1, 1), + prefix='rnn_') + inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)] + outputs, _ = cell.unroll(3, inputs) + outputs = mx.sym.Group(outputs) + assert sorted(cell.params._params.keys()) == ['rnn_h2h_bias', 'rnn_h2h_weight', 'rnn_i2h_bias', 'rnn_i2h_weight'] + assert outputs.list_outputs() == ['rnn_t0_out_output', 'rnn_t1_out_output', 'rnn_t2_out_output'] + + args, outs, auxs = outputs.infer_shape(rnn_t0_data=(1, 3, 16, 10), rnn_t1_data=(1, 3, 16, 10), rnn_t2_data=(1, 3, 16, 10)) + assert outs == [(1, 10, 16, 10), (1, 10, 16, 10), (1, 10, 16, 10)] if __name__ == '__main__': test_rnn() @@ -184,3 +228,6 @@ def test_unfuse(): test_stack() test_bidirectional() test_unfuse() + test_convrnn() + test_convlstm() + test_convgru() From c15902967bfee090a2596410dd346654dca4d145 Mon Sep 17 00:00:00 2001 From: Sergey Kolychev Date: Tue, 4 Jul 2017 19:41:12 -0700 Subject: [PATCH 152/834] reworked cachedop. (#6910) kvstore is indexed via strings not ints from now on. added two more optimizers and reworked sgd optimizer. auto reshape for module->forward. --- perl-package/AI-MXNet/Changes | 3 + perl-package/AI-MXNet/MANIFEST | 1 + perl-package/AI-MXNet/META.json | 4 +- perl-package/AI-MXNet/META.yml | 4 +- perl-package/AI-MXNet/Makefile.PL | 6 +- perl-package/AI-MXNet/README | 2 +- perl-package/AI-MXNet/lib/AI/MXNet.pm | 3 +- perl-package/AI-MXNet/lib/AI/MXNet/Base.pm | 4 +- .../AI-MXNet/lib/AI/MXNet/CachedOp.pm | 89 ++++++ .../AI-MXNet/lib/AI/MXNet/Executor/Group.pm | 8 +- .../AI-MXNet/lib/AI/MXNet/Initializer.pm | 43 ++- perl-package/AI-MXNet/lib/AI/MXNet/KVStore.pm | 18 +- perl-package/AI-MXNet/lib/AI/MXNet/Module.pm | 88 +++++- .../AI-MXNet/lib/AI/MXNet/Module/Base.pm | 23 +- .../AI-MXNet/lib/AI/MXNet/Module/Bucketing.pm | 15 +- perl-package/AI-MXNet/lib/AI/MXNet/NDArray.pm | 33 +- .../AI-MXNet/lib/AI/MXNet/Optimizer.pm | 298 ++++++++++++++++-- .../AI-MXNet/lib/AI/MXNet/RNN/Cell.pm | 5 +- perl-package/AI-MXNet/lib/AI/MXNet/RNN/IO.pm | 14 +- perl-package/AI-MXNet/lib/AI/MXNet/Symbol.pm | 30 ++ .../AI-MXNet/lib/AI/MXNet/TestUtils.pm | 16 +- perl-package/AI-MXNet/t/test_module.t | 165 +++++++++- perl-package/AI-MXNet/t/test_ndarray.t | 19 +- perl-package/AI-MXNet/t/test_optimizers.t | 161 +++++++--- perl-package/AI-MXNetCAPI/Changes | 3 + perl-package/AI-MXNetCAPI/META.json | 2 +- perl-package/AI-MXNetCAPI/META.yml | 2 +- perl-package/AI-MXNetCAPI/README | 2 +- perl-package/AI-MXNetCAPI/lib/AI/MXNetCAPI.pm | 2 +- perl-package/AI-MXNetCAPI/mxnet.i | 62 ++-- perl-package/AI-MXNetCAPI/mxnet_typemaps.i | 8 +- 31 files changed, 966 insertions(+), 167 deletions(-) create mode 100644 perl-package/AI-MXNet/lib/AI/MXNet/CachedOp.pm diff --git a/perl-package/AI-MXNet/Changes b/perl-package/AI-MXNet/Changes index f2663c01254d..5d5c5a280b70 100644 --- a/perl-package/AI-MXNet/Changes +++ b/perl-package/AI-MXNet/Changes @@ -1,5 +1,8 @@ Revision history for Perl extension AI::MXNet +1.0101 Sun Jul 2 17:16:01 PDT 2017 + - reworked CachedOp, two new optimizers, auto module reshape, using strings to index the kvstore. + 1.01 Sat Jun 10 23:57:27 PDT 2017 - sync with python. diff --git a/perl-package/AI-MXNet/MANIFEST b/perl-package/AI-MXNet/MANIFEST index 855aa0a9e883..7a6d78bf0b3f 100644 --- a/perl-package/AI-MXNet/MANIFEST +++ b/perl-package/AI-MXNet/MANIFEST @@ -32,6 +32,7 @@ t/test_executor.t t/test_infer_shape.t lib/AI/MXNet.pm lib/AI/MXNet/Random.pm +lib/AI/MXNet/CachedOp.pm lib/AI/MXNet/Context.pm lib/AI/MXNet/Contrib/AutoGrad.pm lib/AI/MXNet/Contrib/Symbol.pm diff --git a/perl-package/AI-MXNet/META.json b/perl-package/AI-MXNet/META.json index c2f75309c497..54545928e20a 100644 --- a/perl-package/AI-MXNet/META.json +++ b/perl-package/AI-MXNet/META.json @@ -30,7 +30,7 @@ }, "runtime" : { "requires" : { - "AI::MXNetCAPI" : "1.01", + "AI::MXNetCAPI" : "1.0101", "AI::NNVMCAPI" : "1.01", "Function::Parameters" : "1.0705", "GraphViz" : "2.14", @@ -43,5 +43,5 @@ } }, "release_status" : "stable", - "version" : "1.01" + "version" : "1.0101" } diff --git a/perl-package/AI-MXNet/META.yml b/perl-package/AI-MXNet/META.yml index 14d5dc3ec2d9..8c09c96eb685 100644 --- a/perl-package/AI-MXNet/META.yml +++ b/perl-package/AI-MXNet/META.yml @@ -17,10 +17,10 @@ no_index: - t - inc requires: - AI::MXNetCAPI: '1.01' + AI::MXNetCAPI: '1.0101' AI::NNVMCAPI: '1.01' Function::Parameters: '1.0705' GraphViz: '2.14' Mouse: v2.1.0 PDL: '2.007' -version: '1.01' +version: '1.0101' diff --git a/perl-package/AI-MXNet/Makefile.PL b/perl-package/AI-MXNet/Makefile.PL index fc5abc0e5721..4f42af0d7650 100644 --- a/perl-package/AI-MXNet/Makefile.PL +++ b/perl-package/AI-MXNet/Makefile.PL @@ -19,7 +19,7 @@ my %WriteMakefileArgs = ( "LICENSE" => "apache_2_0", "NAME" => "AI::MXNet", "PREREQ_PM" => { - "AI::MXNetCAPI" => "1.01", + "AI::MXNetCAPI" => "1.0101", "AI::NNVMCAPI" => "1.01", "Function::Parameters" => "1.0705", "Mouse" => "2.1.0", @@ -27,7 +27,7 @@ my %WriteMakefileArgs = ( "GraphViz" => "2.14" }, "TEST_REQUIRES" => {}, - "VERSION" => "1.01", + "VERSION" => "1.0101", "test" => { "TESTS" => "t/*.t" } @@ -35,7 +35,7 @@ my %WriteMakefileArgs = ( my %FallbackPrereqs = ( - "AI::MXNetCAPI" => "1.01", + "AI::MXNetCAPI" => "1.0101", "AI::NNVMCAPI" => "1.01", "Function::Parameters" => "1.0705", "Mouse" => "2.1.0", diff --git a/perl-package/AI-MXNet/README b/perl-package/AI-MXNet/README index 85406f604808..f275d08f1a11 100644 --- a/perl-package/AI-MXNet/README +++ b/perl-package/AI-MXNet/README @@ -1,5 +1,5 @@ This archive contains the distribution AI-MXNet, -version 1.01: +version 1.0101: Perl interface to MXNet machine learning library diff --git a/perl-package/AI-MXNet/lib/AI/MXNet.pm b/perl-package/AI-MXNet/lib/AI/MXNet.pm index 41bb1a18b493..54fb6b31e81f 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet.pm @@ -28,7 +28,8 @@ use AI::MXNet::RecordIO; use AI::MXNet::Image; use AI::MXNet::Contrib; use AI::MXNet::Contrib::AutoGrad; -our $VERSION = '1.01'; +use AI::MXNet::CachedOp; +our $VERSION = '1.0101'; sub import { diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Base.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Base.pm index 93859f668a9f..69f8e43af30c 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Base.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Base.pm @@ -3,8 +3,8 @@ use strict; use warnings; use PDL; use PDL::Types qw(); -use AI::MXNetCAPI 0.9506; -use AI::NNVMCAPI 0.95; +use AI::MXNetCAPI 1.0101; +use AI::NNVMCAPI 1.01; use AI::MXNet::Types; use Time::HiRes; use Carp; diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/CachedOp.pm b/perl-package/AI-MXNet/lib/AI/MXNet/CachedOp.pm new file mode 100644 index 000000000000..ede48265970f --- /dev/null +++ b/perl-package/AI-MXNet/lib/AI/MXNet/CachedOp.pm @@ -0,0 +1,89 @@ +package AI::MXNet::CachedOp; + +=head1 NAME + + AI::MXNet::CachedOp - A wrapper around CachedOpHandle +=cut + +use strict; +use warnings; +use AI::MXNet::Base; +use Mouse; +use overload '&{}' => sub { my $self = shift; sub { $self->call(@_) } }; + +has 'handle' => (is => 'ro', isa => 'CachedOpHandle', required => 1); +around BUILDARGS => sub { + my $orig = shift; + my $class = shift; + my ($sym) = @_; + my $handle = check_call( + AI::MXNetCAPI::CreateCachedOp( + $sym->handle + ) + ); + return $class->$orig(handle => $handle); +}; + +sub DEMOLISH +{ + check_call(AI::MXNetCAPI::FreeCachedOp(shift->handle)); +} + +sub call +{ + my $self = shift; + my @args; + my %kwargs; + if(blessed $_[0] and $_[0]->isa('AI::MXNet::NDArray')) + { + while(blessed $_[0] and $_[0]->isa('AI::MXNet::NDArray')) + { + push @args, shift(@_); + } + %kwargs = @_; + } + else + { + %kwargs = @_; + } + my $out = delete $kwargs{out}; + if(%kwargs) + { + confess( + "AI::MXNet::CachedOp::call got unexpected keyword argument(s): ". + join(', ', keys %kwargs) + ); + } + my $original_output; + if(defined $out) + { + $original_output = $out; + if(blessed($out)) + { + $out = [$out]; + } + } + else + { + $out = []; + } + my $output = check_call( + AI::MXNetCAPI::InvokeCachedOp( + $self->handle, + scalar(@args), + [map { $_->handle } @args], + [map { $_->handle } @$out] + ) + ); + return $original_output if defined $original_output; + if(@$output == 1) + { + return AI::MXNet::NDArray->new(handle => $output->[0]); + } + else + { + return [map { AI::MXNet::NDArray->new(handle => $_) } @$output]; + } +} + +1; diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Executor/Group.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Executor/Group.pm index 0ae2db0b1895..35f1b57ee5fa 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Executor/Group.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Executor/Group.pm @@ -203,8 +203,8 @@ use List::Util qw(sum); shared_group : AI::MXNet::DataParallelExecutorGroup Default is undef. This is used in bucketing. When not undef, it should be a executor group corresponding to a different bucket. In other words, it will correspond to a different - symbol but with the same set of parameters (e.g. unrolled RNNs with different lengths). - In this case, many memory will be shared. + symbol with the same set of parameters (e.g. unrolled RNNs with different lengths). + In this case the memory regions of the parameters will be shared. logger : Logger Default is AI::MXNet::Logging->get_logger. fixed_param_names: Maybe[ArrayRef[Str]] @@ -549,9 +549,9 @@ method reshape( A dictionary of name to AI::MXNet::NDArray auxiliary variable mapping. =cut -method set_params(HashRef[AI::MXNet::NDArray] $arg_params, HashRef[AI::MXNet::NDArray] $aux_params) +method set_params(HashRef[AI::MXNet::NDArray] $arg_params, HashRef[AI::MXNet::NDArray] $aux_params, Bool $allow_extra=0) { - $_->copy_params_from($arg_params, $aux_params) for @{ $self->_p->execs }; + $_->copy_params_from($arg_params, $aux_params, $allow_extra) for @{ $self->_p->execs }; } =head2 get_params diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Initializer.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Initializer.pm index c3eee243953b..e6beffb78372 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Initializer.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Initializer.pm @@ -15,8 +15,8 @@ use AI::MXNet::Function::Parameters; attrs : hash ref of str to str attributes of this variable taken from AI::MXNet::Symbol->attr_dict =cut -has 'name' => (is => 'ro', isa => 'Str', required => 1); -has 'attrs' => (is => 'rw', isa => 'HashRef[Str]', lazy => 1, default => sub { +{} }); +has 'name' => (is => 'ro', isa => 'Str', required => 1); +has 'attrs' => (is => 'rw', isa => 'HashRef[Str]', lazy => 1, default => sub { +{} }); use overload '""' => sub { shift->name }; around BUILDARGS => sub { my $orig = shift; @@ -42,6 +42,15 @@ use overload "&{}" => sub { my $self = shift; sub { $self->call(@_) } }, }, fallback => 1; has 'kwargs' => (is => 'rw', init_arg => undef, isa => 'HashRef'); +has '_verbose' => (is => 'rw', isa => 'Bool', lazy => 1, default => 0); +has '_print_func' => (is => 'rw', isa => 'CodeRef', lazy => 1, + default => sub { + return sub { + my $x = shift; + return ($x->norm/sqrt($x->size))->asscalar; + }; + } +); =head1 NAME @@ -52,6 +61,34 @@ has 'kwargs' => (is => 'rw', init_arg => undef, isa => 'HashRef'); Register an initializer class to the AI::MXNet::Initializer factory. =cut +=head2 set_verbosity + + Switch on/off verbose mode + + Parameters + ---------- + $verbose : bool + switch on/off verbose mode + $print_func : CodeRef + A function that computes statistics of initialized arrays. + Takes an AI::MXNet::NDArray and returns a scalar. Defaults to mean + absolute value |x|/size(x) +=cut + +method set_verbosity(Bool $verbose=0, CodeRef $print_func=) +{ + $self->_verbose($verbose); + $self->_print_func($print_func) if defined $print_func; +} + +method _verbose_print($desc, $init, $arr) +{ + if($self->_verbose and defined $self->_print_func) + { + AI::MXNet::Logging->info('Initialized %s as %s: %s', $desc, $init, $self->_print_func->($arr)); + } +} + my %init_registry; method get_init_registry() { @@ -99,6 +136,7 @@ method call(Str|AI::MXNet::InitDesc $desc, AI::MXNet::NDArray $arr) { my ($klass, $kwargs) = @{ decode_json($init) }; $self->get_init_registry->{ lc $klass }->new(%{ $kwargs })->_init_weight("$desc", $arr); + $self->_verbose_print($desc, $init, $arr); } else { @@ -107,6 +145,7 @@ method call(Str|AI::MXNet::InitDesc $desc, AI::MXNet::NDArray $arr) { my $method = "_init_$1"; $self->$method($desc, $arr); + $self->_verbose_print($desc, $1, $arr); } else { diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/KVStore.pm b/perl-package/AI-MXNet/lib/AI/MXNet/KVStore.pm index 9f36cebc0fb7..465cfd65cf07 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/KVStore.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/KVStore.pm @@ -36,7 +36,7 @@ sub DEMOLISH Parameters ---------- - key : int or an array ref of int + key : str or an array ref of str The keys. value : NDArray or an array ref of NDArray objects The values. @@ -59,13 +59,13 @@ sub DEMOLISH =cut method init( - Int|ArrayRef[Int] $key, + Str|ArrayRef[Str] $key, AI::MXNet::NDArray|ArrayRef[AI::MXNet::NDArray]|ArrayRef[ArrayRef[AI::MXNet::NDArray]] $value ) { my ($keys, $vals) = _key_value($key, $value); check_call( - AI::MXNetCAPI::KVStoreInit( + AI::MXNetCAPI::KVStoreInitEx( $self->handle, scalar(@{ $keys }), $keys, $vals ) ); @@ -83,7 +83,7 @@ method init( Parameters ---------- - key : int or array ref of int + key : str or array ref of str value : NDArray or array ref of NDArray or array ref of array refs of NDArray priority : int, optional The priority of the push operation. @@ -127,14 +127,14 @@ method init( =cut method push( - Int|ArrayRef[Int] $key, + Str|ArrayRef[Str] $key, AI::MXNet::NDArray|ArrayRef[AI::MXNet::NDArray]|ArrayRef[ArrayRef[AI::MXNet::NDArray]] $value, Int :$priority=0 ) { my ($keys, $vals) = _key_value($key, $value); check_call( - AI::MXNetCAPI::KVStorePush( + AI::MXNetCAPI::KVStorePushEx( $self->handle, scalar(@{ $keys }), $keys, $vals, $priority ) ); @@ -154,7 +154,7 @@ method push( Parameters ---------- - key : int or array ref of int + key : str or array ref of str Keys out: NDArray or array ref of NDArray or array ref of array refs of NDArray According values @@ -197,14 +197,14 @@ method push( =cut method pull( - Int|ArrayRef[Int] $key, + Str|ArrayRef[Str] $key, AI::MXNet::NDArray|ArrayRef[AI::MXNet::NDArray]|ArrayRef[ArrayRef[AI::MXNet::NDArray]] :$out, Int :$priority=0 ) { my ($keys, $vals) = _key_value($key, $out); check_call( - AI::MXNetCAPI::KVStorePull( + AI::MXNetCAPI::KVStorePullEx( $self->handle, scalar(@{ $keys }), $keys, $vals, $priority ) ); diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Module.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Module.pm index 2c5a2a5fc424..ba70fd059fbe 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Module.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Module.pm @@ -18,6 +18,7 @@ package AI::MXNet::Module; use AI::MXNet::Base; use AI::MXNet::Function::Parameters; use List::Util qw(max); +use Data::Dumper (); use Mouse; func _create_kvstore( @@ -71,10 +72,11 @@ func _initialize_kvstore( { enumerate(sub{ my ($idx, $param_on_devs) = @_; - $kvstore->init($idx, $arg_params->{ $param_names->[$idx] }); + my $name = $param_names->[$idx]; + $kvstore->init($name, $arg_params->{ $name }); if($update_on_kvstore) { - $kvstore->pull($idx, out => $param_on_devs, priority => -$idx); + $kvstore->pull($name, out => $param_on_devs, priority => -$idx); } }, $param_arrays); } @@ -82,7 +84,8 @@ func _initialize_kvstore( func _update_params_on_kvstore( ArrayRef[AI::MXNet::NDArray]|ArrayRef[ArrayRef[AI::MXNet::NDArray]] $param_arrays, ArrayRef[AI::MXNet::NDArray]|ArrayRef[ArrayRef[AI::MXNet::NDArray]] $grad_arrays, - AI::MXNet::KVStore $kvstore + AI::MXNet::KVStore $kvstore, + ArrayRef[Str] $param_names ) { enumerate(sub{ @@ -91,10 +94,11 @@ func _update_params_on_kvstore( { return; } + my $name = $param_names->[$index]; # push gradient, priority is negative index - $kvstore->push($index, $grad_list, priority => -$index); + $kvstore->push($name, $grad_list, priority => -$index); # pull back the weights - $kvstore->pull($index, out => $arg_list, priority => -$index); + $kvstore->pull($name, out => $arg_list, priority => -$index); }, $param_arrays, $grad_arrays); } @@ -103,7 +107,8 @@ func _update_params( ArrayRef[ArrayRef[AI::MXNet::NDArray]] $grad_arrays, AI::MXNet::Updater $updater, Int $num_device, - Maybe[AI::MXNet::KVStore] $kvstore= + Maybe[AI::MXNet::KVStore] $kvstore=, + Maybe[ArrayRef[Str]] $param_names= ) { enumerate(sub{ @@ -114,16 +119,17 @@ func _update_params( } if($kvstore) { + my $name = $param_names->[$index]; # push gradient, priority is negative index - $kvstore->push($index, $grad_list, priority => -$index); + $kvstore->push($name, $grad_list, priority => -$index); # pull back the sum gradients, to the same locations. - $kvstore->pull($index, out => $grad_list, priority => -$index); + $kvstore->pull($name, out => $grad_list, priority => -$index); } enumerate(sub { my ($k, $w, $g) = @_; # faked an index here, to make optimizer create diff # state for the same index but on diff devs, TODO(mli) - # use a better solution latter + # use a better solution later &{$updater}($index*$num_device+$k, $g, $w); }, $arg_list, $grad_list); }, $param_arrays, $grad_arrays); @@ -399,7 +405,8 @@ method init_params( Maybe[HashRef[AI::MXNet::NDArray]] :$arg_params=, Maybe[HashRef[AI::MXNet::NDArray]] :$aux_params=, Bool :$allow_missing=0, - Bool :$force_init=0 + Bool :$force_init=0, + Bool :$allow_extra=0 ) { if($self->params_initialized and not $force_init) @@ -467,21 +474,23 @@ method init_params( $self->_p->_params_dirty(0); # copy the initialized parameters to devices - $self->_p->_exec_group->set_params($self->_p->_arg_params, $self->_p->_aux_params); + $self->_p->_exec_group->set_params($self->_p->_arg_params, $self->_p->_aux_params, $allow_extra); } method set_params( HashRef[AI::MXNet::NDArray] $arg_params, HashRef[AI::MXNet::NDArray] $aux_params, Bool :$allow_missing=0, - Bool :$force_init=1 + Bool :$force_init=1, + Bool :$allow_extra=0 ) { if(not $allow_missing) { $self->init_params( arg_params => $arg_params, aux_params => $aux_params, - allow_missing => $allow_missing, force_init => $force_init + allow_missing => $allow_missing, force_init => $force_init, + allow_extra => $allow_extra ); return; } @@ -494,7 +503,7 @@ method set_params( ); return; } - $self->_p->_exec_group->set_params($arg_params, $aux_params); + $self->_p->_exec_group->set_params($arg_params, $aux_params, $allow_extra); $self->_p->_params_dirty(1); $self->params_initialized(1); } @@ -770,6 +779,51 @@ method forward( ) { assert($self->binded and $self->params_initialized); + # If starting to do the inference, force rebind the module. + if($self->label_shapes and not $data_batch->label) + { + confess( + "If you are trying to do inference, rebind module ". + "with 'force_rebind=True' and 'for_training=False'" + ); + } + + my @curr_data_shapes = map { $_->shape } @{ $self->data_shapes }; + my @new_data_shapes = map { $_->shape } @{ $data_batch->data }; + if(Data::Dumper->Dump(\@curr_data_shapes) ne Data::Dumper->Dump(\@new_data_shapes)) + { + my $new_dshape; + if($data_batch->can('provide_data') and $data_batch->provide_data) + { + $new_dshape = $data_batch->provide_data; + } + else + { + $new_dshape = []; + zip(sub { + my ($i, $shape) = @_; + push @{ $new_dshape }, AI::MXNet::DataDesc->new( + $i->name, $shape, $i->dtype, $i->layout + ); + }, $self->data_shapes, \@new_data_shapes); + } + my $new_lshape; + if($data_batch->can('provide_label') and $data_batch->provide_label) + { + $new_lshape = $data_batch->provide_label; + } + elsif($data_batch->can('label') and $data_batch->label) + { + $new_lshape = []; + zip(sub { + my ($i, $j) = @_; + push @{ $new_lshape }, AI::MXNet::DataDesc->new( + $i->name, $j->shape, $i->dtype, $i->layout + ); + }, $self->label_shapes, $data_batch->label); + } + $self->reshape(data_shapes => $new_dshape, label_shapes => $new_lshape); + } $self->_p->_exec_group->forward($data_batch, $is_train); } @@ -788,7 +842,8 @@ method update() _update_params_on_kvstore( $self->_p->_exec_group->_p->param_arrays, $self->_p->_exec_group->_p->grad_arrays, - $self->_p->_kvstore + $self->_p->_kvstore, + $self->_p->_exec_group->param_names ); } else @@ -798,7 +853,8 @@ method update() $self->_p->_exec_group->_p->grad_arrays, $self->_p->_updater, scalar(@{ $self->_p->_context}), - $self->_p->_kvstore + $self->_p->_kvstore, + $self->_p->_exec_group->param_names ); } } diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Module/Base.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Module/Base.pm index 44df735a15ee..293696db218f 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Module/Base.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Module/Base.pm @@ -677,6 +677,10 @@ method get_params() { confess("NotImplemented") } called to fill those missing params. :$force_init=0 : Bool If true, will force re-initialize even if already initialized. + :$allow_extra=0 : Boolean, optional + Whether allow extra parameters that are not needed by symbol. + If this is True, no error will be thrown when arg_params or aux_params + contain extra parameters that is not needed by the executor. =cut method init_params( @@ -684,7 +688,8 @@ method init_params( Maybe[HashRef[AI::MXNet::NDArray]] :$arg_params=, Maybe[HashRef[AI::MXNet::NDArray]] :$aux_params=, Bool :$allow_missing=0, - Bool :$force_init=0 + Bool :$force_init=0, + Bool :$allow_extra=0 ) { confess("NotImplemented"); @@ -705,13 +710,18 @@ method init_params( called to fill those missing params. :$force_init=0 : Bool If true, will force re-initialize even if already initialized. + :$allow_extra=0 : Bool + Whether allow extra parameters that are not needed by symbol. + If this is True, no error will be thrown when arg_params or aux_params + contain extra parameters that is not needed by the executor. =cut method set_params( Maybe[HashRef[AI::MXNet::NDArray]] $arg_params=, Maybe[HashRef[AI::MXNet::NDArray]] $aux_params=, Bool :$allow_missing=0, - Bool :$force_init=0 + Bool :$force_init=0, + Bool :$allow_extra=0 ) { $self->init_params( @@ -719,7 +729,8 @@ method set_params( arg_params => $arg_params, aux_params => $aux_params, allow_missing => $allow_missing, - force_init => $force_init + force_init => $force_init, + allow_extra => $allow_extra ); } @@ -865,7 +876,11 @@ method prepare(AI::MXNet::DataBatch $data_batch){} =head2 forward - Forward computation. + Forward computation. It supports data batches with different shapes, such as + different batch sizes or different image sizes. + If reshaping of data batch relates to modification of symbol or module, such as + changing image layout ordering or switching from training to predicting, module + rebinding is required. Parameters ---------- diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Module/Bucketing.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Module/Bucketing.pm index 30bdc4378abb..af768f087025 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Module/Bucketing.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Module/Bucketing.pm @@ -210,14 +210,16 @@ method set_params( HashRef[AI::MXNet::NDArray] $arg_params, HashRef[AI::MXNet::NDArray] $aux_params, Bool $allow_missing=0, - Bool $force_init=1 + Bool $force_init=1, + Bool $allow_extra=0 ) { if(not $allow_missing) { $self->init_params( arg_params => $arg_params, aux_params => $aux_params, - allow_missing => $allow_missing, force_init => $force_init + allow_missing => $allow_missing, force_init => $force_init, + allow_extra => $allow_extra ); return; } @@ -232,7 +234,8 @@ method set_params( $self->_curr_module->set_params( $arg_params, $aux_params, allow_missing => $allow_missing, - force_init => $force_init + force_init => $force_init, + allow_extra => $allow_extra ); # because we didn't update self._arg_params, they are dirty now. $self->_params_dirty(1); @@ -244,7 +247,8 @@ method init_params( Maybe[HashRef[AI::MXNet::NDArray]] :$arg_params=, Maybe[HashRef[AI::MXNet::NDArray]] :$aux_params=, Bool :$allow_missing=0, - Bool :$force_init=0 + Bool :$force_init=0, + Bool :$allow_extra=0 ) { return if($self->params_initialized and not $force_init); @@ -254,7 +258,8 @@ method init_params( arg_params => $arg_params, aux_params => $aux_params, allow_missing => $allow_missing, - force_init => $force_init + force_init => $force_init, + allow_extra => $allow_extra ); $self->_params_dirty(0); $self->params_initialized(1); diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/NDArray.pm b/perl-package/AI-MXNet/lib/AI/MXNet/NDArray.pm index 53579b2f1caf..edeb9b1ba1a2 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/NDArray.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/NDArray.pm @@ -12,7 +12,7 @@ use AI::MXNet::NDArray::Slice; use AI::MXNet::Context; use Mouse; use AI::MXNet::Function::Parameters; -use overload +use overload '""' => \&stringify, '+' => \&add, '+=' => \&iadd, @@ -22,6 +22,8 @@ use overload '*=' => \&imultiply, '/' => \÷, '/=' => \&idivide, + '%' => \&modulo, + '%=' => \&imodulo, '**' => \&power, '==' => \&equal, '!=' => \¬_equal, @@ -864,6 +866,24 @@ method true_divide(AI::MXNet::NDArray|Num $other, $reverse=) return $self->divide($other, $reverse); } +method modulo(AI::MXNet::NDArray|Num $other, $reverse=) +{ + return _ufunc_helper( + $self, + $other, + qw/broadcast_mod _mod_scalar _rmod_scalar/, + $reverse + ); +} + +method imodulo(AI::MXNet::NDArray|Num $other, $reverse=) +{ + confess('trying to modulo to a readonly NDArray') unless $self->writable; + return ref $other + ? __PACKAGE__->broadcast_mod($self, $other, { out => $self }) + : __PACKAGE__->_mod_scalar($self, $other, { out => $self }) +} + =head2 empty Creates an empty uninitialized NDArray, with the specified shape. @@ -998,7 +1018,7 @@ method full( Parameters ---------- - $source_array : PDL, PDL::Matrix, Array ref in PDL::pdl format + $source_array : AI::MXNet::NDArray PDL, PDL::Matrix, Array ref in PDL::pdl format Source data to create NDArray from. :$ctx : AI::MXNet::Context, optional @@ -1013,8 +1033,14 @@ method full( The created NDArray. =cut -method array(PDL|PDL::Matrix|ArrayRef $source_array, AI::MXNet::Context :$ctx=AI::MXNet::Context->current_ctx, Dtype :$dtype='float32') +method array(PDL|PDL::Matrix|ArrayRef|AI::MXNet::NDArray $source_array, AI::MXNet::Context :$ctx=AI::MXNet::Context->current_ctx, Dtype :$dtype='float32') { + if(blessed $source_array and $source_array->isa('AI::MXNet::NDArray')) + { + my $arr = __PACKAGE__->empty($source_array->shape, ctx => $ctx, dtype => $dtype); + $arr .= $source_array; + return $arr; + } my $pdl_type = PDL::Type->new(DTYPE_MX_TO_PDL->{ $dtype }); if(not blessed($source_array)) { @@ -1372,6 +1398,7 @@ method backward(Maybe[AI::MXNet::NDArray] $out_grad=, Bool $retain_graph=0) ) } +method CachedOp(@args) { AI::MXNet::CachedOp->new(@args) } my $lvalue_methods = join "\n", map {"use attributes 'AI::MXNet::NDArray', \\&AI::MXNet::NDArray::$_, 'lvalue';"} qw/at slice aspdl asmpdl reshape copy sever T astype as_in_context copyto empty zero ones full diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Optimizer.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Optimizer.pm index 8b60db6c071b..08b9565605eb 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Optimizer.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Optimizer.pm @@ -258,8 +258,15 @@ method _get_wd(Index $index) clip_gradient : float, optional clip gradient in range [-clip_gradient, clip_gradient] - param_idx2name : dict of string/int to float, optional + param_idx2name : hash of string/int to float, optional special treat weight decay in parameter ends with bias, gamma, and beta + + multi_precision: bool, optional + Flag to control the internal precision of the optimizer. + False results in using the same precision as the weights (default), + True makes internal 32-bit copy of the weights and applies gradients + in 32-bit precision even if actual weights used in the model have lower precision. + Turning this on can improve convergence and accuracy when training with float16. =cut package AI::MXNet::SGD; @@ -268,6 +275,7 @@ extends 'AI::MXNet::Optimizer'; has 'kwargs' => (is => "rw", isa => "HashRef[Num]"); has 'momentum' => (is => "rw", isa => "Num", default => 0); +has 'multi_precision' => (is => "ro", isa => "Bool", default => 0); sub BUILD { @@ -285,52 +293,79 @@ sub BUILD method create_state(Index $index, AI::MXNet::NDArray $weight) { - if($self->momentum == 0) + my $momentum; + my $weight_master_copy; + if($self->multi_precision and $weight->dtype eq 'float16') { - return undef; + my $weight_master_copy = AI::MXNet::NDArray->array($weight, ctx => $weight->context, dtype => 'float32'); + if($self->momentum != 0) + { + $momentum = AI::MXNet::NDArray->zeros($weight->shape, ctx => $weight->context, dtype => 'float32'); + } + return [$momentum, $weight_master_copy]; } - else + if($weight->dtype eq 'float16' and not $self->multi_precision) { - return AI::MXNet::NDArray->zeros( - $weight->shape, ctx => $weight->context, dtype => $weight->dtype + AI::MXNet::Logging->warning( + "Accumulating with float16 in optimizer can lead to ". + "poor accuracy or slow convergence. ". + "Consider using multi_precision=True option of the ". + "SGD optimizer" ); } + if($self->momentum != 0) + { + $momentum = AI::MXNet::NDArray->zeros($weight->shape, ctx => $weight->context, dtype => $weight->dtype); + } + return $momentum; } method update( Index $index, AI::MXNet::NDArray $weight, AI::MXNet::NDArray $grad, - Maybe[AI::MXNet::NDArray] $state + Maybe[AI::MXNet::NDArray|ArrayRef[Maybe[AI::MXNet::NDArray]]] $state ) { my $lr = $self->_get_lr($index); my $wd = $self->_get_wd($index); $self->_update_count($index); - if($state) + my $kwargs = { + out => $weight, + lr => $lr, + wd => $wd, + %{ $self->kwargs } + }; + my $use_multi_precision = ref($state) eq 'ARRAY'; + if(not $use_multi_precision) { - AI::MXNet::NDArray->sgd_mom_update( - $weight, $grad, $state, - { - out => $weight, - lr => $lr, - wd => $wd, - %{ $self->kwargs } - } - ); + if(defined $state) + { + AI::MXNet::NDArray->sgd_mom_update( + $weight, $grad, $state, $kwargs + ); + } + else + { + AI::MXNet::NDArray->sgd_update( + $weight, $grad, $kwargs + ); + } } else { - AI::MXNet::NDArray->sgd_update( - $weight, - $grad, - { - out => $weight, - lr => $lr, - wd => $wd, - %{ $self->kwargs } - } - ); + if(defined $state->[0]) + { + AI::MXNet::NDArray->mp_sgd_mom_update( + $weight, $grad, $state->[0], $state->[1], $kwargs + ); + } + else + { + AI::MXNet::NDArray->mp_sgd_update( + $weight, $grad, $state->[1], $kwargs + ); + } } } @@ -1081,6 +1116,184 @@ method update( (($self->beta + $n->sqrt) / $lr + $wd) * ($dn->abs > $self->lamda1); } +__PACKAGE__->register; + +package AI::MXNet::Adamax; + +=head1 NAME + + AI::MXNet::Adamax +=cut + +=head1 DESCRIPTION + + It is a variant of Adam based on the infinity norm + available at http://arxiv.org/abs/1412.6980 Section 7. + + This optimizer accepts the following parameters in addition to those accepted + AI::MXNet::Optimizer. + + Parameters + ---------- + beta1 : float, optional + Exponential decay rate for the first moment estimates. + beta2 : float, optional + Exponential decay rate for the second moment estimates. +=cut + +use Mouse; +extends 'AI::MXNet::Optimizer'; +has '+learning_rate' => (default => 0.002); +has 'beta1' => (is => "ro", isa => "Num", default => 0.9); +has 'beta2' => (is => "ro", isa => "Num", default => 0.999); + +method create_state(Index $index, AI::MXNet::NDArray $weight) +{ + return [ + AI::MXNet::NDArray->zeros( + $weight->shape, + ctx => $weight->context, + dtype => $weight->dtype + ), # mean + AI::MXNet::NDArray->zeros( + $weight->shape, + ctx => $weight->context, + dtype => $weight->dtype + ) # variance + ]; +} + +method update( + Index $index, + AI::MXNet::NDArray $weight, + AI::MXNet::NDArray $grad, + ArrayRef[AI::MXNet::NDArray] $state +) +{ + my $wd = $self->_get_wd($index); + my $lr = $self->_get_lr($index); + $self->_update_count($index); + my $t = $self->_index_update_count->{$index}; + $lr /= (1 - $self->beta1**$t); + + $grad = $grad * $self->rescale_grad + $wd * $weight; + if($self->clip_gradient) + { + $grad = AI::MXNet::NDArray->clip( + $grad, + -$self->clip_gradient, + $self->clip_gradient + ); + } + + # update m_t and u_t + my($m_t, $u_t) = @{ $state }; + $m_t .= $self->beta1 * $m_t + (1 - $self->beta1) * $grad; + $u_t .= AI::MXNet::NDArray->maximum($self->beta2 * $u_t, $grad->abs); + + # update weight + $weight -= $lr * $m_t / $u_t; +} + +__PACKAGE__->register; + +package AI::MXNet::Nadam; + +=head1 NAME + + AI::MXNet::Nadam +=cut + +=head1 DESCRIPTION + + The Nesterov Adam optimizer. + + Much like Adam is essentially RMSprop with momentum, + Nadam is Adam RMSprop with Nesterov momentum available + at http://cs229.stanford.edu/proj2015/054_report.pdf. + + This optimizer accepts the following parameters in addition to those accepted + AI::MXNet::Optimizer. + + Parameters + ---------- + beta1 : float, optional + Exponential decay rate for the first moment estimates. + beta2 : float, optional + Exponential decay rate for the second moment estimates. + epsilon : float, optional + Small value to avoid division by 0. + schedule_decay : float, optional + Exponential decay rate for the momentum schedule +=cut + +use Mouse; +extends 'AI::MXNet::Optimizer'; +has '+learning_rate' => (default => 0.001); +has 'beta1' => (is => "ro", isa => "Num", default => 0.9); +has 'beta2' => (is => "ro", isa => "Num", default => 0.999); +has 'epsilon' => (is => "ro", isa => "Num", default => 1e-8); +has 'schedule_decay' => (is => "ro", isa => "Num", default => 0.004); +has 'm_schedule' => (is => "rw", default => 1, init_arg => undef); + +method create_state(Index $index, AI::MXNet::NDArray $weight) +{ + return [ + AI::MXNet::NDArray->zeros( + $weight->shape, + ctx => $weight->context, + dtype => $weight->dtype + ), # mean + AI::MXNet::NDArray->zeros( + $weight->shape, + ctx => $weight->context, + dtype => $weight->dtype + ) # variance + ]; +} + +method update( + Index $index, + AI::MXNet::NDArray $weight, + AI::MXNet::NDArray $grad, + ArrayRef[AI::MXNet::NDArray] $state +) +{ + my $wd = $self->_get_wd($index); + my $lr = $self->_get_lr($index); + $self->_update_count($index); + my $t = $self->_index_update_count->{$index}; + $grad = $grad * $self->rescale_grad + $wd * $weight; + if($self->clip_gradient) + { + $grad = AI::MXNet::NDArray->clip( + $grad, + -$self->clip_gradient, + $self->clip_gradient + ); + } + # warming momentum schedule + my $momentum_t = $self->beta1 * (1 - 0.5 * (0.96**($t * $self->schedule_decay))); + my $momentum_t_1 = $self->beta1 * (1 - 0.5 * (0.96**(($t + 1) * $self->schedule_decay))); + $self->m_schedule = $self->m_schedule * $momentum_t; + my $m_schedule_next = $self->m_schedule * $momentum_t_1; + + # update m_t and v_t + my ($m_t, $v_t) = @{ $state }; + $m_t .= $self->beta1 * $m_t + (1 - $self->beta1) * $grad; + $v_t .= $self->beta2 * $v_t + (1 - $self->beta2) * $grad * $grad; + + my $grad_prime = $grad / (1 - $self->m_schedule); + my $m_t_prime = $m_t / (1 - $m_schedule_next); + my $v_t_prime = $v_t / (1 - $self->beta2**$t); + my $m_t_bar = (1 - $momentum_t) * $grad_prime + $momentum_t_1 * $m_t_prime; + + # update weight + $weight -= $lr * $m_t_bar / (sqrt($v_t_prime) + $self->epsilon); +} + +__PACKAGE__->register; + # updater for kvstore package AI::MXNet::Updater; use Mouse; @@ -1088,22 +1301,44 @@ use Storable qw(thaw freeze); use overload "&{}" => sub { my $self = shift; sub { $self->call(@_) } }, fallback => 1; -has "optimizer" => (is => "rw", isa => "AI::MXNet::Optimizer"); -has "states" => (is => "rw", isa => "HashRef", default => sub { +{} }); +has "optimizer" => (is => "rw", isa => "AI::MXNet::Optimizer"); +has "states" => (is => "rw", isa => "HashRef", default => sub { +{} }); +has "states_synced" => (is => "rw", isa => "HashRef", default => sub { +{} }); method call(Index $index, AI::MXNet::NDArray $grad, AI::MXNet::NDArray $weight) { if(not exists $self->states->{ $index }) { $self->states->{ $index } = $self->optimizer->create_state($index, $weight); + $self->states_synced->{ $index } = 1; + } + elsif(not $self->states_synced->{ $index }) + { + $self->states->{ $index } = $self->sync_state_context($self->states->{ $index }, $weight->context); + $self->states_synced->{ $index } = 1; } $self->optimizer->update($index, $weight, $grad, $self->states->{ $index }); } *slice = *call; +method sync_state_context(Maybe[AI::MXNet::NDArray|ArrayRef[AI::MXNet::NDArray]] $state, AI::MXNet::Context $context) +{ + if(blessed $state) + { + return $state->as_in_context($context); + } + elsif(ref $state) + { + return [map { $self->sync_state_context($_, $context) } @{ $state }]; + } + return $state; +} + method set_states($states) { - $self->states(thaw($states)); + my $thawed_states = thaw($states); + $self->states($thawed_states); + %{ $self->states_synced } = map { $_ => 0 } keys %{ $thawed_states }; } method get_states() @@ -1113,10 +1348,9 @@ method get_states() package AI::MXNet::Optimizer; - method get_updater(AI::MXNet::Optimizer $optimizer) { return AI::MXNet::Updater->new(optimizer => $optimizer); } -1; \ No newline at end of file +1; diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/RNN/Cell.pm b/perl-package/AI-MXNet/lib/AI/MXNet/RNN/Cell.pm index 89968491e153..c7523aa86bbf 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/RNN/Cell.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/RNN/Cell.pm @@ -981,8 +981,8 @@ method unroll( name => $self->_prefix.'rnn', %states ); - my $outputs; + my %attr = (__layout__ => 'LNC'); if(not $self->_get_next_state) { ($outputs, $states) = ($rnn, []); @@ -990,11 +990,14 @@ method unroll( elsif($self->_mode eq 'lstm') { my @rnn = @{ $rnn }; + $rnn[1]->_set_attr(%attr); + $rnn[2]->_set_attr(%attr); ($outputs, $states) = ($rnn[0], [$rnn[1], $rnn[2]]); } else { my @rnn = @{ $rnn }; + $rnn[1]->_set_attr(%attr); ($outputs, $states) = ($rnn[0], [$rnn[1]]); } if(defined $merge_outputs and not $merge_outputs) diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/RNN/IO.pm b/perl-package/AI-MXNet/lib/AI/MXNet/RNN/IO.pm index 065daded84cf..731f7762b7a0 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/RNN/IO.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/RNN/IO.pm @@ -137,7 +137,7 @@ has 'invalid_label' => (is => 'ro', isa => 'Int', default => -1); has 'data_name' => (is => 'ro', isa => 'Str', default => 'data'); has 'label_name' => (is => 'ro', isa => 'Str', default => 'softmax_label'); has 'dtype' => (is => 'ro', isa => 'Dtype', default => 'float32'); -has 'layout' => (is => 'ro', isa => 'Str', default => 'NTC'); +has 'layout' => (is => 'ro', isa => 'Str', default => 'NT'); has 'buckets' => (is => 'rw', isa => 'Maybe[ArrayRef[Int]]'); has [qw/data nddata ndlabel major_axis default_bucket_key @@ -204,14 +204,16 @@ sub BUILD AI::MXNet::DataDesc->new( name => $self->data_name, shape => $shape, - dtype => $self->dtype + dtype => $self->dtype, + layout => $self->layout ) ]); $self->provide_label([ AI::MXNet::DataDesc->new( name => $self->label_name, shape => $shape, - dtype => $self->dtype + dtype => $self->dtype, + layout => $self->layout ) ]); $self->idx([]); @@ -272,14 +274,16 @@ method next() AI::MXNet::DataDesc->new( name => $self->data_name, shape => $data->shape, - dtype => $self->dtype + dtype => $self->dtype, + layout => $self->layout ) ], provide_label => [ AI::MXNet::DataDesc->new( name => $self->label_name, shape => $label->shape, - dtype => $self->dtype + dtype => $self->dtype, + layout => $self->layout ) ], ); diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Symbol.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Symbol.pm index e22e4189721a..8b14f4e2b1d3 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Symbol.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Symbol.pm @@ -20,6 +20,7 @@ use overload '/' => \÷, '/=' => \&idivide, '**' => \&power, + '%' => \&mod, '==' => \&equal, '!=' => \¬_equal, '>' => \&greater, @@ -169,6 +170,16 @@ method true_divide(AI::MXNet::Symbol|Num $other, $reverse=) return $self->divide($other, $reverse); } +method mod(AI::MXNet::Symbol|Num $other, $reverse=) +{ + return _ufunc_helper( + $self, + $other, + qw/_Mod _ModScalar _RModScalar/, + $reverse + ); +} + method maximum(AI::MXNet::Symbol|Num $other) { return _ufunc_helper( @@ -429,6 +440,25 @@ method list_auxiliary_states() } +=head2 list_inputs + + Lists all arguments and auxiliary states of this Symbol. + + Returns + ------- + inputs : array ref of str + List of all inputs. + + Examples + -------- + >>> my $bn = mx->sym->BatchNorm(name=>'bn'); +=cut + +method list_inputs() +{ + return scalar(check_call(AI::NNVMCAPI::SymbolListInputNames($self->handle, 0))); +} + =head2 infer_type Infer the type of outputs and arguments of given known types of arguments. diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/TestUtils.pm b/perl-package/AI-MXNet/lib/AI/MXNet/TestUtils.pm index e6e3189646d8..52050fa1ebdd 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/TestUtils.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/TestUtils.pm @@ -9,7 +9,7 @@ use Exporter; use base qw(Exporter); @AI::MXNet::TestUtils::EXPORT_OK = qw(same reldiff almost_equal GetMNIST_ubyte GetCifar10 pdl_maximum pdl_minimum mlp2 conv - check_consistency zip assert enumerate same_array); + check_consistency zip assert enumerate same_array dies_like); use constant default_numerical_threshold => 1e-6; =head1 NAME @@ -385,4 +385,18 @@ func same_array( return same($array1->aspdl, $array2->aspdl); } +func dies_like($code, $regexp) +{ + eval { $code->() }; + if($@ =~ $regexp) + { + return 1; + } + else + { + warn $@; + return 0; + } +} + 1; \ No newline at end of file diff --git a/perl-package/AI-MXNet/t/test_module.t b/perl-package/AI-MXNet/t/test_module.t index c6e3c1a8ca0b..4d19a8e7d5df 100644 --- a/perl-package/AI-MXNet/t/test_module.t +++ b/perl-package/AI-MXNet/t/test_module.t @@ -1,9 +1,9 @@ use strict; use warnings; -use Test::More tests => 247; +use Test::More tests => 257; use AI::MXNet qw(mx); use AI::MXNet::Base; -use AI::MXNet::TestUtils qw(almost_equal enumerate same_array); +use AI::MXNet::TestUtils qw(almost_equal enumerate same_array dies_like); use Data::Dumper; sub test_module_layout @@ -451,6 +451,165 @@ sub test_executor_group ); } +sub test_module_set_params +{ + # data iter + mx->random->seed(11); + my $data = mx->nd->array([[0.05, .10]]); + my $label = mx->nd->array([[.01, 0.99]]); + my $train_data = mx->io->NDArrayIter(data => $data, label => $label, batch_size => 1); + + # symbols + my $x = mx->symbol->Variable('data'); + $x = mx->symbol->FullyConnected(name=>'fc_0', data=>$x, num_hidden=>2); + $x = mx->symbol->Activation(name=>"act_0", data=>$x, act_type=>'sigmoid'); + $x = mx->symbol->FullyConnected(name=>'fc_1', data=>$x, num_hidden=>2); + $x = mx->symbol->Activation(name=>"act_1", data=>$x, act_type=>'sigmoid'); + $x = mx->symbol->LinearRegressionOutput(data=>$x, name=>'softmax', grad_scale=>2); + + # create module + my $mod = mx->mod->Module($x, context=>[mx->cpu()]); + $mod->bind(data_shapes => $train_data->provide_data, label_shapes=>$train_data->provide_label, + for_training=>1); + + my $arg_params_correct = {fc_0_weight => mx->nd->array([[.15, .20], [.25, .30]]), + fc_0_bias => mx->nd->array([.35, .35]), + fc_1_weight => mx->nd->array([[.40, .45], [.50, .55]]), + fc_1_bias => mx->nd->array([.60, .60])}; + + my $arg_params_missing = {fc_0_weight => mx->nd->array([[.15, .20], [.25, .30]]), + fc_0_bias => mx->nd->array([.35, .35]), + fc_1_weight => mx->nd->array([[.40, .45], [.50, .55]])}; + + my $arg_params_extra = {fc_0_weight => mx->nd->array([[.15, .20], [.25, .30]]), + fc_0_bias => mx->nd->array([.35, .35]), + fc_1_weight=> mx->nd->array([[.40, .45], [.50, .55]]), + fc_1_bias => mx->nd->array([.60, .60]), + fc_2_weight => mx->nd->array([.60, .60])}; + + my $arg_params_missing_extra = {fc_3_weight => mx->nd->array([.60, .60])}; + + # test regular set_params + $mod->set_params($arg_params_correct, {}, force_init=>1); + + # test allow missing + $mod->set_params($arg_params_missing, {}, allow_missing=>1, force_init=>1); + ok(dies_like(sub { $mod->set_params($arg_params_missing, {}, force_init=>1, allow_missing=>0); }, qr/fc_/)); + + # test allow extra + $mod->set_params($arg_params_extra, {}, force_init=>1, allow_missing=>1, allow_extra=>1); + ok(dies_like(sub { $mod->set_params($arg_params_extra, {}, force_init=>1, allow_missing=>1, allow_extra=>0); }, qr/fc_/)); + + # test allow missing + extra, this will throw a runtime error + ok(dies_like(sub { $mod->set_params($arg_params_missing_extra, {}, force_init=>1, allow_missing=>1, allow_extra=>0); }, qr/fc_/)); +} + +sub test_forward_reshape +{ + my $num_class = 10; + my $data1 = mx->sym->Variable('data1'); + my $data2 = mx->sym->Variable('data2'); + my $conv1 = mx->sym->Convolution(data=>$data1, kernel=>[2, 2], num_filter=>2, stride=>[2, 2]); + my $conv2 = mx->sym->Convolution(data=>$data2, kernel=>[3, 3], num_filter=>3, stride=>[1, 1]); + my $pooling1 = mx->sym->Pooling(data=>$conv1, kernel=>[2, 2], stride=>[1, 1], pool_type=>"avg"); + my $pooling2 = mx->sym->Pooling(data=>$conv2, kernel=>[2, 2], stride=>[1, 1], pool_type=>"max"); + my $flatten1 = mx->sym->flatten(data=>$pooling1); + my $flatten2 = mx->sym->flatten(data=>$pooling2); + my $sum = mx->sym->sum(data=>$flatten1, axis=>1) + mx->sym->sum(data=>$flatten2, axis=>1); + my $fc = mx->sym->FullyConnected(data=>$sum, num_hidden=>$num_class); + my $sym = mx->sym->SoftmaxOutput(data=>$fc, name=>'softmax'); + + my $dshape1 = [10, 3, 64, 64]; + my $dshape2 = [10, 3, 32, 32]; + my $lshape = [10]; + + my $mod = mx->mod->Module(symbol=>$sym, data_names=>['data1', 'data2'], + label_names=>['softmax_label']); + $mod->bind(data_shapes=>[['data1', $dshape1], ['data2', $dshape2]], + label_shapes=>[['softmax_label', $lshape]]); + $mod->init_params(); + $mod->init_optimizer(optimizer_params=>{learning_rate => 0.01}); + + # Train with original data shapes + my $data_batch = mx->io->DataBatch(data=>[mx->nd->random_uniform(0, 9, $dshape1), + mx->nd->random_uniform(5, 15, $dshape2)], + label=>[mx->nd->ones($lshape)]); + $mod->forward($data_batch); + is_deeply($mod->get_outputs->[0]->shape, [$lshape->[0], $num_class]); + $mod->backward(); + $mod->update(); + + # Train with different batch size + $dshape1 = [3, 3, 64, 64]; + $dshape2 = [3, 3, 32, 32]; + $lshape = [3]; + $data_batch = mx->io->DataBatch(data=>[mx->nd->random_uniform(0, 9, $dshape1), + mx->nd->random_uniform(5, 15, $dshape2)], + label=>[mx->nd->ones($lshape)]); + $mod->forward($data_batch); + is_deeply($mod->get_outputs->[0]->shape, [$lshape->[0], $num_class]); + $mod->backward(); + $mod->update(); + + $dshape1 = [20, 3, 64, 64]; + $dshape2 = [20, 3, 32, 32]; + $lshape = [20]; + $data_batch = mx->io->DataBatch(data=>[mx->nd->random_uniform(3, 5, $dshape1), + mx->nd->random_uniform(10, 25, $dshape2)], + label=>[mx->nd->ones($lshape)]); + $mod->forward($data_batch); + is_deeply($mod->get_outputs->[0]->shape, [$lshape->[0], $num_class]); + $mod->backward(); + $mod->update(); + + #Train with both different batch size and data shapes + $dshape1 = [20, 3, 120, 120]; + $dshape2 = [20, 3, 32, 64]; + $lshape = [20]; + $data_batch = mx->io->DataBatch(data=>[mx->nd->random_uniform(0, 9, $dshape1), + mx->nd->random_uniform(5, 15, $dshape2)], + label=>[mx->nd->ones($lshape)]); + $mod->forward($data_batch); + is_deeply($mod->get_outputs->[0]->shape, [$lshape->[0], $num_class]); + $mod->backward(); + $mod->update(); + + $dshape1 = [5, 3, 28, 40]; + $dshape2 = [5, 3, 24, 16]; + $lshape = [5]; + $data_batch = mx->io->DataBatch(data=>[mx->nd->random_uniform(0, 9, $dshape1), + mx->nd->random_uniform(15, 25, $dshape2)], + label=>[mx->nd->ones($lshape)]); + $mod->forward($data_batch); + is_deeply($mod->get_outputs->[0]->shape, [$lshape->[0], $num_class]); + $mod->backward(); + $mod->update(); + + #Test score + my $dataset_shape1 = [30, 3, 30, 30]; + my $dataset_shape2 = [30, 3, 20, 40]; + my $labelset_shape = [30]; + + my $eval_dataiter = mx->io->NDArrayIter(data=>[mx->nd->random_uniform(0, 9, $dataset_shape1), + mx->nd->random_uniform(15, 25, $dataset_shape2)], + label=>[mx->nd->ones($labelset_shape)], + batch_size=>5); + ok(keys %{ $mod->score($eval_dataiter, 'acc') } == 1); + + #Test prediction + $dshape1 = [1, 3, 30, 30]; + $dshape2 = [1, 3, 20, 40]; + $dataset_shape1 = [10, 3, 30, 30]; + $dataset_shape2 = [10, 3, 20, 40]; + + my $pred_dataiter = mx->io->NDArrayIter(data=>[mx->nd->random_uniform(0, 9, $dataset_shape1), + mx->nd->random_uniform(15, 25, $dataset_shape2)]); + $mod->bind(data_shapes=>[['data1', $dshape1], ['data2', $dshape2]], + for_training=>0, force_rebind=>1); + is_deeply($mod->predict($pred_dataiter)->shape, [10, $num_class]); + +} + test_module_input_grads(); test_module_dtype(); test_monitor(); @@ -460,3 +619,5 @@ test_module_states(); test_module_reshape(); test_save_load(); test_executor_group(); +test_module_set_params(); +test_forward_reshape(); \ No newline at end of file diff --git a/perl-package/AI-MXNet/t/test_ndarray.t b/perl-package/AI-MXNet/t/test_ndarray.t index d4e1a4d074b8..4faf464d3b56 100644 --- a/perl-package/AI-MXNet/t/test_ndarray.t +++ b/perl-package/AI-MXNet/t/test_ndarray.t @@ -2,7 +2,7 @@ use strict; use warnings; use AI::MXNet qw(mx); use AI::MXNet::TestUtils qw(almost_equal); -use Test::More tests => 8; +use Test::More tests => 10; sub test_ndarray_reshape { @@ -51,6 +51,23 @@ sub test_output ok(almost_equal($out->aspdl, $ones->aspdl * 2)); } +sub test_cached +{ + my $sym = mx->sym->Convolution(kernel=>[3, 3], num_filter=>10) + 2; + my $op = mx->nd->CachedOp($sym); + my $data = mx->nd->ones([3, 4, 10, 10]); + my $weight = mx->nd->ones([10, 4, 3, 3]); + my $bias = mx->nd->ones([10]); + my $o1 = &{$op}($data, $weight, $bias); + $bias .= 2; + my $o2 = &{$op}($data, $weight, $bias); + ok(almost_equal($o2->aspdl, $o1->aspdl+1)); + $o2 .= 0; + &{$op}($data, $weight, $bias, out=>$o2); + ok(almost_equal($o2->aspdl, $o1->aspdl+1)); +} + test_ndarray_reshape(); test_moveaxis(); test_output(); +test_cached(); diff --git a/perl-package/AI-MXNet/t/test_optimizers.t b/perl-package/AI-MXNet/t/test_optimizers.t index a92a78846ed6..52ff3072d9eb 100644 --- a/perl-package/AI-MXNet/t/test_optimizers.t +++ b/perl-package/AI-MXNet/t/test_optimizers.t @@ -192,12 +192,31 @@ use Mouse; extends 'AI::MXNet::Optimizer'; has '+learning_rate' => (default => 0.01); has 'momentum' => (is => "ro", isa => "Num", default => 0); +has 'multi_precision' => (is => 'ro', isa => 'Bool', default => 0); # Create additional optimizer state: momentum method create_state(Index $index, AI::MXNet::NDArray $weight) { - return undef if $self->momentum == 0; - return mx->nd->zeros($weight->shape, ctx => $weight->context, dtype => $weight->dtype); + my $momentum; + my $weight_master_copy; + my $do_multi_precision = ($self->multi_precision and $weight->dtype eq 'float16'); + if($do_multi_precision) + { + if($self->momentum != 0) + { + $momentum = mx->nd->zeros($weight->shape, ctx => $weight->context, dtype=>'float32'); + } + $weight_master_copy = mx->nd->array($weight, ctx=>$weight->context, dtype=>'float32'); + return [$momentum, $weight_master_copy]; + } + else + { + if($self->momentum != 0) + { + $momentum = mx->nd->zeros($weight->shape, ctx => $weight->context, dtype => $weight->dtype); + } + } + return $momentum; } method update($index, $weight, $grad, $state) @@ -205,48 +224,90 @@ method update($index, $weight, $grad, $state) my $lr = $self->_get_lr($index); my $wd = $self->_get_wd($index); $self->_update_count($index); - if($self->momentum == 0) + my $use_multi_precision = ref($state) eq 'ARRAY'; + + if(not $use_multi_precision) { - if(defined $self->clip_gradient) + if($self->momentum == 0) { - $weight .= ((1 - $lr*$wd)*$weight - - $lr * mx->nd->clip($grad*$self->rescale_grad, -$self->clip_gradient, $self->clip_gradient) - ); + if(defined $self->clip_gradient) + { + $weight .= ((1 - $lr*$wd)*$weight - + $lr * mx->nd->clip($grad*$self->rescale_grad, -$self->clip_gradient, $self->clip_gradient) + ); + } + else + { + $weight .= (1 - $lr*$wd)*$weight - $lr*$self->rescale_grad*$grad; + } } else { - $weight .= (1 - $lr*$wd)*$weight - $lr*$self->rescale_grad*$grad; + my $mom = $state; + if(defined $self->clip_gradient) + { + $mom .= ($self->momentum*$mom - $lr*$wd*$weight - + $lr * mx->nd->clip($grad*$self->rescale_grad, -$self->clip_gradient, $self->clip_gradient) + ); + $weight += $mom; + } + else + { + $mom .= $self->momentum*$mom - $lr*$wd*$weight - $lr*$self->rescale_grad*$grad; + $weight += $mom; + } } } else { - my $mom = $state; - if(defined $self->clip_gradient) + my $grad32 = mx->nd->array($grad, ctx=>$grad->context, dtype=>'float32'); + my $mom = $state->[0]; + my $weight32 = $state->[1]; + if($self->momentum == 0) { - $mom .= ($self->momentum*$mom - $lr*$wd*$weight - - $lr * mx->nd->clip($grad*$self->rescale_grad, -$self->clip_gradient, $self->clip_gradient) - ); - $weight += $mom; + if(defined $self->clip_gradient) + { + $weight32 .= ((1 - $lr*$wd)*$weight32 - + $lr * mx->nd->clip($grad32*$self->rescale_grad, -$self->clip_gradient, $self->clip_gradient) + ); + } + else + { + $weight32 .= (1 - $lr*$wd)*$weight32 - $lr*$self->rescale_grad*$grad32; + } } else { - $mom .= $self->momentum*$mom - $lr*$wd*$weight - $lr*$self->rescale_grad*$grad; - $weight += $mom; + if(defined $self->clip_gradient) + { + $mom .= ($self->momentum*$mom - $lr*$wd*$weight32 - + $lr * mx->nd->clip($grad32*$self->rescale_grad, -$self->clip_gradient, $self->clip_gradient) + ); + $weight32 += $mom; + } + else + { + $mom .= $self->momentum*$mom - $lr*$wd*$weight32 - $lr*$self->rescale_grad*$grad32; + $weight32 += $mom; + } } + my $tmp = $weight32->astype($weight->dtype); + $tmp->copyto($weight); } } + package main; -use Test::More tests => 190; +use Test::More tests => 1314; use AI::MXNet::Base; use PDL::NiceSlice; use AI::MXNet::TestUtils qw(same reldiff almost_equal); use AI::MXNet::Function::Parameters; -func compare_optimizer($opt1, $opt2, $shape) +func compare_optimizer($opt1, $opt2, $shape, $dtype) { - my $w1 = mx->random->uniform({shape => $shape}); - my $g1 = mx->random->uniform({shape => $shape}); + my $w1 = mx->random->uniform({shape => $shape, dtype=>$dtype}); + my $g1 = mx->random->uniform({shape => $shape, dtype=>$dtype}); my $w2 = $w1->copyto(mx->cpu()); my $g2 = $g1->copyto(mx->cpu()); @@ -256,7 +317,7 @@ func compare_optimizer($opt1, $opt2, $shape) zip( sub { my ($s1, $s2) = @_; - ok(same($s1->aspdl, $s2->aspdl)) + ok(same($s1->aspdl, $s2->aspdl)) if defined $s1 and defined $s2; }, ref $state1 eq 'ARRAY' ? $state1 : [$state1], ref $state2 eq 'ARRAY' ? $state2 : [$state2] ) if defined $state1 and defined $state2; @@ -266,7 +327,7 @@ func compare_optimizer($opt1, $opt2, $shape) zip( sub { my ($s1, $s2) = @_; - ok(reldiff($s1->aspdl, $s2->aspdl) < 1e-5) + ok(reldiff($s1->aspdl, $s2->aspdl) < 1e-5) if defined $s1 and defined $s2; }, ref $state1 eq 'ARRAY' ? $state1 : [$state1], ref $state2 eq 'ARRAY' ? $state2 : [$state2] ) if defined $state1 and defined $state2; @@ -285,7 +346,7 @@ func test_adam() {'rescale_grad'=> 0.1}); for my $kwarg (@kwargs) { - compare_optimizer($opt1->new(%$kwarg), $opt2->new(wd => 0.9, %$kwarg), $shape); + compare_optimizer($opt1->new(%$kwarg), $opt2->new(wd => 0.9, %$kwarg), $shape, 'float32'); } } @@ -324,7 +385,7 @@ func test_rms() {rescale_grad => 0.8, wd => 0.05, centered => 1, clip_weights => 0.01}); for my $kwarg (@kwargs) { - compare_optimizer($opt1->new(%$kwarg), $opt2->new(%$kwarg), $shape); + compare_optimizer($opt1->new(%$kwarg), $opt2->new(%$kwarg), $shape, 'float32'); } } @@ -335,25 +396,40 @@ sub test_sgd my $opt1 = 'PerlSGD'; my $opt2 = mx->optimizer->SGD; my $shape = [3, 4, 5]; - my @kwargs = ( - {}, - {momentum => 0.9}, - {clip_gradient => 0.5}, - {clip_gradient => 0.4, rescale_grad => 0.14}, - {rescale_grad => 0.8}, - {clip_gradient => 0.5, wd => 0.07}, - {clip_gradient => 0.4, rescale_grad => 0.14, wd => 0.03}, - {rescale_grad => 0.8, wd => 0.05}, - {clip_gradient => 0.5, momentum => 0.9}, - {clip_gradient => 0.4, rescale_grad => 0.14, momentum => 0.9}, - {rescale_grad => 0.8, momentum => 0.9}, - {clip_gradient => 0.5, wd => 0.07, momentum => 0.9}, - {clip_gradient => 0.4, rescale_grad => 0.14, wd => 0.03, momentum => 0.9}, - {rescale_grad => 0.8, wd => 0.05, momentum => 0.9} - ); - for my $kwarg (@kwargs) + my @mom_options = ({}, {momentum => 0.9}); + my @cg_options = ({}, {clip_gradient => 0.4}, {clip_gradient => 0.5}); + my @rg_options = ({}, {rescale_grad => 0.14}, {rescale_grad => 0.8}); + my @wd_options = ({}, {wd => 0.03}, {wd => 0.05}, {wd => 0.07}); + my @mp_options = ({}, {multi_precision => 0}, {multi_precision => 1}); + for my $dtype(qw/float16 float32 float64/) { - compare_optimizer($opt1->new(%$kwarg), $opt2->new(%$kwarg), $shape); + for my $mom_option (@mom_options) + { + for my $cg_option (@cg_options) + { + for my $rg_option (@rg_options) + { + for my $wd_option (@wd_options) + { + for my $mp_option (@mp_options) + { + my %kwarg; + %kwarg = (%kwarg, %$mom_option); + %kwarg = (%kwarg, %$cg_option); + %kwarg = (%kwarg, %$rg_option); + %kwarg = (%kwarg, %$wd_option); + %kwarg = (%kwarg, %$mp_option); + next if ( + $dtype eq 'float16' + and + (not exists $kwarg{multi_precision} or not $kwarg{multi_precision}) + ); + compare_optimizer($opt1->new(%kwarg), $opt2->new(%kwarg), $shape, $dtype); + } + } + } + } + } } } @@ -392,4 +468,3 @@ test_adam(); test_rms(); test_sgd(); test_lr_wd_mult(); - diff --git a/perl-package/AI-MXNetCAPI/Changes b/perl-package/AI-MXNetCAPI/Changes index df98bd9de411..17595b46e538 100644 --- a/perl-package/AI-MXNetCAPI/Changes +++ b/perl-package/AI-MXNetCAPI/Changes @@ -1,5 +1,8 @@ Revision history for Perl extension AI::MXNetCAPI +1.0101 Sun Jul 2 17:16:01 PDT 2017 + - refactored CachedOp, using strings to index the kvstore. + 1.01 Sat Jun 10 23:57:27 PDT 2017 - sync with python. diff --git a/perl-package/AI-MXNetCAPI/META.json b/perl-package/AI-MXNetCAPI/META.json index 579c81cd8995..a79b1e059107 100644 --- a/perl-package/AI-MXNetCAPI/META.json +++ b/perl-package/AI-MXNetCAPI/META.json @@ -37,5 +37,5 @@ } }, "release_status" : "stable", - "version" : "1.01" + "version" : "1.0101" } diff --git a/perl-package/AI-MXNetCAPI/META.yml b/perl-package/AI-MXNetCAPI/META.yml index a36f94cfeecd..84b7801683a7 100644 --- a/perl-package/AI-MXNetCAPI/META.yml +++ b/perl-package/AI-MXNetCAPI/META.yml @@ -19,4 +19,4 @@ no_index: - inc requires: Test::More: '0' -version: '1.01' +version: '1.0101' diff --git a/perl-package/AI-MXNetCAPI/README b/perl-package/AI-MXNetCAPI/README index 3633756dae1c..07df0c301902 100644 --- a/perl-package/AI-MXNetCAPI/README +++ b/perl-package/AI-MXNetCAPI/README @@ -1,4 +1,4 @@ -AI-MXNetCAPI version 1.01 +AI-MXNetCAPI version 1.0101 ===================== Swig interface to MXNet c api. diff --git a/perl-package/AI-MXNetCAPI/lib/AI/MXNetCAPI.pm b/perl-package/AI-MXNetCAPI/lib/AI/MXNetCAPI.pm index 938146a30b6a..48ebe8090e4f 100644 --- a/perl-package/AI-MXNetCAPI/lib/AI/MXNetCAPI.pm +++ b/perl-package/AI-MXNetCAPI/lib/AI/MXNetCAPI.pm @@ -1,7 +1,7 @@ package AI::MXNetCAPI; use base qw(DynaLoader); bootstrap AI::MXNetCAPI; -our $VERSION = '1.01'; +our $VERSION = '1.0101'; 1; __END__ diff --git a/perl-package/AI-MXNetCAPI/mxnet.i b/perl-package/AI-MXNetCAPI/mxnet.i index d0705d5acc72..bf00e6856d64 100644 --- a/perl-package/AI-MXNetCAPI/mxnet.i +++ b/perl-package/AI-MXNetCAPI/mxnet.i @@ -104,7 +104,7 @@ static void ExecutorMonitor_callback(const char* name, NDArrayHandle handle, voi } } -%} +%} %init %{ /* These SWIG_TypeClientData() calls might break in the future, but @@ -119,6 +119,7 @@ static void ExecutorMonitor_callback(const char* name, NDArrayHandle handle, voi SWIG_TypeClientData(SWIGTYPE_p_MXKVStore, (void *)"KVStoreHandle"); SWIG_TypeClientData(SWIGTYPE_p_MXRecordIO, (void *)"RecordIOHandle"); SWIG_TypeClientData(SWIGTYPE_p_MXRtc, (void *)"RtcHandle"); + SWIG_TypeClientData(SWIGTYPE_p_MXCachedOp, (void *)"CachedOpHandle"); %} /*! \brief manually define unsigned int */ @@ -150,6 +151,8 @@ typedef MXKVStore *KVStoreHandle; typedef MXRecordIO *RecordIOHandle; /*! \brief handle to MXRtc*/ typedef MXRtc *RtcHandle; +/*! \brief handle to cached operator */ +typedef MXCachedOp *CachedOpHandle; typedef void (*ExecutorMonitorCallback)(const char*, NDArrayHandle, @@ -625,6 +628,23 @@ int MXAutogradBackward(mx_uint num_output, NDArrayHandle* in, int retain_graph); + /*! + * \brief create cached operator + */ +int MXCreateCachedOp(SymbolHandle handle, + CachedOpHandle *out); + /*! + * \brief free cached operator + */ +int MXFreeCachedOp(CachedOpHandle handle); + /*! + * \brief invoke cached operator + */ +int MXInvokeCachedOp(CachedOpHandle handle, + int num_inputs, + NDArrayHandle *in, + int *out_size, + NDArrayHandle **out_array); //-------------------------------------------- // Part 3: symbolic configuration generation //-------------------------------------------- @@ -1331,21 +1351,21 @@ int MXKVStoreCreate(const char *type, * \return 0 when success, -1 when failure happens */ int MXKVStoreFree(KVStoreHandle handle); + /*! - * \brief Init a list of (key,value) pairs in kvstore + * \brief Init a list of (key,value) pairs in kvstore, where each key is a string * \param handle handle to the kvstore * \param num the number of key-value pairs * \param keys the list of keys * \param vals the list of values * \return 0 when success, -1 when failure happens */ -int MXKVStoreInit(KVStoreHandle handle, - mx_uint num, - const int* in, - NDArrayHandle* in); - -/*! - * \brief Push a list of (key,value) pairs to kvstore +int MXKVStoreInitEx(KVStoreHandle handle, + mx_uint num, + const char** in, + NDArrayHandle* in); + /*! + * \brief Push a list of (key,value) pairs to kvstore, where each key is a string * \param handle handle to the kvstore * \param num the number of key-value pairs * \param keys the list of keys @@ -1353,13 +1373,13 @@ int MXKVStoreInit(KVStoreHandle handle, * \param priority the priority of the action * \return 0 when success, -1 when failure happens */ -int MXKVStorePush(KVStoreHandle handle, - mx_uint num, - const int* in, - NDArrayHandle* in, - int priority); -/*! - * \brief pull a list of (key, value) pairs from the kvstore +int MXKVStorePushEx(KVStoreHandle handle, + mx_uint num, + const char** in, + NDArrayHandle* in, + int priority); + /*! + * \brief pull a list of (key, value) pairs from the kvstore, where each key is a string * \param handle handle to the kvstore * \param num the number of key-value pairs * \param keys the list of keys @@ -1367,11 +1387,11 @@ int MXKVStorePush(KVStoreHandle handle, * \param priority the priority of the action * \return 0 when success, -1 when failure happens */ -int MXKVStorePull(KVStoreHandle handle, - mx_uint num, - const int* in, - NDArrayHandle* in, - int priority); +int MXKVStorePullEx(KVStoreHandle handle, + mx_uint num, + const char** in, + NDArrayHandle* in, + int priority); /*! * \brief user-defined updater for the kvstore * It's this updater's responsibility to delete \a recv and \a local diff --git a/perl-package/AI-MXNetCAPI/mxnet_typemaps.i b/perl-package/AI-MXNetCAPI/mxnet_typemaps.i index 792f8472d05a..640215fd7792 100644 --- a/perl-package/AI-MXNetCAPI/mxnet_typemaps.i +++ b/perl-package/AI-MXNetCAPI/mxnet_typemaps.i @@ -304,6 +304,7 @@ %typemap(freearg) (mx_float *in) { Safefree($1); } + %typemap(in,numinputs=0) (NDArrayHandle *out) (NDArrayHandle temp), (FunctionHandle* out) (FunctionHandle temp), (SymbolHandle *out) (SymbolHandle temp), @@ -311,12 +312,13 @@ (DataIterHandle *out) (ExecutorHandle temp), (KVStoreHandle *out) (KVStoreHandle temp), (RecordIOHandle *out) (RecordIOHandle temp), - (RtcHandle *out) (RtcHandle temp) + (RtcHandle *out) (RtcHandle temp), + (CachedOpHandle *out) (CachedOpHandle temp) { $1 = &temp; } -%typemap(argout) (NDArrayHandle *out), (FunctionHandle* out), (SymbolHandle *out), (ExecutorHandle *out), (DataIterHandle *out), - (KVStoreHandle *out), (RecordIOHandle *out), (RtcHandle *out) (RtcHandle temp) +%typemap(argout) (NDArrayHandle *out), (FunctionHandle* out), (SymbolHandle *out), (ExecutorHandle *out), (DataIterHandle *out), + (KVStoreHandle *out), (RecordIOHandle *out), (RtcHandle *out) (RtcHandle temp), (CachedOpHandle *out) (CachedOpHandle temp) { if(!result) { From d7852e8c347a8e98cb6084fa5df0d9655c5bead6 Mon Sep 17 00:00:00 2001 From: Zehao Shi Date: Thu, 6 Jul 2017 00:11:54 +0800 Subject: [PATCH 153/834] Fix smooth_l1 comment (#6929) * Fix a spelling mistake. * FIX pad example * fix smooth l1 comment --- src/operator/mshadow_op.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/operator/mshadow_op.h b/src/operator/mshadow_op.h index e2860819ca5c..94bfdb9830c7 100644 --- a/src/operator/mshadow_op.h +++ b/src/operator/mshadow_op.h @@ -950,7 +950,7 @@ MSHADOW_XINLINE double gammaln_grad::Map(double a) { /* Smooth L1 Loss is a loss specific for R-CNN franchise training * Smooth L1 Loss function - * f(x) = 0.5 * (sigma * x) ^ 2, x < 1 / sigma^2 + * f(x) = 0.5 * (sigma * x) ^ 2, |x| < 1 / sigma^2 * = |x| - 0.5 / sigma / sigma, otherwise * When sigma = 1, it is equivalent to Huber Loss evaluated at * delta = 1. @@ -973,7 +973,7 @@ struct smooth_l1_loss { }; // struct smooth_l1_loss /* The derivative of smooth l1 loss is - * f'(x) = sigma^2 * x, x < 1 / sigma^2 + * f'(x) = sigma^2 * x, |x| < 1 / sigma^2 * = sign(x), otherwise */ struct smooth_l1_gradient { From e09cc7c6830f474b7afeeff5f4baba38613493f9 Mon Sep 17 00:00:00 2001 From: Sebastian Bodenstein Date: Wed, 5 Jul 2017 18:13:43 +0200 Subject: [PATCH 154/834] Fix for WarpCTC conflict (#6905) * fix for memcpy bug * moved namespace --- src/operator/contrib/ctc_include/detail/cpu_ctc.h | 4 ++++ src/operator/contrib/ctc_include/detail/gpu_ctc.h | 4 ++++ src/operator/contrib/ctc_loss.cc | 2 +- src/operator/contrib/ctc_loss.cu | 2 +- 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/operator/contrib/ctc_include/detail/cpu_ctc.h b/src/operator/contrib/ctc_include/detail/cpu_ctc.h index 68d74bbc6f83..f31ef62c384e 100644 --- a/src/operator/contrib/ctc_include/detail/cpu_ctc.h +++ b/src/operator/contrib/ctc_include/detail/cpu_ctc.h @@ -10,6 +10,8 @@ #include "ctc_helper.h" +namespace mxnet_warpctc { + template class CpuCTC { public: @@ -484,3 +486,5 @@ ctcStatus_t CpuCTC::score_forward(const ProbT* const activations, return CTC_STATUS_SUCCESS; } + +} // mxnet_warpctc \ No newline at end of file diff --git a/src/operator/contrib/ctc_include/detail/gpu_ctc.h b/src/operator/contrib/ctc_include/detail/gpu_ctc.h index 30fa2cfdacbb..ef71f3cdf956 100644 --- a/src/operator/contrib/ctc_include/detail/gpu_ctc.h +++ b/src/operator/contrib/ctc_include/detail/gpu_ctc.h @@ -1,8 +1,11 @@ #pragma once + #include "ctc_helper.h" #include "gpu_ctc_kernels.h" +namespace mxnet_warpctc { + template class GpuCTC { public: @@ -481,3 +484,4 @@ GpuCTC::score_forward(const ProbT* const activations, label_lengths, input_lengths, true, false); } +} // mxnet_warpctc \ No newline at end of file diff --git a/src/operator/contrib/ctc_loss.cc b/src/operator/contrib/ctc_loss.cc index 918af0e27bbc..c3f3fe1621b4 100644 --- a/src/operator/contrib/ctc_loss.cc +++ b/src/operator/contrib/ctc_loss.cc @@ -18,7 +18,7 @@ ctcStatus_t compute_ctc_cost(const Tensor activations, int minibatch = static_cast(activations.size(1)); int alphabet_size = static_cast(activations.size(2)); int blank_label = 0; - CpuCTC ctc(alphabet_size, minibatch, workspace, blank_label); + mxnet_warpctc::CpuCTC ctc(alphabet_size, minibatch, workspace, blank_label); if (train) return ctc.cost_and_grad(activations.dptr_, grads, costs, labels, label_lengths, input_lengths); diff --git a/src/operator/contrib/ctc_loss.cu b/src/operator/contrib/ctc_loss.cu index 6ef8b4e342bb..ed80eb715516 100644 --- a/src/operator/contrib/ctc_loss.cu +++ b/src/operator/contrib/ctc_loss.cu @@ -18,7 +18,7 @@ ctcStatus_t compute_ctc_cost(const Tensor activations, int minibatch = static_cast(activations.size(1)); int alphabet_size = static_cast(activations.size(2)); int blank_label = 0; - GpuCTC ctc(alphabet_size, minibatch, workspace, + mxnet_warpctc::GpuCTC ctc(alphabet_size, minibatch, workspace, activations.stream_->stream_, blank_label); if (train) return ctc.cost_and_grad(activations.dptr_, grads, costs, labels, From 8003ee188ad971d9e957c7725e0a01e96d3af3f4 Mon Sep 17 00:00:00 2001 From: alues Date: Fri, 7 Jul 2017 01:16:48 +0800 Subject: [PATCH 155/834] Fix Mistake (#6940) --- src/operator/tensor/elemwise_unary_op.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/operator/tensor/elemwise_unary_op.cc b/src/operator/tensor/elemwise_unary_op.cc index 16591c9dd9d3..532963c754ee 100644 --- a/src/operator/tensor/elemwise_unary_op.cc +++ b/src/operator/tensor/elemwise_unary_op.cc @@ -324,7 +324,7 @@ MXNET_OPERATOR_REGISTER_UNARY(square) Example:: - square([2, 3, 4]) = [3, 9, 16] + square([2, 3, 4]) = [4, 9, 16] )code" ADD_FILELINE) .set_attr("FCompute", UnaryCompute) From 45bcc5876c1a309aeaf45ac3b532c05518f1e522 Mon Sep 17 00:00:00 2001 From: Viacheslav Kovalevskyi Date: Thu, 6 Jul 2017 10:26:55 -0700 Subject: [PATCH 156/834] Page title for installing from source has been udpated. (#6942) --- docs/get_started/osx_setup.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/get_started/osx_setup.md b/docs/get_started/osx_setup.md index 2b2858f9d058..8e5439435a59 100644 --- a/docs/get_started/osx_setup.md +++ b/docs/get_started/osx_setup.md @@ -1,6 +1,6 @@ -# Installing MXNet on OS X (Mac) +# Installing MXNet froum source on OS X (Mac) -**NOTE:** For MXNet with Python installation, please refer to the [new install guide](http://mxnet.io/get_started/install.html). +**NOTE:** For prebuild MXNet with Python installation, please refer to the [new install guide](http://mxnet.io/get_started/install.html). Installing MXNet is a two-step process: From 9af2734624d10c49a64d37f2b351244856a42afe Mon Sep 17 00:00:00 2001 From: Tobias Domhan Date: Thu, 6 Jul 2017 19:34:12 +0200 Subject: [PATCH 157/834] Failing with a more descriptive error message when infer_shape does not get passed a tuple. (#6893) --- python/mxnet/symbol.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/python/mxnet/symbol.py b/python/mxnet/symbol.py index ec0eed76fd19..ff98d0238931 100644 --- a/python/mxnet/symbol.py +++ b/python/mxnet/symbol.py @@ -1013,19 +1013,22 @@ def _infer_shape_impl(self, partial, *args, **kwargs): indptr = [0] if len(args) != 0: keys = None - for s in args: + for i, s in enumerate(args): if s is not None: if not isinstance(s, tuple): - raise TypeError('Arguments must be shapes (tuple)') + raise TypeError("Arguments need to be shapes (tuple), " + "but argument %d is %s." % (i, type(s))) sdata.extend(s) indptr.append(len(sdata)) else: keys = [] for k, v in kwargs.items(): - if isinstance(v, tuple): - keys.append(c_str(k)) - sdata.extend(v) - indptr.append(len(sdata)) + if not isinstance(v, tuple): + raise TypeError("Arguments need to be shapes (tuple), " + "but '%s' is %s." % (k, type(v))) + keys.append(c_str(k)) + sdata.extend(v) + indptr.append(len(sdata)) arg_shape_size = mx_uint() arg_shape_ndim = ctypes.POINTER(mx_uint)() arg_shape_data = ctypes.POINTER(ctypes.POINTER(mx_uint))() From 202de02cd713eb17300404b842261f126b6e3c97 Mon Sep 17 00:00:00 2001 From: Kenta Kubo Date: Fri, 7 Jul 2017 02:37:38 +0900 Subject: [PATCH 158/834] Fix Python 3 compatibilities (#6927) --- python/mxnet/metric.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/mxnet/metric.py b/python/mxnet/metric.py index 736864324227..564c727b0c98 100644 --- a/python/mxnet/metric.py +++ b/python/mxnet/metric.py @@ -81,12 +81,12 @@ def update_dict(self, label, pred): if self.output_names is not None: pred = [pred[name] for name in self.output_names] else: - pred = pred.values() + pred = list(pred.values()) if self.label_names is not None: label = [label[name] for name in self.label_names] else: - label = label.values() + label = list(label.values()) self.update(label, pred) From 92428fb7ba8003b3d8d9d098ef20c22123824c89 Mon Sep 17 00:00:00 2001 From: Przemyslaw Tredak Date: Fri, 7 Jul 2017 13:04:33 -0700 Subject: [PATCH 159/834] Fix for pinned memory never being used (#6954) --- src/storage/storage.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/storage/storage.cc b/src/storage/storage.cc index a7cdbf667021..c6e99973cd53 100644 --- a/src/storage/storage.cc +++ b/src/storage/storage.cc @@ -37,11 +37,6 @@ class StorageImpl : public Storage { case Context::kGPU: case Context::kCPUPinned: { #if MXNET_USE_CUDA - num_gpu_device = 0; - cudaError_t e = cudaGetDeviceCount(&num_gpu_device); - if (e != cudaSuccess) { - num_gpu_device = 0; - } if (num_gpu_device > 0) { CUDA_CALL(cudaSetDevice(ctx.dev_id)); } @@ -76,6 +71,11 @@ Storage::Handle StorageImpl::Alloc(size_t size, Context ctx) { } case Context::kCPUPinned: { #if MXNET_USE_CUDA + num_gpu_device = 0; + cudaError_t e = cudaGetDeviceCount(&num_gpu_device); + if (e != cudaSuccess) { + num_gpu_device = 0; + } if (num_gpu_device > 0) { ptr = new storage::NaiveStorageManager(); } else { From 16bff59d1e85126e5d4250e298deda2f7dff6b71 Mon Sep 17 00:00:00 2001 From: "Qiang Kou (KK)" Date: Fri, 7 Jul 2017 20:24:03 -0700 Subject: [PATCH 160/834] [R] negate and more operators for ndarray. close #6768 (#6886) --- R-package/R/ndarray.R | 6 +- R-package/src/ndarray.cc | 74 ++++++++++++++++++++++- R-package/tests/testthat/test_ndarray.R | 79 +++++++++++++++++++++++++ 3 files changed, 157 insertions(+), 2 deletions(-) diff --git a/R-package/R/ndarray.R b/R-package/R/ndarray.R index e30a90117455..da624b01be2d 100644 --- a/R-package/R/ndarray.R +++ b/R-package/R/ndarray.R @@ -145,7 +145,11 @@ is.mx.ndarray <- function(src.array) { #' @param e1 The second operand #' @export Ops.MXNDArray <- function(e1, e2) { - mx.nd.internal.dispatch.Ops(.Generic, e1, e2) + if (missing(e2)) { + mx.nd.internal.dispatch.Ops(.Generic, 0, e1) + } else { + mx.nd.internal.dispatch.Ops(.Generic, e1, e2) + } } #' Dimension operator overload of mx.ndarray diff --git a/R-package/src/ndarray.cc b/R-package/src/ndarray.cc index d270a303fcbf..b289809cca9e 100644 --- a/R-package/src/ndarray.cc +++ b/R-package/src/ndarray.cc @@ -550,6 +550,18 @@ NDArray::RObjectType DispatchOps(SEXP op, SEXP lhs, SEXP rhs) { static OpHandle mod = NDArrayFunction::FindHandle("_mod"); static OpHandle mod_scalar = NDArrayFunction::FindHandle("_mod_scalar"); static OpHandle rmod_scalar = NDArrayFunction::FindHandle("_rmod_scalar"); + static OpHandle equal = NDArrayFunction::FindHandle("_equal"); + static OpHandle equal_scalar = NDArrayFunction::FindHandle("_equal_scalar"); + static OpHandle not_equal = NDArrayFunction::FindHandle("_not_equal"); + static OpHandle not_equal_scalar = NDArrayFunction::FindHandle("_not_equal_scalar"); + static OpHandle greater = NDArrayFunction::FindHandle("_greater"); + static OpHandle greater_scalar = NDArrayFunction::FindHandle("_greater_scalar"); + static OpHandle greater_equal = NDArrayFunction::FindHandle("_greater_equal"); + static OpHandle greater_equal_scalar = NDArrayFunction::FindHandle("_greater_equal_scalar"); + static OpHandle lesser = NDArrayFunction::FindHandle("_lesser"); + static OpHandle lesser_scalar = NDArrayFunction::FindHandle("_lesser_scalar"); + static OpHandle lesser_equal = NDArrayFunction::FindHandle("_lesser_equal"); + static OpHandle lesser_equal_scalar = NDArrayFunction::FindHandle("_lesser_equal_scalar"); // parse the arguments std::string values[2]; NDArrayHandle handles[2]; @@ -610,8 +622,68 @@ NDArray::RObjectType DispatchOps(SEXP op, SEXP lhs, SEXP rhs) { } break; } + case '=': { + if (lhs_nd && rhs_nd) { + out = BinaryOp(equal, handles); + } else if (lhs_nd && !rhs_nd) { + out = BinaryScalarOp(equal_scalar, handles[0], values[1]); + } else { + out = BinaryScalarOp(equal_scalar, handles[1], values[0]); + } + break; + } + case '!': { + if (lhs_nd && rhs_nd) { + out = BinaryOp(not_equal, handles); + } else if (lhs_nd && !rhs_nd) { + out = BinaryScalarOp(not_equal_scalar, handles[0], values[1]); + } else { + out = BinaryScalarOp(not_equal_scalar, handles[1], values[0]); + } + break; + } + case '>': { + if (sop == ">=") { + if (lhs_nd && rhs_nd) { + out = BinaryOp(greater_equal, handles); + } else if (lhs_nd && !rhs_nd) { + out = BinaryScalarOp(greater_equal_scalar, handles[0], values[1]); + } else { + out = BinaryScalarOp(lesser_equal_scalar, handles[1], values[0]); + } + } else { + if (lhs_nd && rhs_nd) { + out = BinaryOp(greater, handles); + } else if (lhs_nd && !rhs_nd) { + out = BinaryScalarOp(greater_scalar, handles[0], values[1]); + } else { + out = BinaryScalarOp(lesser_scalar, handles[1], values[0]); + } + } + break; + } + case '<': { + if (sop == "<=") { + if (lhs_nd && rhs_nd) { + out = BinaryOp(lesser_equal, handles); + } else if (lhs_nd && !rhs_nd) { + out = BinaryScalarOp(lesser_equal_scalar, handles[0], values[1]); + } else { + out = BinaryScalarOp(greater_equal_scalar, handles[1], values[0]); + } + } else { + if (lhs_nd && rhs_nd) { + out = BinaryOp(lesser, handles); + } else if (lhs_nd && !rhs_nd) { + out = BinaryScalarOp(lesser_scalar, handles[0], values[1]); + } else { + out = BinaryScalarOp(greater_scalar, handles[1], values[0]); + } + } + break; + } default: { - RLOG_FATAL << "Operator " << sop << "not supported for MXNDArray"; + RLOG_FATAL << "Operator " << sop << " not supported for MXNDArray"; } } return NDArray::RObject(out, true); diff --git a/R-package/tests/testthat/test_ndarray.R b/R-package/tests/testthat/test_ndarray.R index e574ea74dbd6..142c87e9666b 100644 --- a/R-package/tests/testthat/test_ndarray.R +++ b/R-package/tests/testthat/test_ndarray.R @@ -107,4 +107,83 @@ test_that("ndarray crop", { arr_x[c(1:2), 1 , c(1:3)] <- 0 expect_equal(as.array(x), arr_x) +}) + +test_that("ndarray negate", { + arr <- array(runif(24, -10, 10), dim = c(2, 3, 4)) + nd <- mx.nd.array(arr) + + expect_equal(arr, as.array(nd), tolerance = 1e-6) + expect_equal(-arr, as.array(-nd), tolerance = 1e-6) + expect_equal(arr, as.array(nd), tolerance = 1e-6) +}) + +test_that("ndarray equal", { + x <- mx.nd.zeros(c(2, 3)) + y <- mx.nd.ones(c(2, 3)) + z = x == y + expect_equal(as.array(z), array(0, c(2,3))) + + z = 0 == x + expect_equal(as.array(z), array(1, c(2,3))) +}) + +test_that("ndarray not equal", { + x <- mx.nd.zeros(c(2, 3)) + y <- mx.nd.ones(c(2, 3)) + z = x != y + expect_equal(as.array(z), array(1, c(2,3))) + + z = 0 != x + expect_equal(as.array(z), array(0, c(2,3))) +}) + +test_that("ndarray greater", { + x <- mx.nd.zeros(c(2, 3)) + y <- mx.nd.ones(c(2, 3)) + z = x > y + expect_equal(as.array(z), array(0, c(2,3))) + + z = y > 0 + expect_equal(as.array(z), array(1, c(2,3))) + + z = 0 > y + expect_equal(as.array(z), array(0, c(2,3))) + + z = x >= y + expect_equal(as.array(z), array(0, c(2,3))) + + z = y >= 0 + expect_equal(as.array(z), array(1, c(2,3))) + + z = 0 >= y + expect_equal(as.array(z), array(0, c(2,3))) + + z = y >= 1 + expect_equal(as.array(z), array(1, c(2,3))) +}) + +test_that("ndarray lesser", { + x <- mx.nd.zeros(c(2, 3)) + y <- mx.nd.ones(c(2, 3)) + z = x < y + expect_equal(as.array(z), array(1, c(2,3))) + + z = y < 0 + expect_equal(as.array(z), array(0, c(2,3))) + + z = 0 < y + expect_equal(as.array(z), array(1, c(2,3))) + + z = x <= y + expect_equal(as.array(z), array(1, c(2,3))) + + z = y <= 0 + expect_equal(as.array(z), array(0, c(2,3))) + + z = 0 <= y + expect_equal(as.array(z), array(1, c(2,3))) + + z = y <= 1 + expect_equal(as.array(z), array(1, c(2,3))) }) \ No newline at end of file From 44b8c07ba5f224b9155015eb298cd6f318eaf833 Mon Sep 17 00:00:00 2001 From: Yanbo Liang Date: Mon, 10 Jul 2017 14:05:39 +0800 Subject: [PATCH 161/834] Fix broken link in example/svm_mnist/README.md. (#6972) --- example/svm_mnist/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/example/svm_mnist/README.md b/example/svm_mnist/README.md index 082c2053f27e..408f5108b44a 100644 --- a/example/svm_mnist/README.md +++ b/example/svm_mnist/README.md @@ -1,6 +1,6 @@ # Use case with Support Vector Machine -To ensure that not only the implementation is learning, but is able to outsmart the softmax, as [this article](arxiv.org/pdf/1306.0239.pdf) suggests, I ran svm_mnist.py script. It was based on the MNIST experiment description on the article and [this tutorial](https://github.com/dmlc/mxnet-gtc-tutorial/blob/master/tutorial.ipynb). +To ensure that not only the implementation is learning, but is able to outsmart the softmax, as [this article](https://arxiv.org/pdf/1306.0239.pdf) suggests, I ran svm_mnist.py script. It was based on the MNIST experiment description on the article and [this tutorial](https://github.com/dmlc/mxnet-gtc-tutorial/blob/master/tutorial.ipynb). ## To this you will need @@ -8,4 +8,4 @@ To ensure that not only the implementation is learning, but is able to outsmart * [Numpy](http://www.scipy.org/scipylib/download.html) * [Sklearn](http://scikit-learn.org/stable/install.html) -I recommend installing [matplot](http://matplotlib.org/users/installing.html) to visualize examples \ No newline at end of file +I recommend installing [matplot](http://matplotlib.org/users/installing.html) to visualize examples From f205ffd6041e277750a6a0713ef585c9aae9efb5 Mon Sep 17 00:00:00 2001 From: Hu Shiwen Date: Mon, 10 Jul 2017 14:16:05 +0800 Subject: [PATCH 162/834] ctc example (#6949) --- example/ctc/README.md | 116 ++++++++++++++++++ example/ctc/lstm.py | 89 ++++++++++++++ example/ctc/lstm_ocr.py | 237 +++++++++++++++++++++++++++++++++++++ example/ctc/ocr_predict.py | 88 ++++++++++++++ 4 files changed, 530 insertions(+) create mode 100644 example/ctc/README.md create mode 100644 example/ctc/lstm.py create mode 100644 example/ctc/lstm_ocr.py create mode 100644 example/ctc/ocr_predict.py diff --git a/example/ctc/README.md b/example/ctc/README.md new file mode 100644 index 000000000000..e3a4d8c0857f --- /dev/null +++ b/example/ctc/README.md @@ -0,0 +1,116 @@ +# CTC with Mxnet +this is mx.contrib.sym.ctc_loss example. It was modified from example [warpctc](https://github.com/dmlc/mxnet/tree/master/example/warpctc) + +# Core code +this is core change in lstm.py +```Cython +def lstm_unroll(num_lstm_layer, seq_len, + num_hidden, num_label): + param_cells = [] + last_states = [] + for i in range(num_lstm_layer): + param_cells.append(LSTMParam(i2h_weight=mx.sym.Variable("l%d_i2h_weight" % i), + i2h_bias=mx.sym.Variable("l%d_i2h_bias" % i), + h2h_weight=mx.sym.Variable("l%d_h2h_weight" % i), + h2h_bias=mx.sym.Variable("l%d_h2h_bias" % i))) + state = LSTMState(c=mx.sym.Variable("l%d_init_c" % i), + h=mx.sym.Variable("l%d_init_h" % i)) + last_states.append(state) + assert (len(last_states) == num_lstm_layer) + + # embeding layer + data = mx.sym.Variable('data') + label = mx.sym.Variable('label') + wordvec = mx.sym.SliceChannel(data=data, num_outputs=seq_len, squeeze_axis=1) + + hidden_all = [] + for seqidx in range(seq_len): + hidden = wordvec[seqidx] + for i in range(num_lstm_layer): + next_state = lstm(num_hidden, indata=hidden, + prev_state=last_states[i], + param=param_cells[i], + seqidx=seqidx, layeridx=i) + hidden = next_state.h + last_states[i] = next_state + hidden_all.append(hidden) + + hidden_concat = mx.sym.Concat(*hidden_all, dim=0) + + pred_fc = mx.sym.FullyConnected(data=hidden_concat, num_hidden=11) + pred_ctc = mx.sym.Reshape(data=pred_fc, shape=(-4, seq_len, -1, 0)) + + loss = mx.contrib.sym.ctc_loss(data=pred_ctc, label=label) + ctc_loss = mx.sym.MakeLoss(loss) + + softmax_class = mx.symbol.SoftmaxActivation(data=pred_fc) + softmax_loss = mx.sym.MakeLoss(softmax_class) + softmax_loss = mx.sym.BlockGrad(softmax_loss) + + return mx.sym.Group([softmax_loss, ctc_loss]) +``` +# Some Result +If there were more training, the result would be better + +``` +2017-07-08 13:22:01,155 Epoch[94] Batch [50] Speed: 4273.43 samples/sec Accuracy=0.808747 +2017-07-08 13:22:13,141 Epoch[94] Batch [100] Speed: 4271.84 samples/sec Accuracy=0.786855 +2017-07-08 13:22:25,179 Epoch[94] Batch [150] Speed: 4253.81 samples/sec Accuracy=0.810625 +2017-07-08 13:22:37,198 Epoch[94] Batch [200] Speed: 4259.96 samples/sec Accuracy=0.808809 +2017-07-08 13:22:49,233 Epoch[94] Batch [250] Speed: 4254.13 samples/sec Accuracy=0.806426 +2017-07-08 13:23:01,308 Epoch[94] Batch [300] Speed: 4239.98 samples/sec Accuracy=0.817305 +2017-07-08 13:23:02,030 Epoch[94] Train-Accuracy=0.819336 +2017-07-08 13:23:02,030 Epoch[94] Time cost=73.092 +2017-07-08 13:23:02,101 Saved checkpoint to "ocr-0095.params" +2017-07-08 13:23:07,192 Epoch[94] Validation-Accuracy=0.819417 +2017-07-08 13:23:20,579 Epoch[95] Batch [50] Speed: 4288.76 samples/sec Accuracy=0.817459 +2017-07-08 13:23:32,573 Epoch[95] Batch [100] Speed: 4268.75 samples/sec Accuracy=0.815215 +2017-07-08 13:23:44,635 Epoch[95] Batch [150] Speed: 4244.85 samples/sec Accuracy=0.820215 +2017-07-08 13:23:56,670 Epoch[95] Batch [200] Speed: 4254.38 samples/sec Accuracy=0.823613 +2017-07-08 13:24:08,650 Epoch[95] Batch [250] Speed: 4273.83 samples/sec Accuracy=0.827109 +2017-07-08 13:24:20,680 Epoch[95] Batch [300] Speed: 4256.49 samples/sec Accuracy=0.824961 +2017-07-08 13:24:21,401 Epoch[95] Train-Accuracy=0.840495 +2017-07-08 13:24:21,401 Epoch[95] Time cost=73.008 +2017-07-08 13:24:21,441 Saved checkpoint to "ocr-0096.params" +2017-07-08 13:24:26,508 Epoch[95] Validation-Accuracy=0.834798 +2017-07-08 13:24:39,938 Epoch[96] Batch [50] Speed: 4259.32 samples/sec Accuracy=0.825578 +2017-07-08 13:24:51,987 Epoch[96] Batch [100] Speed: 4249.67 samples/sec Accuracy=0.826562 +2017-07-08 13:25:04,041 Epoch[96] Batch [150] Speed: 4247.44 samples/sec Accuracy=0.831855 +2017-07-08 13:25:16,058 Epoch[96] Batch [200] Speed: 4260.77 samples/sec Accuracy=0.830840 +2017-07-08 13:25:28,109 Epoch[96] Batch [250] Speed: 4248.44 samples/sec Accuracy=0.827168 +2017-07-08 13:25:40,057 Epoch[96] Batch [300] Speed: 4285.23 samples/sec Accuracy=0.832715 +2017-07-08 13:25:40,782 Epoch[96] Train-Accuracy=0.830729 +2017-07-08 13:25:40,782 Epoch[96] Time cost=73.098 +2017-07-08 13:25:40,821 Saved checkpoint to "ocr-0097.params" +2017-07-08 13:25:45,886 Epoch[96] Validation-Accuracy=0.840820 +2017-07-08 13:25:59,283 Epoch[97] Batch [50] Speed: 4271.85 samples/sec Accuracy=0.831648 +2017-07-08 13:26:11,243 Epoch[97] Batch [100] Speed: 4280.89 samples/sec Accuracy=0.835371 +2017-07-08 13:26:23,263 Epoch[97] Batch [150] Speed: 4259.89 samples/sec Accuracy=0.831094 +2017-07-08 13:26:35,230 Epoch[97] Batch [200] Speed: 4278.40 samples/sec Accuracy=0.827129 +2017-07-08 13:26:47,199 Epoch[97] Batch [250] Speed: 4277.77 samples/sec Accuracy=0.834258 +2017-07-08 13:26:59,257 Epoch[97] Batch [300] Speed: 4245.93 samples/sec Accuracy=0.833770 +2017-07-08 13:26:59,971 Epoch[97] Train-Accuracy=0.844727 +2017-07-08 13:26:59,971 Epoch[97] Time cost=72.908 +2017-07-08 13:27:00,020 Saved checkpoint to "ocr-0098.params" +2017-07-08 13:27:05,130 Epoch[97] Validation-Accuracy=0.827962 +2017-07-08 13:27:18,521 Epoch[98] Batch [50] Speed: 4281.06 samples/sec Accuracy=0.834118 +2017-07-08 13:27:30,537 Epoch[98] Batch [100] Speed: 4261.20 samples/sec Accuracy=0.835352 +2017-07-08 13:27:42,542 Epoch[98] Batch [150] Speed: 4264.88 samples/sec Accuracy=0.839395 +2017-07-08 13:27:54,544 Epoch[98] Batch [200] Speed: 4266.31 samples/sec Accuracy=0.836328 +2017-07-08 13:28:06,550 Epoch[98] Batch [250] Speed: 4264.50 samples/sec Accuracy=0.841465 +2017-07-08 13:28:18,622 Epoch[98] Batch [300] Speed: 4241.11 samples/sec Accuracy=0.831680 +2017-07-08 13:28:19,349 Epoch[98] Train-Accuracy=0.833984 +2017-07-08 13:28:19,349 Epoch[98] Time cost=73.018 +2017-07-08 13:28:19,393 Saved checkpoint to "ocr-0099.params" +2017-07-08 13:28:24,472 Epoch[98] Validation-Accuracy=0.818034 +2017-07-08 13:28:37,961 Epoch[99] Batch [50] Speed: 4242.14 samples/sec Accuracy=0.835861 +2017-07-08 13:28:50,031 Epoch[99] Batch [100] Speed: 4241.94 samples/sec Accuracy=0.846543 +2017-07-08 13:29:02,108 Epoch[99] Batch [150] Speed: 4239.22 samples/sec Accuracy=0.850645 +2017-07-08 13:29:14,160 Epoch[99] Batch [200] Speed: 4248.34 samples/sec Accuracy=0.844141 +2017-07-08 13:29:26,225 Epoch[99] Batch [250] Speed: 4243.71 samples/sec Accuracy=0.842129 +2017-07-08 13:29:38,277 Epoch[99] Batch [300] Speed: 4248.07 samples/sec Accuracy=0.851250 +2017-07-08 13:29:38,975 Epoch[99] Train-Accuracy=0.854492 +2017-07-08 13:29:38,976 Epoch[99] Time cost=73.315 +2017-07-08 13:29:39,023 Saved checkpoint to "ocr-0100.params" +2017-07-08 13:29:44,110 Epoch[99] Validation-Accuracy=0.851969 +``` diff --git a/example/ctc/lstm.py b/example/ctc/lstm.py new file mode 100644 index 000000000000..9c493bbfb500 --- /dev/null +++ b/example/ctc/lstm.py @@ -0,0 +1,89 @@ +# pylint:skip-file +import sys + +from mxnet.symbol_doc import SymbolDoc + +sys.path.insert(0, "../../python") +import mxnet as mx +import numpy as np +from collections import namedtuple +import time +import math + +LSTMState = namedtuple("LSTMState", ["c", "h"]) +LSTMParam = namedtuple("LSTMParam", ["i2h_weight", "i2h_bias", + "h2h_weight", "h2h_bias"]) +LSTMModel = namedtuple("LSTMModel", ["rnn_exec", "symbol", + "init_states", "last_states", + "seq_data", "seq_labels", "seq_outputs", + "param_blocks"]) + + +def lstm(num_hidden, indata, prev_state, param, seqidx, layeridx): + """LSTM Cell symbol""" + i2h = mx.sym.FullyConnected(data=indata, + weight=param.i2h_weight, + bias=param.i2h_bias, + num_hidden=num_hidden * 4, + name="t%d_l%d_i2h" % (seqidx, layeridx)) + h2h = mx.sym.FullyConnected(data=prev_state.h, + weight=param.h2h_weight, + bias=param.h2h_bias, + num_hidden=num_hidden * 4, + name="t%d_l%d_h2h" % (seqidx, layeridx)) + gates = i2h + h2h + slice_gates = mx.sym.SliceChannel(gates, num_outputs=4, + name="t%d_l%d_slice" % (seqidx, layeridx)) + in_gate = mx.sym.Activation(slice_gates[0], act_type="sigmoid") + in_transform = mx.sym.Activation(slice_gates[1], act_type="tanh") + forget_gate = mx.sym.Activation(slice_gates[2], act_type="sigmoid") + out_gate = mx.sym.Activation(slice_gates[3], act_type="sigmoid") + next_c = (forget_gate * prev_state.c) + (in_gate * in_transform) + next_h = out_gate * mx.sym.Activation(next_c, act_type="tanh") + return LSTMState(c=next_c, h=next_h) + + +def lstm_unroll(num_lstm_layer, seq_len, + num_hidden, num_label): + param_cells = [] + last_states = [] + for i in range(num_lstm_layer): + param_cells.append(LSTMParam(i2h_weight=mx.sym.Variable("l%d_i2h_weight" % i), + i2h_bias=mx.sym.Variable("l%d_i2h_bias" % i), + h2h_weight=mx.sym.Variable("l%d_h2h_weight" % i), + h2h_bias=mx.sym.Variable("l%d_h2h_bias" % i))) + state = LSTMState(c=mx.sym.Variable("l%d_init_c" % i), + h=mx.sym.Variable("l%d_init_h" % i)) + last_states.append(state) + assert (len(last_states) == num_lstm_layer) + + # embeding layer + data = mx.sym.Variable('data') + label = mx.sym.Variable('label') + wordvec = mx.sym.SliceChannel(data=data, num_outputs=seq_len, squeeze_axis=1) + + hidden_all = [] + for seqidx in range(seq_len): + hidden = wordvec[seqidx] + for i in range(num_lstm_layer): + next_state = lstm(num_hidden, indata=hidden, + prev_state=last_states[i], + param=param_cells[i], + seqidx=seqidx, layeridx=i) + hidden = next_state.h + last_states[i] = next_state + hidden_all.append(hidden) + + hidden_concat = mx.sym.Concat(*hidden_all, dim=0) + + pred_fc = mx.sym.FullyConnected(data=hidden_concat, num_hidden=11) + pred_ctc = mx.sym.Reshape(data=pred_fc, shape=(-4, seq_len, -1, 0)) + + loss = mx.contrib.sym.ctc_loss(data=pred_ctc, label=label) + ctc_loss = mx.sym.MakeLoss(loss) + + softmax_class = mx.symbol.SoftmaxActivation(data=pred_fc) + softmax_loss = mx.sym.MakeLoss(softmax_class) + softmax_loss = mx.sym.BlockGrad(softmax_loss) + + return mx.sym.Group([softmax_loss, ctc_loss]) diff --git a/example/ctc/lstm_ocr.py b/example/ctc/lstm_ocr.py new file mode 100644 index 000000000000..7d437bfdc424 --- /dev/null +++ b/example/ctc/lstm_ocr.py @@ -0,0 +1,237 @@ +# pylint: disable=C0111,too-many-arguments,too-many-instance-attributes,too-many-locals,redefined-outer-name,fixme +# pylint: disable=superfluous-parens, no-member, invalid-name +from __future__ import print_function +import sys, random +sys.path.insert(0, "../../python") +import numpy as np +import mxnet as mx + +from lstm import lstm_unroll + +from captcha.image import ImageCaptcha +import cv2, random + + +class SimpleBatch(object): + def __init__(self, data_names, data, label_names, label): + self.data = data + self.label = label + self.data_names = data_names + self.label_names = label_names + + self.pad = 0 + self.index = None # TODO: what is index? + + @property + def provide_data(self): + return [(n, x.shape) for n, x in zip(self.data_names, self.data)] + + @property + def provide_label(self): + return [(n, x.shape) for n, x in zip(self.label_names, self.label)] + + +def gen_rand(): + buf = "" + max_len = random.randint(3, 4) + for i in range(max_len): + buf += str(random.randint(0, 9)) + return buf + + +def get_label(buf): + ret = np.zeros(4) + for i in range(len(buf)): + ret[i] = 1 + int(buf[i]) + if len(buf) == 3: + ret[3] = 0 + return ret + + +class OCRIter(mx.io.DataIter): + def __init__(self, count, batch_size, num_label, init_states): + super(OCRIter, self).__init__() + global SEQ_LENGTH + # you can get this font from http://font.ubuntu.com/ + self.captcha = ImageCaptcha(fonts=['./data/Xerox.ttf']) + self.batch_size = batch_size + self.count = count + self.num_label = num_label + self.init_states = init_states + self.init_state_arrays = [mx.nd.zeros(x[1]) for x in init_states] + self.provide_data = [('data', (batch_size, 80, 30))] + init_states + self.provide_label = [('label', (self.batch_size, 4))] + self.cache_data = [] + self.cache_label = [] + + def __iter__(self): + print('iter') + init_state_names = [x[0] for x in self.init_states] + for k in range(self.count): + data = [] + label = [] + for i in range(self.batch_size): + num = gen_rand() + img = self.captcha.generate(num) + img = np.fromstring(img.getvalue(), dtype='uint8') + img = cv2.imdecode(img, cv2.IMREAD_GRAYSCALE) + img = cv2.resize(img, (80, 30)) + img = img.transpose(1, 0) + img = img.reshape((80, 30)) + img = np.multiply(img, 1 / 255.0) + data.append(img) + label.append(get_label(num)) + + data_all = [mx.nd.array(data)] + self.init_state_arrays + label_all = [mx.nd.array(label)] + data_names = ['data'] + init_state_names + label_names = ['label'] + + data_batch = SimpleBatch(data_names, data_all, label_names, label_all) + yield data_batch + + def reset(self): + self.cache_data.clear() + self.cache_label.clear() + pass + + +BATCH_SIZE = 1024 +SEQ_LENGTH = 80 + + +def ctc_label(p): + ret = [] + p1 = [0] + p + for i in range(len(p)): + c1 = p1[i] + c2 = p1[i + 1] + if c2 == 0 or c2 == c1: + continue + ret.append(c2) + return ret + + +def remove_blank(l): + ret = [] + for i in range(len(l)): + if l[i] == 0: + break + ret.append(l[i]) + return ret + + +def Accuracy(label, pred): + global BATCH_SIZE + global SEQ_LENGTH + hit = 0. + total = 0. + rp = np.argmax(pred, axis=1) + for i in range(BATCH_SIZE): + l = remove_blank(label[i]) + p = [] + for k in range(SEQ_LENGTH): + p.append(np.argmax(pred[k * BATCH_SIZE + i])) + p = ctc_label(p) + if len(p) == len(l): + match = True + for k in range(len(p)): + if p[k] != int(l[k]): + match = False + break + if match: + hit += 1.0 + total += 1.0 + return hit / total + + +def LCS(p, l): + # Dynamic Programming Finding LCS + if len(p) == 0: + return 0 + P = np.array(list(p)).reshape((1, len(p))) + L = np.array(list(l)).reshape((len(l), 1)) + M = np.int32(P == L) + for i in range(M.shape[0]): + for j in range(M.shape[1]): + up = 0 if i == 0 else M[i - 1, j] + left = 0 if j == 0 else M[i, j - 1] + M[i, j] = max(up, left, M[i, j] if (i == 0 or j == 0) else M[i, j] + M[i - 1, j - 1]) + return M.max() + + +def Accuracy_LCS(label, pred): + global BATCH_SIZE + global SEQ_LENGTH + hit = 0. + total = 0. + for i in range(BATCH_SIZE): + l = remove_blank(label[i]) + p = [] + for k in range(SEQ_LENGTH): + p.append(np.argmax(pred[k * BATCH_SIZE + i])) + p = ctc_label(p) + hit += LCS(p, l) * 1.0 / len(l) + total += 1.0 + return hit / total + + +def asum_stat(x): + """returns |x|/size(x), async execution.""" + # npx = x.asnumpy() + # print(npx) + return x + return mx.ndarray.norm(x) / np.sqrt(x.size) + + +if __name__ == '__main__': + num_hidden = 100 + num_lstm_layer = 2 + + num_epoch = 100 + learning_rate = 0.01 + momentum = 0.9 + num_label = 4 + + contexts = [mx.context.gpu(0)] + + + def sym_gen(seq_len): + return lstm_unroll(num_lstm_layer, seq_len, + num_hidden=num_hidden, + num_label=num_label) + + + init_c = [('l%d_init_c' % l, (BATCH_SIZE, num_hidden)) for l in range(num_lstm_layer)] + init_h = [('l%d_init_h' % l, (BATCH_SIZE, num_hidden)) for l in range(num_lstm_layer)] + init_states = init_c + init_h + + data_train = OCRIter(20000, BATCH_SIZE, num_label, init_states) + data_val = OCRIter(1000, BATCH_SIZE, num_label, init_states) + + symbol = sym_gen(SEQ_LENGTH) + + import logging + + head = '%(asctime)-15s %(message)s' + logging.basicConfig(level=logging.DEBUG, format=head) + + print('begin fit') + + module = mx.mod.Module(symbol, data_names=['data', 'l0_init_c', 'l0_init_h', 'l1_init_c', 'l1_init_h'], + label_names=['label'], + context=contexts) + + module.fit(train_data=data_train, + eval_data=data_val, + eval_metric=mx.metric.np(Accuracy, allow_extra_outputs=True), + optimizer='sgd', + optimizer_params={'learning_rate': learning_rate, + 'momentum': momentum, + 'wd': 0.00001, + }, + initializer=mx.init.Xavier(factor_type="in", magnitude=2.34), + num_epoch=num_epoch, + batch_end_callback=mx.callback.Speedometer(BATCH_SIZE, 50), + epoch_end_callback=mx.callback.do_checkpoint("ocr"), + ) diff --git a/example/ctc/ocr_predict.py b/example/ctc/ocr_predict.py new file mode 100644 index 000000000000..a07733ef55e0 --- /dev/null +++ b/example/ctc/ocr_predict.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python2.7 +# coding=utf-8 +from __future__ import print_function +import sys, os +curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__))) +sys.path.append("../../amalgamation/python/") +sys.path.append("../../python/") + +from mxnet_predict import Predictor +import mxnet as mx + +import numpy as np +import cv2 +import os + +class lstm_ocr_model(object): + # Keep Zero index for blank. (CTC request it) + CONST_CHAR='0123456789' + def __init__(self, path_of_json, path_of_params): + super(lstm_ocr_model, self).__init__() + self.path_of_json = path_of_json + self.path_of_params = path_of_params + self.predictor = None + self.__init_ocr() + + def __init_ocr(self): + num_label = 4 # Set your max length of label, add one more for blank + batch_size = 1 + + num_hidden = 100 + num_lstm_layer = 2 + init_c = [('l%d_init_c'%l, (batch_size, num_hidden)) for l in range(num_lstm_layer)] + init_h = [('l%d_init_h'%l, (batch_size, num_hidden)) for l in range(num_lstm_layer)] + init_states = init_c + init_h + + init_state_arrays = np.zeros((batch_size, num_hidden), dtype="float32") + self.init_state_dict={} + for x in init_states: + self.init_state_dict[x[0]] = init_state_arrays + + all_shapes = [('data', (batch_size, 80 * 30))] + init_states + [('label', (batch_size, num_label))] + all_shapes_dict = {} + for _shape in all_shapes: + all_shapes_dict[_shape[0]] = _shape[1] + self.predictor = Predictor(open(self.path_of_json).read(), + open(self.path_of_params).read(), + all_shapes_dict) + + def forward_ocr(self, img): + img = cv2.resize(img, (80, 30)) + img = img.transpose(1, 0) + img = img.reshape((80 * 30)) + img = np.multiply(img, 1/255.0) + self.predictor.forward(data=img, **self.init_state_dict) + prob = self.predictor.get_output(0) + label_list = [] + for p in prob: + max_index = np.argsort(p)[::-1][0] + label_list.append(max_index) + return self.__get_string(label_list) + + def __get_string(self, label_list): + # Do CTC label rule + # CTC cannot emit a repeated symbol on consecutive timesteps + ret = [] + label_list2 = [0] + list(label_list) + for i in range(len(label_list)): + c1 = label_list2[i] + c2 = label_list2[i+1] + if c2 == 0 or c2 == c1: + continue + ret.append(c2) + # change to ascii + s = '' + for l in ret: + if l > 0 and l < (len(lstm_ocr_model.CONST_CHAR)+1): + c = lstm_ocr_model.CONST_CHAR[l-1] + else: + c = '' + s += c + return s + +if __name__ == '__main__': + _lstm_ocr_model = lstm_ocr_model('ocr-symbol.json', 'ocr-0010.params') + img = cv2.imread('sample.jpg', 0) + _str = _lstm_ocr_model.forward_ocr(img) + print('Result: ', _str) + From 9aa91e09f3800c22d0f0831b5344968bb3406df3 Mon Sep 17 00:00:00 2001 From: Madan Jampani Date: Mon, 10 Jul 2017 11:50:47 -0700 Subject: [PATCH 163/834] Mention mxnet version in tutorial prereq (#6983) --- docs/tutorials/python/mnist.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorials/python/mnist.md b/docs/tutorials/python/mnist.md index c8789e2bd6dc..4fdf372964a1 100644 --- a/docs/tutorials/python/mnist.md +++ b/docs/tutorials/python/mnist.md @@ -11,7 +11,7 @@ MNIST is a widely used dataset for the hand-written digit classification task. I ## Prerequisites To complete this tutorial, we need: -- MXNet. See the instructions for your operating system in [Setup and Installation](http://mxnet.io/get_started/install.html). +- MXNet version 0.10 or later. See the installation instructions for your operating system in [Setup and Installation](http://mxnet.io/get_started/install.html). - [Python Requests](http://docs.python-requests.org/en/master/) and [Jupyter Notebook](http://jupyter.org/index.html). From f5cdd4220222c1c918d185978654d2fa73011f4a Mon Sep 17 00:00:00 2001 From: Indhu Bharathi Date: Mon, 10 Jul 2017 12:26:07 -0700 Subject: [PATCH 164/834] Fix linear regression tutorial (#6978) * Fix linear regression tutorial - Print training logs in notebook. - Switch from accuracy to mse for validation metrics. Accuracy doesn't make much sense for regression. - Lower the number of epochs to 50. Number of epochs in unnecessarily huge (1000) consuming too much time. Training converges in around 30 epochs. * Fix indentation. --- docs/tutorials/python/linear-regression.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/tutorials/python/linear-regression.md b/docs/tutorials/python/linear-regression.md index b904792b8509..c26435dec6cc 100644 --- a/docs/tutorials/python/linear-regression.md +++ b/docs/tutorials/python/linear-regression.md @@ -21,6 +21,9 @@ To begin, the following code imports the necessary packages we'll need for this ```python import mxnet as mx import numpy as np + +import logging +logging.getLogger().setLevel(logging.DEBUG) ``` ## Preparing the Data @@ -153,7 +156,8 @@ parameters of the model to fit the training data. This is accomplished using the ```python model.fit(train_iter, eval_iter, optimizer_params={'learning_rate':0.005, 'momentum': 0.9}, - num_epoch=1000, + num_epoch=50, + eval_metric='mse', batch_end_callback = mx.callback.Speedometer(batch_size, 2)) ``` From c5b78ae1fce2ad6a0d086fcf66e8fb24e1416d58 Mon Sep 17 00:00:00 2001 From: lxn2 Date: Mon, 10 Jul 2017 12:36:17 -0700 Subject: [PATCH 165/834] Change slave labels and add nightly Jenkinsfile (#6957) * Empty commit * Change slave labels and add nightly Jenkinsfile --- Jenkinsfile | 42 +++++++++++++++++++-------------------- tests/nightly/Jenkinsfile | 10 ++++++++++ 2 files changed, 31 insertions(+), 21 deletions(-) create mode 100644 tests/nightly/Jenkinsfile diff --git a/Jenkinsfile b/Jenkinsfile index 881fef529f11..48f6251a0be2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -40,7 +40,7 @@ def init_git_win() { stage("Sanity Check") { timeout(time: max_time, unit: 'MINUTES') { - node('linux') { + node('mxnetlinux') { ws('workspace/sanity') { init_git() make('lint', 'cpplint rcpplint jnilint') @@ -86,7 +86,7 @@ echo ${libs} | sed -e 's/,/ /g' | xargs md5sum stage('Build') { parallel 'CPU: Openblas': { - node('linux') { + node('mxnetlinux') { ws('workspace/build-cpu') { init_git() def flag = """ \ @@ -102,7 +102,7 @@ USE_BLAS=openblas \ } }, 'GPU: CUDA7.5+cuDNN5': { - node('GPU' && 'linux') { + node('mxnetlinux') { ws('workspace/build-gpu') { init_git() def flag = """ \ @@ -122,7 +122,7 @@ USE_CPP_PACKAGE=1 \ } }, 'Amalgamation': { - node('linux') { + node('mxnetlinux') { ws('workspace/amalgamation') { init_git() make('cpu', '-C amalgamation/ USE_BLAS=openblas MIN=1') @@ -130,7 +130,7 @@ USE_CPP_PACKAGE=1 \ } }, 'GPU: MKLML': { - node('GPU' && 'linux') { + node('mxnetlinux') { ws('workspace/build-mklml') { init_git() def flag = """ \ @@ -151,7 +151,7 @@ USE_CPP_PACKAGE=1 \ } }, 'CPU windows':{ - node('windows') { + node('mxnetwindows') { ws('workspace/build-cpu') { withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas', 'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0']) { init_git_win() @@ -181,7 +181,7 @@ del /Q *.7z } }, 'GPU windows':{ - node('windows') { + node('mxnetwindows') { ws('workspace/build-gpu') { withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas', 'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0']) { init_git_win() @@ -232,7 +232,7 @@ def python_gpu_ut(docker_type) { stage('Unit Test') { parallel 'Python2/3: CPU': { - node('linux') { + node('mxnetlinux') { ws('workspace/ut-python-cpu') { init_git() unpack_lib('cpu') @@ -241,7 +241,7 @@ stage('Unit Test') { } }, 'Python2/3: GPU': { - node('GPU' && 'linux') { + node('mxnetlinux') { ws('workspace/ut-python-gpu') { init_git() unpack_lib('gpu', mx_lib) @@ -250,7 +250,7 @@ stage('Unit Test') { } }, 'Python2/3: MKLML': { - node('GPU' && 'linux') { + node('mxnetlinux') { ws('workspace/ut-python-mklml') { init_git() unpack_lib('mklml') @@ -260,7 +260,7 @@ stage('Unit Test') { } }, 'Scala: CPU': { - node('linux') { + node('mxnetlinux') { ws('workspace/ut-scala-cpu') { init_git() unpack_lib('cpu') @@ -272,7 +272,7 @@ stage('Unit Test') { } }, 'R: CPU': { - node('linux') { + node('mxnetlinux') { ws('workspace/ut-r-cpu') { init_git() unpack_lib('cpu') @@ -287,7 +287,7 @@ stage('Unit Test') { } }, 'R: GPU': { - node('GPU' && 'linux') { + node('mxnetlinux') { ws('workspace/ut-r-gpu') { init_git() unpack_lib('gpu') @@ -300,9 +300,9 @@ stage('Unit Test') { } } } - }, + }, 'Python2/3: CPU Win':{ - node('windows') { + node('mxnetwindows') { ws('workspace/ut-python-cpu') { init_git_win() unstash 'vc14_cpu' @@ -322,7 +322,7 @@ C:\\mxnet\\test_cpu.bat""" } }, 'Python2/3: GPU Win':{ - node('windows') { + node('mxnetwindows') { ws('workspace/ut-python-gpu') { init_git_win() unstash 'vc14_gpu' @@ -346,7 +346,7 @@ C:\\mxnet\\test_gpu.bat""" stage('Integration Test') { parallel 'Python': { - node('GPU' && 'linux') { + node('mxnetlinux') { ws('workspace/it-python-gpu') { init_git() unpack_lib('gpu') @@ -357,7 +357,7 @@ stage('Integration Test') { } }, 'Caffe': { - node('GPU' && 'linux') { + node('mxnetlinux') { ws('workspace/it-caffe') { init_git() unpack_lib('gpu') @@ -368,7 +368,7 @@ stage('Integration Test') { } }, 'cpp-package': { - node('GPU' && 'linux') { + node('mxnetlinux') { ws('workspace/it-cpp-package') { init_git() unpack_lib('gpu') @@ -382,7 +382,7 @@ stage('Integration Test') { } stage('Deploy') { - node('linux') { + node('mxnetlinux') { ws('workspace/docs') { if (env.BRANCH_NAME == "master") { init_git() @@ -390,4 +390,4 @@ stage('Deploy') { } } } -} +} \ No newline at end of file diff --git a/tests/nightly/Jenkinsfile b/tests/nightly/Jenkinsfile new file mode 100644 index 000000000000..443c811e2709 --- /dev/null +++ b/tests/nightly/Jenkinsfile @@ -0,0 +1,10 @@ +// -*- mode: groovy -*- +// Jenkins pipeline +// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ +// Runs nightly builds + +stage("Hello World") { + node('mxnetlinux') { + sh "echo 'Hello World'" + } +} \ No newline at end of file From 3a50dcadfdb3fedfc94518460eaa983c42b1752c Mon Sep 17 00:00:00 2001 From: Przemyslaw Tredak Date: Mon, 10 Jul 2017 13:49:08 -0700 Subject: [PATCH 166/834] Fix im2col.h (#6986) --- src/operator/nn/im2col.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/operator/nn/im2col.h b/src/operator/nn/im2col.h index 435d502da77e..ce4d9e31db9c 100644 --- a/src/operator/nn/im2col.h +++ b/src/operator/nn/im2col.h @@ -241,7 +241,7 @@ inline void im2col(mshadow::Stream* s, if (2 == kernel_shape.ndim()) { im2col_cpu(data_im, im_shape[1], im_shape[2], im_shape[3], kernel_shape[0], kernel_shape[1], pad[0], pad[1], - stride[0], stride[1], dilation[1], dilation[1], data_col); + stride[0], stride[1], dilation[0], dilation[1], data_col); } else { im2col_nd_core_cpu(data_im, true, im_shape, col_shape, kernel_shape, pad, stride, dilation, data_col); From b1cb2cd118bca41755274ab2dbc99756669baf4a Mon Sep 17 00:00:00 2001 From: "Qiang Kou (KK)" Date: Tue, 11 Jul 2017 10:19:22 -0700 Subject: [PATCH 167/834] [R] custom iter in model training; MF demo in R (#6673) --- R-package/R/model.R | 96 +++++++++++++++++++-------- R-package/tests/testthat/get_data.R | 11 +++ R-package/tests/testthat/test_model.R | 75 +++++++++++++++++++++ example/recommenders/demo-MF.R | 67 +++++++++++++++++++ 4 files changed, 220 insertions(+), 29 deletions(-) create mode 100644 example/recommenders/demo-MF.R diff --git a/R-package/R/model.R b/R-package/R/model.R index 0f6af3dd61cb..f76e51c4c633 100644 --- a/R-package/R/model.R +++ b/R-package/R/model.R @@ -1,15 +1,31 @@ # slice the shape on the highest dimension mx.model.slice.shape <- function(shape, nsplit) { - ndim <- length(shape) - batchsize <- shape[[ndim]] - step <- as.integer((batchsize + nsplit - 1) / nsplit) - lapply(0:(nsplit - 1), function(k) { - begin = min(k * step, batchsize) - end = min((k + 1) * step, batchsize) - s <- shape - s[[ndim]] = end - begin - return(list(begin=begin, end=end, shape=s)) - }) + if (is.numeric(shape)) { + ndim <- length(shape) + batchsize <- shape[[ndim]] + step <- as.integer((batchsize + nsplit - 1) / nsplit) + lapply(0:(nsplit - 1), function(k) { + begin = min(k * step, batchsize) + end = min((k + 1) * step, batchsize) + s <- shape + s[[ndim]] = end - begin + return(list(begin=begin, end=end, shape=s)) + }) + } else if (is.list(shape)) { + shape.names = names(shape) + ndim <- length(shape[[1]]) + batchsize <- shape[[1]][[ndim]] + step <- as.integer((batchsize + nsplit - 1) / nsplit) + lapply(0:(nsplit - 1), function(k) { + begin = min(k * step, batchsize) + end = min((k + 1) * step, batchsize) + s <- lapply(shape, function(s) { + s[[ndim]] = end - begin + return(s) + }) + return(list(begin=begin, end=end, shape=s)) + }) + } } # get the argument name of data and label @@ -102,12 +118,13 @@ mx.model.train <- function(symbol, ctx, input.shape, output.shape, # create the executors sliceinfo <- mx.model.slice.shape(input.shape, ndevice) sliceinfo2 <- mx.model.slice.shape(output.shape, ndevice) + arg_names <- arguments(symbol) label_name <- arg_names[endsWith(arg_names, "label")] train.execs <- lapply(1:ndevice, function(i) { - arg_lst <- list(symbol = symbol, ctx = ctx[[i]], grad.req = "write", - data = sliceinfo[[i]]$shape) - arg_lst[[label_name]] = sliceinfo2[[i]]$shape + arg_lst <- list(symbol = symbol, ctx = ctx[[i]], grad.req = "write") + arg_lst <- append(arg_lst, sliceinfo[[i]]$shape) + arg_lst <- append(arg_lst, sliceinfo2[[i]]$shape) arg_lst[["fixed.param"]] = fixed.param do.call(mx.simple.bind, arg_lst) }) @@ -135,7 +152,9 @@ mx.model.train <- function(symbol, ctx, input.shape, output.shape, kvstore$init(params.index, train.execs[[1]]$ref.arg.arrays[params.index]) } # Get the input names - input.names <- mx.model.check.arguments(symbol) + # input.names <- mx.model.check.arguments(symbol) + arg_names <- arguments(symbol) + label_name <- arg_names[endsWith(arg_names, "label")] for (iteration in begin.round:end.round) { nbatch <- 0 @@ -147,14 +166,13 @@ mx.model.train <- function(symbol, ctx, input.shape, output.shape, dlist <- train.data$value() slices <- lapply(1:ndevice, function(i) { s <- sliceinfo[[i]] - ret <- list(data=mx.nd.slice(dlist$data, s$begin, s$end), - label=mx.nd.slice(dlist$label, s$begin, s$end)) + ret <- sapply(names(dlist), function(n) {mx.nd.slice(dlist[[n]], s$begin, s$end)}) return(ret) }) # copy data to executor for (i in 1:ndevice) { s <- slices[[i]] - names(s) <- input.names + names(s)[endsWith(names(s), "label")] = label_name mx.exec.update.arg.arrays(train.execs[[i]], s, match.name=TRUE) } for (texec in train.execs) { @@ -218,13 +236,12 @@ mx.model.train <- function(symbol, ctx, input.shape, output.shape, dlist <- eval.data$value() slices <- lapply(1:ndevice, function(i) { s <- sliceinfo[[i]] - ret <- list(data=mx.nd.slice(dlist$data, s$begin, s$end), - label=mx.nd.slice(dlist$label, s$begin, s$end)) + ret <- sapply(names(dlist), function(n) {mx.nd.slice(dlist[[n]], s$begin, s$end)}) return(ret) }) for (i in 1:ndevice) { s <- slices[[i]] - names(s) <- input.names + names(s)[endsWith(names(s), "label")] = label_name mx.exec.update.arg.arrays(train.execs[[i]], s, match.name=TRUE) } for (texec in train.execs) { @@ -265,10 +282,10 @@ mx.model.train <- function(symbol, ctx, input.shape, output.shape, # Initialize parameters mx.model.init.params <- function(symbol, input.shape, output.shape, initializer, ctx) { if (!is.MXSymbol(symbol)) stop("symbol need to be MXSymbol") - arg_names <- arguments(symbol) - label_name <- arg_names[endsWith(arg_names, "label")] - arg_lst <- list(symbol = symbol, data=input.shape) - arg_lst[[label_name]] = output.shape + + arg_lst <- list(symbol = symbol) + arg_lst <- append(arg_lst, input.shape) + arg_lst <- append(arg_lst, output.shape) slist <- do.call(mx.symbol.infer.shape, arg_lst) if (is.null(slist)) stop("Not enough information to get shapes") @@ -393,6 +410,10 @@ mx.model.select.layout.predict <- function(X, model) { #' Model parameter, list of name to NDArray of net's weights. #' @param aux.params list, optional #' Model parameter, list of name to NDArray of net's auxiliary states. +#' @param input.names optional +#' The names of the input symbols. +#' @param output.names optional +#' The names of the output symbols. #' @return model A trained mxnet model. #' #' @export @@ -405,7 +426,9 @@ function(symbol, X, y=NULL, ctx=NULL, begin.round=1, epoch.end.callback=NULL, batch.end.callback=NULL, array.batch.size=128, array.layout="auto", kvstore = "local", verbose = TRUE, - arg.params = NULL, aux.params = NULL, fixed.param = NULL, + arg.params = NULL, aux.params = NULL, + input.names=NULL, output.names = NULL, + fixed.param = NULL, ...) { if (is.array(X) || is.matrix(X)) { if (array.layout == "auto") { @@ -420,8 +443,18 @@ function(symbol, X, y=NULL, ctx=NULL, begin.round=1, X$reset() if (!X$iter.next()) stop("Empty input") } - input.shape <- dim((X$value())$data) - output.shape <- dim((X$value())$label) + if (is.null(input.names)) { + input.names <- "data" + } + input.shape <- sapply(input.names, function(n){dim(X$value()[[n]])}, simplify = FALSE) + if (is.null(output.names)) { + arg_names <- arguments(symbol) + output.names <- arg_names[endsWith(arg_names, "label")] + output.shape <- list() + output.shape[[output.names]] <- dim((X$value())$label) + } else { + output.shape <- sapply(output.names, function(n){dim(X$value()[[n]])}, simplify = FALSE) + } params <- mx.model.init.params(symbol, input.shape, output.shape, initializer, mx.cpu()) if (!is.null(arg.params)) params$arg.params <- arg.params if (!is.null(aux.params)) params$aux.params <- aux.params @@ -431,8 +464,13 @@ function(symbol, X, y=NULL, ctx=NULL, begin.round=1, } if (!is.list(ctx)) stop("ctx must be mx.context or list of mx.context") if (is.character(optimizer)) { - ndim <- length(input.shape) - batchsize = input.shape[[ndim]] + if (is.numeric(input.shape)) { + ndim <- length(input.shape) + batchsize = input.shape[[ndim]] + } else { + ndim <- length(input.shape[[1]]) + batchsize = input.shape[[1]][[ndim]] + } optimizer <- mx.opt.create(optimizer, rescale.grad=(1/batchsize), ...) } if (!is.null(eval.data) && !is.list(eval.data) && !is.mx.dataiter(eval.data)) { diff --git a/R-package/tests/testthat/get_data.R b/R-package/tests/testthat/get_data.R index 021cbfeafb5f..27db31a551f5 100644 --- a/R-package/tests/testthat/get_data.R +++ b/R-package/tests/testthat/get_data.R @@ -37,3 +37,14 @@ GetCifar10 <- function() { unzip('data/cifar10.zip', exdir = 'data/') } } + +GetMovieLens <- function() { + if (!dir.exists("data")) { + dir.create("data/") + } + if (!file.exists('data/ml-100k/u.data')) { + download.file('http://files.grouplens.org/datasets/movielens/ml-100k.zip', + destfile = 'data/ml-100k.zip') + unzip('data/ml-100k.zip', exdir = 'data/') + } +} diff --git a/R-package/tests/testthat/test_model.R b/R-package/tests/testthat/test_model.R index 9b804e16537b..dec1517556ba 100644 --- a/R-package/tests/testthat/test_model.R +++ b/R-package/tests/testthat/test_model.R @@ -108,3 +108,78 @@ test_that("Classification", { eval.metric = mx.metric.accuracy) }) +test_that("Matrix Factorization", { + GetMovieLens() + DF <- read.table("./data/ml-100k/u.data", header = F, sep = "\t") + names(DF) <- c("user", "item", "score", "time") + max_user <- max(DF$user) + max_item <- max(DF$item) + DF_mat_x <- data.matrix(t(DF[, 1:2])) + DF_y <- DF[, 3] + k <- 64 + user <- mx.symbol.Variable("user") + item <- mx.symbol.Variable("item") + score <- mx.symbol.Variable("label") + user1 <- mx.symbol.Embedding(data = mx.symbol.BlockGrad(user), input_dim = max_user, + output_dim = k, name = "user1") + item1 <- mx.symbol.Embedding(data = mx.symbol.BlockGrad(item), input_dim = max_item, + output_dim = k, name = "item1" + ) + pred <- user1 * item1 + pred1 <- mx.symbol.sum_axis(pred, axis = 1, name = "pred1") + pred2 <- mx.symbol.Flatten(pred1, name = "pred2") + pred3 <- mx.symbol.LinearRegressionOutput(data = pred2, label = score, name = "pred3") + devices = lapply(1:2, function(i) { + mx.cpu(i) + }) + mx.set.seed(123) + + CustomIter <- setRefClass( "CustomIter", fields = c("iter1", "iter2"), + contains = "Rcpp_MXArrayDataIter", + methods = list( + initialize = function(iter1, iter2) { + .self$iter1 <- iter1 + .self$iter2 <- iter2 + .self + }, + value = function() { + user <- .self$iter1$value()$data + item <- .self$iter2$value()$data + label <- .self$iter1$value()$label + list(user = user, + item = item, + label = label) + }, + iter.next = function() { + .self$iter1$iter.next() + .self$iter2$iter.next() + }, + reset = function() { + .self$iter1$reset() + .self$iter2$reset() + }, + num.pad = function() { + .self$iter1$num.pad() + }, + finalize = function() { + .self$iter1$finalize() + .self$iter2$finalize() + } + ) + ) + + user_iter = mx.io.arrayiter(data = DF[, 1], label = DF[, 3], batch.size = k) + + item_iter = mx.io.arrayiter(data = DF[, 2], label = DF[, 3], batch.size = k) + + train_iter <- CustomIter$new(user_iter, item_iter) + + model <- mx.model.FeedForward.create(pred3, X = train_iter, ctx = devices, + num.round = 10, initializer = mx.init.uniform(0.07), + learning.rate = 0.07, + eval.metric = mx.metric.rmse, + momentum = 0.9, + epoch.end.callback = mx.callback.log.train.metric(1), + input.names = c("user", "item"), + output.names = "label") +}) diff --git a/example/recommenders/demo-MF.R b/example/recommenders/demo-MF.R new file mode 100644 index 000000000000..509aa04b9e11 --- /dev/null +++ b/example/recommenders/demo-MF.R @@ -0,0 +1,67 @@ +library(mxnet) +DF <- read.table("./ml-100k/u.data", header = F, sep = "\t") +names(DF) <- c("user", "item", "score", "time") +max_user <- max(DF$user) +max_item <- max(DF$item) +DF_mat_x <- data.matrix(t(DF[, 1:2])) +DF_y <- DF[, 3] +k <- 64 +user <- mx.symbol.Variable("user") +item <- mx.symbol.Variable("item") +score <- mx.symbol.Variable("label") +user1 <-mx.symbol.Embedding(data = mx.symbol.BlockGrad(user), input_dim = max_user, + output_dim = k, name = "user1") +item1 <- mx.symbol.Embedding(data = mx.symbol.BlockGrad(item), input_dim = max_item, + output_dim = k, name = "item1") +pred <- user1 * item1 +pred1 <- mx.symbol.sum_axis(pred, axis = 1, name = "pred1") +pred2 <- mx.symbol.Flatten(pred1, name = "pred2") +pred3 <- mx.symbol.LinearRegressionOutput(data = pred2, label = score, name = "pred3") +devices <- mx.cpu() +mx.set.seed(123) + +CustomIter <- setRefClass("CustomIter", fields = c("iter1", "iter2"), + contains = "Rcpp_MXArrayDataIter", + methods = list( + initialize = function(iter1, iter2) { + .self$iter1 <- iter1 + .self$iter2 <- iter2 + .self + }, + value = function() { + user <- .self$iter1$value()$data + item <- .self$iter2$value()$data + label <- .self$iter1$value()$label + list(user = user, + item = item, + label = label) + }, + iter.next = function() { + .self$iter1$iter.next() + .self$iter2$iter.next() + }, + reset = function() { + .self$iter1$reset() + .self$iter2$reset() + }, + num.pad = function() { + .self$iter1$num.pad() + }, + finalize = function() { + .self$iter1$finalize() + .self$iter2$finalize() + } + ) +) + +user_iter = mx.io.arrayiter(data = DF[, 1], label = DF[, 3], batch.size = k) + +item_iter = mx.io.arrayiter(data = DF[, 2], label = DF[, 3], batch.size = k) + +train_iter <- CustomIter$new(user_iter, item_iter) + +model <- mx.model.FeedForward.create(pred3, X = train_iter, ctx = devices, + num.round = 10, initializer = mx.init.uniform(0.07), + learning.rate = 0.07, eval.metric = mx.metric.rmse, + momentum = 0.9, epoch.end.callback = mx.callback.log.train.metric(1), + input.names = c("user", "item"), output.names = "label") From 22f9a0dc96a0ad3741c1f8db7b66f27d9de9971a Mon Sep 17 00:00:00 2001 From: Soonhwan-Kwon Date: Wed, 12 Jul 2017 02:56:24 +0900 Subject: [PATCH 168/834] add bucketing/batchnorm and improved performance for speech_recognition example (#6971) --- example/speech_recognition/README.md | 15 + example/speech_recognition/arch_deepspeech.py | 284 ++++++++++------- example/speech_recognition/deepspeech.cfg | 48 ++- example/speech_recognition/default.cfg | 50 ++- example/speech_recognition/main.py | 300 +++++++++++------- .../speech_recognition/stt_datagenerator.py | 90 ++++-- example/speech_recognition/stt_io_iter.py | 9 +- .../speech_recognition/stt_layer_batchnorm.py | 8 +- example/speech_recognition/stt_layer_conv.py | 12 +- example/speech_recognition/stt_layer_fc.py | 28 +- example/speech_recognition/stt_layer_gru.py | 70 ++-- example/speech_recognition/stt_layer_lstm.py | 82 +++-- example/speech_recognition/stt_metric.py | 11 +- example/speech_recognition/stt_utils.py | 7 +- example/speech_recognition/train.py | 74 +++-- 15 files changed, 685 insertions(+), 403 deletions(-) diff --git a/example/speech_recognition/README.md b/example/speech_recognition/README.md index 69961b1bdc5c..00d166602403 100644 --- a/example/speech_recognition/README.md +++ b/example/speech_recognition/README.md @@ -123,3 +123,18 @@ The new file should implement two functions, prepare_data() and arch(), for buil Run the following line after preparing the files. 
    python main.py --configfile custom.cfg --archfile arch_custom
    + +*** +## **Further more** +You can prepare full LibriSpeech dataset by following the instruction on https://github.com/baidu-research/ba-dls-deepspeech +**Change flac_to_wav.sh script of baidu to flac_to_wav.sh in repository to avoid bug** +```bash +git clone https://github.com/baidu-research/ba-dls-deepspeech +cd ba-dls-deepspeech +./download.sh +cp -f /path/to/example/flac_to_wav.sh ./ +./flac_to_wav.sh +python create_desc_json.py /path/to/ba-dls-deepspeech/LibriSpeech/train-clean-100 train_corpus.json +python create_desc_json.py /path/to/ba-dls-deepspeech/LibriSpeech/dev-clean validation_corpus.json +python create_desc_json.py /path/to/ba-dls-deepspeech/LibriSpeech/test-clean test_corpus.json +``` diff --git a/example/speech_recognition/arch_deepspeech.py b/example/speech_recognition/arch_deepspeech.py index 92f1002a2f01..4288b246f3e5 100644 --- a/example/speech_recognition/arch_deepspeech.py +++ b/example/speech_recognition/arch_deepspeech.py @@ -1,6 +1,12 @@ +# pylint: disable=C0111, too-many-statements, too-many-locals +# pylint: too-many-arguments,too-many-instance-attributes,too-many-locals,redefined-outer-name,fixme +# pylint: disable=superfluous-parens, no-member, invalid-name +""" +architecture file for deep speech 2 model +""" import json import math - +import argparse import mxnet as mx from stt_layer_batchnorm import batchnorm @@ -13,6 +19,9 @@ def prepare_data(args): + """ + set atual shape of data + """ rnn_type = args.config.get("arch", "rnn_type") num_rnn_layer = args.config.getint("arch", "num_rnn_layer") num_hidden_rnn_list = json.loads(args.config.get("arch", "num_hidden_rnn_list")) @@ -20,26 +29,29 @@ def prepare_data(args): batch_size = args.config.getint("common", "batch_size") if rnn_type == 'lstm': - init_c = [('l%d_init_c' % l, (batch_size, num_hidden_rnn_list[l])) for l in range(num_rnn_layer)] - init_h = [('l%d_init_h' % l, (batch_size, num_hidden_rnn_list[l])) for l in range(num_rnn_layer)] + init_c = [('l%d_init_c' % l, (batch_size, num_hidden_rnn_list[l])) + for l in range(num_rnn_layer)] + init_h = [('l%d_init_h' % l, (batch_size, num_hidden_rnn_list[l])) + for l in range(num_rnn_layer)] elif rnn_type == 'bilstm': - forward_init_c = [('forward_l%d_init_c' % l, (batch_size, num_hidden_rnn_list[l])) for l in - range(num_rnn_layer)] - backward_init_c = [('backward_l%d_init_c' % l, (batch_size, num_hidden_rnn_list[l])) for l in - range(num_rnn_layer)] + forward_init_c = [('forward_l%d_init_c' % l, (batch_size, num_hidden_rnn_list[l])) + for l in range(num_rnn_layer)] + backward_init_c = [('backward_l%d_init_c' % l, (batch_size, num_hidden_rnn_list[l])) + for l in range(num_rnn_layer)] init_c = forward_init_c + backward_init_c - forward_init_h = [('forward_l%d_init_h' % l, (batch_size, num_hidden_rnn_list[l])) for l in - range(num_rnn_layer)] - backward_init_h = [('backward_l%d_init_h' % l, (batch_size, num_hidden_rnn_list[l])) for l in - range(num_rnn_layer)] + forward_init_h = [('forward_l%d_init_h' % l, (batch_size, num_hidden_rnn_list[l])) + for l in range(num_rnn_layer)] + backward_init_h = [('backward_l%d_init_h' % l, (batch_size, num_hidden_rnn_list[l])) + for l in range(num_rnn_layer)] init_h = forward_init_h + backward_init_h elif rnn_type == 'gru': - init_h = [('l%d_init_h' % l, (batch_size, num_hidden_rnn_list[l])) for l in range(num_rnn_layer)] + init_h = [('l%d_init_h' % l, (batch_size, num_hidden_rnn_list[l])) + for l in range(num_rnn_layer)] elif rnn_type == 'bigru': - forward_init_h = [('forward_l%d_init_h' % l, (batch_size, num_hidden_rnn_list[l])) for l in - range(num_rnn_layer)] - backward_init_h = [('backward_l%d_init_h' % l, (batch_size, num_hidden_rnn_list[l])) for l in - range(num_rnn_layer)] + forward_init_h = [('forward_l%d_init_h' % l, (batch_size, num_hidden_rnn_list[l])) + for l in range(num_rnn_layer)] + backward_init_h = [('backward_l%d_init_h' % l, (batch_size, num_hidden_rnn_list[l])) + for l in range(num_rnn_layer)] init_h = forward_init_h + backward_init_h else: raise Exception('network type should be one of the lstm,bilstm,gru,bigru') @@ -51,115 +63,143 @@ def prepare_data(args): return init_states -def arch(args): - mode = args.config.get("common", "mode") - if mode == "train": - channel_num = args.config.getint("arch", "channel_num") - conv_layer1_filter_dim = tuple(json.loads(args.config.get("arch", "conv_layer1_filter_dim"))) - conv_layer1_stride = tuple(json.loads(args.config.get("arch", "conv_layer1_stride"))) - conv_layer2_filter_dim = tuple(json.loads(args.config.get("arch", "conv_layer2_filter_dim"))) - conv_layer2_stride = tuple(json.loads(args.config.get("arch", "conv_layer2_stride"))) - - rnn_type = args.config.get("arch", "rnn_type") - num_rnn_layer = args.config.getint("arch", "num_rnn_layer") - num_hidden_rnn_list = json.loads(args.config.get("arch", "num_hidden_rnn_list")) - - is_batchnorm = args.config.getboolean("arch", "is_batchnorm") - - seq_len = args.config.getint('arch', 'max_t_count') - num_label = args.config.getint('arch', 'max_label_length') - - num_rear_fc_layers = args.config.getint("arch", "num_rear_fc_layers") - num_hidden_rear_fc_list = json.loads(args.config.get("arch", "num_hidden_rear_fc_list")) - act_type_rear_fc_list = json.loads(args.config.get("arch", "act_type_rear_fc_list")) - # model symbol generation - # input preparation - data = mx.sym.Variable('data') - label = mx.sym.Variable('label') - - net = mx.sym.Reshape(data=data, shape=(-4, -1, 1, 0, 0)) - net = conv(net=net, - channels=channel_num, - filter_dimension=conv_layer1_filter_dim, - stride=conv_layer1_stride, - no_bias=is_batchnorm - ) - if is_batchnorm: - # batch norm normalizes axis 1 - net = batchnorm(net) - - net = conv(net=net, - channels=channel_num, - filter_dimension=conv_layer2_filter_dim, - stride=conv_layer2_stride, - no_bias=is_batchnorm - ) - if is_batchnorm: - # batch norm normalizes axis 1 - net = batchnorm(net) - net = mx.sym.transpose(data=net, axes=(0, 2, 1, 3)) - net = mx.sym.Reshape(data=net, shape=(0, 0, -3)) - seq_len_after_conv_layer1 = int( - math.floor((seq_len - conv_layer1_filter_dim[0]) / conv_layer1_stride[0])) + 1 - seq_len_after_conv_layer2 = int( - math.floor((seq_len_after_conv_layer1 - conv_layer2_filter_dim[0]) / conv_layer2_stride[0])) + 1 - net = slice_symbol_to_seq_symobls(net=net, seq_len=seq_len_after_conv_layer2, axis=1) - if rnn_type == "bilstm": - net = bi_lstm_unroll(net=net, +def arch(args, seq_len=None): + """ + define deep speech 2 network + """ + if isinstance(args, argparse.Namespace): + mode = args.config.get("common", "mode") + is_bucketing = args.config.getboolean("arch", "is_bucketing") + if mode == "train" or is_bucketing: + channel_num = args.config.getint("arch", "channel_num") + conv_layer1_filter_dim = \ + tuple(json.loads(args.config.get("arch", "conv_layer1_filter_dim"))) + conv_layer1_stride = tuple(json.loads(args.config.get("arch", "conv_layer1_stride"))) + conv_layer2_filter_dim = \ + tuple(json.loads(args.config.get("arch", "conv_layer2_filter_dim"))) + conv_layer2_stride = tuple(json.loads(args.config.get("arch", "conv_layer2_stride"))) + + rnn_type = args.config.get("arch", "rnn_type") + num_rnn_layer = args.config.getint("arch", "num_rnn_layer") + num_hidden_rnn_list = json.loads(args.config.get("arch", "num_hidden_rnn_list")) + + is_batchnorm = args.config.getboolean("arch", "is_batchnorm") + + if seq_len is None: + seq_len = args.config.getint('arch', 'max_t_count') + + num_label = args.config.getint('arch', 'max_label_length') + + num_rear_fc_layers = args.config.getint("arch", "num_rear_fc_layers") + num_hidden_rear_fc_list = json.loads(args.config.get("arch", "num_hidden_rear_fc_list")) + act_type_rear_fc_list = json.loads(args.config.get("arch", "act_type_rear_fc_list")) + # model symbol generation + # input preparation + data = mx.sym.Variable('data') + label = mx.sym.Variable('label') + + net = mx.sym.Reshape(data=data, shape=(-4, -1, 1, 0, 0)) + net = conv(net=net, + channels=channel_num, + filter_dimension=conv_layer1_filter_dim, + stride=conv_layer1_stride, + no_bias=is_batchnorm, + name='conv1') + if is_batchnorm: + # batch norm normalizes axis 1 + net = batchnorm(net, name="conv1_batchnorm") + + net = conv(net=net, + channels=channel_num, + filter_dimension=conv_layer2_filter_dim, + stride=conv_layer2_stride, + no_bias=is_batchnorm, + name='conv2') + if is_batchnorm: + # batch norm normalizes axis 1 + net = batchnorm(net, name="conv2_batchnorm") + + net = mx.sym.transpose(data=net, axes=(0, 2, 1, 3)) + net = mx.sym.Reshape(data=net, shape=(0, 0, -3)) + seq_len_after_conv_layer1 = int( + math.floor((seq_len - conv_layer1_filter_dim[0]) / conv_layer1_stride[0])) + 1 + seq_len_after_conv_layer2 = int( + math.floor((seq_len_after_conv_layer1 - conv_layer2_filter_dim[0]) + / conv_layer2_stride[0])) + 1 + net = slice_symbol_to_seq_symobls(net=net, seq_len=seq_len_after_conv_layer2, axis=1) + if rnn_type == "bilstm": + net = bi_lstm_unroll(net=net, + seq_len=seq_len_after_conv_layer2, + num_hidden_lstm_list=num_hidden_rnn_list, + num_lstm_layer=num_rnn_layer, + dropout=0., + is_batchnorm=is_batchnorm, + is_bucketing=is_bucketing) + elif rnn_type == "gru": + net = gru_unroll(net=net, seq_len=seq_len_after_conv_layer2, - num_hidden_lstm_list=num_hidden_rnn_list, - num_lstm_layer=num_rnn_layer, + num_hidden_gru_list=num_hidden_rnn_list, + num_gru_layer=num_rnn_layer, dropout=0., - is_batchnorm=is_batchnorm) - elif rnn_type == "gru": - net = gru_unroll(net=net, - seq_len=seq_len_after_conv_layer2, - num_hidden_gru_list=num_hidden_rnn_list, - num_gru_layer=num_rnn_layer, - dropout=0., - is_batchnorm=is_batchnorm) - elif rnn_type == "bigru": - net = bi_gru_unroll(net=net, + is_batchnorm=is_batchnorm, + is_bucketing=is_bucketing) + elif rnn_type == "bigru": + net = bi_gru_unroll(net=net, + seq_len=seq_len_after_conv_layer2, + num_hidden_gru_list=num_hidden_rnn_list, + num_gru_layer=num_rnn_layer, + dropout=0., + is_batchnorm=is_batchnorm, + is_bucketing=is_bucketing) + else: + raise Exception('rnn_type should be one of the followings, bilstm,gru,bigru') + + # rear fc layers + net = sequence_fc(net=net, seq_len=seq_len_after_conv_layer2, + num_layer=num_rear_fc_layers, prefix="rear", + num_hidden_list=num_hidden_rear_fc_list, + act_type_list=act_type_rear_fc_list, + is_batchnorm=is_batchnorm) + # warpctc layer + net = warpctc_layer(net=net, seq_len=seq_len_after_conv_layer2, - num_hidden_gru_list=num_hidden_rnn_list, - num_gru_layer=num_rnn_layer, - dropout=0., - is_batchnorm=is_batchnorm) + label=label, + num_label=num_label, + character_classes_count= + (args.config.getint('arch', 'n_classes') + 1)) + args.config.set('arch', 'max_t_count', str(seq_len_after_conv_layer2)) + return net + elif mode == 'load' or mode == 'predict': + conv_layer1_filter_dim = \ + tuple(json.loads(args.config.get("arch", "conv_layer1_filter_dim"))) + conv_layer1_stride = tuple(json.loads(args.config.get("arch", "conv_layer1_stride"))) + conv_layer2_filter_dim = \ + tuple(json.loads(args.config.get("arch", "conv_layer2_filter_dim"))) + conv_layer2_stride = tuple(json.loads(args.config.get("arch", "conv_layer2_stride"))) + if seq_len is None: + seq_len = args.config.getint('arch', 'max_t_count') + seq_len_after_conv_layer1 = int( + math.floor((seq_len - conv_layer1_filter_dim[0]) / conv_layer1_stride[0])) + 1 + seq_len_after_conv_layer2 = int( + math.floor((seq_len_after_conv_layer1 - conv_layer2_filter_dim[0]) + / conv_layer2_stride[0])) + 1 + + args.config.set('arch', 'max_t_count', str(seq_len_after_conv_layer2)) else: - raise Exception('rnn_type should be one of the followings, bilstm,gru,bigru') - - # rear fc layers - net = sequence_fc(net=net, seq_len=seq_len_after_conv_layer2, num_layer=num_rear_fc_layers, prefix="rear", - num_hidden_list=num_hidden_rear_fc_list, act_type_list=act_type_rear_fc_list, - is_batchnorm=is_batchnorm) - if is_batchnorm: - hidden_all = [] - # batch norm normalizes axis 1 - for seq_index in range(seq_len_after_conv_layer2): - hidden = net[seq_index] - hidden = batchnorm(hidden) - hidden_all.append(hidden) - net = hidden_all - - # warpctc layer - net = warpctc_layer(net=net, - seq_len=seq_len_after_conv_layer2, - label=label, - num_label=num_label, - character_classes_count=(args.config.getint('arch', 'n_classes') + 1) - ) - args.config.set('arch', 'max_t_count', str(seq_len_after_conv_layer2)) - return net - else: - conv_layer1_filter_dim = tuple(json.loads(args.config.get("arch", "conv_layer1_filter_dim"))) - conv_layer1_stride = tuple(json.loads(args.config.get("arch", "conv_layer1_stride"))) - conv_layer2_filter_dim = tuple(json.loads(args.config.get("arch", "conv_layer2_filter_dim"))) - conv_layer2_stride = tuple(json.loads(args.config.get("arch", "conv_layer2_stride"))) - seq_len = args.config.getint('arch', 'max_t_count') - seq_len_after_conv_layer1 = int( - math.floor((seq_len - conv_layer1_filter_dim[0]) / conv_layer1_stride[0])) + 1 - seq_len_after_conv_layer2 = int( - math.floor((seq_len_after_conv_layer1 - conv_layer2_filter_dim[0]) / conv_layer2_stride[0])) + 1 - args.config.set('arch', 'max_t_count', str(seq_len_after_conv_layer2)) + raise Exception('mode must be the one of the followings - train,predict,load') + + +class BucketingArch(object): + def __init__(self, args): + self.args = args + def sym_gen(self, seq_len): + args = self.args + net = arch(args, seq_len) + init_states = prepare_data(args) + init_state_names = [x[0] for x in init_states] + init_state_names.insert(0, 'data') + return net, init_state_names, ('label',) + def get_sym_gen(self): + return self.sym_gen diff --git a/example/speech_recognition/deepspeech.cfg b/example/speech_recognition/deepspeech.cfg index 13cf578c679a..4f0f49699771 100644 --- a/example/speech_recognition/deepspeech.cfg +++ b/example/speech_recognition/deepspeech.cfg @@ -3,23 +3,27 @@ mode = train #ex: gpu0,gpu1,gpu2,gpu3 context = gpu0,gpu1,gpu2 +#context = gpu0 # checkpoint prefix, check point will be saved under checkpoints folder with prefix -prefix = deep +prefix = deep_bucket # when mode is load or predict, model will be loaded from the file name with model_file under checkpoints -model_file = deepspeechn_epoch1n_batch-0009 +model_file = deep_bucketn_epoch0n_batch-0018 batch_size = 12 +#batch_size=4 # log will be saved by the log_filename -log_filename = deep.log +log_filename = deep_bucket.log # checkpoint set n to save checkpoints after n epoch save_checkpoint_every_n_epoch = 1 -save_checkpoint_every_n_batch = 1000 +save_checkpoint_every_n_batch = 3000 is_bi_graphemes = True -tensorboard_log_dir = tblog/deep +tensorboard_log_dir = tblog/deep_bucket # if random_seed is -1 then it gets random seed from timestamp mx_random_seed = -1 random_seed = -1 +kvstore_option = device [data] +max_duration = 16.0 train_json = ./train_corpus_all.json test_json = ./test_corpus.json val_json = ./test_corpus.json @@ -50,31 +54,49 @@ rnn_type = bigru #vanilla_lstm or fc_lstm (no effect when network_type is gru, bigru) lstm_type = fc_lstm is_batchnorm = True +is_bucketing = True +buckets = [200, 300, 400, 500, 600, 700, 800, 900, 1000, 1100, 1200, 1300, 1400, 1500, 1600] [train] num_epoch = 70 learning_rate = 0.0003 # constant learning rate annealing by factor learning_rate_annealing = 1.1 -# supports only sgd and adam -optimizer = sgd -# for sgd -momentum = 0.9 -# set to 0 to disable gradient clipping -clip_gradient = 0 initializer = Xavier init_scale = 2 factor_type = in -weight_decay = 0. # show progress every how nth batches show_every = 100 save_optimizer_states = True -normalize_target_k = 13000 +normalize_target_k = 100000 # overwrite meta files(feats_mean,feats_std,unicode_en_baidu_bi_graphemes.csv) overwrite_meta_files = True +overwrite_bi_graphemes_dictionary = False +# save feature extracted from soundfile as csvfile, it can take too much disk space +save_feature_as_csvfile = False enable_logging_train_metric = True enable_logging_validation_metric = True [load] load_optimizer_states = True is_start_from_batch = True + +[optimizer] +optimizer = sgd +# define parameters for optimizer +# optimizer_params_dictionary should use " not ' as string wrapper +# sgd/nag +optimizer_params_dictionary={"momentum":0.9} +# dcasgd +# optimizer_params_dictionary={"momentum":0.9, "lamda":1.0} +# adam +# optimizer_params_dictionary={"beta1":0.9,"beta2":0.999} +# adagrad +# optimizer_params_dictionary={"eps":1e-08} +# rmsprop +# optimizer_params_dictionary={"gamma1":0.9, "gamma2":0.9,"epsilon":1e-08} +# adadelta +# optimizer_params_dictionary={"rho":0.95, "epsilon":1e-08} +# set to 0 to disable gradient clipping +clip_gradient = 100 +weight_decay = 0. diff --git a/example/speech_recognition/default.cfg b/example/speech_recognition/default.cfg index 853a04aebbdd..127c492b6166 100644 --- a/example/speech_recognition/default.cfg +++ b/example/speech_recognition/default.cfg @@ -6,20 +6,22 @@ context = gpu0 # checkpoint prefix, check point will be saved under checkpoints folder with prefix prefix = test_fc # when mode is load or predict, model will be loaded from the file name with model_file under checkpoints -model_file = test_fc-0001 +model_file = test_fc-0040 batch_size = 2 # log will be saved by the log_filename log_filename = test.log # checkpoint set n to save checkpoints after n epoch -save_checkpoint_every_n_epoch = 1 +save_checkpoint_every_n_epoch = 20 save_checkpoint_every_n_batch = 1000 is_bi_graphemes = False tensorboard_log_dir = tblog/libri_sample # if random_seed is -1 then it gets random seed from timestamp -mx_random_seed = -1 -random_seed = -1 +mx_random_seed = 1234 +random_seed = 1234 +kvstore_option = device [data] +max_duration = 16.0 train_json = ./Libri_sample.json test_json = ./Libri_sample.json val_json = ./Libri_sample.json @@ -37,8 +39,8 @@ conv_layer1_stride = [2, 2] conv_layer2_filter_dim = [11, 21] conv_layer2_stride = [1, 2] -num_rnn_layer = 3 -num_hidden_rnn_list = [1760, 1760, 1760] +num_rnn_layer = 1 +num_hidden_rnn_list = [1760] num_hidden_proj = 0 num_rear_fc_layers = 0 @@ -50,33 +52,49 @@ rnn_type = bigru #vanilla_lstm or fc_lstm (no effect when network_type is gru, bigru) lstm_type = fc_lstm is_batchnorm = True +is_bucketing = False +buckets = [] [train] -num_epoch = 70 - +num_epoch = 50 learning_rate = 0.005 # constant learning rate annealing by factor learning_rate_annealing = 1.1 -# supports only sgd and adam -optimizer = adam -# for sgd -momentum = 0.9 -# set to 0 to disable gradient clipping -clip_gradient = 0 - initializer = Xavier init_scale = 2 factor_type = in -weight_decay = 0.00001 # show progress every nth batches show_every = 1 save_optimizer_states = True normalize_target_k = 2 # overwrite meta files(feats_mean,feats_std,unicode_en_baidu_bi_graphemes.csv) overwrite_meta_files = True +overwrite_bi_graphemes_dictionary = False +# save feature extracted from soundfile as csvfile, it can take too much disk space +save_feature_as_csvfile = False enable_logging_train_metric = True enable_logging_validation_metric = True [load] load_optimizer_states = True is_start_from_batch = False + +[optimizer] +optimizer = adam +# define parameters for optimizer +# optimizer_params_dictionary should use " not ' as string wrapper +# sgd/nag +# optimizer_params_dictionary={"momentum":0.9} +# dcasgd +# optimizer_params_dictionary={"momentum":0.9, "lamda":1.0} +# adam +optimizer_params_dictionary={"beta1":0.9,"beta2":0.999} +# adagrad +# optimizer_params_dictionary={"eps":1e-08} +# rmsprop +# optimizer_params_dictionary={"gamma1":0.9, "gamma2":0.9,"epsilon":1e-08} +# adadelta +# optimizer_params_dictionary={"rho":0.95, "epsilon":1e-08} +# set to 0 to disable gradient clipping +clip_gradient = 0 +weight_decay = 0. diff --git a/example/speech_recognition/main.py b/example/speech_recognition/main.py index 398a8a537e01..a425e0a8ab40 100644 --- a/example/speech_recognition/main.py +++ b/example/speech_recognition/main.py @@ -1,34 +1,32 @@ +import json +import os import sys - -sys.path.insert(0, "../../python") +from collections import namedtuple +from datetime import datetime from config_util import parse_args, parse_contexts, generate_file_path from train import do_training import mxnet as mx from stt_io_iter import STTIter from label_util import LabelUtil from log_util import LogUtil - import numpy as np from stt_datagenerator import DataGenerator from stt_metric import STTMetric -from datetime import datetime from stt_bi_graphemes_util import generate_bi_graphemes_dictionary -######################################## -########## FOR JUPYTER NOTEBOOK -import os +from stt_bucketing_module import STTBucketingModule +from stt_io_bucketingiter import BucketSTTIter +sys.path.insert(0, "../../python") # os.environ['MXNET_ENGINE_TYPE'] = "NaiveEngine" os.environ['MXNET_ENGINE_TYPE'] = "ThreadedEnginePerDevice" os.environ['MXNET_ENABLE_GPU_P2P'] = "0" - class WHCS: width = 0 height = 0 channel = 0 stride = 0 - class ConfigLogger(object): def __init__(self, log): self.__log = log @@ -42,9 +40,25 @@ def write(self, data): line = data.strip() self.__log.info(line) +def load_labelutil(labelUtil, is_bi_graphemes, language="en"): + if language == "en": + if is_bi_graphemes: + try: + labelUtil.load_unicode_set("resources/unicodemap_en_baidu_bi_graphemes.csv") + except: + raise Exception("There is no resources/unicodemap_en_baidu_bi_graphemes.csv." + + " Please set overwrite_bi_graphemes_dictionary True at train section") + else: + labelUtil.load_unicode_set("resources/unicodemap_en_baidu.csv") + else: + raise Exception("Error: Language Type: %s" % language) + + def load_data(args): mode = args.config.get('common', 'mode') + if mode not in ['train', 'predict', 'load']: + raise Exception('mode must be the one of the followings - train,predict,load') batch_size = args.config.getint('common', 'batch_size') whcs = WHCS() @@ -56,101 +70,78 @@ def load_data(args): model_name = args.config.get('common', 'prefix') is_bi_graphemes = args.config.getboolean('common', 'is_bi_graphemes') overwrite_meta_files = args.config.getboolean('train', 'overwrite_meta_files') + overwrite_bi_graphemes_dictionary = args.config.getboolean('train', 'overwrite_bi_graphemes_dictionary') + max_duration = args.config.getfloat('data', 'max_duration') language = args.config.get('data', 'language') - is_bi_graphemes = args.config.getboolean('common', 'is_bi_graphemes') + log = LogUtil().getlogger() labelUtil = LabelUtil.getInstance() - if language == "en": - if is_bi_graphemes: - try: - labelUtil.load_unicode_set("resources/unicodemap_en_baidu_bi_graphemes.csv") - except: - raise Exception("There is no resources/unicodemap_en_baidu_bi_graphemes.csv. Please set overwrite_meta_files at train section True") - else: - labelUtil.load_unicode_set("resources/unicodemap_en_baidu.csv") - else: - raise Exception("Error: Language Type: %s" % language) - args.config.set('arch', 'n_classes', str(labelUtil.get_count())) - - if mode == 'predict': - test_json = args.config.get('data', 'test_json') - datagen = DataGenerator(save_dir=save_dir, model_name=model_name) - datagen.load_train_data(test_json) - datagen.get_meta_from_file(np.loadtxt(generate_file_path(save_dir, model_name, 'feats_mean')), - np.loadtxt(generate_file_path(save_dir, model_name, 'feats_std'))) - elif mode =="train" or mode == "load": + if mode == "train" or mode == "load": data_json = args.config.get('data', 'train_json') val_json = args.config.get('data', 'val_json') datagen = DataGenerator(save_dir=save_dir, model_name=model_name) - datagen.load_train_data(data_json) - #test bigramphems - - if overwrite_meta_files and is_bi_graphemes: - generate_bi_graphemes_dictionary(datagen.train_texts) - + datagen.load_train_data(data_json, max_duration=max_duration) + if is_bi_graphemes: + if not os.path.isfile("resources/unicodemap_en_baidu_bi_graphemes.csv") or overwrite_bi_graphemes_dictionary: + load_labelutil(labelUtil=labelUtil, is_bi_graphemes=False, language=language) + generate_bi_graphemes_dictionary(datagen.train_texts) + load_labelutil(labelUtil=labelUtil, is_bi_graphemes=is_bi_graphemes, language=language) args.config.set('arch', 'n_classes', str(labelUtil.get_count())) if mode == "train": if overwrite_meta_files: + log.info("Generate mean and std from samples") normalize_target_k = args.config.getint('train', 'normalize_target_k') datagen.sample_normalize(normalize_target_k, True) else: - datagen.get_meta_from_file(np.loadtxt(generate_file_path(save_dir, model_name, 'feats_mean')), - np.loadtxt(generate_file_path(save_dir, model_name, 'feats_std'))) - datagen.load_validation_data(val_json) + log.info("Read mean and std from meta files") + datagen.get_meta_from_file( + np.loadtxt(generate_file_path(save_dir, model_name, 'feats_mean')), + np.loadtxt(generate_file_path(save_dir, model_name, 'feats_std'))) + datagen.load_validation_data(val_json, max_duration=max_duration) elif mode == "load": # get feat_mean and feat_std to normalize dataset - datagen.get_meta_from_file(np.loadtxt(generate_file_path(save_dir, model_name, 'feats_mean')), - np.loadtxt(generate_file_path(save_dir, model_name, 'feats_std'))) - datagen.load_validation_data(val_json) - else: - raise Exception( - 'Define mode in the cfg file first. train or predict or load can be the candidate for the mode.') + datagen.get_meta_from_file( + np.loadtxt(generate_file_path(save_dir, model_name, 'feats_mean')), + np.loadtxt(generate_file_path(save_dir, model_name, 'feats_std'))) + datagen.load_validation_data(val_json, max_duration=max_duration) + elif mode == 'predict': + test_json = args.config.get('data', 'test_json') + datagen = DataGenerator(save_dir=save_dir, model_name=model_name) + datagen.load_train_data(test_json, max_duration=max_duration) + labelutil = load_labelutil(labelUtil, is_bi_graphemes, language="en") + args.config.set('arch', 'n_classes', str(labelUtil.get_count())) + datagen.get_meta_from_file( + np.loadtxt(generate_file_path(save_dir, model_name, 'feats_mean')), + np.loadtxt(generate_file_path(save_dir, model_name, 'feats_std'))) is_batchnorm = args.config.getboolean('arch', 'is_batchnorm') - if batch_size == 1 and is_batchnorm: + if batch_size == 1 and is_batchnorm and (mode == 'train' or mode == 'load'): raise Warning('batch size 1 is too small for is_batchnorm') # sort file paths by its duration in ascending order to implement sortaGrad - if mode == "train" or mode == "load": max_t_count = datagen.get_max_seq_length(partition="train") - max_label_length = datagen.get_max_label_length(partition="train",is_bi_graphemes=is_bi_graphemes) + max_label_length = \ + datagen.get_max_label_length(partition="train", is_bi_graphemes=is_bi_graphemes) elif mode == "predict": max_t_count = datagen.get_max_seq_length(partition="test") - max_label_length = datagen.get_max_label_length(partition="test",is_bi_graphemes=is_bi_graphemes) - else: - raise Exception( - 'Define mode in the cfg file first. train or predict or load can be the candidate for the mode.') + max_label_length = \ + datagen.get_max_label_length(partition="test", is_bi_graphemes=is_bi_graphemes) args.config.set('arch', 'max_t_count', str(max_t_count)) args.config.set('arch', 'max_label_length', str(max_label_length)) from importlib import import_module prepare_data_template = import_module(args.config.get('arch', 'arch_file')) init_states = prepare_data_template.prepare_data(args) - if mode == "train": - sort_by_duration = True - else: - sort_by_duration = False - - data_loaded = STTIter(partition="train", - count=datagen.count, - datagen=datagen, - batch_size=batch_size, - num_label=max_label_length, - init_states=init_states, - seq_length=max_t_count, - width=whcs.width, - height=whcs.height, - sort_by_duration=sort_by_duration, - is_bi_graphemes=is_bi_graphemes) - - if mode == 'predict': - return data_loaded, args - else: - validation_loaded = STTIter(partition="validation", - count=datagen.val_count, + sort_by_duration = (mode == "train") + is_bucketing = args.config.getboolean('arch', 'is_bucketing') + save_feature_as_csvfile = args.config.getboolean('train', 'save_feature_as_csvfile') + if is_bucketing: + buckets = json.loads(args.config.get('arch', 'buckets')) + data_loaded = BucketSTTIter(partition="train", + count=datagen.count, datagen=datagen, batch_size=batch_size, num_label=max_label_length, @@ -158,37 +149,91 @@ def load_data(args): seq_length=max_t_count, width=whcs.width, height=whcs.height, - sort_by_duration=False, - is_bi_graphemes=is_bi_graphemes) + sort_by_duration=sort_by_duration, + is_bi_graphemes=is_bi_graphemes, + buckets=buckets, + save_feature_as_csvfile=save_feature_as_csvfile) + else: + data_loaded = STTIter(partition="train", + count=datagen.count, + datagen=datagen, + batch_size=batch_size, + num_label=max_label_length, + init_states=init_states, + seq_length=max_t_count, + width=whcs.width, + height=whcs.height, + sort_by_duration=sort_by_duration, + is_bi_graphemes=is_bi_graphemes, + save_feature_as_csvfile=save_feature_as_csvfile) + + if mode == 'train' or mode == 'load': + if is_bucketing: + validation_loaded = BucketSTTIter(partition="validation", + count=datagen.val_count, + datagen=datagen, + batch_size=batch_size, + num_label=max_label_length, + init_states=init_states, + seq_length=max_t_count, + width=whcs.width, + height=whcs.height, + sort_by_duration=False, + is_bi_graphemes=is_bi_graphemes, + buckets=buckets, + save_feature_as_csvfile=save_feature_as_csvfile) + else: + validation_loaded = STTIter(partition="validation", + count=datagen.val_count, + datagen=datagen, + batch_size=batch_size, + num_label=max_label_length, + init_states=init_states, + seq_length=max_t_count, + width=whcs.width, + height=whcs.height, + sort_by_duration=False, + is_bi_graphemes=is_bi_graphemes, + save_feature_as_csvfile=save_feature_as_csvfile) return data_loaded, validation_loaded, args + elif mode == 'predict': + return data_loaded, args def load_model(args, contexts, data_train): # load model from model_name prefix and epoch of model_num_epoch with gpu contexts of contexts mode = args.config.get('common', 'mode') load_optimizer_states = args.config.getboolean('load', 'load_optimizer_states') - is_start_from_batch = args.config.getboolean('load','is_start_from_batch') + is_start_from_batch = args.config.getboolean('load', 'is_start_from_batch') from importlib import import_module symbol_template = import_module(args.config.get('arch', 'arch_file')) - model_loaded = symbol_template.arch(args) + is_bucketing = args.config.getboolean('arch', 'is_bucketing') if mode == 'train': + if is_bucketing: + bucketing_arch = symbol_template.BucketingArch(args) + model_loaded = bucketing_arch.get_sym_gen() + else: + model_loaded = symbol_template.arch(args) model_num_epoch = None - else: + elif mode == 'load' or mode == 'predict': model_file = args.config.get('common', 'model_file') model_name = os.path.splitext(model_file)[0] - model_num_epoch = int(model_name[-4:]) + if is_bucketing: + bucketing_arch = symbol_template.BucketingArch(args) + model_loaded = bucketing_arch.get_sym_gen() + else: + model_path = 'checkpoints/' + str(model_name[:-5]) - model_path = 'checkpoints/' + str(model_name[:-5]) - - data_names = [x[0] for x in data_train.provide_data] - label_names = [x[0] for x in data_train.provide_label] + data_names = [x[0] for x in data_train.provide_data] + label_names = [x[0] for x in data_train.provide_label] - model_loaded = mx.module.Module.load(prefix=model_path, epoch=model_num_epoch, context=contexts, - data_names=data_names, label_names=label_names, - load_optimizer_states=load_optimizer_states) + model_loaded = mx.module.Module.load( + prefix=model_path, epoch=model_num_epoch, context=contexts, + data_names=data_names, label_names=label_names, + load_optimizer_states=load_optimizer_states) if is_start_from_batch: import re model_num_epoch = int(re.findall('\d+', model_file)[0]) @@ -198,7 +243,8 @@ def load_model(args, contexts, data_train): if __name__ == '__main__': if len(sys.argv) <= 1: - raise Exception('cfg file path must be provided. ex)python main.py --configfile examplecfg.cfg') + raise Exception('cfg file path must be provided. ' + + 'ex)python main.py --configfile examplecfg.cfg') args = parse_args(sys.argv[1]) # set parameters from cfg file # give random seed @@ -206,9 +252,9 @@ def load_model(args, contexts, data_train): mx_random_seed = args.config.getint('common', 'mx_random_seed') # random seed for shuffling data list if random_seed != -1: - random.seed(random_seed) + np.random.seed(random_seed) # set mx.random.seed to give seed for parameter initialization - if mx_random_seed !=-1: + if mx_random_seed != -1: mx.random.seed(mx_random_seed) else: mx.random.seed(hash(datetime.now())) @@ -220,22 +266,23 @@ def load_model(args, contexts, data_train): mode = args.config.get('common', 'mode') if mode not in ['train', 'predict', 'load']: raise Exception( - 'Define mode in the cfg file first. train or predict or load can be the candidate for the mode.') + 'Define mode in the cfg file first. ' + + 'train or predict or load can be the candidate for the mode.') # get meta file where character to number conversions are defined contexts = parse_contexts(args) num_gpu = len(contexts) batch_size = args.config.getint('common', 'batch_size') - # check the number of gpus is positive divisor of the batch size for data parallel if batch_size % num_gpu != 0: raise Exception('num_gpu should be positive divisor of batch_size') - - if mode == "predict": - data_train, args = load_data(args) - elif mode == "train" or mode == "load": + if mode == "train" or mode == "load": data_train, data_val, args = load_data(args) + elif mode == "predict": + data_train, args = load_data(args) + is_batchnorm = args.config.getboolean('arch', 'is_batchnorm') + is_bucketing = args.config.getboolean('arch', 'is_bucketing') # log current config config_logger = ConfigLogger(log) @@ -243,28 +290,63 @@ def load_model(args, contexts, data_train): # load model model_loaded, model_num_epoch = load_model(args, contexts, data_train) - # if mode is 'train', it trains the model if mode == 'train': - data_names = [x[0] for x in data_train.provide_data] - label_names = [x[0] for x in data_train.provide_label] - module = mx.mod.Module(model_loaded, context=contexts, data_names=data_names, label_names=label_names) + if is_bucketing: + module = STTBucketingModule( + sym_gen=model_loaded, + default_bucket_key=data_train.default_bucket_key, + context=contexts + ) + else: + data_names = [x[0] for x in data_train.provide_data] + label_names = [x[0] for x in data_train.provide_label] + module = mx.mod.Module(model_loaded, context=contexts, + data_names=data_names, label_names=label_names) do_training(args=args, module=module, data_train=data_train, data_val=data_val) # if mode is 'load', it loads model from the checkpoint and continues the training. elif mode == 'load': - do_training(args=args, module=model_loaded, data_train=data_train, data_val=data_val, begin_epoch=model_num_epoch+1) + do_training(args=args, module=model_loaded, data_train=data_train, data_val=data_val, + begin_epoch=model_num_epoch + 1) # if mode is 'predict', it predict label from the input by the input model elif mode == 'predict': # predict through data - model_loaded.bind(for_training=False, data_shapes=data_train.provide_data, - label_shapes=data_train.provide_label) + if is_bucketing: + max_t_count = args.config.getint('arch', 'max_t_count') + load_optimizer_states = args.config.getboolean('load', 'load_optimizer_states') + model_file = args.config.get('common', 'model_file') + model_name = os.path.splitext(model_file)[0] + model_num_epoch = int(model_name[-4:]) + + model_path = 'checkpoints/' + str(model_name[:-5]) + model = STTBucketingModule( + sym_gen=model_loaded, + default_bucket_key=data_train.default_bucket_key, + context=contexts + ) + + model.bind(data_shapes=data_train.provide_data, + label_shapes=data_train.provide_label, + for_training=True) + _, arg_params, aux_params = mx.model.load_checkpoint(model_path, model_num_epoch) + model.set_params(arg_params, aux_params) + model_loaded = model + else: + model_loaded.bind(for_training=False, data_shapes=data_train.provide_data, + label_shapes=data_train.provide_label) max_t_count = args.config.getint('arch', 'max_t_count') - eval_metric = STTMetric(batch_size=batch_size, num_gpu=num_gpu, seq_length=max_t_count) - is_batchnorm = args.config.getboolean('arch', 'is_batchnorm') - if is_batchnorm : + eval_metric = STTMetric(batch_size=batch_size, num_gpu=num_gpu) + if is_batchnorm: for nbatch, data_batch in enumerate(data_train): - # when is_train = False it leads to high cer when batch_norm - model_loaded.forward(data_batch, is_train=True) + model_loaded.forward(data_batch, is_train=False) model_loaded.update_metric(eval_metric, data_batch.label) - else : - model_loaded.score(eval_data=data_train, num_batch=None, eval_metric=eval_metric, reset=True) + else: + #model_loaded.score(eval_data=data_train, num_batch=None, + # eval_metric=eval_metric, reset=True) + for nbatch, data_batch in enumerate(data_train): + model_loaded.forward(data_batch, is_train=False) + model_loaded.update_metric(eval_metric, data_batch.label) + else: + raise Exception( + 'Define mode in the cfg file first. ' + + 'train or predict or load can be the candidate for the mode') diff --git a/example/speech_recognition/stt_datagenerator.py b/example/speech_recognition/stt_datagenerator.py index 390de432e751..d2a7b4b5cbae 100644 --- a/example/speech_recognition/stt_datagenerator.py +++ b/example/speech_recognition/stt_datagenerator.py @@ -2,7 +2,6 @@ import json import random - import numpy as np from stt_utils import calc_feat_dim, spectrogram_from_file @@ -10,6 +9,7 @@ from log_util import LogUtil from label_util import LabelUtil from stt_bi_graphemes_util import generate_bi_graphemes_label +from multiprocessing import cpu_count, Process, Manager class DataGenerator(object): def __init__(self, save_dir, model_name, step=10, window=20, max_freq=8000, desc_file=None): @@ -32,7 +32,7 @@ def __init__(self, save_dir, model_name, step=10, window=20, max_freq=8000, desc # 1d 161 length of array filled with 1s self.feats_std = np.ones((self.feat_dim,)) self.max_input_length = 0 - self.max_length_list_in_batch =[] + self.max_length_list_in_batch = [] # 1d 161 length of array filled with random value #[0.0, 1.0) self.rng = random.Random() @@ -48,14 +48,15 @@ def get_meta_from_file(self, feats_mean, feats_std): self.feats_mean = feats_mean self.feats_std = feats_std - def featurize(self, audio_clip, overwrite=False): + def featurize(self, audio_clip, overwrite=False, save_feature_as_csvfile=False): """ For a given audio clip, calculate the log of its Fourier Transform Params: audio_clip(str): Path to the audio clip """ return spectrogram_from_file( audio_clip, step=self.step, window=self.window, - max_freq=self.max_freq, overwrite=overwrite) + max_freq=self.max_freq, overwrite=overwrite, + save_feature_as_csvfile=save_feature_as_csvfile) def load_metadata_from_desc_file(self, desc_file, partition='train', max_duration=16.0,): @@ -107,11 +108,11 @@ def load_metadata_from_desc_file(self, desc_file, partition='train', raise Exception("Invalid partition to load metadata. " "Must be train/validation/test") - def load_train_data(self, desc_file): - self.load_metadata_from_desc_file(desc_file, 'train') + def load_train_data(self, desc_file, max_duration): + self.load_metadata_from_desc_file(desc_file, 'train', max_duration=max_duration) - def load_validation_data(self, desc_file): - self.load_metadata_from_desc_file(desc_file, 'validation') + def load_validation_data(self, desc_file, max_duration): + self.load_metadata_from_desc_file(desc_file, 'validation', max_duration=max_duration) @staticmethod def sort_by_duration(durations, audio_paths, texts): @@ -146,10 +147,11 @@ def get_max_seq_length(self, partition): "Must be train/validation/test") max_duration_indexes = durations.index(max(durations)) max_seq_length = self.featurize(audio_paths[max_duration_indexes]).shape[0] - self.max_seq_length=max_seq_length + self.max_seq_length = max_seq_length return max_seq_length - def prepare_minibatch(self, audio_paths, texts, overwrite=False, is_bi_graphemes=False): + def prepare_minibatch(self, audio_paths, texts, overwrite=False, + is_bi_graphemes=False, seq_length=-1, save_feature_as_csvfile=False): """ Featurize a minibatch of audio, zero pad them and return a dictionary Params: audio_paths (list(str)): List of paths to audio files @@ -162,12 +164,15 @@ def prepare_minibatch(self, audio_paths, texts, overwrite=False, is_bi_graphemes # Features is a list of (timesteps, feature_dim) arrays # Calculate the features for each audio clip, as the log of the # Fourier Transform of the audio - features = [self.featurize(a, overwrite=overwrite) for a in audio_paths] + features = [self.featurize(a, overwrite=overwrite, save_feature_as_csvfile=save_feature_as_csvfile) for a in audio_paths] input_lengths = [f.shape[0] for f in features] feature_dim = features[0].shape[1] mb_size = len(features) # Pad all the inputs so that they are all the same length - x = np.zeros((mb_size, self.max_seq_length, feature_dim)) + if seq_length == -1: + x = np.zeros((mb_size, self.max_seq_length, feature_dim)) + else: + x = np.zeros((mb_size, seq_length, feature_dim)) y = np.zeros((mb_size, self.max_label_length)) labelUtil = LabelUtil.getInstance() label_lengths = [] @@ -199,34 +204,59 @@ def iterate_validation(self, minibatch_size=16): return self.iterate(self.val_audio_paths, self.val_texts, minibatch_size) + def preprocess_sample_normalize(self, threadIndex, audio_paths, overwrite, return_dict): + if len(audio_paths) > 0: + audio_clip = audio_paths[0] + feat = self.featurize(audio_clip=audio_clip, overwrite=overwrite) + feat_squared = np.square(feat) + count = float(feat.shape[0]) + dim = feat.shape[1] + if len(audio_paths) > 1: + for audio_path in audio_paths[1:]: + next_feat = self.featurize(audio_clip=audio_path, overwrite=overwrite) + next_feat_squared = np.square(next_feat) + feat_vertically_stacked = np.concatenate((feat, next_feat)).reshape(-1, dim) + feat = np.sum(feat_vertically_stacked, axis=0, keepdims=True) + feat_squared_vertically_stacked = np.concatenate( + (feat_squared, next_feat_squared)).reshape(-1, dim) + feat_squared = np.sum(feat_squared_vertically_stacked, axis=0, keepdims=True) + count += float(next_feat.shape[0]) + return_dict[threadIndex] = {'feat': feat, 'feat_squared': feat_squared, 'count': count} + def sample_normalize(self, k_samples=1000, overwrite=False): """ Estimate the mean and std of the features from the training set Params: k_samples (int): Use this number of samples for estimation """ + log = LogUtil().getlogger() + log.info("Calculating mean and std from samples") # if k_samples is negative then it goes through total dataset if k_samples < 0: - audio_paths_iter = iter(self.audio_paths) + audio_paths = self.audio_paths + # using sample else: k_samples = min(k_samples, len(self.train_audio_paths)) samples = self.rng.sample(self.train_audio_paths, k_samples) - audio_paths_iter = iter(samples) - audio_clip = audio_paths_iter.next() - feat = self.featurize(audio_clip=audio_clip, overwrite=overwrite) - feat_squared = np.square(feat) - count = float(feat.shape[0]) - dim = feat.shape[1] - - for iter_index in range(len(samples) - 1): - next_feat = self.featurize(audio_clip=audio_paths_iter.next(), overwrite=overwrite) - next_feat_squared = np.square(next_feat) - feat_vertically_stacked = np.concatenate((feat, next_feat)).reshape(-1, dim) - feat = np.sum(feat_vertically_stacked, axis=0, keepdims=True) - feat_squared_vertically_stacked = np.concatenate((feat_squared, next_feat_squared)).reshape(-1, dim) - feat_squared = np.sum(feat_squared_vertically_stacked, axis=0, keepdims=True) - count = count + float(next_feat.shape[0]) + audio_paths = samples + manager = Manager() + return_dict = manager.dict() + jobs = [] + for threadIndex in range(cpu_count()): + proc = Process(target=self.preprocess_sample_normalize, args=(threadIndex, audio_paths, overwrite, return_dict)) + jobs.append(proc) + proc.start() + for proc in jobs: + proc.join() + + feat = np.sum(np.vstack([item['feat'] for item in return_dict.values()]), axis=0) + count = sum([item['count'] for item in return_dict.values()]) + feat_squared = np.sum(np.vstack([item['feat_squared'] for item in return_dict.values()]), axis=0) + self.feats_mean = feat / float(count) self.feats_std = np.sqrt(feat_squared / float(count) - np.square(self.feats_mean)) - np.savetxt(generate_file_path(self.save_dir, self.model_name, 'feats_mean'), self.feats_mean) - np.savetxt(generate_file_path(self.save_dir, self.model_name, 'feats_std'), self.feats_std) + np.savetxt( + generate_file_path(self.save_dir, self.model_name, 'feats_mean'), self.feats_mean) + np.savetxt( + generate_file_path(self.save_dir, self.model_name, 'feats_std'), self.feats_std) + log.info("End calculating mean and std from samples") diff --git a/example/speech_recognition/stt_io_iter.py b/example/speech_recognition/stt_io_iter.py index 70c31ce92dde..5ae65191c840 100644 --- a/example/speech_recognition/stt_io_iter.py +++ b/example/speech_recognition/stt_io_iter.py @@ -31,7 +31,8 @@ def provide_label(self): class STTIter(mx.io.DataIter): def __init__(self, count, datagen, batch_size, num_label, init_states, seq_length, width, height, sort_by_duration=True, - is_bi_graphemes=False, partition="train",): + is_bi_graphemes=False, partition="train", + save_feature_as_csvfile=False): super(STTIter, self).__init__() self.batch_size = batch_size self.num_label = num_label @@ -75,6 +76,7 @@ def __init__(self, count, datagen, batch_size, num_label, init_states, seq_lengt self.trainDataIter = iter(self.trainDataList) self.is_first_epoch = True + self.save_feature_as_csvfile = save_feature_as_csvfile def __iter__(self): init_state_names = [x[0] for x in self.init_states] @@ -92,9 +94,9 @@ def __iter__(self): audio_paths.append(audio_path) texts.append(text) if self.is_first_epoch: - data_set = self.datagen.prepare_minibatch(audio_paths, texts, overwrite=True, is_bi_graphemes=self.is_bi_graphemes) + data_set = self.datagen.prepare_minibatch(audio_paths, texts, overwrite=True, is_bi_graphemes=self.is_bi_graphemes, save_feature_as_csvfile=self.save_feature_as_csvfile) else: - data_set = self.datagen.prepare_minibatch(audio_paths, texts, overwrite=False, is_bi_graphemes=self.is_bi_graphemes) + data_set = self.datagen.prepare_minibatch(audio_paths, texts, overwrite=False, is_bi_graphemes=self.is_bi_graphemes, save_feature_as_csvfile=self.save_feature_as_csvfile) data_all = [mx.nd.array(data_set['x'])] + self.init_state_arrays label_all = [mx.nd.array(data_set['y'])] @@ -103,7 +105,6 @@ def __iter__(self): data_batch = SimpleBatch(data_names, data_all, label_names, label_all) yield data_batch - self.is_first_epoch = False def reset(self): pass diff --git a/example/speech_recognition/stt_layer_batchnorm.py b/example/speech_recognition/stt_layer_batchnorm.py index 86e75aa49557..5b73f4f9f890 100644 --- a/example/speech_recognition/stt_layer_batchnorm.py +++ b/example/speech_recognition/stt_layer_batchnorm.py @@ -6,7 +6,7 @@ def batchnorm(net, beta=None, eps=0.001, momentum=0.9, - fix_gamma=True, + fix_gamma=False, use_global_stats=False, output_mean_var=False, name=None): @@ -18,7 +18,8 @@ def batchnorm(net, momentum=momentum, fix_gamma=fix_gamma, use_global_stats=use_global_stats, - output_mean_var=output_mean_var + output_mean_var=output_mean_var, + name=name ) else: net = mx.sym.BatchNorm(data=net, @@ -26,6 +27,7 @@ def batchnorm(net, momentum=momentum, fix_gamma=fix_gamma, use_global_stats=use_global_stats, - output_mean_var=output_mean_var + output_mean_var=output_mean_var, + name=name ) return net diff --git a/example/speech_recognition/stt_layer_conv.py b/example/speech_recognition/stt_layer_conv.py index 5ec292557f04..ab0035e4803b 100644 --- a/example/speech_recognition/stt_layer_conv.py +++ b/example/speech_recognition/stt_layer_conv.py @@ -8,20 +8,22 @@ def conv(net, weight=None, bias=None, act_type="relu", - no_bias=False + no_bias=False, + name=None ): # 2d convolution's input should have the shape of 4D (batch_size,1,seq_len,feat_dim) if weight is None or bias is None: # ex) filter_dimension = (41,11) , stride=(2,2) - net = mx.sym.Convolution(data=net, num_filter=channels, kernel=filter_dimension, stride=stride, no_bias=no_bias) + net = mx.sym.Convolution(data=net, num_filter=channels, kernel=filter_dimension, stride=stride, no_bias=no_bias, + name=name) elif weight is None or bias is not None: net = mx.sym.Convolution(data=net, num_filter=channels, kernel=filter_dimension, stride=stride, bias=bias, - no_bias=no_bias) + no_bias=no_bias, name=name) elif weight is not None or bias is None: net = mx.sym.Convolution(data=net, num_filter=channels, kernel=filter_dimension, stride=stride, weight=weight, - no_bias=no_bias) + no_bias=no_bias, name=name) else: net = mx.sym.Convolution(data=net, num_filter=channels, kernel=filter_dimension, stride=stride, weight=weight, - bias=bias, no_bias=no_bias) + bias=bias, no_bias=no_bias, name=name) net = mx.sym.Activation(data=net, act_type=act_type) return net diff --git a/example/speech_recognition/stt_layer_fc.py b/example/speech_recognition/stt_layer_fc.py index b3db2034a3ad..f435922426c5 100644 --- a/example/speech_recognition/stt_layer_fc.py +++ b/example/speech_recognition/stt_layer_fc.py @@ -8,29 +8,30 @@ def fc(net, act_type, weight=None, bias=None, - no_bias=False + no_bias=False, + name=None ): # when weight and bias doesn't have specific name if weight is None and bias is None: - net = mx.sym.FullyConnected(data=net, num_hidden=num_hidden, no_bias=no_bias) + net = mx.sym.FullyConnected(data=net, num_hidden=num_hidden, no_bias=no_bias, name=name) # when weight doesn't have specific name but bias has elif weight is None and bias is not None: if no_bias: - net = mx.sym.FullyConnected(data=net, num_hidden=num_hidden, no_bias=no_bias) + net = mx.sym.FullyConnected(data=net, num_hidden=num_hidden, no_bias=no_bias, name=name) else: - net = mx.sym.FullyConnected(data=net, num_hidden=num_hidden, bias=bias, no_bias=no_bias) + net = mx.sym.FullyConnected(data=net, num_hidden=num_hidden, bias=bias, no_bias=no_bias, name=name) # when bias doesn't have specific name but weight has elif weight is not None and bias is None: - net = mx.sym.FullyConnected(data=net, num_hidden=num_hidden, weight=weight, no_bias=no_bias) + net = mx.sym.FullyConnected(data=net, num_hidden=num_hidden, weight=weight, no_bias=no_bias, name=name) # when weight and bias specific name else: if no_bias: - net = mx.sym.FullyConnected(data=net, num_hidden=num_hidden, weight=weight, no_bias=no_bias) + net = mx.sym.FullyConnected(data=net, num_hidden=num_hidden, weight=weight, no_bias=no_bias, name=name) else: - net = mx.sym.FullyConnected(data=net, num_hidden=num_hidden, weight=weight, bias=bias, no_bias=no_bias) + net = mx.sym.FullyConnected(data=net, num_hidden=num_hidden, weight=weight, bias=bias, no_bias=no_bias, name=name) # activation if act_type is not None: - net = mx.sym.Activation(data=net, act_type=act_type) + net = mx.sym.Activation(data=net, act_type=act_type, name="%s_activation" % name) return net @@ -41,7 +42,7 @@ def sequence_fc(net, num_hidden_list=[], act_type_list=[], is_batchnorm=False, - dropout_rate=0 + dropout_rate=0, ): if num_layer == len(num_hidden_list) == len(act_type_list): if num_layer > 0: @@ -81,13 +82,16 @@ def sequence_fc(net, num_hidden=num_hidden_list[layer_index], act_type=None, weight=weight_list[layer_index], - no_bias=is_batchnorm + no_bias=is_batchnorm, + name="%s_t%d_l%d_fc" % (prefix, seq_index, layer_index) ) # last layer doesn't have batchnorm hidden = batchnorm(net=hidden, gamma=gamma_list[layer_index], - beta=beta_list[layer_index]) - hidden = mx.sym.Activation(data=hidden, act_type=act_type_list[layer_index]) + beta=beta_list[layer_index], + name="%s_t%d_l%d_batchnorm" % (prefix, seq_index, layer_index)) + hidden = mx.sym.Activation(data=hidden, act_type=act_type_list[layer_index], + name="%s_t%d_l%d_activation" % (prefix, seq_index, layer_index)) else: hidden = fc(net=hidden, num_hidden=num_hidden_list[layer_index], diff --git a/example/speech_recognition/stt_layer_gru.py b/example/speech_recognition/stt_layer_gru.py index 8b044746dfcf..89af1c72216d 100644 --- a/example/speech_recognition/stt_layer_gru.py +++ b/example/speech_recognition/stt_layer_gru.py @@ -15,7 +15,7 @@ "param_blocks"]) -def gru(num_hidden, indata, prev_state, param, seqidx, layeridx, dropout=0., is_batchnorm=False, gamma=None, beta=None): +def gru(num_hidden, indata, prev_state, param, seqidx, layeridx, dropout=0., is_batchnorm=False, gamma=None, beta=None, name=None): """ GRU Cell symbol Reference: @@ -31,7 +31,10 @@ def gru(num_hidden, indata, prev_state, param, seqidx, layeridx, dropout=0., is_ name="t%d_l%d_gates_i2h" % (seqidx, layeridx)) if is_batchnorm: - i2h = batchnorm(net=i2h, gamma=gamma, beta=beta) + if name is not None: + i2h = batchnorm(net=i2h, gamma=gamma, beta=beta, name="%s_batchnorm" % name) + else: + i2h = batchnorm(net=i2h, gamma=gamma, beta=beta) h2h = mx.sym.FullyConnected(data=prev_state.h, weight=param.gates_h2h_weight, bias=param.gates_h2h_bias, @@ -53,15 +56,15 @@ def gru(num_hidden, indata, prev_state, param, seqidx, layeridx, dropout=0., is_ weight=param.trans_h2h_weight, bias=param.trans_h2h_bias, num_hidden=num_hidden, - name="t%d_l%d_trans_i2h" % (seqidx, layeridx)) + name="t%d_l%d_trans_h2h" % (seqidx, layeridx)) h_trans = htrans_i2h + htrans_h2h h_trans_active = mx.sym.Activation(h_trans, act_type="tanh") next_h = prev_state.h + update_gate * (h_trans_active - prev_state.h) return GRUState(h=next_h) -def gru_unroll(net, num_gru_layer, seq_len, num_hidden_gru_list, dropout=0., is_batchnorm=False, prefix="", - direction="forward"): +def gru_unroll(net, num_gru_layer, seq_len, num_hidden_gru_list, dropout=0., is_batchnorm=False, prefix="", + direction="forward", is_bucketing=False): if num_gru_layer > 0: param_cells = [] last_states = [] @@ -81,9 +84,14 @@ def gru_unroll(net, num_gru_layer, seq_len, num_hidden_gru_list, dropout=0., is_ if is_batchnorm: batchnorm_gamma = [] batchnorm_beta = [] - for seqidx in range(seq_len): - batchnorm_gamma.append(mx.sym.Variable(prefix + "t%d_i2h_gamma" % seqidx)) - batchnorm_beta.append(mx.sym.Variable(prefix + "t%d_i2h_beta" % seqidx)) + if is_bucketing: + for l in range(num_gru_layer): + batchnorm_gamma.append(mx.sym.Variable(prefix + "l%d_i2h_gamma" % l)) + batchnorm_beta.append(mx.sym.Variable(prefix + "l%d_i2h_beta" % l)) + else: + for seqidx in range(seq_len): + batchnorm_gamma.append(mx.sym.Variable(prefix + "t%d_i2h_gamma" % seqidx)) + batchnorm_beta.append(mx.sym.Variable(prefix + "t%d_i2h_beta" % seqidx)) hidden_all = [] for seqidx in range(seq_len): @@ -103,19 +111,33 @@ def gru_unroll(net, num_gru_layer, seq_len, num_hidden_gru_list, dropout=0., is_ else: dp_ratio = dropout if is_batchnorm: - next_state = gru(num_hidden_gru_list[i], indata=hidden, - prev_state=last_states[i], - param=param_cells[i], - seqidx=k, layeridx=i, dropout=dp_ratio, - is_batchnorm=is_batchnorm, - gamma=batchnorm_gamma[k], - beta=batchnorm_beta[k]) + if is_bucketing: + next_state = gru(num_hidden_gru_list[i], indata=hidden, + prev_state=last_states[i], + param=param_cells[i], + seqidx=k, layeridx=i, dropout=dp_ratio, + is_batchnorm=is_batchnorm, + gamma=batchnorm_gamma[i], + beta=batchnorm_beta[i], + name=prefix + ("t%d_l%d" % (seqidx, i)) + ) + else: + next_state = gru(num_hidden_gru_list[i], indata=hidden, + prev_state=last_states[i], + param=param_cells[i], + seqidx=k, layeridx=i, dropout=dp_ratio, + is_batchnorm=is_batchnorm, + gamma=batchnorm_gamma[k], + beta=batchnorm_beta[k], + name=prefix + ("t%d_l%d" % (seqidx, i)) + ) else: next_state = gru(num_hidden_gru_list[i], indata=hidden, prev_state=last_states[i], param=param_cells[i], seqidx=k, layeridx=i, dropout=dp_ratio, - is_batchnorm=is_batchnorm) + is_batchnorm=is_batchnorm, + name=prefix) hidden = next_state.h last_states[i] = next_state # decoder @@ -133,7 +155,7 @@ def gru_unroll(net, num_gru_layer, seq_len, num_hidden_gru_list, dropout=0., is_ return net -def bi_gru_unroll(net, num_gru_layer, seq_len, num_hidden_gru_list, dropout=0., is_batchnorm=False): +def bi_gru_unroll(net, num_gru_layer, seq_len, num_hidden_gru_list, dropout=0., is_batchnorm=False, is_bucketing=False): if num_gru_layer > 0: net_forward = gru_unroll(net=net, num_gru_layer=num_gru_layer, @@ -142,7 +164,8 @@ def bi_gru_unroll(net, num_gru_layer, seq_len, num_hidden_gru_list, dropout=0., dropout=dropout, is_batchnorm=is_batchnorm, prefix="forward_", - direction="forward") + direction="forward", + is_bucketing=is_bucketing) net_backward = gru_unroll(net=net, num_gru_layer=num_gru_layer, seq_len=seq_len, @@ -150,7 +173,8 @@ def bi_gru_unroll(net, num_gru_layer, seq_len, num_hidden_gru_list, dropout=0., dropout=dropout, is_batchnorm=is_batchnorm, prefix="backward_", - direction="backward") + direction="backward", + is_bucketing=is_bucketing) hidden_all = [] for i in range(seq_len): hidden_all.append(mx.sym.Concat(*[net_forward[i], net_backward[i]], dim=1)) @@ -159,7 +183,7 @@ def bi_gru_unroll(net, num_gru_layer, seq_len, num_hidden_gru_list, dropout=0., def bi_gru_unroll_two_input_two_output(net1, net2, num_gru_layer, seq_len, num_hidden_gru_list, dropout=0., - is_batchnorm=False): + is_batchnorm=False, is_bucketing=False): if num_gru_layer > 0: net_forward = gru_unroll(net=net1, num_gru_layer=num_gru_layer, @@ -168,7 +192,8 @@ def bi_gru_unroll_two_input_two_output(net1, net2, num_gru_layer, seq_len, num_h dropout=dropout, is_batchnorm=is_batchnorm, prefix="forward_", - direction="forward") + direction="forward", + is_bucketing=is_bucketing) net_backward = gru_unroll(net=net2, num_gru_layer=num_gru_layer, seq_len=seq_len, @@ -176,7 +201,8 @@ def bi_gru_unroll_two_input_two_output(net1, net2, num_gru_layer, seq_len, num_h dropout=dropout, is_batchnorm=is_batchnorm, prefix="backward_", - direction="backward") + direction="backward", + is_bucketing=is_bucketing) return net_forward, net_backward else: return net1, net2 diff --git a/example/speech_recognition/stt_layer_lstm.py b/example/speech_recognition/stt_layer_lstm.py index 19e37369b1b0..93b4ca09b908 100644 --- a/example/speech_recognition/stt_layer_lstm.py +++ b/example/speech_recognition/stt_layer_lstm.py @@ -16,7 +16,7 @@ "param_blocks"]) -def vanilla_lstm(num_hidden, indata, prev_state, param, seqidx, layeridx, is_batchnorm=False, gamma=None, beta=None): +def vanilla_lstm(num_hidden, indata, prev_state, param, seqidx, layeridx, is_batchnorm=False, gamma=None, beta=None, name=None): """LSTM Cell symbol""" i2h = mx.sym.FullyConnected(data=indata, weight=param.i2h_weight, @@ -24,7 +24,10 @@ def vanilla_lstm(num_hidden, indata, prev_state, param, seqidx, layeridx, is_bat num_hidden=num_hidden * 4, name="t%d_l%d_i2h" % (seqidx, layeridx)) if is_batchnorm: - i2h = batchnorm(net=i2h, gamma=gamma, beta=beta) + if name is not None: + i2h = batchnorm(net=i2h, gamma=gamma, beta=beta, name="%s_batchnorm" % name) + else: + i2h = batchnorm(net=i2h, gamma=gamma, beta=beta) h2h = mx.sym.FullyConnected(data=prev_state.h, weight=param.h2h_weight, bias=param.h2h_bias, @@ -43,7 +46,7 @@ def vanilla_lstm(num_hidden, indata, prev_state, param, seqidx, layeridx, is_bat def lstm(num_hidden, indata, prev_state, param, seqidx, layeridx, dropout=0., num_hidden_proj=0, is_batchnorm=False, - gamma=None, beta=None): + gamma=None, beta=None, name=None): """LSTM Cell symbol""" # dropout input if dropout > 0.: @@ -55,7 +58,10 @@ def lstm(num_hidden, indata, prev_state, param, seqidx, layeridx, dropout=0., nu num_hidden=num_hidden * 4, name="t%d_l%d_i2h" % (seqidx, layeridx)) if is_batchnorm: - i2h = batchnorm(net=i2h, gamma=gamma, beta=beta) + if name is not None: + i2h = batchnorm(net=i2h, gamma=gamma, beta=beta, name="%s_batchnorm" % name) + else: + i2h = batchnorm(net=i2h, gamma=gamma, beta=beta) h2h = mx.sym.FullyConnected(data=prev_state.h, weight=param.h2h_weight, @@ -96,7 +102,7 @@ def lstm(num_hidden, indata, prev_state, param, seqidx, layeridx, dropout=0., nu def lstm_unroll(net, num_lstm_layer, seq_len, num_hidden_lstm_list, dropout=0., num_hidden_proj=0, - lstm_type='fc_lstm', is_batchnorm=False, prefix="", direction="forward"): + lstm_type='fc_lstm', is_batchnorm=False, prefix="", direction="forward", is_bucketing=False): if num_lstm_layer > 0: param_cells = [] last_states = [] @@ -121,9 +127,14 @@ def lstm_unroll(net, num_lstm_layer, seq_len, num_hidden_lstm_list, dropout=0., if is_batchnorm: batchnorm_gamma = [] batchnorm_beta = [] - for seqidx in range(seq_len): - batchnorm_gamma.append(mx.sym.Variable(prefix + "t%d_i2h_gamma" % seqidx)) - batchnorm_beta.append(mx.sym.Variable(prefix + "t%d_i2h_beta" % seqidx)) + if is_bucketing: + for l in range(num_lstm_layer): + batchnorm_gamma.append(mx.sym.Variable(prefix + "l%d_i2h_gamma" % l)) + batchnorm_beta.append(mx.sym.Variable(prefix + "l%d_i2h_beta" % l)) + else: + for seqidx in range(seq_len): + batchnorm_gamma.append(mx.sym.Variable(prefix + "t%d_i2h_gamma" % seqidx)) + batchnorm_beta.append(mx.sym.Variable(prefix + "t%d_i2h_beta" % seqidx)) hidden_all = [] for seqidx in range(seq_len): @@ -145,18 +156,20 @@ def lstm_unroll(net, num_lstm_layer, seq_len, num_hidden_lstm_list, dropout=0., if lstm_type == 'fc_lstm': if is_batchnorm: - next_state = lstm(num_hidden_lstm_list[i], - indata=hidden, - prev_state=last_states[i], - param=param_cells[i], - seqidx=k, - layeridx=i, - dropout=dp, - num_hidden_proj=num_hidden_proj, - is_batchnorm=is_batchnorm, - gamma=batchnorm_gamma[k], - beta=batchnorm_beta[k] - ) + if is_bucketing: + next_state = lstm(num_hidden_lstm_list[i], + indata=hidden, + prev_state=last_states[i], + param=param_cells[i], + seqidx=k, + layeridx=i, + dropout=dp, + num_hidden_proj=num_hidden_proj, + is_batchnorm=is_batchnorm, + gamma=batchnorm_gamma[i], + beta=batchnorm_beta[i], + name=prefix + ("t%d_l%d" % (seqidx, i)) + ) else: next_state = lstm(num_hidden_lstm_list[i], indata=hidden, @@ -166,7 +179,8 @@ def lstm_unroll(net, num_lstm_layer, seq_len, num_hidden_lstm_list, dropout=0., layeridx=i, dropout=dp, num_hidden_proj=num_hidden_proj, - is_batchnorm=is_batchnorm + is_batchnorm=is_batchnorm, + name=prefix + ("t%d_l%d" % (seqidx, i)) ) elif lstm_type == 'vanilla_lstm': if is_batchnorm: @@ -175,15 +189,17 @@ def lstm_unroll(net, num_lstm_layer, seq_len, num_hidden_lstm_list, dropout=0., param=param_cells[i], seqidx=k, layeridx=i, is_batchnorm=is_batchnorm, - gamma=batchnorm_gamma[k], - beta=batchnorm_beta[k] + gamma=batchnorm_gamma[i], + beta=batchnorm_beta[i], + name=prefix + ("t%d_l%d" % (seqidx, i)) ) else: next_state = vanilla_lstm(num_hidden_lstm_list[i], indata=hidden, prev_state=last_states[i], param=param_cells[i], seqidx=k, layeridx=i, - is_batchnorm=is_batchnorm + is_batchnorm=is_batchnorm, + name=prefix + ("t%d_l%d" % (seqidx, i)) ) else: raise Exception("lstm type %s error" % lstm_type) @@ -206,7 +222,7 @@ def lstm_unroll(net, num_lstm_layer, seq_len, num_hidden_lstm_list, dropout=0., def bi_lstm_unroll(net, num_lstm_layer, seq_len, num_hidden_lstm_list, dropout=0., num_hidden_proj=0, - lstm_type='fc_lstm', is_batchnorm=False): + lstm_type='fc_lstm', is_batchnorm=False, is_bucketing=False): if num_lstm_layer > 0: net_forward = lstm_unroll(net=net, num_lstm_layer=num_lstm_layer, @@ -217,7 +233,8 @@ def bi_lstm_unroll(net, num_lstm_layer, seq_len, num_hidden_lstm_list, dropout=0 lstm_type=lstm_type, is_batchnorm=is_batchnorm, prefix="forward_", - direction="forward") + direction="forward", + is_bucketing=is_bucketing) net_backward = lstm_unroll(net=net, num_lstm_layer=num_lstm_layer, @@ -228,7 +245,8 @@ def bi_lstm_unroll(net, num_lstm_layer, seq_len, num_hidden_lstm_list, dropout=0 lstm_type=lstm_type, is_batchnorm=is_batchnorm, prefix="backward_", - direction="backward") + direction="backward", + is_bucketing=is_bucketing) hidden_all = [] for i in range(seq_len): hidden_all.append(mx.sym.Concat(*[net_forward[i], net_backward[i]], dim=1)) @@ -239,7 +257,9 @@ def bi_lstm_unroll(net, num_lstm_layer, seq_len, num_hidden_lstm_list, dropout=0 # bilistm_2to1 def bi_lstm_unroll_two_input_two_output(net1, net2, num_lstm_layer, seq_len, num_hidden_lstm_list, dropout=0., num_hidden_proj=0, - lstm_type='fc_lstm', is_batchnorm=False): + lstm_type='fc_lstm', + is_batchnorm=False, + is_bucketing=False): if num_lstm_layer > 0: net_forward = lstm_unroll(net=net1, num_lstm_layer=num_lstm_layer, @@ -250,7 +270,8 @@ def bi_lstm_unroll_two_input_two_output(net1, net2, num_lstm_layer, seq_len, num lstm_type=lstm_type, is_batchnorm=is_batchnorm, prefix="forward_", - direction="forward") + direction="forward", + is_bucketing=is_bucketing) net_backward = lstm_unroll(net=net2, num_lstm_layer=num_lstm_layer, @@ -261,7 +282,8 @@ def bi_lstm_unroll_two_input_two_output(net1, net2, num_lstm_layer, seq_len, num lstm_type=lstm_type, is_batchnorm=is_batchnorm, prefix="backward_", - direction="backward") + direction="backward", + is_bucketing=is_bucketing) return net_forward, net_backward else: return net1, net2 diff --git a/example/speech_recognition/stt_metric.py b/example/speech_recognition/stt_metric.py index 0fc2bd11d906..1c5f4408a60e 100644 --- a/example/speech_recognition/stt_metric.py +++ b/example/speech_recognition/stt_metric.py @@ -19,12 +19,11 @@ def check_label_shapes(labels, preds, shape=0): class STTMetric(mx.metric.EvalMetric): - def __init__(self, batch_size, num_gpu, seq_length, is_epoch_end=False, is_logging=True): + def __init__(self, batch_size, num_gpu, is_epoch_end=False, is_logging=True): super(STTMetric, self).__init__('STTMetric') self.batch_size = batch_size self.num_gpu = num_gpu - self.seq_length = seq_length self.total_n_label = 0 self.total_l_dist = 0 self.is_epoch_end = is_epoch_end @@ -37,15 +36,17 @@ def update(self, labels, preds): log = LogUtil().getlogger() labelUtil = LabelUtil.getInstance() self.batch_loss = 0. + for label, pred in zip(labels, preds): label = label.asnumpy() pred = pred.asnumpy() - for i in range(int(int(self.batch_size) / int(self.num_gpu))): + seq_length = len(pred) / int(int(self.batch_size) / int(self.num_gpu)) + for i in range(int(int(self.batch_size) / int(self.num_gpu))): l = remove_blank(label[i]) p = [] - for k in range(int(self.seq_length)): + for k in range(int(seq_length)): p.append(np.argmax(pred[k * int(int(self.batch_size) / int(self.num_gpu)) + i])) p = pred_best(p) @@ -60,7 +61,7 @@ def update(self, labels, preds): self.num_inst += 1 self.sum_metric += this_cer if self.is_epoch_end: - loss = ctc_loss(l, pred, i, int(self.seq_length), int(self.batch_size), int(self.num_gpu)) + loss = ctc_loss(l, pred, i, int(seq_length), int(self.batch_size), int(self.num_gpu)) self.batch_loss += loss if self.is_logging: log.info("loss: %f " % loss) diff --git a/example/speech_recognition/stt_utils.py b/example/speech_recognition/stt_utils.py index 6a32f0e57c2d..3c7ffce0f980 100644 --- a/example/speech_recognition/stt_utils.py +++ b/example/speech_recognition/stt_utils.py @@ -92,7 +92,7 @@ def spectrogram(samples, fft_length=256, sample_rate=2, hop_length=128): def spectrogram_from_file(filename, step=10, window=20, max_freq=None, - eps=1e-14, overwrite=False): + eps=1e-14, overwrite=False, save_feature_as_csvfile=False): """ Calculate the log of linear spectrogram from FFT energy Params: filename (str): Path to the audio file @@ -104,7 +104,7 @@ def spectrogram_from_file(filename, step=10, window=20, max_freq=None, """ csvfilename = filename.replace(".wav", ".csv") - if (os.path.isfile(csvfilename) is False) or overwrite: + if (os.path.isfile(csvfilename) is False) or overwrite: with soundfile.SoundFile(filename) as sound_file: audio = sound_file.read(dtype='float32') sample_rate = sound_file.samplerate @@ -126,7 +126,8 @@ def spectrogram_from_file(filename, step=10, window=20, max_freq=None, ind = np.where(freqs <= max_freq)[0][-1] + 1 res = np.transpose(np.log(pxx[:ind, :] + eps)) - np.savetxt(csvfilename, res) + if save_feature_as_csvfile: + np.savetxt(csvfilename, res) return res else: return np.loadtxt(csvfilename) diff --git a/example/speech_recognition/train.py b/example/speech_recognition/train.py index 37f00fc4dd90..f3a7555529e3 100644 --- a/example/speech_recognition/train.py +++ b/example/speech_recognition/train.py @@ -7,7 +7,9 @@ from stt_metric import STTMetric #tensorboard setting from tensorboard import SummaryWriter -import numpy as np +import json +from stt_bucketing_module import STTBucketingModule + def get_initializer(args): @@ -28,6 +30,7 @@ def __init__(self, learning_rate=0.001): def __call__(self, num_update): return self.learning_rate + def do_training(args, module, data_train, data_val, begin_epoch=0): from distutils.dir_util import mkpath from log_util import LogUtil @@ -35,7 +38,7 @@ def do_training(args, module, data_train, data_val, begin_epoch=0): log = LogUtil().getlogger() mkpath(os.path.dirname(get_checkpoint_path(args))) - seq_len = args.config.get('arch', 'max_t_count') + #seq_len = args.config.get('arch', 'max_t_count') batch_size = args.config.getint('common', 'batch_size') save_checkpoint_every_n_epoch = args.config.getint('common', 'save_checkpoint_every_n_epoch') save_checkpoint_every_n_batch = args.config.getint('common', 'save_checkpoint_every_n_batch') @@ -44,27 +47,48 @@ def do_training(args, module, data_train, data_val, begin_epoch=0): contexts = parse_contexts(args) num_gpu = len(contexts) - eval_metric = STTMetric(batch_size=batch_size, num_gpu=num_gpu, seq_length=seq_len,is_logging=enable_logging_validation_metric,is_epoch_end=True) + eval_metric = STTMetric(batch_size=batch_size, num_gpu=num_gpu, is_logging=enable_logging_validation_metric,is_epoch_end=True) # tensorboard setting - loss_metric = STTMetric(batch_size=batch_size, num_gpu=num_gpu, seq_length=seq_len,is_logging=enable_logging_train_metric,is_epoch_end=False) + loss_metric = STTMetric(batch_size=batch_size, num_gpu=num_gpu, is_logging=enable_logging_train_metric,is_epoch_end=False) - optimizer = args.config.get('train', 'optimizer') - momentum = args.config.getfloat('train', 'momentum') + optimizer = args.config.get('optimizer', 'optimizer') learning_rate = args.config.getfloat('train', 'learning_rate') learning_rate_annealing = args.config.getfloat('train', 'learning_rate_annealing') mode = args.config.get('common', 'mode') num_epoch = args.config.getint('train', 'num_epoch') - clip_gradient = args.config.getfloat('train', 'clip_gradient') - weight_decay = args.config.getfloat('train', 'weight_decay') + clip_gradient = args.config.getfloat('optimizer', 'clip_gradient') + weight_decay = args.config.getfloat('optimizer', 'weight_decay') save_optimizer_states = args.config.getboolean('train', 'save_optimizer_states') show_every = args.config.getint('train', 'show_every') + optimizer_params_dictionary = json.loads(args.config.get('optimizer', 'optimizer_params_dictionary')) + kvstore_option = args.config.get('common', 'kvstore_option') n_epoch=begin_epoch + is_bucketing = args.config.getboolean('arch', 'is_bucketing') if clip_gradient == 0: clip_gradient = None + if is_bucketing and mode == 'load': + model_file = args.config.get('common', 'model_file') + model_name = os.path.splitext(model_file)[0] + model_num_epoch = int(model_name[-4:]) + + model_path = 'checkpoints/' + str(model_name[:-5]) + symbol, data_names, label_names = module(1600) + model = STTBucketingModule( + sym_gen=module, + default_bucket_key=data_train.default_bucket_key, + context=contexts) + data_train.reset() - module.bind(data_shapes=data_train.provide_data, + model.bind(data_shapes=data_train.provide_data, + label_shapes=data_train.provide_label, + for_training=True) + _, arg_params, aux_params = mx.model.load_checkpoint(model_path, model_num_epoch) + model.set_params(arg_params, aux_params) + module = model + else: + module.bind(data_shapes=data_train.provide_data, label_shapes=data_train.provide_label, for_training=True) @@ -75,41 +99,32 @@ def do_training(args, module, data_train, data_val, begin_epoch=0): lr_scheduler = SimpleLRScheduler(learning_rate=learning_rate) def reset_optimizer(force_init=False): - if optimizer == "sgd": - module.init_optimizer(kvstore='device', - optimizer=optimizer, - optimizer_params={'lr_scheduler': lr_scheduler, - 'momentum': momentum, - 'clip_gradient': clip_gradient, - 'wd': weight_decay}, - force_init=force_init) - elif optimizer == "adam": - module.init_optimizer(kvstore='device', - optimizer=optimizer, - optimizer_params={'lr_scheduler': lr_scheduler, - #'momentum': momentum, - 'clip_gradient': clip_gradient, - 'wd': weight_decay}, - force_init=force_init) - else: - raise Exception('Supported optimizers are sgd and adam. If you want to implement others define them in train.py') + optimizer_params = {'lr_scheduler': lr_scheduler, + 'clip_gradient': clip_gradient, + 'wd': weight_decay} + optimizer_params.update(optimizer_params_dictionary) + module.init_optimizer(kvstore=kvstore_option, + optimizer=optimizer, + optimizer_params=optimizer_params, + force_init=force_init) if mode == "train": reset_optimizer(force_init=True) else: reset_optimizer(force_init=False) + data_train.reset() + data_train.is_first_epoch = True #tensorboard setting tblog_dir = args.config.get('common', 'tensorboard_log_dir') summary_writer = SummaryWriter(tblog_dir) + while True: if n_epoch >= num_epoch: break - loss_metric.reset() log.info('---------train---------') for nbatch, data_batch in enumerate(data_train): - module.forward_backward(data_batch) module.update() # tensorboard setting @@ -136,6 +151,7 @@ def reset_optimizer(force_init=False): assert curr_acc is not None, 'cannot find Acc_exclude_padding in eval metric' data_train.reset() + data_train.is_first_epoch = False # tensorboard setting train_cer, train_n_label, train_l_dist, train_ctc_loss = loss_metric.get_name_value() From 66b00b54b3803187241f191ab7a62ecc6f5ebb95 Mon Sep 17 00:00:00 2001 From: Yao Wang Date: Tue, 11 Jul 2017 10:56:55 -0700 Subject: [PATCH 169/834] Fix tutorial test (#6988) * Nightly Test for tutorial notebook * Remove shell script --- tests/nightly/test_tutorial.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/nightly/test_tutorial.py b/tests/nightly/test_tutorial.py index 56f6ecd8afc5..19f297878ab3 100644 --- a/tests/nightly/test_tutorial.py +++ b/tests/nightly/test_tutorial.py @@ -52,10 +52,18 @@ def test_tutorial_nb(file_path): try: eprocessor.preprocess(notebook, {'metadata': {}}) except Exception as err: - err_msg = "Python script successfully run without error or warning " \ - "but notebook returned error:\n%s\nSomething weird happened." \ - % (str(err)) + err_msg = str(err) fail_dict[tutorial_name] = err_msg + finally: + output_nb = open("output.txt", mode='w') + nbformat.write(notebook, output_nb) + output_nb.close() + output_nb = open("output.txt", mode='r') + for line in output_nb: + if "Warning:" in line: + fail_dict[tutorial_name] = "%s has warning." % (tutorial_name) + return + if __name__ == "__main__": tutorial_dir = '../../docs/_build/html/tutorials/' @@ -64,7 +72,7 @@ def test_tutorial_nb(file_path): for line in config_file: tutorial_list.append(line.lstrip().rstrip()) file_dir = tutorial_dir + line.lstrip().rstrip() - test_tutorial(file_dir) + test_tutorial_nb(file_dir) fail_num = len(fail_dict) success_num = len(tutorial_list) - fail_num From a8a2505dab96369e06b08caca27ac942fc270cad Mon Sep 17 00:00:00 2001 From: Tobias Domhan Date: Tue, 11 Jul 2017 19:57:54 +0200 Subject: [PATCH 170/834] CMake based compilation for Mac OS X. (#6870) --- CMakeLists.txt | 11 +++++++++-- cmake/Modules/FindAccelerate.cmake | 28 ++++++++++++++++++++++++++++ mshadow | 2 +- 3 files changed, 38 insertions(+), 3 deletions(-) create mode 100644 cmake/Modules/FindAccelerate.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 179c420bcf37..ab29b6a7aaaf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -183,7 +183,11 @@ if(USE_OPENCV) message(STATUS "OpenCV found (${OpenCV_CONFIG_PATH})") add_definitions(-DMXNET_USE_OPENCV=1) if(NOT MSVC) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,--no-undefined") + if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,-undefined,error") + else() + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,--no-undefined") + endif() endif() else(USE_OPENCV) message(STATUS "OpenCV Disabled") @@ -477,7 +481,10 @@ if(USE_CPP_PACKAGE) add_subdirectory(cpp-package) endif() -add_subdirectory(example/image-classification/predict-cpp) +# Problems on Mac OS X: 1. librt not available 2. mxnet built as MODULE library, which can't be linked. +if(!${CMAKE_SYSTEM_NAME} STREQUAL "Darwin") + add_subdirectory(example/image-classification/predict-cpp) +endif() # ---[ Linter target if(MSVC) diff --git a/cmake/Modules/FindAccelerate.cmake b/cmake/Modules/FindAccelerate.cmake new file mode 100644 index 000000000000..8c9938246e54 --- /dev/null +++ b/cmake/Modules/FindAccelerate.cmake @@ -0,0 +1,28 @@ +# Find the Apple Accelerate framework +# +# The following are set after configuration is done: +# Accelerate_FOUND +# Accelerate_INCLUDE_DIRS +# Accelerate_LIBRARIES + +set(Accelerate_INCLUDE_SEARCH_PATHS + /System/Library/Frameworks/Accelerate.framework/Versions/Current/Frameworks/vecLib.framework/Versions/Current/Headers/ +) + +find_path(Accelerate_CBLAS_INCLUDE_DIR NAMES cblas.h PATHS ${Accelerate_INCLUDE_SEARCH_PATHS}) + +set(LOOKED_FOR + Accelerate_CBLAS_INCLUDE_DIR +) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(Accelerate DEFAULT_MSG ${LOOKED_FOR}) + +if(Accelerate_FOUND) + set(Accelerate_INCLUDE_DIR ${Accelerate_CBLAS_INCLUDE_DIR}) + set(Accelerate_LIBRARIES "-framework Accelerate") + mark_as_advanced(${LOOKED_FOR}) + + message(STATUS "Found Accelerate (include: ${Accelerate_CBLAS_INCLUDE_DIR}, library: ${Accelerate_BLAS_LIBRARY})") +endif(Accelerate_FOUND) + diff --git a/mshadow b/mshadow index 20b54f068c10..d32b5dacf2bb 160000 --- a/mshadow +++ b/mshadow @@ -1 +1 @@ -Subproject commit 20b54f068c1035f0319fa5e5bbfb129c450a5256 +Subproject commit d32b5dacf2bb5af4121df5fd60eb7775704f9131 From 96902b70c461396c5db41f5b5461f7caf6f8eb1e Mon Sep 17 00:00:00 2001 From: Andrei Paleyes Date: Tue, 11 Jul 2017 19:14:51 +0100 Subject: [PATCH 171/834] Implemeneted reciprocal operator (#6719) Improved documentation for negative operator Adjusted indentation in operator tests --- docs/api/python/symbol.md | 1 + src/operator/mshadow_op.h | 14 ++++++++++++ src/operator/tensor/elemwise_unary_op.cc | 20 +++++++++++++++++- src/operator/tensor/elemwise_unary_op.cu | 8 +++++++ tests/python/unittest/test_operator.py | 27 ++++++++++++++++-------- 5 files changed, 60 insertions(+), 10 deletions(-) diff --git a/docs/api/python/symbol.md b/docs/api/python/symbol.md index f99bee2bd79b..0ebb869290bf 100644 --- a/docs/api/python/symbol.md +++ b/docs/api/python/symbol.md @@ -253,6 +253,7 @@ Composite multiple symbols into a new one by an operator. broadcast_div broadcast_mod negative + reciprocal dot batch_dot add_n diff --git a/src/operator/mshadow_op.h b/src/operator/mshadow_op.h index 94bfdb9830c7..8d867043dccd 100644 --- a/src/operator/mshadow_op.h +++ b/src/operator/mshadow_op.h @@ -63,6 +63,20 @@ struct negation { } }; +struct reciprocal { + template + MSHADOW_XINLINE static DType Map(DType a) { + return DType(1.0f/a); + } +}; + +struct reciprocal_grad { + template + MSHADOW_XINLINE static DType Map(DType a) { + return DType(-(DType(1.0f) / (a * a))); + } +}; + /*! \brief sigmoid unit */ struct sigmoid { template diff --git a/src/operator/tensor/elemwise_unary_op.cc b/src/operator/tensor/elemwise_unary_op.cc index 532963c754ee..ff03846ab5b3 100644 --- a/src/operator/tensor/elemwise_unary_op.cc +++ b/src/operator/tensor/elemwise_unary_op.cc @@ -203,10 +203,28 @@ NNVM_REGISTER_OP(_backward_cast) // negative MXNET_OPERATOR_REGISTER_UNARY(negative) -.MXNET_DESCRIBE("Negate src") +.MXNET_DESCRIBE("Numerical negative of the argument, element-wise.") .set_attr("FCompute", UnaryCompute) .set_attr("FGradient", ElemwiseGradUseNone{"negative"}); +// reciprocal +MXNET_OPERATOR_REGISTER_UNARY(reciprocal) +.describe(R"code(Returns the reciprocal of the argument, element-wise. + +Calculates 1/x. + +Example:: + + reciprocal([-2, 1, 3, 1.6, 0.2]) = [-0.5, 1.0, 0.33333334, 0.625, 5.0] + +)code" ADD_FILELINE) +.set_attr("FCompute", UnaryCompute) +.set_attr("FGradient", ElemwiseGradUseIn{"_backward_reciprocal"}); + +MXNET_OPERATOR_REGISTER_BINARY(_backward_reciprocal) +.set_attr("FCompute", + BinaryCompute >); + // abs MXNET_OPERATOR_REGISTER_UNARY(abs) .describe(R"code(Returns element-wise absolute value of the input. diff --git a/src/operator/tensor/elemwise_unary_op.cu b/src/operator/tensor/elemwise_unary_op.cu index a3ecc8fab638..67ceb1ce5093 100644 --- a/src/operator/tensor/elemwise_unary_op.cu +++ b/src/operator/tensor/elemwise_unary_op.cu @@ -47,6 +47,14 @@ NNVM_REGISTER_OP(_backward_cast) NNVM_REGISTER_OP(negative) .set_attr("FCompute", UnaryCompute); +// reciprocal +NNVM_REGISTER_OP(reciprocal) +.set_attr("FCompute", UnaryCompute); + +NNVM_REGISTER_OP(_backward_reciprocal) +.set_attr("FCompute", + BinaryCompute >); + // abs NNVM_REGISTER_OP(abs) .set_attr("FCompute", UnaryCompute); diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py index 7eca5650786a..e345326632f3 100644 --- a/tests/python/unittest/test_operator.py +++ b/tests/python/unittest/test_operator.py @@ -3165,18 +3165,27 @@ def test_ctc_loss(): def test_quantization_op(): - min0 = mx.nd.array([0.0]) - max0 = mx.nd.array([1.0]) - a = mx.nd.array([[0.1392, 0.5928], [0.6027, 0.8579]]) - qa, min1, max1 = mx.contrib.nd.quantize(a, min0, max0, out_type='uint8') - a_ = mx.contrib.nd.dequantize(qa, min1, max1, out_type='float32') + min0 = mx.nd.array([0.0]) + max0 = mx.nd.array([1.0]) + a = mx.nd.array([[0.1392, 0.5928], [0.6027, 0.8579]]) + qa, min1, max1 = mx.contrib.nd.quantize(a, min0, max0, out_type='uint8') + a_ = mx.contrib.nd.dequantize(qa, min1, max1, out_type='float32') - qa_real = mx.nd.array([[35, 151], [154, 219]]) - a_real = mx.nd.array([[0.13725491, 0.59215689], [0.60392159, 0.8588236]]) + qa_real = mx.nd.array([[35, 151], [154, 219]]) + a_real = mx.nd.array([[0.13725491, 0.59215689], [0.60392159, 0.8588236]]) - assert same(qa.asnumpy(), qa_real.asnumpy()) - assert same(a_.asnumpy(), a_real.asnumpy()) + assert same(qa.asnumpy(), qa_real.asnumpy()) + assert same(a_.asnumpy(), a_real.asnumpy()) +def test_reciprocal_op(): + data_tmp = np.random.rand(3, 4) * 10 - 5 + # Avoid possible division by 0 errors + data_tmp[data_tmp == 0] = 1.0 + data = mx.symbol.Variable('data') + test = mx.sym.reciprocal(data) + + check_numeric_gradient(test, [data_tmp]) + check_symbolic_forward(test, [data_tmp], [np.reciprocal(data_tmp)]) def test_custom_op(): class Sqr(mx.operator.CustomOp): From 5aa3034c0151389dd24cf64ac579dc45d41cbb15 Mon Sep 17 00:00:00 2001 From: "Qiang Kou (KK)" Date: Tue, 11 Jul 2017 15:02:25 -0700 Subject: [PATCH 172/834] [R] update the cats vs dogs example. close #6788 (#6935) --- R-package/R/model.R | 8 +- R-package/tests/testthat/get_data.R | 26 ++ R-package/tests/testthat/test_model.R | 45 +++- R-package/vignettes/CatsDogsFinetune.rmd | 251 +++++++++++++++++++ docs/tutorials/r/CatsDogsFinetune.rmd | 305 ----------------------- 5 files changed, 325 insertions(+), 310 deletions(-) create mode 100644 R-package/vignettes/CatsDogsFinetune.rmd delete mode 100644 docs/tutorials/r/CatsDogsFinetune.rmd diff --git a/R-package/R/model.R b/R-package/R/model.R index f76e51c4c633..998156d1a110 100644 --- a/R-package/R/model.R +++ b/R-package/R/model.R @@ -561,8 +561,8 @@ predict.MXFeedForwardModel <- function(model, X, ctx=NULL, array.batch.size=128, #' #' @export mx.model.load <- function(prefix, iteration) { - symbol <- mx.symbol.load(paste0(prefix, "-symbol.json")) - save.dict <- mx.nd.load(sprintf("%s-%04d.params", prefix, iteration)) + symbol <- mx.symbol.load(path.expand(paste0(prefix, "-symbol.json"))) + save.dict <- mx.nd.load(path.expand(sprintf("%s-%04d.params", prefix, iteration))) names <- names(save.dict) arg.index <- as.integer(mx.util.filter.null(lapply(1:length(names), function(i) { if (startsWith(names[[i]], "arg:")) i else NULL @@ -608,8 +608,8 @@ mx.model.save <- function(model, prefix, iteration) { paste0("aux:", nm) })) save.dict <- append(arg.params, aux.params) - mx.symbol.save(model$symbol, paste0(prefix, "-symbol.json")) - mx.nd.save(save.dict, sprintf("%s-%04d.params", prefix, iteration)) + mx.symbol.save(model$symbol, path.expand(paste0(prefix, "-symbol.json"))) + mx.nd.save(save.dict, path.expand(sprintf("%s-%04d.params", prefix, iteration))) } #' Check if the model has been serialized into RData-compatiable format. diff --git a/R-package/tests/testthat/get_data.R b/R-package/tests/testthat/get_data.R index 27db31a551f5..89b04476011e 100644 --- a/R-package/tests/testthat/get_data.R +++ b/R-package/tests/testthat/get_data.R @@ -38,6 +38,32 @@ GetCifar10 <- function() { } } +GetInception <- function() { + if (!dir.exists("model")) { + dir.create("model/") + } + if (!file.exists('model/Inception-BN-0126.params')) { + download.file('http://data.dmlc.ml/models/imagenet/inception-bn/Inception-BN-0126.params', + destfile = 'model/Inception-BN-0126.params') + } + if (!file.exists('model/Inception-BN-symbol.json')) { + download.file('http://data.dmlc.ml/models/imagenet/inception-bn/Inception-BN-symbol.json', + destfile = 'model/Inception-BN-symbol.json') + } +} + +GetCatDog <- function() { + if (!dir.exists("data")) { + dir.create("data/") + } + if (!file.exists('data/cats_dogs/cats_dogs_train.rec') | + !file.exists('data/cats_dogs/cats_dogs_val.rec')) { + download.file('https://s3-us-west-2.amazonaws.com/apache-mxnet/R/data/cats_dogs.zip', + destfile = 'data/cats_dogs.zip') + unzip('data/cats_dogs.zip', exdir = 'data/') + } +} + GetMovieLens <- function() { if (!dir.exists("data")) { dir.create("data/") diff --git a/R-package/tests/testthat/test_model.R b/R-package/tests/testthat/test_model.R index dec1517556ba..fcf8daee70a4 100644 --- a/R-package/tests/testthat/test_model.R +++ b/R-package/tests/testthat/test_model.R @@ -91,6 +91,7 @@ test_that("Regression", { }) + test_that("Classification", { data(Sonar, package = "mlbench") Sonar[, 61] <- as.numeric(Sonar[, 61]) - 1 @@ -103,11 +104,53 @@ test_that("Classification", { model <- mx.mlp(train.x, train.y, hidden_node = 10, out_node = 2, out_activation = "softmax", num.round = 20, array.batch.size = 15, - learning.rate = 0.07, + learning.rate = 0.07, momentum = 0.9, eval.metric = mx.metric.accuracy) }) +test_that("Fine-tune", { + GetInception() + GetCatDog() + train_iter <- mx.io.ImageRecordIter(path.imgrec = "./data/cats_dogs/cats_dogs_train.rec", + batch.size = 8, data.shape = c(224, 224, 3), + rand.crop = TRUE, rand.mirror = TRUE) + val_iter <- mx.io.ImageRecordIter(path.imgrec = "./data/cats_dogs/cats_dogs_val.rec", + batch.size = 8, data.shape = c(224, 224, 3), + rand.crop = FALSE, rand.mirror = FALSE) + inception_bn <- mx.model.load("./model/Inception-BN", iteration = 126) + symbol <- inception_bn$symbol + internals <- symbol$get.internals() + outputs <- internals$outputs + + flatten <- internals$get.output(which(outputs == "flatten_output")) + + new_fc <- mx.symbol.FullyConnected(data = flatten, num_hidden = 2, name = "fc1") + new_soft <- mx.symbol.SoftmaxOutput(data = new_fc, name = "softmax") + arg_params_new <- mxnet:::mx.model.init.params(symbol = new_soft, + input.shape = list("data" = c(224, 224, 3, 8)), + output.shape = NULL, + initializer = mx.init.uniform(0.1), + ctx = mx.cpu())$arg.params + fc1_weights_new <- arg_params_new[["fc1_weight"]] + fc1_bias_new <- arg_params_new[["fc1_bias"]] + + arg_params_new <- inception_bn$arg.params + + arg_params_new[["fc1_weight"]] <- fc1_weights_new + arg_params_new[["fc1_bias"]] <- fc1_bias_new + + #model <- mx.model.FeedForward.create(symbol = new_soft, X = train_iter, eval.data = val_iter, + # ctx = mx.cpu(), eval.metric = mx.metric.accuracy, + # num.round = 2, learning.rate = 0.05, momentum = 0.9, + # wd = 0.00001, kvstore = "local", + # batch.end.callback = mx.callback.log.train.metric(50), + # initializer = mx.init.Xavier(factor_type = "in", magnitude = 2.34), + # optimizer = "sgd", + # arg.params = arg_params_new, + # aux.params = inception_bn$aux.params) +}) + test_that("Matrix Factorization", { GetMovieLens() DF <- read.table("./data/ml-100k/u.data", header = F, sep = "\t") diff --git a/R-package/vignettes/CatsDogsFinetune.rmd b/R-package/vignettes/CatsDogsFinetune.rmd new file mode 100644 index 000000000000..c137ee8b7cef --- /dev/null +++ b/R-package/vignettes/CatsDogsFinetune.rmd @@ -0,0 +1,251 @@ +Dogs vs. Cats classification with mxnet and R +============================================= + +## Packages and prerequisites + +In this tutorial, we mainly use the following three packages: + +* `mxnet`: model training +* `imager`: image processing +* `abind`: manipulations with arrays. + +It is an end-to-end R solution for the dogs vs cats Kaggle competition (https://www.kaggle.com/c/dogs-vs-cats-redux-kernels-edition/) +and it can be used as an example for fine-tuning. +All the code has been test on Ubuntu 16.04. + + +```{r, echo=FALSE} +knitr::opts_chunk$set(eval = FALSE) +``` + +```{r} +library(imager) +library(mxnet) +library(abind) +``` + + +## Image processing + +### Renaming train files + +```{r} +files <- list.files("./train") +old_names <- sapply(files, strsplit, split = ".", fixed = TRUE) +max_length <- max(sapply(old_names, function(x) nchar(x[[2]]))) +zeros <- max_length - sapply(old_names, function(x) nchar(x[[2]])) +zeros <- sapply(zeros, function(x) paste(rep(0, x), collapse = "")) +new_names <- Map(function(x, y) {paste0("./train/", x[1], "/", y, x[2], ".jpg")}, + x = old_names, y = zeros) + +# Full names +files <- paste0("./train/", files) + +dir.create("./train/cat") +dir.create("./train/dog") + +# New names will be in 00001.jpg format +Map(function(x, y) file.rename(from = x, to = y), files, new_names) +``` + +### Training images: 224x224, padded with empty space + +```{r} +files <- list.files("./train", recursive = TRUE) +new_names <- paste0("./train_pad_224x224/", files) +files <- paste0("./train/", files) +dir.create("./train_pad_224x224/") +dir.create("./train_pad_224x224/cat") +dir.create("./train_pad_224x224/dog") + +padImage <- function(x) { + long_side <- max(dim(x)[1:2]) + short_side <- min(dim(x)[1:2]) + pad_img <- pad(x, + nPix = long_side - short_side, + axes = ifelse(dim(x)[1] < dim(x)[2], "x", "y")) + return(pad_img) +} + +Map(function(x, y) { + pad_img <- padImage(load.image(x)) + res_img <- resize(pad_img, size_x = 224, size_y = 224) + imager::save.image(res_img, y) + }, x = files, y = new_names) +``` + +### Renaming test files + +```{r} +files <- list.files("./test") +max_length <- max(sapply(files, nchar)) +zeros <- max_length - sapply(files, nchar) +zeros <- sapply(zeros, function(x) paste(rep(0, x), collapse = "")) +newnames <- paste0("./test/", zeros, files) + +files <- paste0("./test/", files) + +Map(function(x, y) file.rename(from = x, to = y), files, newnames) +``` + + +### Test images: 224x224, padded with empty space + +```{r} +files <- list.files("./test") +new_names <- paste0("./test_pad_224x224/", files) +files <- paste0("./test/", files) +dir.create("./test_pad_224x224/") + +Map(function(x, y) { + pad_img <- padImage(load.image(x)) + res_img <- resize(pad_img, size_x = 224, size_y = 224) + imager::save.image(res_img, y) +}, x = files, y = new_names) +``` + +### Creating .rec files using im2rec.py + +```{bash, eval = FALSE} +python im2rec.py --list=1 --recursive=1 --train-ratio=0.8 cats_dogs train_pad_224x224 +python im2rec.py --num-thread=4 --pass-through=1 cats_dogs_train.lst train_pad_224x224 +python im2rec.py --num-thread=4 --pass-through=1 cats_dogs_val.lst train_pad_224x224 +``` + +## The data iterator + +```{r} +get_iterator <- function(data_shape, train_data, val_data, batch_size = 128) { + train <- mx.io.ImageRecordIter(path.imgrec = train_data, + batch.size = batch_size, + data.shape = data_shape, + rand.crop = TRUE, + rand.mirror = TRUE) + + val <- mx.io.ImageRecordIter(path.imgrec = val_data, + batch.size = batch_size, + data.shape = data_shape, + rand.crop = FALSE, + rand.mirror = FALSE) + + return(list(train = train, val = val)) +} +``` + + +```{r} +data <- get_iterator(data_shape = c(224, 224, 3), + train_data = "cats_dogs_train.rec", + val_data = "cats_dogs_val.rec", + batch_size = 8) +train <- data$train +val <- data$val +``` + + +## Load pretrained model + +Here we use the pretrained model from http://data.dmlc.ml/models/imagenet/. +There are 1000 classes in imagenet, +and we need to replace the last fully connected layer with a new layer for 2 classes. + + +```{r} +inception_bn <- mx.model.load("./Inception-BN", iteration = 126) + +symbol <- inception_bn$symbol +# check symbol$arguments for layer names +internals <- symbol$get.internals() +outputs <- internals$outputs + +flatten <- internals$get.output(which(outputs == "flatten_output")) + +new_fc <- mx.symbol.FullyConnected(data = flatten, + num_hidden = 2, + name = "fc1") +# set name to original name in symbol$arguments +new_soft <- mx.symbol.SoftmaxOutput(data = new_fc, + name = "softmax") +# set name to original name in symbol$arguments + +arg_params_new <- mxnet:::mx.model.init.params(symbol = new_soft, + input.shape = list("data" = c(224, 224, 3, 8)), + output.shape = NULL, + initializer = mx.init.uniform(0.1), + ctx = mx.cpu())$arg.params +fc1_weights_new <- arg_params_new[["fc1_weight"]] +fc1_bias_new <- arg_params_new[["fc1_bias"]] + +arg_params_new <- inception_bn$arg.params + +arg_params_new[["fc1_weight"]] <- fc1_weights_new +arg_params_new[["fc1_bias"]] <- fc1_bias_new +``` + + +## Fine-tuning + +```{r} +model <- mx.model.FeedForward.create( + symbol = new_soft, + X = train, + eval.data = val, + ctx = mx.gpu(0), + eval.metric = mx.metric.accuracy, + num.round = 2, + learning.rate = 0.05, + momentum = 0.9, + wd = 0.00001, + kvstore = "local", + array.batch.size = 128, + epoch.end.callback = mx.callback.save.checkpoint("inception_bn"), + batch.end.callback = mx.callback.log.train.metric(150), + initializer = mx.init.Xavier(factor_type = "in", magnitude = 2.34), + optimizer = "sgd", + arg.params = arg_params_new, + aux.params = inception_bn$aux.params +) +``` +## Making predictions + +```{r} +preprocImage<- function(src, # URL or file location + height = 224, + width = 224, + num_channels = 3, # 3 for RGB, 1 for grayscale + mult_by = 1, # set to 255 for normalized image + crop = FALSE) { # no crop by default + + im <- load.image(src) + + if (crop) { + shape <- dim(im) + short_edge <- min(shape[1:2]) + xx <- floor((shape[1] - short_edge) / 2) + yy <- floor((shape[2] - short_edge) / 2) + im <- crop.borders(im, xx, yy) + } + + resized <- resize(im, size_x = width, size_y = height) + arr <- as.array(resized) * mult_by + dim(arr) <- c(width, height, num_channels, 1) + return(arr) +} +``` + +```{r} +files <- list.files("./test_pad_224x224/") +files <- paste0("./test_pad_224x224/", files) + +files <- split(files, rep(1:1250, each = 10)) +probs <- lapply(files, function(x) { + images <- lapply(x, preprocImage, mult_by = 255) + images <- do.call(abind, images) + probs <- predict(model, X = images, ctx = mx.gpu(0)) +}) +saveRDS(probs, "probs.rds") +probs <- t(do.call(cbind, probs)) + +preds <- data.frame(id = 1:12500, label = probs[, 2]) +write.csv(preds, "subm.csv", row.names = FALSE, quote = FALSE) +``` diff --git a/docs/tutorials/r/CatsDogsFinetune.rmd b/docs/tutorials/r/CatsDogsFinetune.rmd deleted file mode 100644 index a99e7042804e..000000000000 --- a/docs/tutorials/r/CatsDogsFinetune.rmd +++ /dev/null @@ -1,305 +0,0 @@ ---- -title: "Dogs vs. Cats classification with mxnet and R" -author: "Andrey Ogurtsov (https://github.com/statist-bhfz/)" -date: "February 25, 2017" ---- - -## 1. Packages and prerequisites - -Ubuntu 16, **mxnet** 0.9.4 (compiled with GPU support), **imager** for image processind, **abind** for manipulations with arrays. It is almost end-to-end R solution for Kaggle competition https://www.kaggle.com/c/dogs-vs-cats-redux-kernels-edition/, we will use Python only for creating .rec-files. - -Thanks to [jeremiedb](https://github.com/jeremiedb), my code for fine-tuning is largely based on his [answers](https://github.com/dmlc/mxnet/issues/4817). - -```{r} -knitr::opts_chunk$set(eval = FALSE) -``` - -```{r} -library(imager) -library(mxnet) -library(abind) -``` - - -## 2. Image processing - -### 2.1. Renaming train files - -```{r} -files <- list.files("train") -old_names <- sapply(files, strsplit, split = ".", fixed = TRUE) -max_length <- max(sapply(old_names, function(x) nchar(x[[2]]))) -zeros <- max_length - sapply(old_names, function(x) nchar(x[[2]])) -zeros <- sapply(zeros, function(x) paste(rep(0, x), collapse = "")) -new_names <- Map(function(x, y) {paste0("./train/", - x[1], - "/", - y, - x[2], - ".jpg")}, - x = old_names, y = zeros - ) - -# Full names -files <- paste0("./train/", files) - -dir.create("./train/cat") -dir.create("./train/dog") - -# New names will be in 00001.jpg format -Map(function(x, y) file.rename(from = x, to = y), files, new_names) -``` - -### 2.2. Train images: 224x224, padded with empty space - -```{r} -files <- list.files("train", recursive = TRUE) -new_names <- paste0("train_pad_224x224/", files) -files <- paste0("./train/", files) -dir.create("./train_pad_224x224/") -dir.create("./train_pad_224x224/cat") -dir.create("./train_pad_224x224/dog") - -padImage <- function(x) { - long_side <- max(dim(x)[1:2]) - short_side <- min(dim(x)[1:2]) - pad_img <- pad(x, - nPix = long_side - short_side, - axes = ifelse(dim(x)[1] < dim(x)[2], "x", "y")) - return(pad_img) -} - -Map(function(x, y) { - pad_img <- padImage(load.image(x)) - res_img <- resize(pad_img, size_x = 224, size_y = 224) - imager::save.image(res_img, y) -}, x = files, y = new_names) -``` - -### 2.3. Renaming test files - -```{r} -files <- list.files("test") -max_length <- max(sapply(files, nchar)) -zeros <- max_length - sapply(files, nchar) -zeros <- sapply(zeros, function(x) paste(rep(0, x), collapse = "")) -newnames <- paste0("./test/", zeros, files) - -files <- paste0("./test/", files) - -Map(function(x, y) file.rename(from = x, to = y), files, newnames) -``` - - -### 2.4. Test images: 224x224, padded with empty space - -```{r} -files <- list.files("test") -new_names <- paste0("test_pad_224x224/", files) -files <- paste0("./test/", files) -dir.create("./test_pad_224x224/") - -Map(function(x, y) { - pad_img <- padImage(load.image(x)) - res_img <- resize(pad_img, size_x = 224, size_y = 224) - imager::save.image(res_img, y) -}, x = files, y = new_names) -``` - -### 2.5. Creating .rec files - -```{bash, eval = FALSE} -python ~/mxnet/tools/im2rec.py --list=1 --recursive=1 --train-ratio=0.8 cats_dogs train_pad_224x224 -python ~/mxnet/tools/im2rec.py --num-thread=4 --pass-through=1 cats_dogs_train.lst train_pad_224x224 -python ~/mxnet/tools/im2rec.py --num-thread=4 --pass-through=1 cats_dogs_val.lst train_pad_224x224 -``` - - -## 3. Iterators - -```{r} -get_iterator <- function(data_shape, - train_data, - val_data, - batch_size = 128) { - train <- mx.io.ImageRecordIter( - path.imgrec = train_data, - batch.size = batch_size, - data.shape = data_shape, - rand.crop = TRUE, - rand.mirror = TRUE) - - val <- mx.io.ImageRecordIter( - path.imgrec = val_data, - batch.size = batch_size, - data.shape = data_shape, - rand.crop = FALSE, - rand.mirror = FALSE - ) - - return(list(train = train, val = val)) -} -``` - - -```{r} -data <- get_iterator(data_shape = c(224, 224, 3), - train_data = "/media/andrey/Data/KAGGLE/cats_dogs/cats_dogs_train.rec", - val_data = "/media/andrey/Data/KAGGLE/cats_dogs/cats_dogs_val.rec", - batch_size = 8) -train <- data$train -val <- data$val -``` - - -## 4. Load pretrained model - -Model from http://data.dmlc.ml/models/imagenet/ -Last fully connected layes for 1000 classes replaced with new layer for 2 classes. - - -```{r} -inception_bn <- mx.model.load("models/inception_bn/Inception-BN", - iteration = 126) - -symbol <- inception_bn$symbol -# check symbol$arguments for layer names -internals <- symbol$get.internals() -outputs <- internals$outputs - -flatten <- internals$get.output(which(outputs == "flatten_output")) - -new_fc <- mx.symbol.FullyConnected(data = flatten, - num_hidden = 2, - name = "fc1") - # set name to original name in symbol$arguments -new_soft <- mx.symbol.SoftmaxOutput(data = new_fc, - name = "softmax") - # set name to original name in symbol$arguments - -arg_params_new <- mxnet:::mx.model.init.params( - symbol = new_soft, - input.shape = c(224, 224, 3, 8), - initializer = mxnet:::mx.init.uniform(0.1), - ctx = mx.gpu(0) - )$arg.params -fc1_weights_new <- arg_params_new[["fc1_weight"]] -fc1_bias_new <- arg_params_new[["fc1_bias"]] - -arg_params_new <- inception_bn$arg.params - -arg_params_new[["fc1_weight"]] <- fc1_weights_new -arg_params_new[["fc1_bias"]] <- fc1_bias_new -``` - - -## 5. Train (fine-tune) model - -```{r} -model <- mx.model.FeedForward.create( - symbol = new_soft, - X = train, - eval.data = val, - ctx = mx.gpu(0), - eval.metric = mx.metric.accuracy, - num.round = 1, - learning.rate = 0.05, - momentum = 0.9, - wd = 0.00001, - kvstore = "local", - array.batch.size = 128, - epoch.end.callback = mx.callback.save.checkpoint("inception_bn"), - batch.end.callback = mx.callback.log.train.metric(150), - initializer = mx.init.Xavier(factor_type = "in", magnitude = 2.34), - optimizer = "sgd", - arg.params = arg_params_new, - aux.params = inception_bn$aux.params -) -``` - -```{r} -model <- mx.model.load("inception_bn", 1) -``` - -Continue training with decreased speed (`learning.rate = 0.03`): - -```{r} -model <- mx.model.FeedForward.create( - symbol = model$symbol, - X = train, - eval.data = val, - ctx = mx.gpu(0), - eval.metric = mx.metric.accuracy, - num.round = 5, - learning.rate = 0.03, - momentum = 0.9, - wd = 0.00001, - kvstore = "local", - array.batch.size = 100, - epoch.end.callback = mx.callback.save.checkpoint("inception_bn"), - batch.end.callback = mx.callback.log.train.metric(150), - initializer = mx.init.Xavier(factor_type = "in", magnitude = 2.34), - optimizer = "sgd", - arg.params = model$arg.params, - aux.params = model$aux.params -) -``` - -```{r} -model <- mx.model.load("inception_bn", 1) -``` - -My R session crashed after each iteration, so I made some iterations manually. - - -## 6. Make predictions - -```{r} -preprocImage<- function(src, # URL or file location - height = 224, - width = 224, - num_channels = 3, # 3 for RGB, 1 for grayscale - mult_by = 1, # set to 255 for normalized image - crop = FALSE) { # no crop by default - - im <- load.image(src) - - if (crop) { - shape <- dim(im) - short_edge <- min(shape[1:2]) - xx <- floor((shape[1] - short_edge) / 2) - yy <- floor((shape[2] - short_edge) / 2) - im <- crop.borders(im, xx, yy) - } - - resized <- resize(im, size_x = width, size_y = height) - arr <- as.array(resized) * mult_by - dim(arr) <- c(width, height, num_channels, 1) - return(arr) -} -``` - -```{r} -files <- list.files("test_pad_224x224/") -files <- paste0("./test_pad_224x224/", files) - -# ind <- seq(1, 12500, 1250) -# probs <- numeric() -# for (i in ind) { -# images <- lapply(files[i:i+1249], preprocImage, mult_by = 255) -# images <- do.call(abind, images) -# probs[i:i+1249] <- predict(model, X = images, ctx = mx.gpu(0)) -# } - -files <- split(files, rep(1:1250, each = 10)) -probs <- lapply(files, function(x) { - images <- lapply(x, preprocImage, mult_by = 255) - images <- do.call(abind, images) - probs <- predict(model, X = images, ctx = mx.gpu(0)) -}) -saveRDS(probs, "probs.rds") -probs <- t(do.call(cbind, probs)) - -preds <- data.frame(id = 1:12500, label = probs[, 2]) -write.csv(preds, "subm.csv", row.names = FALSE, quote = FALSE) -``` From 65d0d47928c5081b7585182c04081e835111d3db Mon Sep 17 00:00:00 2001 From: Przemyslaw Tredak Date: Tue, 11 Jul 2017 16:43:36 -0700 Subject: [PATCH 173/834] Fix for codes not using pinned memory (#7001) * Fix for codes not using pinned memory * Fix from review * Fix indentation --- src/storage/storage.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/storage/storage.cc b/src/storage/storage.cc index c6e99973cd53..353d72d41cd9 100644 --- a/src/storage/storage.cc +++ b/src/storage/storage.cc @@ -88,6 +88,8 @@ Storage::Handle StorageImpl::Alloc(size_t size, Context ctx) { } case Context::kGPU: { #if MXNET_USE_CUDA + CUDA_CALL(cudaGetDeviceCount(&num_gpu_device)); + CHECK_GT(num_gpu_device, 0) << "GPU usage requires at least 1 GPU"; ptr = new storage::GPUPooledStorageManager(); #else LOG(FATAL) << "Compile with USE_CUDA=1 to enable GPU usage"; From ed190957bb57abd29aca1d22d201a87fd871a272 Mon Sep 17 00:00:00 2001 From: Eric Junyuan Xie Date: Wed, 12 Jul 2017 10:04:40 -0700 Subject: [PATCH 174/834] Refactor Stateful operator and custom op (#6928) * refactor create layer * fix * refactor custom op * fix * fix * fix * fix * fix OpState * remove superfluous infershape * fix * fix * fix lint * fix * fix * fix * Update CMakeLists.txt * delete * fix * fix scala --- Jenkinsfile | 12 +- cpp-package/include/mxnet-cpp/MxNetCpp.h | 6 +- cpp-package/include/mxnet-cpp/base.h | 6 +- cpp-package/include/mxnet-cpp/executor.h | 6 +- cpp-package/include/mxnet-cpp/executor.hpp | 6 +- cpp-package/include/mxnet-cpp/initializer.h | 6 +- cpp-package/include/mxnet-cpp/io.h | 6 +- cpp-package/include/mxnet-cpp/io.hpp | 6 +- cpp-package/include/mxnet-cpp/kvstore.h | 6 +- cpp-package/include/mxnet-cpp/kvstore.hpp | 6 +- cpp-package/include/mxnet-cpp/metric.h | 6 +- cpp-package/include/mxnet-cpp/model.h | 6 +- cpp-package/include/mxnet-cpp/monitor.h | 6 +- cpp-package/include/mxnet-cpp/monitor.hpp | 6 +- cpp-package/include/mxnet-cpp/ndarray.h | 6 +- cpp-package/include/mxnet-cpp/ndarray.hpp | 6 +- cpp-package/include/mxnet-cpp/op_map.h | 6 +- cpp-package/include/mxnet-cpp/op_suppl.h | 6 +- cpp-package/include/mxnet-cpp/op_util.h | 6 +- cpp-package/include/mxnet-cpp/operator.h | 6 +- cpp-package/include/mxnet-cpp/operator.hpp | 6 +- cpp-package/include/mxnet-cpp/optimizer.h | 6 +- cpp-package/include/mxnet-cpp/optimizer.hpp | 6 +- cpp-package/include/mxnet-cpp/shape.h | 6 +- cpp-package/include/mxnet-cpp/symbol.h | 6 +- cpp-package/include/mxnet-cpp/symbol.hpp | 6 +- cpp-package/scripts/OpWrapperGenerator.py | 6 +- dmlc-core | 2 +- include/mxnet/base.h | 6 + include/mxnet/c_api.h | 8 +- include/mxnet/op_attr_types.h | 160 +++++- include/mxnet/operator.h | 70 +-- python/mxnet/ndarray.py | 4 +- python/mxnet/operator.py | 2 +- .../scala/ml/dmlc/mxnet/OperatorSuite.scala | 2 + src/c_api/c_api.cc | 2 +- src/c_api/c_api_ndarray.cc | 106 ++-- src/common/utils.h | 17 + src/engine/naive_engine.cc | 8 +- src/engine/stream_manager.h | 14 +- src/engine/threaded_engine_perdevice.cc | 38 +- src/executor/attach_op_execs_pass.cc | 228 +++++---- src/executor/exec_pass.h | 6 +- src/executor/graph_executor.cc | 30 +- src/executor/graph_executor.h | 7 +- src/ndarray/autograd.cc | 18 +- src/ndarray/autograd.h | 8 +- src/ndarray/ndarray.cc | 6 +- src/ndarray/ndarray_function.cu | 8 +- src/nnvm/legacy_op_util.cc | 116 ++++- src/operator/activation.cc | 4 - src/operator/batch_norm.cc | 16 - src/operator/bilinear_sampler.cc | 4 - src/operator/convolution.cc | 3 - src/operator/cross_device_copy.cc | 12 +- src/operator/custom/custom-inl.h | 270 +--------- src/operator/custom/custom.cc | 463 +++++++++++++----- src/operator/custom/ndarray_op-inl.h | 8 +- src/operator/deconvolution.cc | 2 - src/operator/dropout-inl.h | 4 +- src/operator/dropout.cc | 4 - src/operator/fully_connected.cc | 2 - src/operator/grid_generator.cc | 4 - src/operator/instance_norm.cc | 4 - src/operator/lrn.cc | 14 +- src/operator/pad.cc | 4 - src/operator/pooling.cc | 4 - src/operator/rnn.cc | 4 - src/operator/roi_pooling.cc | 4 - src/operator/sequence_last.cc | 4 - src/operator/sequence_mask.cc | 4 - src/operator/sequence_reverse.cc | 4 - src/operator/softmax_output.cc | 4 - src/operator/spatial_transformer.cc | 4 - src/operator/svm_output.cc | 5 - src/operator/swapaxis.cc | 4 - src/operator/upsampling.cc | 4 - tests/cpp/include/test_op.h | 6 +- tests/cpp/include/test_perf.h | 6 +- tests/cpp/include/test_util.h | 6 +- tests/python/unittest/test_operator.py | 16 +- 81 files changed, 1012 insertions(+), 919 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 48f6251a0be2..26a96a4843bc 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -215,9 +215,11 @@ del /Q *.7z // Python unittest for CPU def python_ut(docker_type) { timeout(time: max_time, unit: 'MINUTES') { + sh "${docker_run} ${docker_type} find . -name '*.pyc' -type f -delete" sh "${docker_run} ${docker_type} PYTHONPATH=./python/ nosetests --with-timer --verbose tests/python/unittest" - sh "${docker_run} ${docker_type} PYTHONPATH=./python/ nosetests-3.4 --with-timer --verbose tests/python/unittest" sh "${docker_run} ${docker_type} PYTHONPATH=./python/ nosetests --with-timer --verbose tests/python/train" + sh "${docker_run} ${docker_type} find . -name '*.pyc' -type f -delete" + sh "${docker_run} ${docker_type} PYTHONPATH=./python/ nosetests-3.4 --with-timer --verbose tests/python/unittest" } } @@ -225,7 +227,9 @@ def python_ut(docker_type) { // both CPU and GPU def python_gpu_ut(docker_type) { timeout(time: max_time, unit: 'MINUTES') { + sh "${docker_run} ${docker_type} find . -name '*.pyc' -type f -delete" sh "${docker_run} ${docker_type} PYTHONPATH=./python/ nosetests --with-timer --verbose tests/python/gpu" + sh "${docker_run} ${docker_type} find . -name '*.pyc' -type f -delete" sh "${docker_run} ${docker_type} PYTHONPATH=./python/ nosetests-3.4 --with-timer --verbose tests/python/gpu" } } @@ -312,11 +316,13 @@ stage('Unit Test') { xcopy C:\\mxnet\\model model /E /I /Y call activate py3 set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_cpu\\python +del /S /Q ${env.WORKSPACE}\\pkg_vc14_cpu\\python\\*.pyc C:\\mxnet\\test_cpu.bat""" bat """xcopy C:\\mxnet\\data data /E /I /Y xcopy C:\\mxnet\\model model /E /I /Y call activate py2 set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_cpu\\python +del /S /Q ${env.WORKSPACE}\\pkg_vc14_cpu\\python\\*.pyc C:\\mxnet\\test_cpu.bat""" } } @@ -332,11 +338,13 @@ C:\\mxnet\\test_cpu.bat""" xcopy C:\\mxnet\\model model /E /I /Y call activate py3 set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_gpu\\python +del /S /Q ${env.WORKSPACE}\\pkg_vc14_gpu\\python\\*.pyc C:\\mxnet\\test_gpu.bat""" bat """xcopy C:\\mxnet\\data data /E /I /Y xcopy C:\\mxnet\\model model /E /I /Y call activate py2 set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_gpu\\python +del /S /Q ${env.WORKSPACE}\\pkg_vc14_gpu\\python\\*.pyc C:\\mxnet\\test_gpu.bat""" } } @@ -390,4 +398,4 @@ stage('Deploy') { } } } -} \ No newline at end of file +} diff --git a/cpp-package/include/mxnet-cpp/MxNetCpp.h b/cpp-package/include/mxnet-cpp/MxNetCpp.h index 8ed90e3c751a..5d61b823baa2 100644 --- a/cpp-package/include/mxnet-cpp/MxNetCpp.h +++ b/cpp-package/include/mxnet-cpp/MxNetCpp.h @@ -5,8 +5,8 @@ * \author Chuntao Hong, Zhang Chen */ -#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_MXNETCPP_H_ -#define CPP_PACKAGE_INCLUDE_MXNET_CPP_MXNETCPP_H_ +#ifndef MXNET_CPP_MXNETCPP_H_ +#define MXNET_CPP_MXNETCPP_H_ #include "mxnet-cpp/executor.hpp" #include "mxnet-cpp/symbol.hpp" @@ -21,4 +21,4 @@ #include "mxnet-cpp/metric.h" #include "mxnet-cpp/initializer.h" -#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_MXNETCPP_H_ +#endif // MXNET_CPP_MXNETCPP_H_ diff --git a/cpp-package/include/mxnet-cpp/base.h b/cpp-package/include/mxnet-cpp/base.h index 18f268a8a85a..b684986a6f54 100644 --- a/cpp-package/include/mxnet-cpp/base.h +++ b/cpp-package/include/mxnet-cpp/base.h @@ -5,8 +5,8 @@ * \author Chuntao Hong, Zhang Chen */ -#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_BASE_H_ -#define CPP_PACKAGE_INCLUDE_MXNET_CPP_BASE_H_ +#ifndef MXNET_CPP_BASE_H_ +#define MXNET_CPP_BASE_H_ #include #include "mxnet/c_api.h" @@ -35,4 +35,4 @@ enum OpReqType { } // namespace cpp } // namespace mxnet -#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_BASE_H_ +#endif // MXNET_CPP_BASE_H_ diff --git a/cpp-package/include/mxnet-cpp/executor.h b/cpp-package/include/mxnet-cpp/executor.h index e4343a19a50d..822344b7efee 100644 --- a/cpp-package/include/mxnet-cpp/executor.h +++ b/cpp-package/include/mxnet-cpp/executor.h @@ -5,8 +5,8 @@ * \author Chuntao Hong, Zhang Chen */ -#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_EXECUTOR_H_ -#define CPP_PACKAGE_INCLUDE_MXNET_CPP_EXECUTOR_H_ +#ifndef MXNET_CPP_EXECUTOR_H_ +#define MXNET_CPP_EXECUTOR_H_ #include #include @@ -135,4 +135,4 @@ class Executor { }; } // namespace cpp } // namespace mxnet -#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_EXECUTOR_H_ +#endif // MXNET_CPP_EXECUTOR_H_ diff --git a/cpp-package/include/mxnet-cpp/executor.hpp b/cpp-package/include/mxnet-cpp/executor.hpp index 4cae684f8881..1a452a1610db 100644 --- a/cpp-package/include/mxnet-cpp/executor.hpp +++ b/cpp-package/include/mxnet-cpp/executor.hpp @@ -5,8 +5,8 @@ * \author Zhang Chen, Chuntao Hong */ -#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_EXECUTOR_HPP_ -#define CPP_PACKAGE_INCLUDE_MXNET_CPP_EXECUTOR_HPP_ +#ifndef MXNET_CPP_EXECUTOR_HPP_ +#define MXNET_CPP_EXECUTOR_HPP_ #include #include @@ -89,4 +89,4 @@ inline void Executor::UpdateAll(Optimizer *opt, float lr, float wd, } // namespace cpp } // namespace mxnet -#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_EXECUTOR_HPP_ +#endif // MXNET_CPP_EXECUTOR_HPP_ diff --git a/cpp-package/include/mxnet-cpp/initializer.h b/cpp-package/include/mxnet-cpp/initializer.h index 843965256df1..f28656577482 100644 --- a/cpp-package/include/mxnet-cpp/initializer.h +++ b/cpp-package/include/mxnet-cpp/initializer.h @@ -5,8 +5,8 @@ * \author Zhang Chen */ -#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_INITIALIZER_H_ -#define CPP_PACKAGE_INCLUDE_MXNET_CPP_INITIALIZER_H_ +#ifndef MXNET_CPP_INITIALIZER_H_ +#define MXNET_CPP_INITIALIZER_H_ #include #include @@ -179,4 +179,4 @@ class Xavier : public Initializer { } // namespace cpp } // namespace mxnet -#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_INITIALIZER_H_ +#endif // MXNET_CPP_INITIALIZER_H_ diff --git a/cpp-package/include/mxnet-cpp/io.h b/cpp-package/include/mxnet-cpp/io.h index 171803831109..727a96467c63 100644 --- a/cpp-package/include/mxnet-cpp/io.h +++ b/cpp-package/include/mxnet-cpp/io.h @@ -4,8 +4,8 @@ * \brief definition of io, such as DataIter * \author Zhang Chen */ -#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_IO_H_ -#define CPP_PACKAGE_INCLUDE_MXNET_CPP_IO_H_ +#ifndef MXNET_CPP_IO_H_ +#define MXNET_CPP_IO_H_ #include #include @@ -124,5 +124,5 @@ class MXDataIter : public DataIter { } // namespace cpp } // namespace mxnet -#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_IO_H_ +#endif // MXNET_CPP_IO_H_ diff --git a/cpp-package/include/mxnet-cpp/io.hpp b/cpp-package/include/mxnet-cpp/io.hpp index 61e575e949a9..1be7993fbe4f 100644 --- a/cpp-package/include/mxnet-cpp/io.hpp +++ b/cpp-package/include/mxnet-cpp/io.hpp @@ -4,8 +4,8 @@ * \brief implementation of data iter * \author Zhang Chen */ -#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_IO_HPP_ -#define CPP_PACKAGE_INCLUDE_MXNET_CPP_IO_HPP_ +#ifndef MXNET_CPP_IO_HPP_ +#define MXNET_CPP_IO_HPP_ #include #include @@ -86,5 +86,5 @@ inline MXDataIter MXDataIter::CreateDataIter() { } // namespace cpp } // namespace mxnet -#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_IO_HPP_ +#endif // MXNET_CPP_IO_HPP_ diff --git a/cpp-package/include/mxnet-cpp/kvstore.h b/cpp-package/include/mxnet-cpp/kvstore.h index 6d3987ecf030..9bb33a4733dd 100644 --- a/cpp-package/include/mxnet-cpp/kvstore.h +++ b/cpp-package/include/mxnet-cpp/kvstore.h @@ -5,8 +5,8 @@ * \author Chuntao Hong */ -#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_KVSTORE_H_ -#define CPP_PACKAGE_INCLUDE_MXNET_CPP_KVSTORE_H_ +#ifndef MXNET_CPP_KVSTORE_H_ +#define MXNET_CPP_KVSTORE_H_ #include #include @@ -46,4 +46,4 @@ class KVStore { } // namespace cpp } // namespace mxnet -#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_KVSTORE_H_ +#endif // MXNET_CPP_KVSTORE_H_ diff --git a/cpp-package/include/mxnet-cpp/kvstore.hpp b/cpp-package/include/mxnet-cpp/kvstore.hpp index d9effcf82f3c..4f66c1d637a5 100644 --- a/cpp-package/include/mxnet-cpp/kvstore.hpp +++ b/cpp-package/include/mxnet-cpp/kvstore.hpp @@ -14,8 +14,8 @@ #include "mxnet-cpp/kvstore.h" #include "mxnet-cpp/optimizer.h" -#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_KVSTORE_HPP_ -#define CPP_PACKAGE_INCLUDE_MXNET_CPP_KVSTORE_HPP_ +#ifndef MXNET_CPP_KVSTORE_HPP_ +#define MXNET_CPP_KVSTORE_HPP_ namespace mxnet { namespace cpp { @@ -175,4 +175,4 @@ inline std::string KVStore::GetRole() { } // namespace cpp } // namespace mxnet -#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_KVSTORE_HPP_ +#endif // MXNET_CPP_KVSTORE_HPP_ diff --git a/cpp-package/include/mxnet-cpp/metric.h b/cpp-package/include/mxnet-cpp/metric.h index 24b3d73bae00..eda927199ca8 100644 --- a/cpp-package/include/mxnet-cpp/metric.h +++ b/cpp-package/include/mxnet-cpp/metric.h @@ -5,8 +5,8 @@ * \author Zhang Chen */ -#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_METRIC_H_ -#define CPP_PACKAGE_INCLUDE_MXNET_CPP_METRIC_H_ +#ifndef MXNET_CPP_METRIC_H_ +#define MXNET_CPP_METRIC_H_ #include #include @@ -187,5 +187,5 @@ class PSNR : public EvalMetric { } // namespace cpp } // namespace mxnet -#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_METRIC_H_ +#endif // MXNET_CPP_METRIC_H_ diff --git a/cpp-package/include/mxnet-cpp/model.h b/cpp-package/include/mxnet-cpp/model.h index 7bfe1980f095..e4cb1a9aee95 100644 --- a/cpp-package/include/mxnet-cpp/model.h +++ b/cpp-package/include/mxnet-cpp/model.h @@ -5,8 +5,8 @@ * \author Zhang Chen */ -#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_MODEL_H_ -#define CPP_PACKAGE_INCLUDE_MXNET_CPP_MODEL_H_ +#ifndef MXNET_CPP_MODEL_H_ +#define MXNET_CPP_MODEL_H_ #include #include @@ -54,5 +54,5 @@ class FeedForward { } // namespace cpp } // namespace mxnet -#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_MODEL_H_ +#endif // MXNET_CPP_MODEL_H_ diff --git a/cpp-package/include/mxnet-cpp/monitor.h b/cpp-package/include/mxnet-cpp/monitor.h index 2ce4e9590794..afe030cbd5d8 100644 --- a/cpp-package/include/mxnet-cpp/monitor.h +++ b/cpp-package/include/mxnet-cpp/monitor.h @@ -5,8 +5,8 @@ * \author Xin Li */ -#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_MONITOR_H_ -#define CPP_PACKAGE_INCLUDE_MXNET_CPP_MONITOR_H_ +#ifndef MXNET_CPP_MONITOR_H_ +#define MXNET_CPP_MONITOR_H_ #include #include @@ -85,4 +85,4 @@ class Monitor { } // namespace cpp } // namespace mxnet -#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_MONITOR_H_ +#endif // MXNET_CPP_MONITOR_H_ diff --git a/cpp-package/include/mxnet-cpp/monitor.hpp b/cpp-package/include/mxnet-cpp/monitor.hpp index d37652dd2c05..eef218bff41d 100644 --- a/cpp-package/include/mxnet-cpp/monitor.hpp +++ b/cpp-package/include/mxnet-cpp/monitor.hpp @@ -5,8 +5,8 @@ * \author Xin Li */ -#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_MONITOR_HPP_ -#define CPP_PACKAGE_INCLUDE_MXNET_CPP_MONITOR_HPP_ +#ifndef MXNET_CPP_MONITOR_HPP_ +#define MXNET_CPP_MONITOR_HPP_ #include #include @@ -103,4 +103,4 @@ inline void Monitor::executor_callback(const char *name, NDArrayHandle handle, } // namespace cpp } // namespace mxnet -#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_MONITOR_HPP_ +#endif // MXNET_CPP_MONITOR_HPP_ diff --git a/cpp-package/include/mxnet-cpp/ndarray.h b/cpp-package/include/mxnet-cpp/ndarray.h index f908b4ff38eb..52451faa94cc 100644 --- a/cpp-package/include/mxnet-cpp/ndarray.h +++ b/cpp-package/include/mxnet-cpp/ndarray.h @@ -5,8 +5,8 @@ * \author Chuntao Hong, Zhang Chen */ -#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_NDARRAY_H_ -#define CPP_PACKAGE_INCLUDE_MXNET_CPP_NDARRAY_H_ +#ifndef MXNET_CPP_NDARRAY_H_ +#define MXNET_CPP_NDARRAY_H_ #include #include @@ -428,4 +428,4 @@ std::ostream& operator<<(std::ostream& out, const NDArray &ndarray); } // namespace cpp } // namespace mxnet -#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_NDARRAY_H_ +#endif // MXNET_CPP_NDARRAY_H_ diff --git a/cpp-package/include/mxnet-cpp/ndarray.hpp b/cpp-package/include/mxnet-cpp/ndarray.hpp index 6157a6600cb4..ba0954b3f815 100644 --- a/cpp-package/include/mxnet-cpp/ndarray.hpp +++ b/cpp-package/include/mxnet-cpp/ndarray.hpp @@ -5,8 +5,8 @@ * \author Zhang Chen, Chuntao Hong */ -#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_NDARRAY_HPP_ -#define CPP_PACKAGE_INCLUDE_MXNET_CPP_NDARRAY_HPP_ +#ifndef MXNET_CPP_NDARRAY_HPP_ +#define MXNET_CPP_NDARRAY_HPP_ #include #include @@ -378,4 +378,4 @@ inline std::ostream & operator<<(std::ostream &out, const NDArray &ndarray) { } // namespace cpp } // namespace mxnet -#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_NDARRAY_HPP_ +#endif // MXNET_CPP_NDARRAY_HPP_ diff --git a/cpp-package/include/mxnet-cpp/op_map.h b/cpp-package/include/mxnet-cpp/op_map.h index 2a2ae50a4e84..ea75a8ca7b4c 100644 --- a/cpp-package/include/mxnet-cpp/op_map.h +++ b/cpp-package/include/mxnet-cpp/op_map.h @@ -5,8 +5,8 @@ * \author Chuntao Hong */ -#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_OP_MAP_H_ -#define CPP_PACKAGE_INCLUDE_MXNET_CPP_OP_MAP_H_ +#ifndef MXNET_CPP_OP_MAP_H_ +#define MXNET_CPP_OP_MAP_H_ #include #include @@ -89,4 +89,4 @@ class OpMap { } // namespace cpp } // namespace mxnet -#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_OP_MAP_H_ +#endif // MXNET_CPP_OP_MAP_H_ diff --git a/cpp-package/include/mxnet-cpp/op_suppl.h b/cpp-package/include/mxnet-cpp/op_suppl.h index c40449cc9f89..b66521bc0654 100644 --- a/cpp-package/include/mxnet-cpp/op_suppl.h +++ b/cpp-package/include/mxnet-cpp/op_suppl.h @@ -5,8 +5,8 @@ * \author Zhang Chen, zhubuntu, Xin Li */ -#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_OP_SUPPL_H_ -#define CPP_PACKAGE_INCLUDE_MXNET_CPP_OP_SUPPL_H_ +#ifndef MXNET_CPP_OP_SUPPL_H_ +#define MXNET_CPP_OP_SUPPL_H_ #include #include @@ -157,5 +157,5 @@ inline Symbol Activation(const std::string& symbol_name, } // namespace cpp } // namespace mxnet -#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_OP_SUPPL_H_ +#endif // MXNET_CPP_OP_SUPPL_H_ diff --git a/cpp-package/include/mxnet-cpp/op_util.h b/cpp-package/include/mxnet-cpp/op_util.h index bf67eab4c1ae..5a737480d469 100644 --- a/cpp-package/include/mxnet-cpp/op_util.h +++ b/cpp-package/include/mxnet-cpp/op_util.h @@ -5,8 +5,8 @@ * \author Chris Olivier */ -#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_OP_UTIL_H_ -#define CPP_PACKAGE_INCLUDE_MXNET_CPP_OP_UTIL_H_ +#ifndef MXNET_CPP_OP_UTIL_H_ +#define MXNET_CPP_OP_UTIL_H_ #include @@ -43,4 +43,4 @@ inline StreamType& operator << (StreamType& os, const ::caffe::LayerParameter& o } // namespace cpp } // namespace mxnet -#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_OP_UTIL_H_ +#endif // MXNET_CPP_OP_UTIL_H_ diff --git a/cpp-package/include/mxnet-cpp/operator.h b/cpp-package/include/mxnet-cpp/operator.h index 9a492576d104..4fc45bbc9f04 100644 --- a/cpp-package/include/mxnet-cpp/operator.h +++ b/cpp-package/include/mxnet-cpp/operator.h @@ -5,8 +5,8 @@ * \author Chuntao Hong, Zhang Chen */ -#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_OPERATOR_H_ -#define CPP_PACKAGE_INCLUDE_MXNET_CPP_OPERATOR_H_ +#ifndef MXNET_CPP_OPERATOR_H_ +#define MXNET_CPP_OPERATOR_H_ #include #include @@ -188,4 +188,4 @@ class Operator { } // namespace cpp } // namespace mxnet -#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_OPERATOR_H_ +#endif // MXNET_CPP_OPERATOR_H_ diff --git a/cpp-package/include/mxnet-cpp/operator.hpp b/cpp-package/include/mxnet-cpp/operator.hpp index 8a421d7b6b4f..17f4885133fc 100644 --- a/cpp-package/include/mxnet-cpp/operator.hpp +++ b/cpp-package/include/mxnet-cpp/operator.hpp @@ -5,8 +5,8 @@ * \author Chuntao Hong, Zhang Chen */ -#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_OPERATOR_HPP_ -#define CPP_PACKAGE_INCLUDE_MXNET_CPP_OPERATOR_HPP_ +#ifndef MXNET_CPP_OPERATOR_HPP_ +#define MXNET_CPP_OPERATOR_HPP_ #include #include @@ -155,4 +155,4 @@ inline Operator &Operator::SetInput(const std::string &name, NDArray ndarray) { } // namespace cpp } // namespace mxnet -#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_OPERATOR_HPP_ +#endif // MXNET_CPP_OPERATOR_HPP_ diff --git a/cpp-package/include/mxnet-cpp/optimizer.h b/cpp-package/include/mxnet-cpp/optimizer.h index 8dbbbf7f39ea..76f8a3564fbe 100644 --- a/cpp-package/include/mxnet-cpp/optimizer.h +++ b/cpp-package/include/mxnet-cpp/optimizer.h @@ -5,8 +5,8 @@ * \author Chuntao Hong, Zhang Chen */ -#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_OPTIMIZER_H_ -#define CPP_PACKAGE_INCLUDE_MXNET_CPP_OPTIMIZER_H_ +#ifndef MXNET_CPP_OPTIMIZER_H_ +#define MXNET_CPP_OPTIMIZER_H_ #include #include @@ -176,4 +176,4 @@ class AdaDeltaOptimizer : public Optimizer { } // namespace cpp } // namespace mxnet -#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_OPTIMIZER_H_ +#endif // MXNET_CPP_OPTIMIZER_H_ diff --git a/cpp-package/include/mxnet-cpp/optimizer.hpp b/cpp-package/include/mxnet-cpp/optimizer.hpp index c86476f65417..9dcb158b9e14 100644 --- a/cpp-package/include/mxnet-cpp/optimizer.hpp +++ b/cpp-package/include/mxnet-cpp/optimizer.hpp @@ -5,8 +5,8 @@ * \author Chuntao Hong, Zhang Chen */ -#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_OPTIMIZER_HPP_ -#define CPP_PACKAGE_INCLUDE_MXNET_CPP_OPTIMIZER_HPP_ +#ifndef MXNET_CPP_OPTIMIZER_HPP_ +#define MXNET_CPP_OPTIMIZER_HPP_ #include #include @@ -387,4 +387,4 @@ inline void AdaDeltaOptimizer::CreateState_(int index, NDArray weight) { } // namespace cpp } // namespace mxnet -#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_OPTIMIZER_HPP_ +#endif // MXNET_CPP_OPTIMIZER_HPP_ diff --git a/cpp-package/include/mxnet-cpp/shape.h b/cpp-package/include/mxnet-cpp/shape.h index d8e3f2c95282..d30ea9df2531 100644 --- a/cpp-package/include/mxnet-cpp/shape.h +++ b/cpp-package/include/mxnet-cpp/shape.h @@ -5,8 +5,8 @@ * \author Chuntao Hong, Zhang Chen */ -#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_SHAPE_H_ -#define CPP_PACKAGE_INCLUDE_MXNET_CPP_SHAPE_H_ +#ifndef MXNET_CPP_SHAPE_H_ +#define MXNET_CPP_SHAPE_H_ #include #include @@ -386,4 +386,4 @@ inline std::istream &operator>>(std::istream &is, Shape &shape) { } // namespace cpp } // namespace mxnet -#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_SHAPE_H_ +#endif // MXNET_CPP_SHAPE_H_ diff --git a/cpp-package/include/mxnet-cpp/symbol.h b/cpp-package/include/mxnet-cpp/symbol.h index e853c2617ea4..c04ae2a03d29 100644 --- a/cpp-package/include/mxnet-cpp/symbol.h +++ b/cpp-package/include/mxnet-cpp/symbol.h @@ -5,8 +5,8 @@ * \author Chuntao Hong, Zhang Chen */ -#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_SYMBOL_H_ -#define CPP_PACKAGE_INCLUDE_MXNET_CPP_SYMBOL_H_ +#ifndef MXNET_CPP_SYMBOL_H_ +#define MXNET_CPP_SYMBOL_H_ #include #include @@ -257,4 +257,4 @@ Symbol operator/(mx_float lhs, const Symbol &rhs); Symbol operator%(mx_float lhs, const Symbol &rhs); } // namespace cpp } // namespace mxnet -#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_SYMBOL_H_ +#endif // MXNET_CPP_SYMBOL_H_ diff --git a/cpp-package/include/mxnet-cpp/symbol.hpp b/cpp-package/include/mxnet-cpp/symbol.hpp index 26962ba5c99b..7f88e485830f 100644 --- a/cpp-package/include/mxnet-cpp/symbol.hpp +++ b/cpp-package/include/mxnet-cpp/symbol.hpp @@ -5,8 +5,8 @@ * \author Zhang Chen, Chuntao Hong */ -#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_SYMBOL_HPP_ -#define CPP_PACKAGE_INCLUDE_MXNET_CPP_SYMBOL_HPP_ +#ifndef MXNET_CPP_SYMBOL_HPP_ +#define MXNET_CPP_SYMBOL_HPP_ #include #include @@ -347,4 +347,4 @@ inline Symbol operator%(mx_float lhs, const Symbol &rhs) { } // namespace cpp } // namespace mxnet -#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_SYMBOL_HPP_ +#endif // MXNET_CPP_SYMBOL_HPP_ diff --git a/cpp-package/scripts/OpWrapperGenerator.py b/cpp-package/scripts/OpWrapperGenerator.py index 392e07f9caa4..8f762368d0a4 100644 --- a/cpp-package/scripts/OpWrapperGenerator.py +++ b/cpp-package/scripts/OpWrapperGenerator.py @@ -372,8 +372,8 @@ def ParseAllOps(): "* \\author Chuntao Hong, Xin Li\n" "*/\n" "\n" - "#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_OP_H_\n" - "#define CPP_PACKAGE_INCLUDE_MXNET_CPP_OP_H_\n" + "#ifndef MXNET_CPP_OP_H_\n" + "#define MXNET_CPP_OP_H_\n" "\n" "#include \n" "#include \n" @@ -389,7 +389,7 @@ def ParseAllOps(): "%s" "} //namespace cpp\n" "} //namespace mxnet\n" - "#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_OP_H_\n") + "#endif // MXNET_CPP_OP_H_\n") # Generate a temporary file name tf = tempfile.NamedTemporaryFile() diff --git a/dmlc-core b/dmlc-core index a6c5701219e6..b647be2dee98 160000 --- a/dmlc-core +++ b/dmlc-core @@ -1 +1 @@ -Subproject commit a6c5701219e635fea808d264aefc5b03c3aec314 +Subproject commit b647be2dee985d77a12e8e41bc27382221938290 diff --git a/include/mxnet/base.h b/include/mxnet/base.h index 0c4c9d3daa77..739105b388bc 100644 --- a/include/mxnet/base.h +++ b/include/mxnet/base.h @@ -211,6 +211,8 @@ struct Context { * The information needed in runtime for actual execution. */ struct RunContext { + /*! \brief base Context */ + Context ctx; /*! * \brief the stream of the device, can be NULL or Stream* in GPU mode */ @@ -224,6 +226,10 @@ struct RunContext { inline mshadow::Stream* get_stream() const { return static_cast*>(stream); } + /*! \brief get the base Context from RunContext */ + inline const Context& get_ctx() const { + return ctx; + } }; } // namespace mxnet diff --git a/include/mxnet/c_api.h b/include/mxnet/c_api.h index b8f8411353bf..8bc1451ba90d 100644 --- a/include/mxnet/c_api.h +++ b/include/mxnet/c_api.h @@ -139,12 +139,12 @@ typedef int (*CustomOpBwdDepFunc)(const int* /*out_grad*/, const int* /*in_data* const int* /*out_data*/, int* /*num_deps*/, int** /*rdeps*/, void* /*state*/); typedef int (*CustomOpCreateFunc)(const char* /*ctx*/, int /*num_inputs*/, - unsigned** /*shapes*/, int* /*ndims*/, - int* /*dtypes*/, struct MXCallbackList* /*ret*/, + unsigned** /*shapes*/, const int* /*ndims*/, + const int* /*dtypes*/, struct MXCallbackList* /*ret*/, void* /*state*/); typedef int (*CustomOpPropCreator)(const char* /*op_type*/, const int /*num_kwargs*/, - const char** /*keys*/, const char** /*values*/, - struct MXCallbackList* /*ret*/); + const char** /*keys*/, const char** /*values*/, + struct MXCallbackList* /*ret*/); /*! * \brief return str message of the last error diff --git a/include/mxnet/op_attr_types.h b/include/mxnet/op_attr_types.h index 316a90fe0841..dbf9a07e0bcb 100644 --- a/include/mxnet/op_attr_types.h +++ b/include/mxnet/op_attr_types.h @@ -15,27 +15,173 @@ #include #include "./base.h" -#include "./operator.h" #include "./ndarray.h" +#include "./engine.h" namespace mxnet { using nnvm::NodeAttrs; + +/*! \brief operation request type to Forward and Backward */ +enum OpReqType { + /*! \brief no operation, do not write anything */ + kNullOp, + /*! \brief write gradient to provided space */ + kWriteTo, + /*! + * \brief perform an inplace write, + * Target shares memory with one of input arguments. + * This option only happen when + */ + kWriteInplace, + /*! \brief add to the provided space */ + kAddTo +}; + +/*! + * \brief All the possible information needed by Operator.Forward and Backward + * This is the superset of RunContext. + * We use this data structure to bookkeep everything needed by Forward and Backward. + * \sa Resource + */ +struct OpContext { + /*! \brief whether it is training phase */ + int is_train; + /*! \brief RunContext related resources */ + RunContext run_ctx; + /*! \brief the callback when operation completes, used by asynchronize ops */ + engine::CallbackOnComplete async_on_complete; + /*! \brief Resources requested by the operator */ + std::vector requested; + /*! + * \brief get mshadow stream from Context + * \return the mshadow stream + * \tparam xpu the device type of the stream + */ + template + inline mshadow::Stream* get_stream() const { + return run_ctx.get_stream(); + } +}; + +/*! \brief the execution type of the operator */ +enum class ExecType { + /*! \brief Forward/Backward are synchronize calls */ + kSync, + /*! + * \brief Forward/Backward are asynchronize, + * will call OpContext.async_on_complete when operation finishes. + */ + kAsync, + /*! \brief Run this operator on the scheduling thread without pushing to engine. */ + kLocal, + /*! + * \brief Cross device copy operation, this is a special operator + * That indicates copy across devices, the input and output can sit on different device. + * In current implementation, copy operator is specially handled by executor. + * This flag is used for special case treatment and future extension of different copy ops. + */ + kCrossDeviceCopy +}; + +/*! + * \brief Operator state. This is a pointer type, its content is mutable + * even if OpStatePtr is const. + */ +class OpStatePtr { + public: + /* \brief Create a OpStatePtr with state of type T. + * \param args Arguments passed to T's constructor. + */ + template + static OpStatePtr Create(Args&&... args) { + OpStatePtr ret; + ret.ptr_ = std::make_shared(); + ret.ptr_->var_ = Engine::Get()->NewVariable(); + ret.ptr_->state_.construct(std::forward(args)...); + + return ret; + } + /* \brief Get engine variable associated with this state */ + engine::VarHandle get_var() const { + return ptr_->var_; + } + /* \brief Get state of type T */ + template + T& get_state() const { + return dmlc::get(ptr_->state_); + } + /* \brief clear state */ + void reset() { + ptr_.reset(); + } + /* \brief Whether state is empty */ + explicit operator bool() const { + return ptr_ ? true : false; + } + + private: + /* \brief state structure */ + struct OpState { + OpState() {} + OpState(const OpState& other) = delete; + OpState& operator=(const OpState& other) = delete; + + ~OpState() { + Engine::Get()->DeleteVariable([](RunContext s) {}, Context::CPU(), var_); + } + + engine::VarHandle var_; + dmlc::any state_; + }; + /* \brief shared pointer to state */ + std::shared_ptr ptr_; +}; + /*! * \brief Create a Layer style, forward/backward operator. * This is easy to write code that contains state. + * OpStatePtr is a pointer type, it's content is mutable even if + * OpStatePtr is constant. + * * * This is not the only way to register an op execution function. * More simpler or specialized operator form can be registered * * \note Register under "FCreateLayerOp" */ -using FCreateLayerOp = std::function< - Operator* (const NodeAttrs& n, - Context ctx, - const std::vector& in_shape, - const std::vector& in_type)>; - +using FCreateOpState = std::function& in_shape, + const std::vector& in_type)>; +/*! + * \brief Execution mode of this operator. + */ +using FExecType = std::function; +/*! + * \brief Resiger a compute function for stateful operator. + * OpStatePtr is a pointer type, it's content is mutable even if + * OpStatePtr is constant. + * + * \note Register under "FStatefulCompute" and "FStatefulCompute" + */ +using FStatefulCompute = std::function& inputs, + const std::vector& req, + const std::vector& outputs)>; +/*! + * \brief Resiger a compute function for stateful operator using NDArray interface. + * OpStatePtr is a pointer type, it's content is mutable even if + * OpStatePtr is constant. + * + * \note Register under "FStatefulComputeEx" and "FStatefulComputeEx" + */ +using FStatefulComputeEx = std::function& inputs, + const std::vector& req, + const std::vector& outputs)>; /*! * \brief The resource request from the operator * diff --git a/include/mxnet/operator.h b/include/mxnet/operator.h index fe5c3de0279f..09a643390342 100644 --- a/include/mxnet/operator.h +++ b/include/mxnet/operator.h @@ -18,50 +18,9 @@ #include #include "./base.h" #include "./resource.h" +#include "./op_attr_types.h" namespace mxnet { -/*! \brief operation request type to Forward and Backward */ -enum OpReqType { - /*! \brief no operation, do not write anything */ - kNullOp, - /*! \brief write gradient to provided space */ - kWriteTo, - /*! - * \brief perform an inplace write, - * Target shares memory with one of input arguments. - * This option only happen when - */ - kWriteInplace, - /*! \brief add to the provided space */ - kAddTo -}; - -/*! - * \brief All the possible information needed by Operator.Forward and Backward - * This is the superset of RunContext. - * We use this data structure to bookkeep everything needed by Forward and Backward. - * \sa Resource - */ -struct OpContext { - /*! \brief whether it is training phase */ - int is_train; - /*! \brief RunContext related resources */ - RunContext run_ctx; - /*! \brief the callback when operation completes, used by asynchronize ops */ - engine::CallbackOnComplete async_on_complete; - /*! \brief Resources requested by the operator */ - std::vector requested; - /*! - * \brief get mshadow stream from Context - * \return the mshadow stream - * \tparam xpu the device type of the stream - */ - template - inline mshadow::Stream* get_stream() const { - return run_ctx.get_stream(); - } -}; - /*! * \brief Operator interface. * Operator defines basic operation unit of optimized computation graph in mxnet. @@ -76,23 +35,6 @@ struct OpContext { */ class Operator { public: - /*! \brief the execution type of the operator */ - enum ExecType { - /*! \brief Forward/Backward are synchronize calls */ - kSync, - /*! - * \brief Forward/Backward are asynchronize, - * will call OpContext.async_on_complete when operation finishes. - */ - kAsync, - /*! - * \brief Cross device copy operation, this is a special operator - * That indicates copy across devices, the input and output can sit on different device. - * In current implementation, copy operator is specially handled by executor. - * This flag is used for special case treatment and future extension of different copy ops. - */ - kCrossDeviceCopy - }; /*! \brief destructor */ virtual ~Operator() {} /*! @@ -148,9 +90,9 @@ class Operator { const std::vector &aux_states) { LOG(FATAL) << "Backward is not implemented"; } - /*! \return execution type of the operator */ - virtual ExecType exec_type() const { - return kSync; + /*! \return [Deprecated] execution type of the operator */ + virtual ExecType exec_type() const final { // NOLINT(*) exec_type has been moved to OperatorProperty + return ExecType::kSync; } }; @@ -478,6 +420,10 @@ class OperatorProperty { * \return a new constructed OperatorProperty */ static OperatorProperty *Create(const char* type_name); + /*! \return execution type of the operator */ + virtual ExecType exec_type() const { + return ExecType::kSync; + } }; /*! \brief typedef the factory function of operator property */ diff --git a/python/mxnet/ndarray.py b/python/mxnet/ndarray.py index 31b7d7cfb944..001400db95b8 100644 --- a/python/mxnet/ndarray.py +++ b/python/mxnet/ndarray.py @@ -258,7 +258,9 @@ def __le__(self, other): return lesser_equal(self, other) def __bool__(self): - raise ValueError("The truth value of an NDArray with more than one element is ambiguous.") + raise ValueError("The truth value of an NDArray is ambiguous. " \ + "Please convert to number with asscalar() first.") + __nonzero__ = __bool__ def __getstate__(self): diff --git a/python/mxnet/operator.py b/python/mxnet/operator.py index a08e764088a5..d57ee717fcf6 100644 --- a/python/mxnet/operator.py +++ b/python/mxnet/operator.py @@ -471,7 +471,7 @@ def infer_shape(self, in_shape): List of aux shapes calculated from in_shape, in the same order as declared in list_auxiliary_states. """ - return in_shape, [in_shape[0]], [] + return in_shape, (in_shape[0],)*len(self.list_outputs()), () def infer_type(self, in_type): """infer_type interface. override to create new operators diff --git a/scala-package/core/src/test/scala/ml/dmlc/mxnet/OperatorSuite.scala b/scala-package/core/src/test/scala/ml/dmlc/mxnet/OperatorSuite.scala index a870cabb568b..dfbc864785f1 100644 --- a/scala-package/core/src/test/scala/ml/dmlc/mxnet/OperatorSuite.scala +++ b/scala-package/core/src/test/scala/ml/dmlc/mxnet/OperatorSuite.scala @@ -881,6 +881,7 @@ class OperatorSuite extends FunSuite with BeforeAndAfterAll NDArray.zeros(Shape(numFilter, inputShape(1), kernel._1, kernel._2))) val exeConv = conv.bind(Context.cpu(), args = convArgs, argsGrad = convArgsGrad) val convOutGrad = Random.normal(0, 2, exeConv.outputs.head.shape) + exeConv.forward() exeConv.backward(convOutGrad) val deconvData = convOutGrad @@ -889,6 +890,7 @@ class OperatorSuite extends FunSuite with BeforeAndAfterAll NDArray.zeros(Shape(numFilter, inputShape(1), kernel._1, kernel._2))) val exeDeconv = deconv.bind(Context.cpu(), args = deconvArgs, argsGrad = deconvArgsGrad) val deconvOutGrad = convData + exeDeconv.forward() exeDeconv.backward(deconvOutGrad) assert(reldiff(convArgsGrad(1), deconvArgsGrad(1)) < 1e-5) } diff --git a/src/c_api/c_api.cc b/src/c_api/c_api.cc index bea6437b4c64..a376b3b6802c 100644 --- a/src/c_api/c_api.cc +++ b/src/c_api/c_api.cc @@ -974,6 +974,6 @@ int MXRtcFree(RtcHandle handle) { int MXCustomOpRegister(const char* op_type, CustomOpPropCreator creator) { API_BEGIN(); - mxnet::op::CustomOpProp::Register(op_type, creator); + mxnet::op::custom::Registry::Get()->Register(op_type, creator); API_END(); } diff --git a/src/c_api/c_api_ndarray.cc b/src/c_api/c_api_ndarray.cc index dfdd46b6aa90..98fbe760854e 100644 --- a/src/c_api/c_api_ndarray.cc +++ b/src/c_api/c_api_ndarray.cc @@ -279,59 +279,70 @@ void PushFCompute(const FCompute& fn, 0, PROFILER_MESSAGE(op->name.c_str())); } -void PushOperator(std::shared_ptr opr, +void PushOperator(const OpStatePtr& state, const nnvm::Op* op, const nnvm::NodeAttrs& attrs, const Context& ctx, const std::vector& read_vars, const std::vector& write_vars, const std::vector& requested, - const std::vector& auxidx, const std::vector& ndinputs, const std::vector& ndoutputs) { - struct Capture { - engine::CallbackOnComplete on_complete; - std::shared_ptr opr; - }; + static auto& fexec_type = nnvm::Op::GetAttr("FExecType"); bool is_train = AutogradRuntime::Get()->IsTraining(); - Engine::Get()->PushAsync( - [ctx, opr, auxidx, ndinputs, ndoutputs, requested, is_train]( - RunContext rctx, - engine::CallbackOnComplete on_complete) { - std::vector input_blobs, aux_blobs, output_blobs; - auto atop = auxidx.begin(); - for (size_t i = 0; i < ndinputs.size(); ++i) { - if (atop != auxidx.end() && i == *atop) { - aux_blobs.push_back(ndinputs[i].data()); - ++atop; - } else { - input_blobs.push_back(ndinputs[i].data()); + ExecType exec_type = ExecType::kSync; + if (fexec_type.count(op)) { + exec_type = fexec_type[op](attrs); + } + + auto fcompute = common::GetFCompute(op, "FStatefulCompute", ctx); + if (fcompute != nullptr) { + CHECK(exec_type == ExecType::kSync || exec_type == ExecType::kAsync); + Engine::Get()->PushAsync( + [state, fcompute, ndinputs, ndoutputs, requested, is_train, exec_type]( + RunContext rctx, + engine::CallbackOnComplete on_complete) { + OpContext opctx{is_train, rctx, on_complete, requested}; + std::vector input_blobs, output_blobs; + for (const auto& i : ndinputs) input_blobs.push_back(i.data()); + for (const auto& i : ndoutputs) output_blobs.push_back(i.data()); + std::vector req(output_blobs.size(), kWriteTo); + fcompute(state, opctx, input_blobs, req, output_blobs); + if (exec_type == ExecType::kSync) { + if (rctx.get_ctx().dev_mask() == gpu::kDevMask) { + rctx.get_stream()->Wait(); + } + on_complete(); } - } - for (auto& i : ndoutputs) { - output_blobs.push_back(i.data()); - } - Capture* capture = new Capture({on_complete, opr}); - OpContext opctx{is_train, rctx, - Engine::Get()->CreateCallback( - [](Engine* engine, void *cpt_handle) { - Capture* cpt = static_cast(cpt_handle); - cpt->on_complete(); - delete cpt; - }, static_cast(capture)), - requested}; - std::vector req(output_blobs.size(), kWriteTo); - opr->Forward(opctx, input_blobs, req, output_blobs, aux_blobs); - if (opr->exec_type() != Operator::kAsync) { - if (ctx.dev_mask() == gpu::kDevMask) { - rctx.get_stream()->Wait(); + }, ctx, read_vars, write_vars, FnProperty::kNormal, + 0, PROFILER_MESSAGE(op->name.c_str())); + } else { + auto fcompute_ex = common::GetFCompute( + op, "FStatefulComputeEx", ctx); + CHECK(fcompute_ex != nullptr) + << "One of FStatefulCompute and FStatefulComputeEx must be registered " + << "for stateful operator " << op->name; + const auto& run = [state, fcompute_ex, ndinputs, ndoutputs, requested, is_train, exec_type]( + RunContext rctx, + engine::CallbackOnComplete on_complete) { + OpContext opctx{is_train, rctx, on_complete, requested}; + std::vector req(ndoutputs.size(), kWriteTo); + fcompute_ex(state, opctx, ndinputs, req, ndoutputs); + if (exec_type == ExecType::kSync) { + if (rctx.get_ctx().dev_mask() == gpu::kDevMask) { + rctx.get_stream()->Wait(); + } + on_complete(); } - delete capture; - on_complete(); - } - }, ctx, read_vars, write_vars, FnProperty::kNormal, - 0, PROFILER_MESSAGE(op->name.c_str())); + }; + if (exec_type == ExecType::kLocal) { + run(RunContext{ctx, nullptr}, engine::CallbackOnComplete()); + } else { + Engine::Get()->PushAsync(run, ctx, read_vars, write_vars, FnProperty::kNormal, + 0, PROFILER_MESSAGE(op->name.c_str())); + } + } } void ImperativeInvokeImpl(const Context& default_ctx, @@ -341,7 +352,7 @@ void ImperativeInvokeImpl(const Context& default_ctx, static auto& fcpu = nnvm::Op::GetAttr("FCompute"); static auto& fgpu = nnvm::Op::GetAttr("FCompute"); static auto& ndfunc = nnvm::Op::GetAttr("FNDArrayFunction"); - static auto& createop = nnvm::Op::GetAttr("FCreateLayerOp"); + static auto& createop = nnvm::Op::GetAttr("FCreateOpState"); MXAPIThreadLocalEntry *ret = MXAPIThreadLocalStore::Get(); const nnvm::Op *op = attrs.op; @@ -378,14 +389,15 @@ void ImperativeInvokeImpl(const Context& default_ctx, PushFCompute(fn, op, attrs, ctx, read_vars, write_vars, requested, ndinputs, ndoutputs); } else if (createop.count(op)) { - std::shared_ptr opr( - createop[op](attrs, ctx, ret->arg_shapes, ret->arg_types)); + auto state = + createop[op](attrs, ctx, ret->arg_shapes, ret->arg_types); if (AutogradRuntime::Get()->IsTraining()) { - AutogradRuntime::Get()->RecordImperativeOperator(opr, op, + AutogradRuntime::Get()->RecordImperativeOperator(state, op, attrs, &ndinputs, &ndoutputs); } - PushOperator(opr, op, attrs, ctx, read_vars, write_vars, - requested, auxidx, ndinputs, ndoutputs); + write_vars.push_back(state.get_var()); + PushOperator(state, op, attrs, ctx, read_vars, write_vars, + requested, ndinputs, ndoutputs); } else { LOG(FATAL) << "Operator " << op->name << " is not implemented for " diff --git a/src/common/utils.h b/src/common/utils.h index 789b4d14b9f2..5f50aab4781f 100644 --- a/src/common/utils.h +++ b/src/common/utils.h @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #endif // DMLC_USE_CXX11 @@ -124,6 +125,22 @@ typename helper::UniqueIf::UnknownBound MakeUnique(size_t n) { template typename helper::UniqueIf::KnownBound MakeUnique(Args&&... args) = delete; +template +FCompType GetFCompute(const nnvm::Op* op, const std::string& name, + const Context& ctx) { + static auto& fcompute_cpu = nnvm::Op::GetAttr(name + ""); + static auto& fcompute_gpu = nnvm::Op::GetAttr(name + ""); + + if (ctx.dev_mask() == cpu::kDevMask) { + return fcompute_cpu.get(op, nullptr); + } else if (ctx.dev_mask() == gpu::kDevMask) { + return fcompute_gpu.get(op, nullptr); + } else { + LOG(FATAL) << "Unknown device mask"; + return nullptr; + } +} + #endif // DMLC_USE_CXX11 } // namespace common diff --git a/src/engine/naive_engine.cc b/src/engine/naive_engine.cc index efb7bd44981b..11ff7c8138bf 100644 --- a/src/engine/naive_engine.cc +++ b/src/engine/naive_engine.cc @@ -138,14 +138,12 @@ class NaiveEngine final : public Engine { if (streams_[dev_id] == nullptr) { streams_[dev_id] = mshadow::NewStream(true, MXNET_USE_CUDNN != 0); } - ctx_.stream = streams_[dev_id]; - exec_fun(ctx_, callback); + exec_fun(RunContext{exec_ctx, streams_[dev_id]}, callback); #else LOG(FATAL) << "GPU is not enabled"; #endif } else { - ctx_.stream = &cpu_stream_; - exec_fun(ctx_, callback); + exec_fun(RunContext{exec_ctx, &cpu_stream_}, callback); } CHECK(this->req_completed_) << "NaiveEngine only support synchronize Push so far"; @@ -176,8 +174,6 @@ class NaiveEngine final : public Engine { static void OnComplete(Engine *engine, void *param) { static_cast(engine)->req_completed_ = true; } - // runtime contetxt - RunContext ctx_; // whether action is completed bool req_completed_; // counter diff --git a/src/engine/stream_manager.h b/src/engine/stream_manager.h index 313db6d2010b..2d684bbb7b9a 100644 --- a/src/engine/stream_manager.h +++ b/src/engine/stream_manager.h @@ -46,9 +46,10 @@ template RunContext StreamManager::GetRunContext( Context const& ctx) { RunContext ret; - ret.stream = nullptr; switch (ctx.dev_mask()) { - case cpu::kDevMask: break; + case cpu::kDevMask: + ret = RunContext{ctx, nullptr}; + break; case gpu::kDevMask: { #if MXNET_USE_CUDA std::size_t use_counter; @@ -65,7 +66,7 @@ RunContext StreamManager::GetRunContext( use_counter = counter; counter = (counter + 1) % kStreams; } - ret.stream = gpu_streams_.at(ctx.dev_id).at(use_counter); + ret = RunContext{ctx, gpu_streams_.at(ctx.dev_id).at(use_counter)}; break; #else LOG(FATAL) << MXNET_GPU_NOT_ENABLED_ERROR; @@ -79,9 +80,10 @@ template RunContext StreamManager::GetIORunContext( Context const& ctx) { RunContext ret; - ret.stream = nullptr; switch (ctx.dev_mask()) { - case cpu::kDevMask: break; + case cpu::kDevMask: + ret = RunContext{ctx, nullptr}; + break; case gpu::kDevMask: { #if MXNET_USE_CUDA CUDA_CALL(cudaSetDevice(ctx.dev_id)); @@ -91,7 +93,7 @@ RunContext StreamManager::GetIORunContext( gpu_io_streams_.at(ctx.dev_id) = mshadow::NewStream(false, false); } } - ret.stream = gpu_io_streams_.at(ctx.dev_id); + ret = RunContext{ctx, gpu_io_streams_.at(ctx.dev_id)}; break; #else LOG(FATAL) << MXNET_GPU_NOT_ENABLED_ERROR; diff --git a/src/engine/threaded_engine_perdevice.cc b/src/engine/threaded_engine_perdevice.cc index 2b333d60647a..97356ae91e0d 100644 --- a/src/engine/threaded_engine_perdevice.cc +++ b/src/engine/threaded_engine_perdevice.cc @@ -39,7 +39,7 @@ class ThreadedEnginePerDevice : public ThreadedEngine { cpu_priority_worker_.reset(new ThreadWorkerBlock()); cpu_priority_worker_->pool.reset(new ThreadPool( cpu_priority_nthreads, [this]() { - this->CPUWorker(cpu_priority_worker_.get()); + this->CPUWorker(Context(), cpu_priority_worker_.get()); })); // GPU tasks will be created lazily } @@ -60,9 +60,7 @@ class ThreadedEnginePerDevice : public ThreadedEngine { MSHADOW_CATCH_ERROR(mshadow::SetDevice(ctx.dev_id)); #endif } - RunContext run_ctx; - run_ctx.stream = nullptr; - this->ExecuteOprBlock(run_ctx, opr_block); + this->ExecuteOprBlock(RunContext{ctx, nullptr}, opr_block); } else { if (ctx.dev_mask() == cpu::kDevMask) { if (opr_block->opr->prop == FnProperty::kCPUPrioritized) { @@ -71,10 +69,10 @@ class ThreadedEnginePerDevice : public ThreadedEngine { int dev_id = ctx.dev_id; int nthread = cpu_worker_nthreads_; auto ptr = - cpu_normal_workers_.Get(dev_id, [this, dev_id, nthread]() { + cpu_normal_workers_.Get(dev_id, [this, ctx, nthread]() { auto blk = new ThreadWorkerBlock(); - blk->pool.reset(new ThreadPool(nthread, [this, blk] () { - this->CPUWorker(blk); + blk->pool.reset(new ThreadPool(nthread, [this, ctx, blk] () { + this->CPUWorker(ctx, blk); })); return blk; }); @@ -89,16 +87,15 @@ class ThreadedEnginePerDevice : public ThreadedEngine { bool is_copy = (prop == FnProperty::kCopyFromGPU || prop == FnProperty::kCopyToGPU); int nthread = gpu_worker_nthreads_; - int dev_id = ctx.dev_id; if (is_copy) { auto ptr = - gpu_copy_workers_.Get(dev_id, [this, dev_id, is_copy, nthread]() { + gpu_copy_workers_.Get(ctx.dev_id, [this, ctx, is_copy, nthread]() { auto blk = new ThreadWorkerBlock(); blk->pool.reset(new ThreadPool( nthread, - [this, dev_id, is_copy, blk] + [this, ctx, is_copy, blk] (std::shared_ptr ready_event) { - this->GPUWorker(dev_id, is_copy, blk, ready_event); + this->GPUWorker(ctx, is_copy, blk, ready_event); }, true)); return blk; }); @@ -106,13 +103,13 @@ class ThreadedEnginePerDevice : public ThreadedEngine { ptr->task_queue.Push(opr_block, opr_block->priority); } } else { - auto ptr = gpu_normal_workers_.Get(dev_id, [this, dev_id, is_copy, nthread]() { + auto ptr = gpu_normal_workers_.Get(ctx.dev_id, [this, ctx, is_copy, nthread]() { auto blk = new ThreadWorkerBlock(); blk->pool.reset(new ThreadPool( nthread, - [this, dev_id, is_copy, blk] + [this, ctx, is_copy, blk] (std::shared_ptr ready_event) { - this->GPUWorker(dev_id, is_copy, blk, ready_event); + this->GPUWorker(ctx, is_copy, blk, ready_event); }, true)); return blk; }); @@ -157,26 +154,25 @@ class ThreadedEnginePerDevice : public ThreadedEngine { * \param block The task block of the worker. */ template - inline void GPUWorker(int dev_id, + inline void GPUWorker(Context ctx, bool is_copy_worker, ThreadWorkerBlock *block, std::shared_ptr ready_event) { #if MXNET_USE_CUDA mshadow::Stream *stream; - RunContext run_ctx; do { ThreadPool::SimpleEvent::SetReadyOnDestroy setReady(ready_event); // allocate stream - mshadow::SetDevice(dev_id); + mshadow::SetDevice(ctx.dev_id); if (is_copy_worker) { stream = mshadow::NewStream(false, false); } else { stream = mshadow::NewStream(true, MXNET_USE_CUDNN != 0); } - run_ctx.stream = stream; } while (false); // execute task OprBlock* opr_block; + RunContext run_ctx{ctx, stream}; auto* task_queue = &(block->task_queue); while (task_queue->Pop(&opr_block)) { this->ExecuteOprBlock(run_ctx, opr_block); @@ -192,10 +188,10 @@ class ThreadedEnginePerDevice : public ThreadedEngine { * \param block The task block of the worker. */ template - inline void CPUWorker(ThreadWorkerBlock *block) { + inline void CPUWorker(Context ctx, + ThreadWorkerBlock *block) { auto* task_queue = &(block->task_queue); - RunContext run_ctx; - run_ctx.stream = nullptr; + RunContext run_ctx{ctx, nullptr}; // execute task OprBlock* opr_block; while (task_queue->Pop(&opr_block)) { diff --git a/src/executor/attach_op_execs_pass.cc b/src/executor/attach_op_execs_pass.cc index 16b55adc15e8..6a0c489a1ec5 100644 --- a/src/executor/attach_op_execs_pass.cc +++ b/src/executor/attach_op_execs_pass.cc @@ -7,6 +7,7 @@ #include #include #include +#include "../common/utils.h" #include "./exec_pass.h" #if MXNET_USE_MKL2017 == 1 #include @@ -22,116 +23,81 @@ const OperatorProperty* OpPropGetOpProperty(const NodeAttrs& attrs); namespace exec { // forward executor -class ForwardOpExecutor : public OpExecutor { +class StatefulComputeExecutor : public OpExecutor { public: void Run(RunContext rctx) override { op_ctx.run_ctx = rctx; - op_->Forward(op_ctx, in_data_, req, out_data_, aux_data_); + fcompute_(state_, op_ctx, in_data_, req, out_data_); #if MKL_EXPERIMENTAL == 1 mkl_tblobs_prv_to_cpu(in_data_); mkl_tblobs_prv_to_cpu(out_data_); - mkl_tblobs_prv_to_cpu(aux_data_); #endif } void Setup() override { - in_data_.clear(); aux_data_.clear(); + in_data_.clear(); for (size_t i = 0; i < in_array.size(); ++i) { - if (!std::binary_search(aux_index_.begin(), aux_index_.end(), i)) { - in_data_.push_back(in_array[i].data()); - } else { - aux_data_.push_back(in_array[i].data()); - } + in_data_.push_back(in_array[i].data()); + } + out_data_.clear(); + for (size_t i = 0; i < out_array.size(); ++i) { + out_data_.push_back(out_array[i].data()); } - out_data_.resize(out_array.size()); - std::transform(out_array.begin(), out_array.end(), out_data_.begin(), [](const NDArray& nd) { - return nd.data(); - }); } - Operator::ExecType exec_type() const override { - return op_->exec_type(); + + ExecType exec_type() const override { + return exec_type_; } - explicit ForwardOpExecutor(std::shared_ptr op, - std::vector aux_index) - : op_(op), aux_index_(aux_index) { - std::sort(aux_index_.begin(), aux_index_.end()); + + virtual engine::VarHandle var() const { + return state_.get_var(); } + explicit StatefulComputeExecutor(const OpStatePtr& state, + const FStatefulCompute& fcompute, + ExecType exec_type) + : state_(state), fcompute_(fcompute), exec_type_(exec_type) {} + private: friend Graph AttachOpExecs(Graph g); - std::shared_ptr op_; - std::vector aux_index_; - std::vector in_data_, out_data_, aux_data_; + OpStatePtr state_; + FStatefulCompute fcompute_; + ExecType exec_type_; + std::vector in_data_, out_data_; }; -// backward executor -class BackwardOpExecutor : public OpExecutor { + +// forward executor +class StatefulComputeExExecutor : public OpExecutor { public: void Run(RunContext rctx) override { op_ctx.run_ctx = rctx; - op_->Backward(op_ctx, out_grad_, in_data_, out_data_, - req, in_grad_, aux_data_); -#if MKL_EXPERIMENTAL == 1 - mkl_tblobs_prv_to_cpu(out_grad_); - mkl_tblobs_prv_to_cpu(in_data_); - mkl_tblobs_prv_to_cpu(out_data_); - mkl_tblobs_prv_to_cpu(in_grad_); - mkl_tblobs_prv_to_cpu(aux_data_); -#endif + fcompute_(state_, op_ctx, in_array, req, out_array); } - void Setup() override { - size_t arg_top = 0, aux_top = 0; - aux_data_.resize(aux_index_.size()); - for (size_t i = 0; i < in_array.size(); ++i) { - if (!std::binary_search(aux_index_.begin(), aux_index_.end(), i)) { - CHECK_GT(arg_data_ptr_.size(), arg_top); - *arg_data_ptr_[arg_top++] = in_array[i].data(); - } else { - aux_data_.at(aux_top++) = in_array[i].data(); - } - } - CHECK_EQ(out_array.size(), in_grad_.size()); - std::transform(out_array.begin(), out_array.end(), - in_grad_.begin(), [](const NDArray& nd) { - return nd.data(); - }); - } - Operator::ExecType exec_type() const override { - return op_->exec_type(); + + void Setup() override {} + + ExecType exec_type() const override { + return exec_type_; } - explicit BackwardOpExecutor(std::shared_ptr op, - const OperatorProperty* prop, - std::vector aux_index) - : op_(op), aux_index_(aux_index) { - std::sort(aux_index_.begin(), aux_index_.end()); - out_grad_.resize(prop->NumVisibleOutputs()); - in_data_.resize(prop->ListArguments().size()); - in_grad_.resize(in_data_.size()); - out_data_.resize(prop->NumOutputs()); - - std::vector out_grad_ptr(out_grad_.size()); - for (size_t i = 0; i < out_grad_.size(); ++i) { - out_grad_ptr[i] = &out_grad_[i]; - } - std::vector in_data_ptr(in_data_.size()); - for (size_t i = 0; i < in_data_.size(); ++i) { - in_data_ptr[i] = &in_data_[i]; - } - std::vector out_data_ptr(out_data_.size()); - for (size_t i = 0; i < out_data_.size(); ++i) { - out_data_ptr[i] = &out_data_[i]; - } - arg_data_ptr_ = prop->BackwardInputs( - out_grad_ptr, in_data_ptr, out_data_ptr); + + virtual engine::VarHandle var() const { + return state_.get_var(); } + explicit StatefulComputeExExecutor(const OpStatePtr& state, + const FStatefulComputeEx& fcompute, + ExecType exec_type) + : state_(state), fcompute_(fcompute), exec_type_(exec_type) {} + private: - std::shared_ptr op_; - std::vector aux_index_; - std::vector out_grad_, in_grad_, in_data_, out_data_, aux_data_; - std::vector arg_data_ptr_; + friend Graph AttachOpExecs(Graph g); + OpStatePtr state_; + FStatefulComputeEx fcompute_; + ExecType exec_type_; }; + // fcompute executor executor class FComputeExecutor : public OpExecutor { public: @@ -143,6 +109,7 @@ class FComputeExecutor : public OpExecutor { mkl_tblobs_prv_to_cpu(out_data_); #endif } + void Setup() override { in_data_.resize(in_array.size()); out_data_.resize(out_array.size()); @@ -152,29 +119,20 @@ class FComputeExecutor : public OpExecutor { std::transform(in_array.begin(), in_array.end(), in_data_.begin(), get_blob); std::transform(out_array.begin(), out_array.end(), out_data_.begin(), get_blob); } - Operator::ExecType exec_type() const override { - return Operator::kSync; - } - explicit FComputeExecutor(FCompute fcompute, const NodeAttrs& attrs) - : fcompute_(fcompute), attrs_(attrs) { + + ExecType exec_type() const override { + return exec_type_; } - static FCompute GetFCompute(const Op* op, Context ctx) { - static auto& fcompute_cpu = nnvm::Op::GetAttr("FCompute"); - static auto& fcompute_gpu = nnvm::Op::GetAttr("FCompute"); - if (ctx.dev_mask() == cpu::kDevMask) { - return fcompute_cpu.get(op, nullptr); - } else if (ctx.dev_mask() == gpu::kDevMask) { - return fcompute_gpu.get(op, nullptr); - } else { - LOG(FATAL) << "Unknown device mask"; - return nullptr; - } + explicit FComputeExecutor(const NodeAttrs& attrs, FCompute fcompute, + ExecType exec_type) + : attrs_(attrs), fcompute_(fcompute), exec_type_(exec_type) { } private: - FCompute fcompute_; NodeAttrs attrs_; + FCompute fcompute_; + ExecType exec_type_; std::vector in_data_, out_data_; }; @@ -184,15 +142,16 @@ Graph AttachOpExecs(Graph g) { using nnvm::ShapeVector; using nnvm::FMutateInputs; - auto& fcreate_layer_op = nnvm::Op::GetAttr("FCreateLayerOp"); + auto& fcreate_op_state = nnvm::Op::GetAttr("FCreateOpState"); auto& fmutate_inputs = nnvm::Op::GetAttr("FMutateInputs"); + auto& fexec_type = nnvm::Op::GetAttr("FExecType"); auto& is_layer_backward = nnvm::Op::GetAttr("TIsLayerOpBackward"); const auto& vdtype = g.GetAttr("dtype"); const auto& vshape = g.GetAttr("shape"); const auto& vctx = g.GetAttr("context"); - const auto& saved_opr = g.GetAttr< - std::unordered_map>>("saved_opr"); + const auto& saved_states = g.GetAttr< + std::unordered_map >("saved_states"); // get the graph const auto& idx = g.indexed_graph(); @@ -202,39 +161,72 @@ Graph AttachOpExecs(Graph g) { for (size_t i = 0; i < idx.num_nodes(); ++i) { const auto& inode = idx[i]; if (inode.source->is_variable()) continue; + const nnvm::Op *op = inode.source->op(); + ExecType exec_type = ExecType::kSync; std::vector mutate_index; - if (fmutate_inputs.count(inode.source->op())) { - mutate_index = fmutate_inputs[inode.source->op()](inode.source->attrs); + if (fmutate_inputs.count(op)) { + mutate_index = fmutate_inputs[op](inode.source->attrs); + } + if (fexec_type.count(op)) { + exec_type = fexec_type[op](inode.source->attrs); } - FCompute fcompute = FComputeExecutor::GetFCompute(inode.source->op(), vctx[i]); - if (fcreate_layer_op.count(inode.source->op())) { + + if (fcreate_op_state.count(op)) { std::vector ishape; std::vector itype; for (const auto& e : inode.inputs) { ishape.emplace_back(vshape[idx.entry_id(e)]); itype.emplace_back(vdtype[idx.entry_id(e)]); } - std::shared_ptr opr; - if (saved_opr.count(inode.source)) { - opr = saved_opr.at(inode.source); + + OpStatePtr state; + if (saved_states.count(inode.source)) { + state = saved_states.at(inode.source); } else { - opr.reset(fcreate_layer_op[inode.source->op()]( - inode.source->attrs, vctx[i], ishape, itype)); + state = fcreate_op_state[op]( + inode.source->attrs, vctx[i], ishape, itype); } - ret[i] = std::make_shared(opr, mutate_index); - } else if (is_layer_backward.get(inode.source->op(), false)) { + FStatefulCompute fcompute = common::GetFCompute( + op, "FStatefulCompute", vctx[i]); + if (fcompute != nullptr) { + ret[i] = std::make_shared(state, fcompute, exec_type); + } else { + FStatefulComputeEx fcompute_ex = common::GetFCompute( + op, "FStatefulComputeEx", vctx[i]); + CHECK(fcompute_ex != nullptr) + << "One of FStatefulCompute and FStatefulComputeEx must be registered " + << "for stateful operator " << op->name; + ret[i] = std::make_shared(state, fcompute_ex, exec_type); + } + } else if (is_layer_backward.get(op, false)) { CHECK_GE(inode.control_deps.size(), 1); uint32_t fwd_id = inode.control_deps[0]; CHECK(vctx[fwd_id] == vctx[i]); CHECK(ret[fwd_id] != nullptr); - ret[i] = std::make_shared( - dynamic_cast(ret[fwd_id].get())->op_, - mxnet::op::OpPropGetOpProperty(inode.source->attrs), - mutate_index); - } else if (fcompute != nullptr) { - ret[i] = std::make_shared(fcompute, inode.source->attrs); + FStatefulCompute fcompute = common::GetFCompute( + op, "FStatefulCompute", vctx[i]); + if (fcompute != nullptr) { + ret[i] = std::make_shared( + dynamic_cast(ret[fwd_id].get())->state_, + fcompute, exec_type); + } else { + FStatefulComputeEx fcompute_ex = common::GetFCompute( + op, "FStatefulComputeEx", vctx[i]); + CHECK(fcompute_ex != nullptr) + << "One of FStatefulCompute and FStatefulComputeEx must be registered " + << "for stateful operator " << op->name; + ret[i] = std::make_shared( + dynamic_cast(ret[fwd_id].get())->state_, + fcompute_ex, exec_type); + } } else { - LOG(INFO) << "FCompute not registered " << inode.source->op()->name; + FCompute fcompute = common::GetFCompute(op, "FCompute", vctx[i]); + if (fcompute != nullptr) { + ret[i] = std::make_shared( + inode.source->attrs, fcompute, exec_type); + } else { + LOG(INFO) << "FCompute not registered " << op->name; + } } } g.attrs["op_execs"] = std::make_shared(ret); diff --git a/src/executor/exec_pass.h b/src/executor/exec_pass.h index 8df6a3c5d3bb..76b02de736e9 100644 --- a/src/executor/exec_pass.h +++ b/src/executor/exec_pass.h @@ -49,7 +49,11 @@ class OpExecutor { */ virtual void Run(RunContext rctx) = 0; /*! \return the execution type */ - virtual Operator::ExecType exec_type() const = 0; + virtual ExecType exec_type() const = 0; + /*! \return return engine variable for operator states */ + virtual engine::VarHandle var() const { + return nullptr; + } }; /*! diff --git a/src/executor/graph_executor.cc b/src/executor/graph_executor.cc index 2be000112711..add1d36434a8 100644 --- a/src/executor/graph_executor.cc +++ b/src/executor/graph_executor.cc @@ -707,7 +707,7 @@ void GraphExecutor::FinishInitGraph(nnvm::Symbol symbol, } g = DetectInplaceAddTo(g); - g.attrs["saved_opr"] = std::make_shared(std::move(saved_opr_)); + g.attrs["saved_states"] = std::make_shared(std::move(saved_states_)); g = AttachOpExecs(g); g = AttachOpResources(g); graph_ = std::move(g); @@ -1037,7 +1037,7 @@ void GraphExecutor::InitCachedOps() { if (inode.source->is_variable()) continue; if (op_nodes_[nid].skip_exec_node) continue; auto& exec = op_nodes_[nid].exec; - bool is_async = op_nodes_[nid].exec->exec_type() == Operator::kAsync; + bool is_async = op_nodes_[nid].exec->exec_type() == ExecType::kAsync; bool is_gpu = op_nodes_[nid].ctx.dev_mask() == gpu::kDevMask; // the variables @@ -1052,6 +1052,9 @@ void GraphExecutor::InitCachedOps() { for (auto& nd : exec->out_array) { mutate_vars.push_back(nd.var()); } + if (exec->var() != nullptr) { + mutate_vars.push_back(exec->var()); + } // dedup vars Engine::Get()->DeduplicateVarHandle(&use_vars, &mutate_vars); // all vars include both mutate vars and use vars @@ -1100,16 +1103,15 @@ void GraphExecutor::InitOpSegs() { // Generate segments based on the graph structure bool prefer_bulk_exec_inference = dmlc::GetEnv("MXNET_EXEC_BULK_EXEC_INFERENCE", true); - if (prefer_bulk_exec_inference && num_forward_nodes_ == total_num_nodes) { - // bulk the whole graph for inference - cached_seg_opr_[0] = this->CreateCachedSegOpr(0, num_forward_nodes_); - return; - } - // Whether to perform bulk exec for training bool prefer_bulk_exec = dmlc::GetEnv("MXNET_EXEC_BULK_EXEC_TRAIN", 1); // The maximum number of node in a segment executed in bulk size_t num_nodes_threshold = dmlc::GetEnv("MXNET_EXEC_BULK_EXEC_MAX_NODE_TRAIN", 15); + if (prefer_bulk_exec_inference && num_forward_nodes_ == total_num_nodes) { + // bulk the whole graph for inference + num_nodes_threshold = std::numeric_limits::max(); + } + // create forward segments for training if (prefer_bulk_exec > 0) { size_t topo_start = 0; @@ -1119,7 +1121,7 @@ void GraphExecutor::InitOpSegs() { // check if the segment relies on external input, or exceeds maxinum number of node, // or requires async ops if (node->is_variable() || nid - topo_start > num_nodes_threshold || - op_node.exec->exec_type() != Operator::kSync) { + op_node.exec->exec_type() != ExecType::kSync) { // create a new segment for the previous nodes if the current one cannot be bulked cached_seg_opr_[topo_start] = this->CreateCachedSegOpr(topo_start, nid); topo_start = nid + 1; @@ -1146,7 +1148,7 @@ void GraphExecutor::InitOpSegs() { continue; } if (idx[nid].source->is_variable() || nid - topo_start > num_nodes_threshold || - op_node.exec->exec_type() != Operator::kSync) { + op_node.exec->exec_type() != ExecType::kSync) { cached_seg_opr_[topo_start] = this->CreateCachedSegOpr(topo_start, nid); topo_start = nid + 1; } else { @@ -1224,11 +1226,13 @@ void GraphExecutor::RunOps(bool is_train, size_t topo_start, size_t topo_end) { OpNode& opnode = op_nodes_[nid]; if (op_nodes_[nid].skip_exec_node) continue; opnode.exec->op_ctx.is_train = is_train; - if (opnode.exec->exec_type() == Operator::kCrossDeviceCopy) { + if (opnode.exec->exec_type() == ExecType::kCrossDeviceCopy) { CHECK_EQ(inode.inputs.size(), 1U); CHECK_EQ(opnode.exec->in_array.size(), 1U); CHECK_EQ(opnode.exec->out_array.size(), 1U); CopyFromTo(opnode.exec->in_array[0], &(opnode.exec->out_array[0])); + } else if (opnode.exec->exec_type() == ExecType::kLocal) { + opnode.exec->Run(RunContext{opnode.ctx, nullptr}); } else if (opnode.cached_opr != nullptr) { #if MXNET_USE_PROFILER bool profiling = engine::Profiler::Get()->GetState() == engine::Profiler::kRunning; @@ -1271,7 +1275,7 @@ GraphExecutor::CachedSegOpr GraphExecutor::CreateCachedSegOpr(size_t topo_start, OpNode& op_node = op_nodes_[nid]; if (op_node.skip_exec_node) continue; if (inode.source->is_variable()) continue; - if (op_node.exec->exec_type() != Operator::kSync) { + if (op_node.exec->exec_type() != ExecType::kSync) { return ret; } if (pctx == nullptr) pctx = &(op_node.ctx); @@ -1283,7 +1287,7 @@ GraphExecutor::CachedSegOpr GraphExecutor::CreateCachedSegOpr(size_t topo_start, std::inserter(mutate_vars, mutate_vars.end())); std::copy(op_node.use_vars.begin(), op_node.use_vars.end(), std::inserter(use_vars, use_vars.end())); - ret.exec_list.push_back(exec.get()); + ret.exec_list.push_back(exec); #if MXNET_USE_PROFILER opr_names += inode.source->op()->name + ","; #endif diff --git a/src/executor/graph_executor.h b/src/executor/graph_executor.h index d5a4e8c3aa6c..5b6fa395b242 100644 --- a/src/executor/graph_executor.h +++ b/src/executor/graph_executor.h @@ -21,9 +21,6 @@ namespace mxnet { -using NodeOperatorMap = std::unordered_map>; - // forward declaration namespace exec { class GraphExecutor; @@ -120,7 +117,7 @@ class GraphExecutor : public Executor { // the cached operator Engine::OprHandle opr = nullptr; // list of op executors - std::vector exec_list; + std::vector > exec_list; }; // Initialize in_args, arg_grads, and aux_states void InitArguments(const nnvm::IndexedGraph& idx, @@ -211,7 +208,7 @@ class GraphExecutor : public Executor { // number of forward nodes size_t num_forward_nodes_{0}; // saved operator for autograd - NodeOperatorMap saved_opr_; + std::unordered_map saved_states_; // monitor call back std::function monitor_callback_{nullptr}; // whether to enable bulk execution diff --git a/src/ndarray/autograd.cc b/src/ndarray/autograd.cc index ce1b98f095d8..b35364d0c70f 100644 --- a/src/ndarray/autograd.cc +++ b/src/ndarray/autograd.cc @@ -83,15 +83,15 @@ void AutogradRuntime::RecordImperativeFCompute(const nnvm::Op* op, const nnvm::NodeAttrs& attrs, std::vector *p_inputs, std::vector *p_outputs) { - RecordOp(op, attrs, p_inputs, p_outputs, nullptr); + RecordOp(op, attrs, p_inputs, p_outputs, OpStatePtr()); } -void AutogradRuntime::RecordImperativeOperator(const std::shared_ptr& opr, +void AutogradRuntime::RecordImperativeOperator(const OpStatePtr& state, const nnvm::Op* op, const nnvm::NodeAttrs& attrs, std::vector *p_inputs, std::vector *p_outputs) { - RecordOp(op, attrs, p_inputs, p_outputs, opr); + RecordOp(op, attrs, p_inputs, p_outputs, state); } std::shared_ptr AutogradRuntime::_GetSharedRef() { @@ -108,7 +108,7 @@ AGNodePtr AutogradRuntime::RecordOp(const nnvm::Op* op, const nnvm::NodeAttrs& attrs, std::vector *p_inputs, std::vector *p_outputs, - const std::shared_ptr& opr) { + const OpStatePtr& state) { std::vector& inputs = *p_inputs; std::vector& outputs = *p_outputs; @@ -117,7 +117,7 @@ AGNodePtr AutogradRuntime::RecordOp(const nnvm::Op* op, nn_node->attrs.name = "node_" + std::to_string(node_count_++); AGNodePtr ag_node = AGNode::Create(nn_node); - ag_node->opr = opr; + ag_node->state = state; for (uint32_t i = 0; i < outputs.size(); ++i) { CHECK(outputs[i].entry_.is_none()) @@ -167,13 +167,13 @@ void AutogradRuntime::ComputeGradient(const std::vector& outputs, std::vector args, args_grad; std::vector aux_states; std::vector grad_reqs; - std::unordered_map> saved_opr; + std::unordered_map saved_states; AGDFSVisit(heads, [&](const AGNodePtr& n) { if (n->nn_node->is_variable()) { vlist.push_back(n); } else { - if (n->opr != nullptr) { - saved_opr.insert({n->nn_node.get(), n->opr}); + if (n->state) { + saved_states.insert({n->nn_node.get(), n->state}); } if (fmutate_inputs.count(n->nn_node->op())) { for (uint32_t i : fmutate_inputs[n->nn_node->op()](n->nn_node->attrs)) { @@ -203,7 +203,7 @@ void AutogradRuntime::ComputeGradient(const std::vector& outputs, std::map ctx_map; auto exec = new exec::GraphExecutor(); // (TODO) too hack here - exec->saved_opr_ = saved_opr; + exec->saved_states_ = saved_states; exec->Init(sym, args[0].ctx(), ctx_map, args, args_grad, grad_reqs, aux_states, nullptr, feed_dict); diff --git a/src/ndarray/autograd.h b/src/ndarray/autograd.h index e6868064ca0d..baf843dbd4e1 100644 --- a/src/ndarray/autograd.h +++ b/src/ndarray/autograd.h @@ -25,7 +25,7 @@ class AGNode { public: OpReqType grad_req; nnvm::NodePtr nn_node; - std::shared_ptr opr; + OpStatePtr state; std::vector inputs; std::vector outputs; std::vector out_grads; @@ -40,7 +40,7 @@ class AGNode { void clear_history() { if (out_grads.size()) return; - opr.reset(); + state.reset(); outputs.clear(); nn_node.reset(); for (auto& i : inputs) i.ag_node->clear_history(); @@ -73,7 +73,7 @@ class AutogradRuntime { std::vector* p_inputs, std::vector* p_outputs); /*! \brief record imperative operator which is executed by operator. */ - void RecordImperativeOperator(const std::shared_ptr& opr, + void RecordImperativeOperator(const OpStatePtr& state, const nnvm::Op* op, const nnvm::NodeAttrs& attrs, std::vector* p_inputs, @@ -103,7 +103,7 @@ class AutogradRuntime { const nnvm::NodeAttrs& attrs, std::vector* p_inputs, std::vector* p_outputs, - const std::shared_ptr& opr); + const OpStatePtr& state); /*! \brief AutogradRuntime singleton. */ static AutogradRuntime* instance_; /*! \brief indicate whether is training. */ diff --git a/src/ndarray/ndarray.cc b/src/ndarray/ndarray.cc index 6f1795d6f368..9999f9c8307b 100644 --- a/src/ndarray/ndarray.cc +++ b/src/ndarray/ndarray.cc @@ -757,8 +757,7 @@ void NDArray::SyncCopyFromCPU(const void *data, size_t size) const { if (this->ctx().dev_mask() == cpu::kDevMask) { this->WaitToWrite(); - RunContext rctx; - rctx.stream = nullptr; + RunContext rctx{this->ctx(), nullptr}; TBlob dst = this->data(); ndarray::Copy(src, &dst, Context::CPU(), Context::CPU(), rctx); } else { @@ -786,8 +785,7 @@ void NDArray::SyncCopyToCPU(void *data, size_t size) const { if (this->ctx().dev_mask() == cpu::kDevMask) { this->WaitToRead(); - RunContext rctx; - rctx.stream = nullptr; + RunContext rctx{this->ctx(), nullptr}; ndarray::Copy(this->data(), &dst, Context::CPU(), Context::CPU(), rctx); } else { diff --git a/src/ndarray/ndarray_function.cu b/src/ndarray/ndarray_function.cu index ff6702f2f41b..13d36a2c4293 100644 --- a/src/ndarray/ndarray_function.cu +++ b/src/ndarray/ndarray_function.cu @@ -20,7 +20,7 @@ void Copy(const TBlob &from, TBlob *to, MSHADOW_TYPE_SWITCH(to->type_flag_, DType, { mshadow::Copy(to->FlatTo1D(), from.FlatTo1D(), - static_cast*>(ctx.stream)); + ctx.get_stream()); }); } @@ -33,7 +33,7 @@ void Copy(const TBlob &from, TBlob *to, MSHADOW_TYPE_SWITCH(to->type_flag_, DType, { mshadow::Copy(to->FlatTo1D(), from.FlatTo1D(), - static_cast*>(ctx.stream)); + ctx.get_stream()); }); } @@ -42,7 +42,7 @@ void Copy(const TBlob &from, TBlob *to, Context from_ctx, Context to_ctx, RunContext ctx) { if (from_ctx.dev_id == to_ctx.dev_id) { - mshadow::Stream* s = static_cast*>(ctx.stream); + mshadow::Stream* s = ctx.get_stream(); MSHADOW_TYPE_SWITCH(to->type_flag_, DType, { if (to->type_flag_ == from.type_flag_) { mshadow::Copy(to->FlatTo1D(s), @@ -60,7 +60,7 @@ void Copy(const TBlob &from, TBlob *to, << "copy across only support continugous memory"; CHECK_EQ(to->type_flag_, from.type_flag_) << "Source and target must have the same data type when copying across devices."; - mshadow::Stream *s = static_cast*>(ctx.stream); + mshadow::Stream *s = ctx.get_stream(); CHECK(s != NULL) << "need stream in GPU context"; cudaMemcpyPeerAsync(to->dptr_, to_ctx.dev_id, diff --git a/src/nnvm/legacy_op_util.cc b/src/nnvm/legacy_op_util.cc index 9b39794b4782..2bba5f1c3655 100644 --- a/src/nnvm/legacy_op_util.cc +++ b/src/nnvm/legacy_op_util.cc @@ -54,6 +54,97 @@ class ParsedOpProp { } }; +class OperatorState { + public: + OperatorState(Operator *opr, const OperatorProperty *prop) { + opr_ = opr; + fwd_init_ = bwd_init_ = false; + + in_data_.resize(prop->ListArguments().size()); + out_data_.resize(prop->NumOutputs()); + aux_data_.resize(prop->ListAuxiliaryStates().size()); + in_grad_.resize(in_data_.size()); + out_grad_.resize(prop->NumVisibleOutputs()); + + std::vector out_grad_ptr(out_grad_.size()); + for (size_t i = 0; i < out_grad_.size(); ++i) { + out_grad_ptr[i] = &out_grad_[i]; + } + std::vector in_data_ptr(in_data_.size()); + for (size_t i = 0; i < in_data_.size(); ++i) { + in_data_ptr[i] = &in_data_[i]; + } + std::vector out_data_ptr(out_data_.size()); + for (size_t i = 0; i < out_data_.size(); ++i) { + out_data_ptr[i] = &out_data_[i]; + } + arg_data_ptr_ = prop->BackwardInputs( + out_grad_ptr, in_data_ptr, out_data_ptr); + } + + ~OperatorState() { delete opr_; } + + void Forward(const OpContext &ctx, + const std::vector& inputs, + const std::vector& req, + const std::vector& outputs) { + if (!fwd_init_) { + CHECK_EQ(inputs.size(), in_data_.size() + aux_data_.size()); + CHECK_EQ(outputs.size(), out_data_.size()); + for (size_t i = 0; i < in_data_.size(); ++i) in_data_[i] = inputs[i]; + for (size_t i = 0; i < aux_data_.size(); ++i) { + aux_data_[i] = inputs[i + in_data_.size()]; + } + for (size_t i = 0; i < out_data_.size(); ++i) out_data_[i] = outputs[i]; + fwd_init_ = true; + } + opr_->Forward(ctx, in_data_, req, out_data_, aux_data_); + } + + void Backward(const OpContext &ctx, + const std::vector& inputs, + const std::vector& req, + const std::vector& outputs) { + if (!bwd_init_) { + CHECK(fwd_init_); + CHECK_EQ(arg_data_ptr_.size() + aux_data_.size(), inputs.size()); + for (size_t i = 0; i < arg_data_ptr_.size(); ++i) { + *arg_data_ptr_[i] = inputs[i]; + } + for (size_t i = 0; i < aux_data_.size(); ++i) { + aux_data_[i] = inputs[inputs.size() - aux_data_.size() + i]; + } + CHECK_EQ(outputs.size(), in_grad_.size()); + for (size_t i = 0; i < outputs.size(); ++i) in_grad_[i] = outputs[i]; + bwd_init_ = true; + } + opr_->Backward(ctx, out_grad_, in_data_, out_data_, req, in_grad_, aux_data_); + } + + private: + Operator *opr_; + bool fwd_init_, bwd_init_; + std::vector in_data_, aux_data_, out_data_, in_grad_, out_grad_; + std::vector arg_data_ptr_; +}; + +void LegacyOpForward(const OpStatePtr& state, + const OpContext& ctx, + const std::vector& inputs, + const std::vector& req, + const std::vector& outputs) { + auto& op = state.get_state(); + op.Forward(ctx, inputs, req, outputs); +} + +void LegacyOpBackward(const OpStatePtr& state, + const OpContext& ctx, + const std::vector& inputs, + const std::vector& req, + const std::vector& outputs) { + auto& op = state.get_state(); + op.Backward(ctx, inputs, req, outputs); +} // function to use operator property to infer attr // get op property from the attribute @@ -182,14 +273,15 @@ std::vector OpBackResourceRequest(const NodeAttrs& attrs) { return prop.ptr->BackwardResource(ishape); } -Operator* OpPropCreateLayerOp(const NodeAttrs& attrs, - Context ctx, - const std::vector& ishape, - const std::vector& itype) { +OpStatePtr OpPropCreateLayerOp(const NodeAttrs& attrs, + Context ctx, + const std::vector& ishape, + const std::vector& itype) { auto& prop = nnvm::get(attrs.parsed); std::vector is(ishape.begin(), ishape.begin() + prop.arguments.size()); std::vector it(itype.begin(), itype.begin() + prop.arguments.size()); - return prop.ptr->CreateOperatorEx(ctx, &is, &it); + return OpStatePtr::Create(prop.ptr->CreateOperatorEx(ctx, &is, &it), + prop.ptr.get()); } inline std::vector OpPropGradient( @@ -300,6 +392,11 @@ std::vector > OpBackInplaceOption(const NodeAttrs& attrs) { return remap; } +inline ExecType OpExecType(const NodeAttrs& attrs) { + auto& prop = nnvm::get(attrs.parsed); + return prop.ptr->exec_type(); +} + // register the legacy operator properties under NNVM registry. void RegisterLegacyOpProp() { for (auto reg : dmlc::Registry::List()) { @@ -328,10 +425,14 @@ void RegisterLegacyOpProp() { op.set_attr("FMutateInputs", OpPropMutateInputs); op.set_attr("FInplaceOption", OpPropInplaceOption); op.set_attr("FResourceRequest", OpPropResourceRequest); - op.set_attr("FCreateLayerOp", OpPropCreateLayerOp); + op.set_attr("FExecType", OpExecType); + op.set_attr("FCreateOpState", OpPropCreateLayerOp); + op.set_attr("FStatefulCompute", LegacyOpForward); + op.set_attr("FStatefulCompute", LegacyOpForward); if (reg->key_var_num_args.length() != 0) { op.set_attr("key_var_num_args", reg->key_var_num_args); } + // register BackwardOps std::string back_op_name = "_backward_" + reg->name; Op& back_op = ::dmlc::Registry<::nnvm::Op>::Get()->__REGISTER__(back_op_name); @@ -348,6 +449,9 @@ void RegisterLegacyOpProp() { "FResourceRequest", OpBackResourceRequest); back_op.set_attr("TIsLayerOpBackward", true); back_op.set_attr("TIsBackward", true); + back_op.set_attr("FExecType", OpExecType); + back_op.set_attr("FStatefulCompute", LegacyOpBackward); + back_op.set_attr("FStatefulCompute", LegacyOpBackward); } } diff --git a/src/operator/activation.cc b/src/operator/activation.cc index 0b1562925398..c8b8c3b5acb2 100644 --- a/src/operator/activation.cc +++ b/src/operator/activation.cc @@ -55,10 +55,6 @@ Operator *CreateOp(ActivationParam param, int dtype) { // DO_BIND_DISPATCH comes from operator_common.h Operator *ActivationProp::CreateOperatorEx(Context ctx, std::vector *in_shape, std::vector *in_type) const { - std::vector out_shape, aux_shape; - std::vector out_type, aux_type; - CHECK(InferType(in_type, &out_type, &aux_type)); - CHECK(InferShape(in_shape, &out_shape, &aux_shape)); DO_BIND_DISPATCH(CreateOp, param_, (*in_type)[0]); } diff --git a/src/operator/batch_norm.cc b/src/operator/batch_norm.cc index 0ef5733f9f8c..1bc6fd08e2ea 100644 --- a/src/operator/batch_norm.cc +++ b/src/operator/batch_norm.cc @@ -315,21 +315,11 @@ Operator *CreateOp(BatchNormParam param, const int dtype, const TShape& sha break; } } -#define BATCHNORM_LOG_MKL_INFO() \ - do { \ - if (!mxnet::op::batchnorm::disable_mkl) { \ - LOG(INFO) << MKLBatchNormOp::getName() \ - << " Skipping MKL optimization (unsupported dimension, axis or type)"; \ - } \ - } while (0) -#else -#define BATCHNORM_LOG_MKL_INFO() ((void)0) #endif if (!op) { MSHADOW_REAL_TYPE_SWITCH_EX(dtype, DType, AccReal, { - BATCHNORM_LOG_MKL_INFO(); op = new BatchNormOp(param); }); } return op; @@ -338,11 +328,6 @@ Operator *CreateOp(BatchNormParam param, const int dtype, const TShape& sha // DO_BIND_DISPATCH comes from operator_common.h Operator *BatchNormProp::CreateOperatorEx(Context ctx, std::vector *in_shape, std::vector *in_type) const { - std::vector out_shape, aux_shape; - std::vector out_type, aux_type; - CHECK(InferType(in_type, &out_type, &aux_type)); - CHECK(InferShape(in_shape, &out_shape, &aux_shape)); - CHECK_GE(in_shape->size(), 1U); DO_BIND_DISPATCH(CreateOp, param_, (*in_type)[0], (*in_shape)[0]); } @@ -415,4 +400,3 @@ NNVM_REGISTER_OP(BatchNorm) } // namespace op } // namespace mxnet - diff --git a/src/operator/bilinear_sampler.cc b/src/operator/bilinear_sampler.cc index 7cc94c50982f..fd2bff824fd7 100644 --- a/src/operator/bilinear_sampler.cc +++ b/src/operator/bilinear_sampler.cc @@ -142,10 +142,6 @@ Operator* CreateOp(BilinearSamplerParam param, int dtype) { Operator *BilinearSamplerProp::CreateOperatorEx(Context ctx, std::vector *in_shape, std::vector *in_type) const { - std::vector out_shape, aux_shape; - std::vector out_type, aux_type; - CHECK(InferType(in_type, &out_type, &aux_type)); - CHECK(InferShape(in_shape, &out_shape, &aux_shape)); DO_BIND_DISPATCH(CreateOp, param_, (*in_type)[0]); } diff --git a/src/operator/convolution.cc b/src/operator/convolution.cc index b8fc49021d77..fd604d90c546 100644 --- a/src/operator/convolution.cc +++ b/src/operator/convolution.cc @@ -44,7 +44,6 @@ Operator* CreateOp(ConvolutionParam param, int dtype, break; } } - LOG(INFO) << MKLConvolutionOp::getName() << " Skip MKL optimization"; #endif #if MXNET_USE_NNPACK == 1 const size_t batch_size = (*in_shape)[0][0]; @@ -72,8 +71,6 @@ Operator *ConvolutionProp::CreateOperatorEx(Context ctx, std::vector *in_shape, std::vector *in_type) const { std::vector out_shape, aux_shape; - std::vector out_type, aux_type; - CHECK(InferType(in_type, &out_type, &aux_type)); CHECK(InferShape(in_shape, &out_shape, &aux_shape)); DO_BIND_DISPATCH(CreateOp, param_, (*in_type)[0], in_shape, &out_shape, ctx); } diff --git a/src/operator/cross_device_copy.cc b/src/operator/cross_device_copy.cc index ce618c97fa05..a9a5f475f0bc 100644 --- a/src/operator/cross_device_copy.cc +++ b/src/operator/cross_device_copy.cc @@ -20,12 +20,6 @@ class CrossDeviceCopyOp : public Operator { // We still re-use things such as InferShape in OperatorProperty LOG(FATAL) << "Not Reached"; } - - ExecType exec_type() const override { - // TODO(tianqi) Think of other way to blend cross device op into operator interface. - // declare the op as cross device, - return kCrossDeviceCopy; - } }; class CrossDeviceCopyProp : public OperatorProperty { @@ -58,6 +52,12 @@ class CrossDeviceCopyProp : public OperatorProperty { Operator* CreateOperator(Context ctx) const override { return new CrossDeviceCopyOp(); } + + ExecType exec_type() const override { + // TODO(tianqi) Think of other way to blend cross device op into operator interface. + // declare the op as cross device, + return ExecType::kCrossDeviceCopy; + } }; diff --git a/src/operator/custom/custom-inl.h b/src/operator/custom/custom-inl.h index f640c3abd7a6..3c688feb05a1 100644 --- a/src/operator/custom/custom-inl.h +++ b/src/operator/custom/custom-inl.h @@ -25,273 +25,33 @@ namespace mxnet { namespace op { +namespace custom { -struct CustomOpParam { - std::string op_type; - std::vector > kwargs; -}; - -template -class CustomOp : public Operator { - public: - explicit CustomOp(MXCallbackList* op_info) { - op_info_.reset(op_info, [](MXCallbackList *ptr){ - reinterpret_cast(ptr->callbacks[kCustomOpDelete])( - ptr->contexts[kCustomOpDelete]); - delete ptr; - }); - if (std::string("NaiveEngine") == dmlc::GetEnv("MXNET_ENGINE_TYPE", std::string())) { - sync_mode_ = true; - } else { - sync_mode_ = false; - destructing_ = false; - worker_ = std::thread([&]() { - std::unique_lock lock(mtx_); - while (!q_.empty() || !destructing_) { - cv_.wait(lock, [&] {return !q_.empty() || destructing_;}); - while (!q_.empty()) { - q_.front()(); - q_.pop(); - } - } - }); - } - } - - ~CustomOp() { - if (!sync_mode_) { - { - std::unique_lock lock(mtx_); - destructing_ = true; - cv_.notify_all(); - } - worker_.join(); - } - } - - virtual void Forward(const OpContext &ctx, - const std::vector &in_data, - const std::vector &req, - const std::vector &out_data, - const std::vector &aux_args); - - virtual void Backward(const OpContext &ctx, - const std::vector &out_grad, - const std::vector &in_data, - const std::vector &out_data, - const std::vector &req, - const std::vector &in_grad, - const std::vector &aux_args); - - virtual ExecType exec_type() const { - return kAsync; - } - - private: - Context get_ctx(); - std::shared_ptr op_info_; - std::mutex mtx_; - std::condition_variable cv_; - std::thread worker_; - std::queue > q_; - bool destructing_; - bool sync_mode_; -}; // CustomOp - -template -Operator* CreateOp(MXCallbackList *op_info); - -class CustomOpProp : public OperatorProperty { +class Registry { public: - static void Register(const std::string &op_type, CustomOpPropCreator creator) { + void Register(const std::string &op_type, CustomOpPropCreator creator) { + std::lock_guard lock(mutex_); if (registry_.find(op_type) != registry_.end()) { LOG(WARNING) << "New registration is overriding existing custom operator " << op_type; } registry_[op_type] = creator; } - void Init(const std::vector >& kwargs) override { - kwargs_ = kwargs; - param_.op_type = ""; - param_.kwargs.clear(); - std::vector keys, vals; - for (auto &p : kwargs) { - if (p.first == "op_type") { - param_.op_type = p.second; - } else { - param_.kwargs.push_back(p); - keys.push_back(p.first.c_str()); - vals.push_back(p.second.c_str()); - } - } - CHECK_NE(param_.op_type, "") << "Custom operator type missing"; - CHECK(registry_.find(param_.op_type) != registry_.end()) - << "Cannot find custom operator type " << param_.op_type; - CustomOpPropCreator creator = registry_[param_.op_type]; - info_.reset(new MXCallbackList, [](MXCallbackList* ptr){ - reinterpret_cast(ptr->callbacks[kCustomOpPropDelete])( - ptr->contexts[kCustomOpPropDelete]); - delete ptr; - }); - CHECK(creator(param_.op_type.c_str(), keys.size(), keys.data(), vals.data(), info_.get())); - num_inputs_ = ListArguments().size(); - num_outputs_ = ListOutputs().size(); - num_auxs_ = ListAuxiliaryStates().size(); - } - - std::vector ListArguments() const override { - char ** args = NULL; - CHECK(reinterpret_cast(info_->callbacks[kCustomOpPropListArguments])( - &args, info_->contexts[kCustomOpPropListArguments])); - std::vector ret; - for (int i = 0; args[i] != NULL; ++i) { - ret.push_back(args[i]); - } - return ret; - } - - std::vector ListOutputs() const override { - char ** args = NULL; - CHECK(reinterpret_cast(info_->callbacks[kCustomOpPropListOutputs])( - &args, info_->contexts[kCustomOpPropListOutputs])); - std::vector ret; - for (int i = 0; args[i] != NULL; ++i) { - ret.push_back(args[i]); - } - return ret; - } - - std::vector ListAuxiliaryStates() const override { - char ** args = NULL; - CHECK(reinterpret_cast(info_->callbacks[kCustomOpPropListAuxiliaryStates])( - &args, info_->contexts[kCustomOpPropListAuxiliaryStates])); - std::vector ret; - for (int i = 0; args[i] != NULL; ++i) { - ret.push_back(args[i]); - } - return ret; - } - - int NumOutputs() const override { - return ListOutputs().size(); - } - - std::map GetParams() const override { - return std::map(kwargs_.begin(), kwargs_.end()); + CustomOpPropCreator Find(const std::string &op_type) { + std::lock_guard lock(mutex_); + auto it = registry_.find(op_type); + if (it != registry_.end()) return it->second; + return nullptr; } - - bool InferShape(std::vector *in_shape, - std::vector *out_shape, - std::vector *aux_shape) const override { - std::vector shapes; - std::vector ndims; - size_t size = 0; - for (const auto& s : *in_shape) size += s.ndim(); - std::vector shapes_buffer(size); - shapes_buffer.resize(size); - uint32_t *ptr = shapes_buffer.data(); - for (auto iter = in_shape->begin(); iter != in_shape->end(); ++iter) { - shapes.push_back(ptr); - ndims.push_back(iter->ndim()); - ptr = nnvm::ShapeTypeCast(iter->begin(), iter->end(), ptr); - } - shapes.resize(num_inputs_+num_outputs_+num_auxs_); - ndims.resize(num_inputs_+num_outputs_+num_auxs_); - - CHECK(reinterpret_cast(info_->callbacks[kCustomOpPropInferShape])( - shapes.size(), ndims.data(), shapes.data(), info_->contexts[kCustomOpPropInferShape])); - for (unsigned i = 0; i < in_shape->size(); ++i) { - SHAPE_ASSIGN_CHECK(*in_shape, i, TShape(shapes[i], shapes[i]+ndims[i])); - } - out_shape->clear(); - for (unsigned i = num_inputs_; i < num_inputs_+num_outputs_; ++i) { - out_shape->push_back(TShape(shapes[i], shapes[i]+ndims[i])); - } - aux_shape->clear(); - for (unsigned i = num_inputs_+num_outputs_; i < shapes.size(); ++i) { - aux_shape->push_back(TShape(shapes[i], shapes[i]+ndims[i])); - } - return true; - } - - bool InferType(std::vector *in_type, - std::vector *out_type, - std::vector *aux_type) const override { - if (info_->num_callbacks <= kCustomOpPropInferType) { - return OperatorProperty::InferType(in_type, out_type, aux_type); - } - - std::vector types; - for (const auto &i : *in_type) types.push_back(i); - for (const auto &i : *out_type) types.push_back(i); - for (const auto &i : *aux_type) types.push_back(i); - - CHECK(reinterpret_cast(info_->callbacks[kCustomOpPropInferType])( - types.size(), types.data(), info_->contexts[kCustomOpPropInferType])); - for (unsigned i = 0; i < num_inputs_; ++i) { - TYPE_ASSIGN_CHECK(*in_type, i, types[i]); - } - for (unsigned i = 0; i < num_outputs_; ++i) { - TYPE_ASSIGN_CHECK(*out_type, i, types[i+num_inputs_]); - } - for (unsigned i = 0; i < num_auxs_; ++i) { - TYPE_ASSIGN_CHECK(*aux_type, i, types[i+num_inputs_+num_outputs_]); - } - return true; - } - - - OperatorProperty* Copy() const override { - CustomOpProp *prop_sym = new CustomOpProp(); - prop_sym->Init(kwargs_); - return prop_sym; - } - - std::string TypeString() const override { - return "Custom"; - } - - std::vector DeclareBackwardDependency( - const std::vector &out_grad, - const std::vector &in_data, - const std::vector &out_data) const override { - int num_dep; - int *rdeps; - CHECK(reinterpret_cast( - info_->callbacks[kCustomOpPropDeclareBackwardDependency])( - out_grad.data(), in_data.data(), out_data.data(), &num_dep, - &rdeps, info_->contexts[kCustomOpPropDeclareBackwardDependency])); - std::vector deps; - deps.insert(deps.end(), rdeps, rdeps+num_dep); - return deps; - } - - std::vector > BackwardInplaceOption( - const std::vector &out_grad, - const std::vector &in_data, - const std::vector &out_data, - const std::vector &in_grad) const override { - return {}; - } - - Operator* CreateOperator(Context ctx) const override { - LOG(FATAL) << "Not Implemented."; - return NULL; - } - - Operator* CreateOperatorEx(Context ctx, std::vector *in_shape, - std::vector *in_type) const override; - + static Registry* Get(); private: - static std::map registry_; + Registry() {} + std::mutex mutex_; + std::map registry_; +}; - CustomOpParam param_; - std::shared_ptr info_; - std::vector > kwargs_; - unsigned num_inputs_, num_outputs_, num_auxs_; - mutable std::vector shapes_buffer_; -}; // class CustomOpProp +} // namespace custom } // namespace op } // namespace mxnet #endif // MXNET_OPERATOR_CUSTOM_CUSTOM_INL_H_ diff --git a/src/operator/custom/custom.cc b/src/operator/custom/custom.cc index 8fb324c1f5c2..1854bb7f05d0 100644 --- a/src/operator/custom/custom.cc +++ b/src/operator/custom/custom.cc @@ -8,196 +8,387 @@ #include #include +#include "../../ndarray/autograd.h" +#include "../elemwise_op_common.h" + namespace mxnet { namespace op { -std::map CustomOpProp::registry_; +namespace custom { + +Registry* Registry::Get() { + static Registry inst; + return &inst; +} + +struct CustomParam { + std::string op_type; + size_t num_args, num_outs, num_auxs; + std::vector bwd_idx; + std::shared_ptr info; +}; + + +template +std::vector List(const NodeAttrs& attrs) { + const CustomParam& params = nnvm::get(attrs.parsed); + char ** args = NULL; + CHECK(reinterpret_cast( + params.info->callbacks[Type])( + &args, params.info->contexts[Type])); + std::vector ret; + for (int i = 0; args[i] != NULL; ++i) { + ret.push_back(args[i]); + } + return ret; +} + +void AttrParser(NodeAttrs* attrs) { + attrs->parsed = CustomParam(); + CustomParam& params = nnvm::get(attrs->parsed); + + std::vector keys, vals; + for (auto &p : attrs->dict) { + if (p.first == "op_type") { + params.op_type = p.second; + } else { + keys.push_back(p.first.c_str()); + vals.push_back(p.second.c_str()); + } + } + CHECK(!params.op_type.empty()) << "Required argument `op_type` is missing."; + CustomOpPropCreator creator = Registry::Get()->Find(params.op_type); + CHECK(Registry::Get()->Find(params.op_type) != nullptr) + << "Cannot find custom operator " << params.op_type; + params.info.reset(new MXCallbackList, [](MXCallbackList* ptr){ + reinterpret_cast(ptr->callbacks[kCustomOpPropDelete])( + ptr->contexts[kCustomOpPropDelete]); + delete ptr; + }); + CHECK(creator(params.op_type.c_str(), keys.size(), keys.data(), + vals.data(), params.info.get())); + + params.num_args = List(*attrs).size(); + params.num_outs = List(*attrs).size(); + params.num_auxs = List(*attrs).size(); + + int num_dep, *rdeps, counter = 0; + std::vector out_grad, in_data, out_data; + for (size_t i = 0; i < params.num_outs; ++i) out_grad.push_back(counter++); + for (size_t i = 0; i < params.num_args; ++i) in_data.push_back(counter++); + for (size_t i = 0; i < params.num_outs; ++i) out_data.push_back(counter++); + CHECK(reinterpret_cast( + params.info->callbacks[kCustomOpPropDeclareBackwardDependency])( + out_grad.data(), in_data.data(), out_data.data(), &num_dep, + &rdeps, params.info->contexts[kCustomOpPropDeclareBackwardDependency])); + params.bwd_idx.insert(params.bwd_idx.end(), rdeps, rdeps+num_dep); +} + +bool InferShape(const NodeAttrs& attrs, + std::vector *in_shape, + std::vector *out_shape) { + const CustomParam& params = nnvm::get(attrs.parsed); + + size_t total = params.num_args + params.num_outs + params.num_auxs; + std::vector shapes(total); + std::vector ndims(total); + size_t buff_size = 0; + for (const auto& i : *in_shape) buff_size += i.ndim(); + std::vector buff(buff_size); + uint32_t *ptr = buff.data(); + for (size_t i = 0; i < in_shape->size(); ++i) { + shapes[i] = ptr; + ndims[i] = (*in_shape)[i].ndim(); + for (size_t j = 0; j < (*in_shape)[i].ndim(); ++j, ++ptr) { + *ptr = static_cast((*in_shape)[i][j]); + } + } + + CHECK(reinterpret_cast( + params.info->callbacks[kCustomOpPropInferShape])( + shapes.size(), ndims.data(), shapes.data(), + params.info->contexts[kCustomOpPropInferShape])); + + for (size_t i = 0; i < params.num_args; ++i) { + SHAPE_ASSIGN_CHECK(*in_shape, i, TShape(shapes[i], shapes[i]+ndims[i])); + } -template<> -Context CustomOp::get_ctx() { - return Context::CPU(); + size_t base = params.num_args; + for (size_t i = 0; i < params.num_outs; ++i) { + SHAPE_ASSIGN_CHECK(*out_shape, i, + TShape(shapes[base+i], shapes[base+i]+ndims[base+i])); + } + + base = params.num_args + params.num_outs; + for (size_t i = 0; i < params.num_auxs; ++i) { + SHAPE_ASSIGN_CHECK(*in_shape, params.num_args+i, + TShape(shapes[base+i], shapes[base+i]+ndims[base+i])); + } + return true; } -template<> -Operator *CreateOp(MXCallbackList *op_info) { - return new CustomOp(op_info); +bool InferType(const NodeAttrs& attrs, + std::vector *in_type, + std::vector *out_type) { + const CustomParam& params = nnvm::get(attrs.parsed); + + if (params.info->num_callbacks <= kCustomOpPropInferType) { + return ElemwiseAttr( + attrs, in_type, out_type, -1); + } + + std::vector types; + types.reserve(params.num_args + params.num_outs + params.num_auxs); + for (size_t i = 0; i < params.num_args; ++i) { + types.push_back((*in_type)[i]); + } + for (const auto &i : *out_type) { + types.push_back(i); + } + for (size_t i = 0; i < params.num_auxs; ++i) { + types.push_back((*in_type)[params.num_args+i]); + } + + CHECK(reinterpret_cast( + params.info->callbacks[kCustomOpPropInferType])( + types.size(), types.data(), params.info->contexts[kCustomOpPropInferType])); + + for (size_t i = 0; i < params.num_args; ++i) { + TYPE_ASSIGN_CHECK(*in_type, i, types[i]); + } + for (size_t i = 0; i < params.num_outs; ++i) { + TYPE_ASSIGN_CHECK(*out_type, i, types[params.num_args+i]); + } + for (size_t i = 0; i < params.num_auxs; ++i) { + TYPE_ASSIGN_CHECK(*in_type, params.num_args+i, + types[params.num_args+params.num_outs+i]); + } + return true; } -#if MXNET_USE_CUDA -template<> -Context CustomOp::get_ctx() { - int dev_id; - CHECK_EQ(cudaGetDevice(&dev_id), cudaSuccess); - return Context::GPU(dev_id); +std::vector Gradient( + const nnvm::NodePtr& n, + const std::vector& out_grads) { + const CustomParam& params = nnvm::get(n->attrs.parsed); + + nnvm::NodePtr g = nnvm::Node::Create(); + g->attrs.op = nnvm::Op::Get("_backward_Custom"); + g->attrs.name = n->attrs.name; + g->attrs.parsed = params; + g->control_deps.emplace_back(n); + + g->inputs.reserve(params.bwd_idx.size()); + for (const int& t : params.bwd_idx) { + size_t i = static_cast(t); + if (i >= params.num_outs + params.num_args) { + uint32_t idx = static_cast(i-params.num_outs-params.num_args); + g->inputs.push_back(nnvm::NodeEntry{n, idx, 0}); + } else if (i >= params.num_outs) { + g->inputs.push_back(n->inputs[i-params.num_outs]); + } else { + g->inputs.push_back(out_grads[i]); + } + } + + for (size_t i = 0; i < params.num_auxs; ++i) { + g->inputs.push_back(n->inputs[i+params.num_args]); + } + + std::vector ret; + for (index_t i = 0; i < g->num_outputs(); ++i) { + ret.emplace_back(nnvm::NodeEntry{g, i, 0}); + } + + return ret; } -template<> -Operator* CreateOp(MXCallbackList *op_info) { - return new CustomOp(op_info); + +OpStatePtr CreateState(const NodeAttrs& attrs, Context ctx, + const std::vector& in_shape, + const std::vector& in_type) { + const CustomParam& params = nnvm::get(attrs.parsed); + + size_t total = params.num_args + params.num_outs + params.num_auxs; + std::vector shapes(total); + std::vector ndims(total); + size_t buff_size = 0; + for (const auto& i : in_shape) buff_size += i.ndim(); + std::vector buff(buff_size); + uint32_t *ptr = buff.data(); + for (size_t i = 0; i < in_shape.size(); ++i) { + shapes[i] = ptr; + ndims[i] = in_shape[i].ndim(); + for (size_t j = 0; j < in_shape[i].ndim(); ++j, ++ptr) { + *ptr = static_cast(in_shape[i][j]); + } + } + + std::string str_ctx; + if (ctx.dev_mask() == cpu::kDevMask) { + str_ctx = "cpu"; + } else { + str_ctx = "gpu"; + } + + MXCallbackList *op_info = new MXCallbackList; + CHECK(reinterpret_cast( + params.info->callbacks[kCustomOpPropCreateOperator])( + str_ctx.c_str(), shapes.size(), shapes.data(), ndims.data(), in_type.data(), + op_info, params.info->contexts[kCustomOpPropCreateOperator])); + + CustomParam state = params; + state.info.reset(op_info, [](MXCallbackList *ptr){ + reinterpret_cast(ptr->callbacks[kCustomOpDelete])( + ptr->contexts[kCustomOpDelete]); + delete ptr; + }); + + return OpStatePtr::Create(state); } -#endif // MXNET_USE_CUDA - -template -void CustomOp::Forward(const OpContext &ctx, - const std::vector &in_data, - const std::vector &req, - const std::vector &out_data, - const std::vector &aux_args) { - using namespace mshadow; - Context ndctx = get_ctx(); + +void Forward(const OpStatePtr& state, + const OpContext& ctx, + const std::vector& inputs, + const std::vector& req, + const std::vector& outputs) { + const CustomParam& params = state.get_state(); std::vector ptrs; - std::vector ndcpy; - std::vector ndvar; std::vector tags; - std::vector reqs(req.begin(), req.end()); - for (auto& blob : in_data) { - ptrs.push_back(reinterpret_cast(new NDArray(blob, ndctx.dev_id))); + for (size_t i = 0; i < params.num_args; ++i) { + NDArray *nd = new NDArray(inputs[i].Detach()); + ptrs.push_back(reinterpret_cast(nd)); tags.push_back(0); } - for (auto& blob : out_data) { - NDArray* nd = new NDArray(blob, ndctx.dev_id); + + for (size_t i = 0; i < params.num_outs; ++i) { + NDArray *nd = new NDArray(outputs[i].Detach()); ptrs.push_back(reinterpret_cast(nd)); - ndcpy.push_back(*nd); - ndvar.push_back(nd->var()); tags.push_back(1); } - for (auto& blob : aux_args) { - NDArray* nd = new NDArray(blob, ndctx.dev_id); + + for (size_t i = 0; i < params.num_auxs; ++i) { + NDArray *nd = new NDArray(inputs[i+params.num_args].Detach()); ptrs.push_back(reinterpret_cast(nd)); - ndcpy.push_back(*nd); - ndvar.push_back(nd->var()); tags.push_back(4); } - std::sort(ndvar.begin(), ndvar.end()); - ndvar.resize(std::unique(ndvar.begin(), ndvar.end()) - ndvar.begin()); - auto compute = [=]() mutable { - CHECK(reinterpret_cast(op_info_->callbacks[kCustomOpForward])( - ptrs.size(), ptrs.data(), tags.data(), reqs.data(), - static_cast(ctx.is_train), op_info_->contexts[kCustomOpForward])); + bool old = autograd::AutogradRuntime::Get()->SetIsTraining(false); - // NDArray* in ptrs is freed by frontend side. We keep a copy in ndcpy to keep ndvar alive - Engine::Get()->PushSync([ndcpy, ctx](RunContext rctx) { - ctx.async_on_complete(); - }, ndctx, ndvar, {}, - FnProperty::kNormal, 0, PROFILER_MESSAGE("CustomOpForward")); - }; + CHECK(reinterpret_cast(params.info->callbacks[kCustomOpForward])( + ptrs.size(), ptrs.data(), tags.data(), reinterpret_cast(req.data()), + static_cast(ctx.is_train), params.info->contexts[kCustomOpForward])); - if (sync_mode_) { - compute(); - } else { - std::unique_lock lock(mtx_); - q_.push(compute); - cv_.notify_all(); - } + autograd::AutogradRuntime::Get()->SetIsTraining(old); } -template -void CustomOp::Backward(const OpContext &ctx, - const std::vector &out_grad, - const std::vector &in_data, - const std::vector &out_data, - const std::vector &req, - const std::vector &in_grad, - const std::vector &aux_args) { - using namespace mshadow; - Context ndctx = get_ctx(); - std::vector ptrs; - std::vector ndcpy; - std::vector ndvar; + +void Backward(const OpStatePtr& state, + const OpContext& ctx, + const std::vector& inputs, + const std::vector& req, + const std::vector& outputs) { + const CustomParam& params = state.get_state(); + + size_t total = 2*params.num_args + 2*params.num_outs + params.num_auxs; + std::vector ptrs(params.num_args + 2*params.num_outs, nullptr); std::vector tags; - std::vector reqs(req.begin(), req.end()); + ptrs.reserve(total); + tags.reserve(total); + for (size_t i = 0; i < params.num_outs; ++i) tags.push_back(3); + for (size_t i = 0; i < params.num_args; ++i) tags.push_back(0); + for (size_t i = 0; i < params.num_outs; ++i) tags.push_back(1); - for (auto& blob : in_data) { - ptrs.push_back(reinterpret_cast(new NDArray(blob, ndctx.dev_id))); - tags.push_back(0); + for (size_t i = 0; i < params.bwd_idx.size(); ++i) { + NDArray *nd = new NDArray(inputs[i].Detach()); + ptrs[params.bwd_idx[i]] = reinterpret_cast(nd); } - for (auto& blob : out_data) { - ptrs.push_back(reinterpret_cast(new NDArray(blob, ndctx.dev_id))); - tags.push_back(1); + for (size_t i = 0; i < ptrs.size(); ++i) { + if (ptrs[i] == nullptr) ptrs[i] = reinterpret_cast(new NDArray()); } - for (auto& blob : in_grad) { - NDArray* nd = new NDArray(blob, ndctx.dev_id); + for (const auto& i : outputs) { + NDArray* nd = new NDArray(i.Detach()); ptrs.push_back(reinterpret_cast(nd)); - ndcpy.push_back(*nd); - ndvar.push_back(nd->var()); tags.push_back(2); } - for (auto& blob : aux_args) { - NDArray* nd = new NDArray(blob, ndctx.dev_id); + for (size_t i = 0; i < params.num_auxs; ++i) { + NDArray* nd = new NDArray(inputs[inputs.size()-params.num_auxs+i].Detach()); ptrs.push_back(reinterpret_cast(nd)); - ndcpy.push_back(*nd); - ndvar.push_back(nd->var()); tags.push_back(4); } - std::sort(ndvar.begin(), ndvar.end()); - ndvar.resize(std::unique(ndvar.begin(), ndvar.end()) - ndvar.begin()); - for (auto& blob : out_grad) { - ptrs.push_back(reinterpret_cast(new NDArray(blob, ndctx.dev_id))); - tags.push_back(3); - } - auto compute = [=]() mutable { - CHECK(reinterpret_cast(op_info_->callbacks[kCustomOpBackward])( - ptrs.size(), ptrs.data(), tags.data(), reqs.data(), 1, - op_info_->contexts[kCustomOpBackward])); + bool old = autograd::AutogradRuntime::Get()->SetIsTraining(false); - // NDArray* in ptrs is freed by frontend side. We keep a copy in ndcpy to keep ndvar alive - Engine::Get()->PushSync([ndcpy, ctx](RunContext rctx){ - ctx.async_on_complete(); - }, ndctx, ndvar, {}, - FnProperty::kNormal, 0, PROFILER_MESSAGE("CustomOpBackward")); - }; + CHECK(reinterpret_cast(params.info->callbacks[kCustomOpBackward])( + ptrs.size(), ptrs.data(), tags.data(), reinterpret_cast(req.data()), 1, + params.info->contexts[kCustomOpBackward])); - if (sync_mode_) { - compute(); - } else { - std::unique_lock lock(mtx_); - q_.push(compute); - cv_.notify_all(); - } + autograd::AutogradRuntime::Get()->SetIsTraining(old); } -Operator* CustomOpProp::CreateOperatorEx(Context ctx, std::vector *in_shape, - std::vector *in_type) const { - std::vector shapes; - std::vector ndims; - size_t size = 0; - for (const auto& s : *in_shape) size += s.ndim(); - shapes_buffer_.resize(size); - uint32_t *ptr = shapes_buffer_.data(); - for (auto iter = in_shape->begin(); iter != in_shape->end(); ++iter) { - shapes.push_back(ptr); - ndims.push_back(iter->ndim()); - ptr = nnvm::ShapeTypeCast(iter->begin(), iter->end(), ptr); - } - std::string str_ctx; - if (ctx.dev_mask() == cpu::kDevMask) { - str_ctx = "cpu"; - } else { - str_ctx = "gpu"; - } - MXCallbackList *op_info = new MXCallbackList; - CHECK(reinterpret_cast(info_->callbacks[kCustomOpPropCreateOperator])( - str_ctx.c_str(), shapes.size(), shapes.data(), ndims.data(), in_type->data(), op_info, - info_->contexts[kCustomOpPropCreateOperator])); - DO_BIND_DISPATCH(CreateOp, op_info); -} -MXNET_REGISTER_OP_PROPERTY(Custom, CustomOpProp) +NNVM_REGISTER_OP(Custom) .describe(R"code(Apply a custom operator implemented in a frontend language (like Python). Custom operators should override required methods like `forward` and `backward`. The custom operator must be registered before it can be used. Please check the tutorial here: http://mxnet.io/how_to/new_op.html. -)code") +)code" ADD_FILELINE) +.set_num_inputs([](const NodeAttrs& attrs){ + const CustomParam& params = nnvm::get(attrs.parsed); + return params.num_args + params.num_auxs; + }) +.set_num_outputs([](const NodeAttrs& attrs){ + const CustomParam& params = nnvm::get(attrs.parsed); + return params.num_outs; + }) +.set_attr_parser(AttrParser) +.set_attr("FInferShape", InferShape) +.set_attr("FInferType", InferType) +.set_attr("FListInputNames", [](const NodeAttrs& attrs) { + std::vector args = List(attrs); + std::vector auxs = List(attrs); + args.insert(args.end(), auxs.begin(), auxs.end()); + return args; + }) +.set_attr("FListOutputNames", List) +.set_attr("FMutateInputs", [](const NodeAttrs& attrs) { + const CustomParam& params = nnvm::get(attrs.parsed); + std::vector ret; + for (size_t i = 0; i < params.num_auxs; ++i) ret.push_back(i+params.num_args); + return ret; + }) +.set_attr("FExecType", [](const NodeAttrs& attrs) { + return ExecType::kLocal; + }) +.set_attr("FGradient", Gradient) +.set_attr("FCreateOpState", CreateState) +.set_attr("FStatefulComputeEx", Forward) +.set_attr("FStatefulComputeEx", Forward) .add_argument("data", "NDArray-or-Symbol[]", "Input data for the custom operator.") .add_argument("op_type", "string", "Name of the custom operator. " "This is the name that is passed to `mx.operator.register` " "to register the operator."); +NNVM_REGISTER_OP(_backward_Custom) +.set_num_inputs([](const NodeAttrs& attrs){ + const CustomParam& params = nnvm::get(attrs.parsed); + return params.bwd_idx.size(); + }) +.set_num_outputs([](const NodeAttrs& attrs){ + const CustomParam& params = nnvm::get(attrs.parsed); + return params.num_args; + }) +.set_attr("TIsLayerOpBackward", true) +.set_attr("TIsBackward", true) +.set_attr("FExecType", [](const NodeAttrs& attrs) { + return ExecType::kLocal; + }) +.set_attr("FStatefulComputeEx", Backward) +.set_attr("FStatefulComputeEx", Backward); + +} // namespace custom } // namespace op } // namespace mxnet diff --git a/src/operator/custom/ndarray_op-inl.h b/src/operator/custom/ndarray_op-inl.h index 05b1a3a902e8..fa4208f1da89 100644 --- a/src/operator/custom/ndarray_op-inl.h +++ b/src/operator/custom/ndarray_op-inl.h @@ -52,10 +52,6 @@ class NDArrayOp : public Operator { const std::vector &in_grad, const std::vector &aux_args); - virtual ExecType exec_type() const { - return kAsync; - } - private: NDArrayOpParam param_; Context get_ctx(); @@ -169,6 +165,10 @@ class NDArrayOpProp : public OperatorProperty { Operator* CreateOperator(Context ctx) const override; + ExecType exec_type() const override { + return ExecType::kAsync; + } + private: NDArrayOpParam param_; }; // class PythonProp diff --git a/src/operator/deconvolution.cc b/src/operator/deconvolution.cc index 83af00903919..397bd0065f80 100644 --- a/src/operator/deconvolution.cc +++ b/src/operator/deconvolution.cc @@ -24,8 +24,6 @@ Operator* CreateOp(DeconvolutionParam param, int dtype, Operator* DeconvolutionProp::CreateOperatorEx(Context ctx, std::vector *in_shape, std::vector *in_type) const { std::vector out_shape, aux_shape; - std::vector out_type, aux_type; - CHECK(InferType(in_type, &out_type, &aux_type)); CHECK(InferShape(in_shape, &out_shape, &aux_shape)); DO_BIND_DISPATCH(CreateOp, param_, in_type->at(0), in_shape, &out_shape, ctx); } diff --git a/src/operator/dropout-inl.h b/src/operator/dropout-inl.h index 47bb0a3dffd3..e77d61351be0 100644 --- a/src/operator/dropout-inl.h +++ b/src/operator/dropout-inl.h @@ -88,7 +88,7 @@ class DropoutOp : public Operator { Tensor out = out_data[dropout::kOut].FlatTo2D(s); if (ctx.is_train) { Tensor mask = out_data[dropout::kMask].FlatTo2D(s); -#if defined(USE_MKL) && defined(_OPENMP) +#if !defined(__CUDACC__) && defined(USE_MKL) && defined(_OPENMP) DType* outptr = out.dptr_; DType* dataptr = data.dptr_; int* maskptr = reinterpret_cast(mask.dptr_); @@ -124,7 +124,7 @@ class DropoutOp : public Operator { Tensor grad = out_grad[dropout::kOut].FlatTo2D(s); Tensor mask = out_data[dropout::kMask].FlatTo2D(s); Tensor gdata = in_grad[dropout::kData].FlatTo2D(s); -#if defined(USE_MKL) && defined(_OPENMP) +#if !defined(__CUDACC__) && defined(USE_MKL) && defined(_OPENMP) DType* ingradptr = gdata.dptr_; DType* outgradptr = grad.dptr_; int* maskptr = reinterpret_cast(mask.dptr_); diff --git a/src/operator/dropout.cc b/src/operator/dropout.cc index 20afef2c63c8..74a50baf80a4 100644 --- a/src/operator/dropout.cc +++ b/src/operator/dropout.cc @@ -21,10 +21,6 @@ Operator *CreateOp(DropoutParam param, int dtype) { // DO_BIND_DISPATCH comes from operator_common.h Operator *DropoutProp::CreateOperatorEx(Context ctx, std::vector *in_shape, std::vector *in_type) const { - std::vector out_shape, aux_shape; - std::vector out_type, aux_type; - CHECK(InferType(in_type, &out_type, &aux_type)); - CHECK(InferShape(in_shape, &out_shape, &aux_shape)); DO_BIND_DISPATCH(CreateOp, param_, in_type->at(0)); } diff --git a/src/operator/fully_connected.cc b/src/operator/fully_connected.cc index cec2015425c6..56cf4f6dbdde 100644 --- a/src/operator/fully_connected.cc +++ b/src/operator/fully_connected.cc @@ -49,8 +49,6 @@ Operator* CreateOp(FullyConnectedParam param, int dtype, Operator *FullyConnectedProp::CreateOperatorEx(Context ctx, std::vector *in_shape, std::vector *in_type) const { std::vector out_shape(1, TShape()), aux_shape; - std::vector out_type(1, -1), aux_type; - CHECK(InferType(in_type, &out_type, &aux_type)); CHECK(InferShape(in_shape, &out_shape, &aux_shape)); DO_BIND_DISPATCH(CreateOp, param_, (*in_type)[0], in_shape, &out_shape, ctx); } diff --git a/src/operator/grid_generator.cc b/src/operator/grid_generator.cc index 8625d1ba971a..62ff75a88359 100644 --- a/src/operator/grid_generator.cc +++ b/src/operator/grid_generator.cc @@ -22,10 +22,6 @@ Operator* CreateOp(GridGeneratorParam param, int dtype) { Operator *GridGeneratorProp::CreateOperatorEx(Context ctx, std::vector *in_shape, std::vector *in_type) const { - std::vector out_shape, aux_shape; - std::vector out_type, aux_type; - CHECK(InferType(in_type, &out_type, &aux_type)); - CHECK(InferShape(in_shape, &out_shape, &aux_shape)); DO_BIND_DISPATCH(CreateOp, param_, (*in_type)[0]); } diff --git a/src/operator/instance_norm.cc b/src/operator/instance_norm.cc index bf3285a7a9d0..cc2bd6b93e8b 100644 --- a/src/operator/instance_norm.cc +++ b/src/operator/instance_norm.cc @@ -18,10 +18,6 @@ Operator* CreateOp(InstanceNormParam param, int dtype) { Operator* InstanceNormProp::CreateOperatorEx(Context ctx, std::vector* in_shape, std::vector* in_type) const { - std::vector out_shape, aux_shape; - std::vector out_type, aux_type; - CHECK(InferType(in_type, &out_type, &aux_type)); - CHECK(InferShape(in_shape, &out_shape, &aux_shape)); DO_BIND_DISPATCH(CreateOp, param_, (*in_type)[0]); } diff --git a/src/operator/lrn.cc b/src/operator/lrn.cc index e896e16b443a..ac4a309cbe05 100644 --- a/src/operator/lrn.cc +++ b/src/operator/lrn.cc @@ -28,10 +28,6 @@ Operator* CreateOp(LRNParam param, int dtype) { // DO_BIND_DISPATCH comes from operator_common.h Operator* LocalResponseNormProp::CreateOperatorEx(Context ctx, std::vector *in_shape, std::vector *in_type) const { - std::vector out_shape, aux_shape; - std::vector out_type, aux_type; - CHECK(InferType(in_type, &out_type, &aux_type)); - CHECK(InferShape(in_shape, &out_shape, &aux_shape)); DO_BIND_DISPATCH(CreateOp, param_, (*in_type)[0]); } @@ -42,14 +38,14 @@ MXNET_REGISTER_OP_PROPERTY(LRN, LocalResponseNormProp) .add_arguments(LRNParam::__FIELDS__()) .describe(R"code(Applies local response normalization to the input. -The local response normalization layer performs "lateral inhibition" by normalizing -over local input regions. +The local response normalization layer performs "lateral inhibition" by normalizing +over local input regions. If :math:`a_{x,y}^{i}` is the activity of a neuron computed by applying kernel :math:`i` at position -:math:`(x, y)` and then applying the ReLU nonlinearity, the response-normalized -activity :math:`b_{x,y}^{i}` is given by the expression: +:math:`(x, y)` and then applying the ReLU nonlinearity, the response-normalized +activity :math:`b_{x,y}^{i}` is given by the expression: -.. math:: +.. math:: b_{x,y}^{i} = \frac{a_{x,y}^{i}}{\Bigg({k + \alpha \sum_{j=max(0, i-\frac{n}{2})}^{min(N-1, i+\frac{n}{2})} (a_{x,y}^{j})^{2}}\Bigg)^{\beta}} where the sum runs over :math:`n` "adjacent" kernel maps at the same spatial position, and :math:`N` is the total diff --git a/src/operator/pad.cc b/src/operator/pad.cc index ded48c99f608..5d1afca588fb 100644 --- a/src/operator/pad.cc +++ b/src/operator/pad.cc @@ -634,10 +634,6 @@ Operator *CreateOp(PadParam param, int dtype) { // DO_BIND_DISPATCH comes from operator_common.h Operator *PadProp::CreateOperatorEx(Context ctx, std::vector *in_shape, std::vector *in_type) const { - std::vector out_shape, aux_shape; - std::vector out_type, aux_type; - CHECK(InferType(in_type, &out_type, &aux_type)); - CHECK(InferShape(in_shape, &out_shape, &aux_shape)); DO_BIND_DISPATCH(CreateOp, param_, (*in_type)[0]); } diff --git a/src/operator/pooling.cc b/src/operator/pooling.cc index c66543d711bf..f26c2e8b199e 100644 --- a/src/operator/pooling.cc +++ b/src/operator/pooling.cc @@ -70,10 +70,6 @@ Operator *CreateOp(PoolingParam param, int dtype) { // DO_BIND_DISPATCH comes from operator_common.h Operator* PoolingProp::CreateOperatorEx(Context ctx, std::vector *in_shape, std::vector *in_type) const { - std::vector out_shape, aux_shape; - std::vector out_type, aux_type; - CHECK(InferType(in_type, &out_type, &aux_type)); - CHECK(InferShape(in_shape, &out_shape, &aux_shape)); DO_BIND_DISPATCH(CreateOp, param_, (*in_type)[0]); } diff --git a/src/operator/rnn.cc b/src/operator/rnn.cc index f43379fdd8dd..f19c3bbad04b 100644 --- a/src/operator/rnn.cc +++ b/src/operator/rnn.cc @@ -22,10 +22,6 @@ Operator *CreateOp(RNNParam param, int dtype) { Operator *RNNProp::CreateOperatorEx(Context ctx, std::vector *in_shape, std::vector *in_type) const { - std::vector out_shape, aux_shape; - std::vector out_type, aux_type; - CHECK(InferType(in_type, &out_type, &aux_type)); - CHECK(InferShape(in_shape, &out_shape, &aux_shape)); DO_BIND_DISPATCH(CreateOp, param_, (*in_type)[0]); } diff --git a/src/operator/roi_pooling.cc b/src/operator/roi_pooling.cc index 35fe94c33242..0faca1e463bc 100644 --- a/src/operator/roi_pooling.cc +++ b/src/operator/roi_pooling.cc @@ -217,10 +217,6 @@ Operator *CreateOp(ROIPoolingParam param, int dtype) { Operator *ROIPoolingProp::CreateOperatorEx(Context ctx, std::vector *in_shape, std::vector *in_type) const { - std::vector out_shape, aux_shape; - std::vector out_type, aux_type; - CHECK(InferType(in_type, &out_type, &aux_type)); - CHECK(InferShape(in_shape, &out_shape, &aux_shape)); DO_BIND_DISPATCH(CreateOp, param_, in_type->at(0)); } diff --git a/src/operator/sequence_last.cc b/src/operator/sequence_last.cc index 7c796613efa8..8a50ff73ec64 100644 --- a/src/operator/sequence_last.cc +++ b/src/operator/sequence_last.cc @@ -20,10 +20,6 @@ Operator *CreateOp(SequenceLastParam param, int dtype) { Operator *SequenceLastProp::CreateOperatorEx(Context ctx, std::vector *in_shape, std::vector *in_type) const { - std::vector out_shape, aux_shape; - std::vector out_type, aux_type; - CHECK(InferType(in_type, &out_type, &aux_type)); - CHECK(InferShape(in_shape, &out_shape, &aux_shape)); DO_BIND_DISPATCH(CreateOp, param_, (*in_type)[0]); } diff --git a/src/operator/sequence_mask.cc b/src/operator/sequence_mask.cc index 763bc17171ae..0ac782e51c3c 100644 --- a/src/operator/sequence_mask.cc +++ b/src/operator/sequence_mask.cc @@ -33,10 +33,6 @@ Operator *CreateOp(SequenceMaskParam param, int dtype) { Operator *SequenceMaskProp::CreateOperatorEx(Context ctx, std::vector *in_shape, std::vector *in_type) const { - std::vector out_shape, aux_shape; - std::vector out_type, aux_type; - CHECK(InferType(in_type, &out_type, &aux_type)); - CHECK(InferShape(in_shape, &out_shape, &aux_shape)); DO_BIND_DISPATCH(CreateOp, param_, (*in_type)[0]); } diff --git a/src/operator/sequence_reverse.cc b/src/operator/sequence_reverse.cc index 871db9b3d486..01dcb6810e62 100644 --- a/src/operator/sequence_reverse.cc +++ b/src/operator/sequence_reverse.cc @@ -20,10 +20,6 @@ Operator *CreateOp(SequenceReverseParam param, int dtype) { Operator *SequenceReverseProp::CreateOperatorEx( Context ctx, std::vector *in_shape, std::vector *in_type) const { - std::vector out_shape, aux_shape; - std::vector out_type, aux_type; - CHECK(InferType(in_type, &out_type, &aux_type)); - CHECK(InferShape(in_shape, &out_shape, &aux_shape)); DO_BIND_DISPATCH(CreateOp, param_, (*in_type)[0]); } diff --git a/src/operator/softmax_output.cc b/src/operator/softmax_output.cc index 06225a3b0be7..08580e9328cd 100644 --- a/src/operator/softmax_output.cc +++ b/src/operator/softmax_output.cc @@ -20,10 +20,6 @@ Operator *CreateOp(SoftmaxOutputParam param, int dtype) { // DO_BIND_DISPATCH comes from operator_common.h Operator *SoftmaxOutputProp::CreateOperatorEx(Context ctx, std::vector *in_shape, std::vector *in_type) const { - std::vector out_shape, aux_shape; - std::vector out_type, aux_type; - CHECK(InferType(in_type, &out_type, &aux_type)); - CHECK(InferShape(in_shape, &out_shape, &aux_shape)); DO_BIND_DISPATCH(CreateOp, param_, (*in_type)[0]); } diff --git a/src/operator/spatial_transformer.cc b/src/operator/spatial_transformer.cc index fe91a143c23e..45c1d8588776 100644 --- a/src/operator/spatial_transformer.cc +++ b/src/operator/spatial_transformer.cc @@ -116,10 +116,6 @@ Operator* CreateOp(SpatialTransformerParam param, int dtype) { Operator *SpatialTransformerProp::CreateOperatorEx(Context ctx, std::vector *in_shape, std::vector *in_type) const { - std::vector out_shape, aux_shape; - std::vector out_type, aux_type; - CHECK(InferType(in_type, &out_type, &aux_type)); - CHECK(InferShape(in_shape, &out_shape, &aux_shape)); DO_BIND_DISPATCH(CreateOp, param_, (*in_type)[0]); } diff --git a/src/operator/svm_output.cc b/src/operator/svm_output.cc index ead853e214b8..5f1f77ad9fc1 100644 --- a/src/operator/svm_output.cc +++ b/src/operator/svm_output.cc @@ -62,10 +62,6 @@ Operator *CreateOp(SVMOutputParam param, int dtype) { // DO_BIND_DISPATCH comes from operator_common.h Operator *SVMOutputProp::CreateOperatorEx(Context ctx, std::vector *in_shape, std::vector *in_type) const { - std::vector out_shape, aux_shape; - std::vector out_type, aux_type; - CHECK(InferType(in_type, &out_type, &aux_type)); - CHECK(InferShape(in_shape, &out_shape, &aux_shape)); DO_BIND_DISPATCH(CreateOp, param_, (*in_type)[0]); } @@ -84,4 +80,3 @@ This tutorial demonstrates using SVM as output layer for classification instead } // namespace op } // namespace mxnet - diff --git a/src/operator/swapaxis.cc b/src/operator/swapaxis.cc index 24ea807ef9ce..097f9837025f 100644 --- a/src/operator/swapaxis.cc +++ b/src/operator/swapaxis.cc @@ -21,10 +21,6 @@ Operator* CreateOp(SwapAxisParam param, int dtype) { Operator* SwapAxisProp::CreateOperatorEx(Context ctx, std::vector *in_shape, std::vector *in_type) const { - std::vector out_shape, aux_shape; - std::vector out_type, aux_type; - CHECK(InferShape(in_shape, &out_shape, &aux_shape)); - CHECK(InferType(in_type, &out_type, &aux_type)); DO_BIND_DISPATCH(CreateOp, param_, in_type->at(0)); } diff --git a/src/operator/upsampling.cc b/src/operator/upsampling.cc index cc9861346825..ad89d4ace137 100644 --- a/src/operator/upsampling.cc +++ b/src/operator/upsampling.cc @@ -44,10 +44,6 @@ Operator *CreateOp(UpSamplingParam param, int dtype) { Operator* UpSamplingProp::CreateOperatorEx(Context ctx, std::vector *in_shape, std::vector *in_type) const { - std::vector out_shape, aux_shape; - std::vector out_type, aux_type; - CHECK(InferType(in_type, &out_type, &aux_type)); - CHECK(InferShape(in_shape, &out_shape, &aux_shape)); DO_BIND_DISPATCH(CreateOp, param_, in_type->at(0)); } diff --git a/tests/cpp/include/test_op.h b/tests/cpp/include/test_op.h index 2c96092db81c..57fda19e4c9e 100644 --- a/tests/cpp/include/test_op.h +++ b/tests/cpp/include/test_op.h @@ -17,8 +17,8 @@ * test_perf.h: Performance-related classes * test_op.h: Operator-specific testing classes */ -#ifndef TESTS_CPP_INCLUDE_TEST_OP_H_ -#define TESTS_CPP_INCLUDE_TEST_OP_H_ +#ifndef TEST_OP_H_ +#define TEST_OP_H_ #include "test_perf.h" #include "test_util.h" @@ -705,4 +705,4 @@ static test::op::OpInfo createOpAndInfoF(const boo } // namespace test } // namespace mxnet -#endif // TESTS_CPP_INCLUDE_TEST_OP_H_ +#endif // TEST_OP_H_ diff --git a/tests/cpp/include/test_perf.h b/tests/cpp/include/test_perf.h index 6343863db16e..93b7863de694 100644 --- a/tests/cpp/include/test_perf.h +++ b/tests/cpp/include/test_perf.h @@ -5,8 +5,8 @@ * \author Chris Olivier */ -#ifndef TESTS_CPP_INCLUDE_TEST_PERF_H_ -#define TESTS_CPP_INCLUDE_TEST_PERF_H_ +#ifndef TEST_PERF_H_ +#define TEST_PERF_H_ #include #include @@ -286,4 +286,4 @@ class TimingItem { } // namespace test } // namespace mxnet -#endif // TESTS_CPP_INCLUDE_TEST_PERF_H_ +#endif // TEST_PERF_H_ diff --git a/tests/cpp/include/test_util.h b/tests/cpp/include/test_util.h index b0e4c866f9de..3fa82688c115 100644 --- a/tests/cpp/include/test_util.h +++ b/tests/cpp/include/test_util.h @@ -4,8 +4,8 @@ * \brief unit test performance analysis functions * \author Chris Olivier */ -#ifndef TESTS_CPP_INCLUDE_TEST_UTIL_H_ -#define TESTS_CPP_INCLUDE_TEST_UTIL_H_ +#ifndef TEST_UTIL_H_ +#define TEST_UTIL_H_ #include #include @@ -413,4 +413,4 @@ struct ScopeSet { } // namespace test } // namespace mxnet -#endif // TESTS_CPP_INCLUDE_TEST_UTIL_H_ +#endif // TEST_UTIL_H_ diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py index e345326632f3..7a958f7de01b 100644 --- a/tests/python/unittest/test_operator.py +++ b/tests/python/unittest/test_operator.py @@ -1633,7 +1633,7 @@ def dot_sym_xT_yT(data_type): def test_batch_dot(): dtypes = ['float32', 'float64'] - + for data_type in dtypes: for batch_size in range(1, 5): for m in range(1, 5): @@ -3227,6 +3227,12 @@ def create_operator(self, ctx, shapes, dtypes): x = mx.nd.array(np.random.uniform(-1, 1, size=(4, 10))) check_numeric_gradient(op, [x]) + dx = mx.nd.zeros_like(x) + mx.contrib.autograd.mark_variables([x], [dx]) + with mx.contrib.autograd.train_section(): + y = mx.nd.Custom(x, op_type='sqr') + y.backward() + def test_psroipooling(): for num_rois in [1, 2]: @@ -3306,10 +3312,10 @@ def test_deformable_psroipooling(): im_data_var = mx.symbol.Variable(name="im_data") rois_data_var = mx.symbol.Variable(name="rois_data") offset_data_var = mx.symbol.Variable(name="offset_data") - op = mx.contrib.sym.DeformablePSROIPooling(data=im_data_var, rois=rois_data_var, - trans=offset_data_var, spatial_scale=spatial_scale, - sample_per_part=4, group_size=num_group, - pooled_size=num_group, output_dim=num_classes, + op = mx.contrib.sym.DeformablePSROIPooling(data=im_data_var, rois=rois_data_var, + trans=offset_data_var, spatial_scale=spatial_scale, + sample_per_part=4, group_size=num_group, + pooled_size=num_group, output_dim=num_classes, trans_std=0.1, no_trans=False, name='test_op') if grad_nodes[0] == 'offset_data': # wider tolerance needed for coordinate differential From b88e0d47528f90ec3abb9fbb6b589098db269dc4 Mon Sep 17 00:00:00 2001 From: Eric Junyuan Xie Date: Wed, 31 May 2017 09:56:32 -0700 Subject: [PATCH 175/834] nn interface (#6501) * mixed nn interface * fix loss * fix * fix prefix kwargs * refactor names * fix * fix * rnn lint * fix * fix * ignore invalid grad in optimizer * fix loss shape * fix * fix * post rebase fix * move nn to mxnet. * fix * use cached * dcgan * fix * fix * lint --- example/autograd/data.py | 69 ++ example/autograd/dcgan.py | 126 ++++ example/autograd/mnist.py | 65 ++ example/autograd/resnet.py | 197 ++++++ python/mxnet/__init__.py | 2 + python/mxnet/module/base_module.py | 51 +- python/mxnet/module/executor_group.py | 14 +- python/mxnet/module/module.py | 20 +- python/mxnet/nn/__init__.py | 15 + python/mxnet/nn/conv_layers.py | 743 ++++++++++++++++++++++ python/mxnet/nn/layer.py | 411 ++++++++++++ python/mxnet/nn/loss.py | 257 ++++++++ python/mxnet/nn/optim.py | 109 ++++ python/mxnet/nn/parameter.py | 358 +++++++++++ python/mxnet/nn/utils.py | 65 ++ python/mxnet/symbol.py | 8 +- src/operator/tensor/broadcast_reduce_op.h | 55 +- src/operator/tensor/matrix_op-inl.h | 18 +- tests/python/gpu/test_operator_gpu.py | 47 +- tests/python/unittest/test_loss.py | 209 ++++++ tests/python/unittest/test_nn.py | 204 ++++++ tests/python/unittest/test_rnn.py | 111 +++- 22 files changed, 3042 insertions(+), 112 deletions(-) create mode 100644 example/autograd/data.py create mode 100644 example/autograd/dcgan.py create mode 100644 example/autograd/mnist.py create mode 100644 example/autograd/resnet.py create mode 100644 python/mxnet/nn/__init__.py create mode 100644 python/mxnet/nn/conv_layers.py create mode 100644 python/mxnet/nn/layer.py create mode 100644 python/mxnet/nn/loss.py create mode 100644 python/mxnet/nn/optim.py create mode 100644 python/mxnet/nn/parameter.py create mode 100644 python/mxnet/nn/utils.py create mode 100644 tests/python/unittest/test_loss.py create mode 100644 tests/python/unittest/test_nn.py diff --git a/example/autograd/data.py b/example/autograd/data.py new file mode 100644 index 000000000000..d913c9df69eb --- /dev/null +++ b/example/autograd/data.py @@ -0,0 +1,69 @@ +# pylint: skip-file +""" data iterator for mnist """ +import sys +import os +# code to automatically download dataset +curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__))) +sys.path.append(os.path.join(curr_path, "../../tests/python/common")) +import get_data +import mxnet as mx + +def mnist_iterator(batch_size, input_shape): + """return train and val iterators for mnist""" + # download data + get_data.GetMNIST_ubyte() + flat = False if len(input_shape) == 3 else True + + train_dataiter = mx.io.MNISTIter( + image="data/train-images-idx3-ubyte", + label="data/train-labels-idx1-ubyte", + input_shape=input_shape, + batch_size=batch_size, + shuffle=True, + flat=flat) + + val_dataiter = mx.io.MNISTIter( + image="data/t10k-images-idx3-ubyte", + label="data/t10k-labels-idx1-ubyte", + input_shape=input_shape, + batch_size=batch_size, + flat=flat) + + return (train_dataiter, val_dataiter) + + +def cifar10_iterator(batch_size, data_shape, resize=-1): + train = mx.io.ImageRecordIter( + path_imgrec = "data/cifar/train.rec", + # mean_img = "data/cifar/mean.bin", + resize = resize, + data_shape = data_shape, + batch_size = batch_size, + rand_crop = True, + rand_mirror = True) + + val = mx.io.ImageRecordIter( + path_imgrec = "data/cifar/test.rec", + # mean_img = "data/cifar/mean.bin", + resize = resize, + rand_crop = False, + rand_mirror = False, + data_shape = data_shape, + batch_size = batch_size) + + return train, val + +class DummyIter(mx.io.DataIter): + def __init__(self, batch_size, data_shape): + self.data_shape = (batch_size,) + data_shape + self.label_shape = (batch_size,) + self.provide_data = [('data', self.data_shape)] + self.provide_label = [('softmax_label', self.label_shape)] + + def next(self): + return mx.io.DataBatch(data=[mx.nd.zeros(self.data_shape)], + label=[mx.nd.zeros(self.label_shape)]) + + +def dummy_iterator(batch_size, data_shape): + return DummyIter(batch_size, data_shape), DummyIter(batch_size, data_shape) diff --git a/example/autograd/dcgan.py b/example/autograd/dcgan.py new file mode 100644 index 000000000000..db827b471e3d --- /dev/null +++ b/example/autograd/dcgan.py @@ -0,0 +1,126 @@ +import argparse +import mxnet as mx +from mxnet import nn +from mxnet.contrib import autograd +from data import cifar10_iterator + + +parser = argparse.ArgumentParser() +parser.add_argument('--batchSize', type=int, default=64, help='input batch size') +parser.add_argument('--imageSize', type=int, default=64, help='the height / width of the input image to network') +parser.add_argument('--nz', type=int, default=100, help='size of the latent z vector') +parser.add_argument('--ngf', type=int, default=64) +parser.add_argument('--ndf', type=int, default=64) +parser.add_argument('--niter', type=int, default=25, help='number of epochs to train for') +parser.add_argument('--lr', type=float, default=0.0002, help='learning rate, default=0.0002') +parser.add_argument('--beta1', type=float, default=0.5, help='beta1 for adam. default=0.5') +parser.add_argument('--cuda', action='store_true', help='enables cuda') +parser.add_argument('--ngpu', type=int, default=1, help='number of GPUs to use') +parser.add_argument('--netG', default='', help="path to netG (to continue training)") +parser.add_argument('--netD', default='', help="path to netD (to continue training)") +parser.add_argument('--outf', default='.', help='folder to output images and model checkpoints') +parser.add_argument('--manualSeed', type=int, help='manual seed') + +opt = parser.parse_args() +print(opt) + +ngpu = int(opt.ngpu) +nz = int(opt.nz) +ngf = int(opt.ngf) +ndf = int(opt.ndf) +nc = 3 +ctx = mx.gpu(0) + +train_iter, val_iter = cifar10_iterator(opt.batchSize, (3, 64, 64), 64) + + +netG = nn.Sequential() +# input is Z, going into a convolution +netG.add(nn.Conv2DTranspose(ngf * 8, 4, 1, 0, in_filters=nz, use_bias=False)) +netG.add(nn.BatchNorm(num_features=ngf * 8)) +netG.add(nn.Activation('relu')) +# state size. (ngf*8) x 4 x 4 +netG.add(nn.Conv2DTranspose(ngf * 4, 4, 2, 1, in_filters=ngf * 8, use_bias=False)) +netG.add(nn.BatchNorm(num_features=ngf * 4)) +netG.add(nn.Activation('relu')) +# state size. (ngf*8) x 8 x 8 +netG.add(nn.Conv2DTranspose(ngf * 2, 4, 2, 1, in_filters=ngf * 4, use_bias=False)) +netG.add(nn.BatchNorm(num_features=ngf * 2)) +netG.add(nn.Activation('relu')) +# state size. (ngf*8) x 16 x 16 +netG.add(nn.Conv2DTranspose(ngf, 4, 2, 1, in_filters=ngf * 2, use_bias=False)) +netG.add(nn.BatchNorm(num_features=ngf)) +netG.add(nn.Activation('relu')) +# state size. (ngf*8) x 32 x 32 +netG.add(nn.Conv2DTranspose(nc, 4, 2, 1, in_filters=ngf, use_bias=False)) +netG.add(nn.Activation('tanh')) +# state size. (nc) x 64 x 64 + + +netD = nn.Sequential() +# input is (nc) x 64 x 64 +netD.add(nn.Conv2D(ndf, 4, 2, 1, in_filters=nc, use_bias=False)) +netD.add(nn.LeakyReLU(0.2)) +# state size. (ndf) x 32 x 32 +netD.add(nn.Conv2D(ndf * 2, 4, 2, 1, in_filters=ndf, use_bias=False)) +netD.add(nn.BatchNorm(num_features=ndf * 2)) +netD.add(nn.LeakyReLU(0.2)) +# state size. (ndf) x 16 x 16 +netD.add(nn.Conv2D(ndf * 4, 4, 2, 1, in_filters=ndf * 2, use_bias=False)) +netD.add(nn.BatchNorm(num_features=ndf * 4)) +netD.add(nn.LeakyReLU(0.2)) +# state size. (ndf) x 8 x 8 +netD.add(nn.Conv2D(ndf * 8, 4, 2, 1, in_filters=ndf * 4, use_bias=False)) +netD.add(nn.BatchNorm(num_features=ndf * 8)) +netD.add(nn.LeakyReLU(0.2)) +# state size. (ndf) x 4 x 4 +netD.add(nn.Conv2D(2, 4, 1, 0, in_filters=ndf * 8, use_bias=False)) +# netD.add(nn.Activation('sigmoid')) + + +netG.params.initialize(mx.init.Normal(0.02), ctx=ctx) +netD.params.initialize(mx.init.Normal(0.02), ctx=ctx) + + +optimizerG = nn.Optim(netG.params, 'adam', {'learning_rate': opt.lr, 'beta1': opt.beta1}) +optimizerD = nn.Optim(netD.params, 'adam', {'learning_rate': opt.lr, 'beta1': opt.beta1}) + + +real_label = mx.nd.ones((opt.batchSize,), ctx=ctx) +fake_label = mx.nd.zeros((opt.batchSize,), ctx=ctx) + +for epoch in range(opt.niter): + for batch in train_iter: + ############################ + # (1) Update D network: maximize log(D(x)) + log(1 - D(G(z))) + ########################### + # train with real_t + data = batch.data[0].copyto(ctx) + noise = mx.nd.random_normal(0, 1, shape=(opt.batchSize, nz, 1, 1), ctx=ctx) + + with autograd.train_section(): + output = netD(data) + output = output.reshape((opt.batchSize, 2)) + errD_real = nn.loss.softmax_cross_entropy_loss(output, real_label) + + fake = netG(noise) + output = netD(fake.detach()) + output = output.reshape((opt.batchSize, 2)) + errD_fake = nn.loss.softmax_cross_entropy_loss(output, fake_label) + errD = errD_real + errD_fake + errD.backward() + + optimizerD.step(opt.batchSize) + + ############################ + # (2) Update G network: maximize log(D(G(z))) + ########################### + with autograd.train_section(): + output = netD(fake) + output = output.reshape((opt.batchSize, 2)) + errG = nn.loss.softmax_cross_entropy_loss(output, real_label) + errG.backward() + + optimizerG.step(opt.batchSize) + + print mx.nd.mean(errD).asscalar(), mx.nd.mean(errG).asscalar() diff --git a/example/autograd/mnist.py b/example/autograd/mnist.py new file mode 100644 index 000000000000..b574d332f548 --- /dev/null +++ b/example/autograd/mnist.py @@ -0,0 +1,65 @@ +# pylint: skip-file +from data import mnist_iterator +import mxnet as mx +from mxnet import nn +import numpy as np +import logging +from mxnet.contrib import autograd as ag +logging.basicConfig(level=logging.DEBUG) + +# define network + +net = nn.Sequential() +net.add(nn.Dense(128, in_units=784, activation='relu')) +net.add(nn.Dense(64, in_units=128, activation='relu')) +net.add(nn.Dense(10, in_units=64)) + +# data + +train_data, val_data = mnist_iterator(batch_size=100, input_shape = (784,)) + +# train + +def test(ctx): + metric = mx.metric.Accuracy() + val_data.reset() + for batch in val_data: + data = nn.utils.load_data(batch.data[0], ctx_list=ctx, batch_axis=0) + label = nn.utils.load_data(batch.label[0], ctx_list=ctx, batch_axis=0) + outputs = [] + for x in data: + outputs.append(net(x)) + metric.update(label, outputs) + print 'validation acc: %s=%f'%metric.get() + +def train(epoch, ctx): + if isinstance(ctx, mx.Context): + ctx = [ctx] + net.params.initialize(mx.init.Xavier(magnitude=2.24), ctx=ctx) + optim = nn.Optim(net.params, 'sgd', {'learning_rate': 0.1}) + metric = mx.metric.Accuracy() + + for i in range(epoch): + train_data.reset() + for batch in train_data: + data = nn.utils.load_data(batch.data[0], ctx_list=ctx, batch_axis=0) + label = nn.utils.load_data(batch.label[0], ctx_list=ctx, batch_axis=0) + outputs = [] + with ag.train_section(): + for x, y in zip(data, label): + z = net(x) + loss = nn.loss.softmax_cross_entropy_loss(z, y) + ag.compute_gradient([loss]) + outputs.append(z) + metric.update(label, outputs) + optim.step(batch.data[0].shape[0]) + name, acc = metric.get() + metric.reset() + print 'training acc at epoch %d: %s=%f'%(i, name, acc) + test(ctx) + + net.params.save('mnist.params') + + +if __name__ == '__main__': + train(10, [mx.cpu(0), mx.cpu(1)]) diff --git a/example/autograd/resnet.py b/example/autograd/resnet.py new file mode 100644 index 000000000000..cd69d82a0ba3 --- /dev/null +++ b/example/autograd/resnet.py @@ -0,0 +1,197 @@ +from __future__ import division + +import time +import mxnet as mx +from mxnet import nn +from mxnet.contrib import autograd as ag +from data import * + +def conv3x3(filters, stride, in_filters): + return nn.Conv2D(filters, kernel_size=3, strides=stride, padding=1, + use_bias=False, in_filters=in_filters) + +class BasicBlock(nn.Layer): + def __init__(self, filters, stride, downsample=False, in_filters=0, **kwargs): + super(BasicBlock, self).__init__(**kwargs) + with self.scope: + self.bn1 = nn.BatchNorm(num_features=in_filters) + self.conv1 = conv3x3(filters, stride, in_filters) + self.bn2 = nn.BatchNorm(num_features=filters) + self.conv2 = conv3x3(filters, 1, filters) + if downsample: + self.downsample = nn.Conv2D(filters, 1, stride, use_bias=False, + in_filters=in_filters) + else: + self.downsample = None + + def generic_forward(self, domain, x): + if not self.downsample: + residual = x + x = self.bn1(x) + x = domain.Activation(x, act_type='relu') + if self.downsample: + residual = self.downsample(x) + x = self.conv1(x) + + x = self.bn2(x) + x = domain.Activation(x, act_type='relu') + x = self.conv2(x) + + return x + residual + + +class Bottleneck(nn.Layer): + def __init__(self, filters, stride, downsample=False, in_filters=0, **kwargs): + super(Bottleneck, self).__init__(**kwargs) + with self.scope: + self.bn1 = nn.BatchNorm(num_features=in_filters) + self.conv1 = conv3x3(filters//4, 1, in_filters) + self.bn2 = nn.BatchNorm(num_features=filters//4) + self.conv2 = conv3x3(filters//4, stride, filters//4) + self.bn3 = nn.BatchNorm(num_features=filters//4) + self.conv3 = conv3x3(filters, 1, filters//4) + if downsample: + self.downsample = nn.Conv2D(filters, 1, stride, use_bias=False, + in_filters=in_filters) + else: + self.downsample = None + + def generic_forward(self, domain, x): + if not self.downsample: + residual = x + x = self.bn1(x) + x = domain.Activation(x, act_type='relu') + if self.downsample: + residual = self.downsample(x) + x = self.conv1(x) + + x = self.bn2(x) + x = domain.Activation(x, act_type='relu') + x = self.conv2(x) + + x = self.bn3(x) + x = domain.Activation(x, act_type='relu') + x = self.conv3(x) + + return x + residual + +class Resnet(nn.Layer): + def __init__(self, block, classes, layers, filters, thumbnail=False, **kwargs): + super(Resnet, self).__init__(**kwargs) + with self.scope: + assert len(layers) == len(filters) - 1 + self._thumbnail = thumbnail + self.bn_data = nn.BatchNorm(num_features=3, scale=False, center=False) + if thumbnail: + self.conv0 = conv3x3(filters[0], 1, 3) + else: + self.conv0 = nn.Conv2D(filters[0], 7, 2, 3, use_bias=False, + in_filters=3) + self.bn0 = nn.BatchNorm(num_features=filters[0]) + self.pool0 = nn.MaxPool2D(3, 2, 1) + + self.body = nn.Sequential() + in_filters = filters[0] + for i in range(len(layers)): + stride = 1 if i == 0 else 2 + self.body.add(self._make_layer(block, layers[i], filters[i+1], + stride, in_filters=in_filters)) + in_filters = filters[i+1] + + self.bn1 = nn.BatchNorm(num_features=in_filters) + self.pool1 = nn.GlobalAvgPool2D() + self.dense1 = nn.Dense(classes, in_units=in_filters) + + def _make_layer(self, block, layers, filters, stride, in_filters=0): + layer = nn.Sequential() + layer.add(block(filters, stride, True, in_filters=in_filters)) + for i in range(layers-1): + layer.add(block(filters, 1, False, in_filters=filters)) + return layer + + def generic_forward(self, domain, x): + x = self.bn_data(x) + x = self.conv0(x) + if not self._thumbnail: + x = self.bn0(x) + x = domain.Activation(x, act_type='relu') + x = self.pool0(x) + + x = self.body(x) + + x = self.bn1(x) + x = domain.Activation(x, act_type='relu') + x = self.pool1(x) + x = x.reshape((0, -1)) + x = self.dense1(x) + + return x + + +def resnet18_cifar(classes): + return Resnet(BasicBlock, classes, [2, 2, 2], [16, 16, 32, 64], True) + +def resnet50_imagenet(classes): + return Resnet(Bottleneck, classes, [3, 4, 6, 3], [64, 256, 512, 1024, 2048], False) + +net = resnet18_cifar(10) +batch_size = 32*8 +train_data, val_data = cifar10_iterator(batch_size, (3, 32, 32)) + + +def test(ctx): + metric = mx.metric.Accuracy() + val_data.reset() + for batch in val_data: + data = nn.utils.load_data(batch.data[0], ctx_list=ctx, batch_axis=0) + label = nn.utils.load_data(batch.label[0], ctx_list=ctx, batch_axis=0) + outputs = [] + for x in data: + outputs.append(net(x)) + metric.update(label, outputs) + print 'validation acc: %s=%f'%metric.get() + + +def train(epoch, ctx): + if isinstance(ctx, mx.Context): + ctx = [ctx] + net.params.initialize(mx.init.Xavier(magnitude=2.24), ctx=ctx) + optim = nn.Optim(net.params, 'sgd', {'learning_rate': 0.1}) + metric = mx.metric.Accuracy() + + for i in range(epoch): + tic = time.time() + train_data.reset() + btic = time.time() + for batch in train_data: + data = nn.utils.load_data(batch.data[0], ctx_list=ctx, batch_axis=0) + label = nn.utils.load_data(batch.label[0], ctx_list=ctx, batch_axis=0) + outputs = [] + with ag.train_section(): + for x, y in zip(data, label): + z = net(x) + loss = nn.loss.softmax_cross_entropy_loss(z, y) + ag.compute_gradient([loss]) + outputs.append(z) + optim.step(batch.data[0].shape[0]) + metric.update(label, outputs) + print batch_size/(time.time()-btic) + btic = time.time() + + name, acc = metric.get() + metric.reset() + print 'training acc at epoch %d: %s=%f'%(i, name, acc) + print 'time: %f'%(time.time()-tic) + test(ctx) + + net.params.save('mnist.params') + +if __name__ == '__main__': + train(200, [mx.gpu(i) for i in range(2)]) + import logging + logging.basicConfig(level=logging.DEBUG) + data = mx.sym.var('data') + out = net(data) + softmax = mx.sym.SoftmaxOutput(out, name='softmax') + mod = mx.mod.Module(softmax, context=[mx.gpu(i) for i in range(1)]) + mod.fit(train_data, num_epoch=100, batch_end_callback = mx.callback.Speedometer(batch_size, 10)) diff --git a/python/mxnet/__init__.py b/python/mxnet/__init__.py index ff5f6cd6be7e..133b5ffd6187 100644 --- a/python/mxnet/__init__.py +++ b/python/mxnet/__init__.py @@ -60,4 +60,6 @@ from . import rnn +from . import nn + __version__ = base.__version__ diff --git a/python/mxnet/module/base_module.py b/python/mxnet/module/base_module.py index cb6cfccb2759..8b545994b417 100644 --- a/python/mxnet/module/base_module.py +++ b/python/mxnet/module/base_module.py @@ -1,4 +1,5 @@ # pylint: disable=fixme, too-many-arguments, too-many-locals, too-many-public-methods, too-many-branches +# pylint: disable=too-many-lines """`BaseModule` defines an API for modules.""" import time @@ -76,6 +77,43 @@ def _parse_data_desc(data_names, label_names, data_shapes, label_shapes): return data_shapes, label_shapes +def _parse_metric(sym, metrics): + output_names = [] + if not metrics: + metrics = [] + elif isinstance(metrics, (str, metric.EvalMetric)): + metrics = [metric.create(metrics)] + else: + metrics = [metric.create(i) for i in metrics] + + sym_metrics = [] + loss_metrics = [] + for i in sym: + tag = i.attr('__output__') + if tag is None or tag == 'pred': + output_names.append(i.list_outputs()[0]) + elif tag == 'loss': + name = i.list_outputs()[0] + loss_metrics.append( + metric.Loss(name=name, output_names=[name], + label_names=[])) + + str_metric = i.attr('__metric__') + if str_metric: + sym_metrics.append(metric.create(str_metric)) + + for m in metrics: + m.output_names = output_names + metrics += sym_metrics + metrics += loss_metrics + if len(metrics) > 1: + return metric.CompositeEvalMetric(metrics) + elif len(metrics) == 1: + return metrics[0] + else: + return None + + class BaseModule(object): """The base class of a module. @@ -190,7 +228,7 @@ def forward_backward(self, data_batch): self.forward(data_batch, is_train=True) self.backward() - def score(self, eval_data, eval_metric, num_batch=None, batch_end_callback=None, + def score(self, eval_data, eval_metric=None, num_batch=None, batch_end_callback=None, score_end_callback=None, reset=True, epoch=0): """Runs prediction on ``eval_data`` and evaluates the performance according to @@ -230,8 +268,7 @@ def score(self, eval_data, eval_metric, num_batch=None, batch_end_callback=None, if reset: eval_data.reset() - if not isinstance(eval_metric, metric.EvalMetric): - eval_metric = metric.create(eval_metric) + eval_metric = _parse_metric(self._symbol, eval_metric) eval_metric.reset() actual_num_batch = 0 @@ -372,7 +409,7 @@ def predict(self, eval_data, num_batch=None, merge_batches=True, reset=True, return output_list - def fit(self, train_data, eval_data=None, eval_metric='acc', + def fit(self, train_data, eval_data=None, eval_metric=None, epoch_end_callback=None, batch_end_callback=None, kvstore='local', optimizer='sgd', optimizer_params=(('learning_rate', 0.01),), eval_end_callback=None, @@ -466,8 +503,10 @@ def fit(self, train_data, eval_data=None, eval_metric='acc', if validation_metric is None: validation_metric = eval_metric - if not isinstance(eval_metric, metric.EvalMetric): - eval_metric = metric.create(eval_metric) + eval_metric = _parse_metric(self._symbol, eval_metric) + if eval_metric is None: + eval_metric = metric.create('acc') + validation_metric = 'acc' ################################################################################ # training loop diff --git a/python/mxnet/module/executor_group.py b/python/mxnet/module/executor_group.py index 169e81ee326e..063b00732b81 100755 --- a/python/mxnet/module/executor_group.py +++ b/python/mxnet/module/executor_group.py @@ -152,7 +152,7 @@ def __init__(self, symbol, contexts, workload, data_shapes, label_shapes, param_ grad_req = 'null' data_shapes = [x if isinstance(x, DataDesc) else DataDesc(*x) for x in data_shapes] - if label_shapes is not None: + if label_shapes: label_shapes = [x if isinstance(x, DataDesc) else DataDesc(*x) for x in label_shapes] data_names = [x.name for x in data_shapes] @@ -248,7 +248,7 @@ def _collect_arrays(self): self.state_arrays = [[e.arg_dict[name] for e in self.execs] for name in self.state_names] - if self.label_shapes is not None: + if self.label_shapes: self.label_arrays = [[(self.slices[i], e.arg_dict[name]) for i, e in enumerate(self.execs)] for name, _ in self.label_shapes] @@ -291,13 +291,13 @@ def bind_exec(self, data_shapes, label_shapes, shared_group=None, reshape=False) # calculate workload and bind executors self.data_layouts = self.decide_slices(data_shapes) - if label_shapes is not None: + if label_shapes: # call it to make sure labels has the same batch size as data self.label_layouts = self.decide_slices(label_shapes) for i in range(len(self.contexts)): data_shapes_i = self._sliced_shape(data_shapes, i, self.data_layouts) - if label_shapes is not None: + if label_shapes: label_shapes_i = self._sliced_shape(label_shapes, i, self.label_layouts) else: label_shapes_i = [] @@ -312,7 +312,7 @@ def bind_exec(self, data_shapes, label_shapes, shared_group=None, reshape=False) self.data_shapes = data_shapes self.label_shapes = label_shapes self.data_names = [i.name for i in self.data_shapes] - if label_shapes is not None: + if label_shapes: self.label_names = [i.name for i in self.label_shapes] self._collect_arrays() @@ -573,11 +573,11 @@ def _bind_ith_exec(self, i, data_shapes, label_shapes, shared_group): shared_data_arrays = self.shared_data_arrays[i] input_shapes = dict(data_shapes) - if label_shapes is not None: + if label_shapes: input_shapes.update(dict(label_shapes)) input_types = {x.name: x.dtype for x in data_shapes} - if label_shapes is not None: + if label_shapes: input_types.update({x.name: x.dtype for x in label_shapes}) executor = self.symbol.simple_bind(ctx=context, grad_req=self.grad_req, diff --git a/python/mxnet/module/module.py b/python/mxnet/module/module.py index 75201292010c..65c277c7f54b 100644 --- a/python/mxnet/module/module.py +++ b/python/mxnet/module/module.py @@ -9,6 +9,7 @@ from .. import context as ctx from .. import ndarray as nd +from .. import symbol as _sym from .. import optimizer as opt from .executor_group import DataParallelExecutorGroup @@ -57,6 +58,7 @@ def __init__(self, symbol, data_names=('data',), label_names=('softmax_label',), self._work_load_list = work_load_list self._symbol = symbol + self._pred_symbol = _sym.Group([i for i in symbol if i.attr('__output__') != 'loss']) data_names = list(data_names) if data_names is not None else [] label_names = list(label_names) if label_names is not None else [] @@ -371,16 +373,14 @@ def bind(self, data_shapes, label_shapes=None, for_training=True, self.binded = True self._grad_req = grad_req - if not for_training: - assert not inputs_need_grad + if not for_training and self._label_names and not label_shapes: + symbol = self._pred_symbol + self._data_shapes, self._label_shapes = _parse_data_desc( + self.data_names, [], data_shapes, []) else: - pass - # this is not True, as some module might not contains a loss function - # that consumes the labels - # assert label_shapes is not None - - self._data_shapes, self._label_shapes = _parse_data_desc( - self.data_names, self.label_names, data_shapes, label_shapes) + symbol = self._symbol + self._data_shapes, self._label_shapes = _parse_data_desc( + self.data_names, self.label_names, data_shapes, label_shapes) if shared_module is not None: assert isinstance(shared_module, Module) and \ @@ -389,7 +389,7 @@ def bind(self, data_shapes, label_shapes=None, for_training=True, else: shared_group = None - self._exec_group = DataParallelExecutorGroup(self._symbol, self._context, + self._exec_group = DataParallelExecutorGroup(symbol, self._context, self._work_load_list, self._data_shapes, self._label_shapes, self._param_names, for_training, inputs_need_grad, diff --git a/python/mxnet/nn/__init__.py b/python/mxnet/nn/__init__.py new file mode 100644 index 000000000000..aa34f1c1f6c0 --- /dev/null +++ b/python/mxnet/nn/__init__.py @@ -0,0 +1,15 @@ +# coding: utf-8 +# pylint: disable=wildcard-import +"""Neural network module.""" + +from .parameter import * + +from .layer import * + +from .optim import * + +from .conv_layers import * + +from . import loss + +from . import utils diff --git a/python/mxnet/nn/conv_layers.py b/python/mxnet/nn/conv_layers.py new file mode 100644 index 000000000000..d81613970f25 --- /dev/null +++ b/python/mxnet/nn/conv_layers.py @@ -0,0 +1,743 @@ +# coding: utf-8 +# pylint: disable= arguments-differ +"""Convolutional neural network layers.""" +from .layer import Layer +from .. import symbol +from ..base import numeric_types + +def _infer_weight_shape(op, data_shape): + sym = symbol.invoke(op, [symbol.var('data', shape=data_shape)]) + return sym.infer_shape_partial()[0] + + +class _Conv(Layer): + """Abstract nD convolution layer (private, used as implementation base). + + This layer creates a convolution kernel that is convolved + with the layer input to produce a tensor of outputs. + If `use_bias` is True, a bias vector is created and added to the outputs. + Finally, if `activation` is not `None`, + it is applied to the outputs as well. + + Parameters + ---------- + filters: Integer, the dimensionality of the output space + (i.e. the number output of filters in the convolution). + kernel_size: An integer or tuple/list of n integers, specifying the + dimensions of the convolution window. + strides: An integer or tuple/list of n integers, + specifying the strides of the convolution. + Specifying any stride value != 1 is incompatible with specifying + any `dilation_rate` value != 1. + padding: An integer or a tuple/list of n integers, + If padding is non-zero, then the input is implicitly zero-padded + on both sides for padding number of points + dilation: An integer or tuple/list of n integers, specifying + the dilation rate to use for dilated convolution. + groups: int + controls the connections between inputs and outputs. + At groups=1, all inputs are convolved to all outputs. + At groups=2, the operation becomes equivalent to having two conv + layers side by side, each seeing half the input channels, and producing + half the output channels, and both subsequently concatenated. + layout: A string, + Can be 'NCW', 'NWC', 'NCHW', 'NHWC', 'NCDHW', 'NDHWC', etc. + 'N', 'C', 'H', 'W', 'D' stands for batch, channel, height, width and + depth dimensions respectively. + in_filters: int, default 0 + The number of input channels to this layer. Only required when using + NDArray API. + activation: Activation function to use + see mx.sym.Activation. + If you don't specify anything, no activation is applied + (ie. "linear" activation: `a(x) = x`). + use_bias: Boolean, whether the layer uses a bias vector. + kernel_initializer: Initializer for the `kernel` weights matrix + see Initializer. + bias_initializer: Initializer for the bias vector + see Initializer. + """ + def __init__(self, filters, kernel_size, strides, padding, dilation, + groups, layout, in_filters=0, activation=None, use_bias=True, + kernel_initializer=None, bias_initializer=None, + op_name='Convolution', prefix=None, params=None, **kwargs): + super(_Conv, self).__init__(prefix=prefix, params=params) + with self.scope: + self._filters = filters + self._in_filters = in_filters + if isinstance(strides, numeric_types): + strides = (strides,)*len(kernel_size) + if isinstance(padding, numeric_types): + padding = (padding,)*len(kernel_size) + if isinstance(dilation, numeric_types): + dilation = (dilation,)*len(kernel_size) + attrs = { + 'kernel': kernel_size, 'stride': strides, 'dilate': dilation, + 'pad': padding, 'num_filter': filters, 'num_group': groups, + 'no_bias': not use_bias, 'layout': layout} + attrs.update(kwargs) + self._op = symbol.CachedOp(op_name, 3 if use_bias else 2, **attrs) + + dshape = [0]*(len(kernel_size) + 2) + dshape[layout.find('N')] = 1 + dshape[layout.find('C')] = in_filters + wshapes = _infer_weight_shape(self._op, dshape) + self.weight = self.params.get('weight', shape=wshapes[1], + init=kernel_initializer) + if use_bias: + self.bias = self.params.get('bias', shape=wshapes[2], + init=bias_initializer) + + if activation is not None: + self.act = Activation(activation) + else: + self.act = None + + def generic_forward(self, F, x, weight, bias=None): + if bias is None: + act = F.invoke(self._op, [x, weight]) + else: + act = F.invoke(self._op, [x, weight, bias]) + if self.act is not None: + act = self.act(act) + return act + + +class Conv1D(_Conv): + """1D convolution layer (e.g. temporal convolution). + + This layer creates a convolution kernel that is convolved + with the layer input over a single spatial (or temporal) dimension + to produce a tensor of outputs. + If `use_bias` is True, a bias vector is created and added to the outputs. + Finally, if `activation` is not `None`, + it is applied to the outputs as well. + + When using this layer with NDArray API, + provide an `in_filters` argument + (integers, the number of input channels). + + + Parameters + ---------- + filters: Integer, the dimensionality of the output space + (i.e. the number output of filters in the convolution). + kernel_size: An integer or tuple/list of 1 integers, specifying the + dimensions of the convolution window. + strides: An integer or tuple/list of 1 integers, + specifying the strides of the convolution. + Specifying any stride value != 1 is incompatible with specifying + any `dilation_rate` value != 1. + padding: An integer or a tuple/list of 1 integers, + If padding is non-zero, then the input is implicitly zero-padded + on both sides for padding number of points + dilation: An integer or tuple/list of 1 integers, specifying + the dilation rate to use for dilated convolution. + groups: int + controls the connections between inputs and outputs. + At groups=1, all inputs are convolved to all outputs. + At groups=2, the operation becomes equivalent to having two conv + layers side by side, each seeing half the input channels, and producing + half the output channels, and both subsequently concatenated. + layout: A string, + Can be 'NCW', 'NWC', etc. + 'N', 'C', 'W' stands for batch, channel, and width (time) dimensions + respectively. + in_filters: int, default 0 + The number of input channels to this layer. Only required when using + NDArray API. + activation: Activation function to use + see mx.sym.Activation. + If you don't specify anything, no activation is applied + (ie. "linear" activation: `a(x) = x`). + use_bias: Boolean, whether the layer uses a bias vector. + kernel_initializer: Initializer for the `kernel` weights matrix + see Initializer. + bias_initializer: Initializer for the bias vector + see Initializer. + """ + def __init__(self, filters, kernel_size, strides=1, padding=0, dilation=1, + groups=1, layout='NCW', in_filters=0, activation=None, use_bias=True, + kernel_initializer=None, bias_initializer=None, **kwargs): + if isinstance(kernel_size, numeric_types): + kernel_size = (kernel_size,) + assert len(kernel_size) == 1, "kernel_size must be a number or a list of 1 ints" + super(Conv1D, self).__init__( + filters, kernel_size, strides, padding, dilation, groups, layout, + in_filters, activation, use_bias, kernel_initializer, bias_initializer, **kwargs) + + +class Conv2D(_Conv): + """2D convolution layer (e.g. spatial convolution over images). + + This layer creates a convolution kernel that is convolved + with the layer input to produce a tensor of + outputs. If `use_bias` is True, + a bias vector is created and added to the outputs. Finally, if + `activation` is not `None`, it is applied to the outputs as well. + + When using this layer with NDArray API, + provide an `in_filters` argument + (integers, the number of input channels). + + + Parameters + ---------- + filters: Integer, the dimensionality of the output space + (i.e. the number output of filters in the convolution). + kernel_size: An integer or tuple/list of 2 integers, specifying the + dimensions of the convolution window. + strides: An integer or tuple/list of 2 integers, + specifying the strides of the convolution. + Specifying any stride value != 1 is incompatible with specifying + any `dilation_rate` value != 1. + padding: An integer or a tuple/list of 2 integers, + If padding is non-zero, then the input is implicitly zero-padded + on both sides for padding number of points + dilation: An integer or tuple/list of 2 integers, specifying + the dilation rate to use for dilated convolution. + groups: int + controls the connections between inputs and outputs. + At groups=1, all inputs are convolved to all outputs. + At groups=2, the operation becomes equivalent to having two conv + layers side by side, each seeing half the input channels, and producing + half the output channels, and both subsequently concatenated. + layout: A string, + Can be 'NCHW', 'NHWC', etc. + 'N', 'C', 'H', 'W' stands for batch, channel, height, and width + dimensions respectively. + in_filters: int, default 0 + The number of input channels to this layer. Only required when using + NDArray API. + activation: Activation function to use + see mx.sym.Activation. + If you don't specify anything, no activation is applied + (ie. "linear" activation: `a(x) = x`). + use_bias: Boolean, whether the layer uses a bias vector. + kernel_initializer: Initializer for the `kernel` weights matrix + see Initializer. + bias_initializer: Initializer for the bias vector + see Initializer. + """ + def __init__(self, filters, kernel_size, strides=(1, 1), padding=(0, 0), + dilation=(1, 1), groups=1, layout='NCHW', in_filters=0, + activation=None, use_bias=True, + kernel_initializer=None, bias_initializer=None, **kwargs): + if isinstance(kernel_size, numeric_types): + kernel_size = (kernel_size,)*2 + assert len(kernel_size) == 2, "kernel_size must be a number or a list of 2 ints" + super(Conv2D, self).__init__( + filters, kernel_size, strides, padding, dilation, groups, layout, + in_filters, activation, use_bias, kernel_initializer, bias_initializer, **kwargs) + + +class Conv3D(_Conv): + """3D convolution layer (e.g. spatial convolution over volumes). + + This layer creates a convolution kernel that is convolved + with the layer input to produce a tensor of + outputs. If `use_bias` is True, + a bias vector is created and added to the outputs. Finally, if + `activation` is not `None`, it is applied to the outputs as well. + + When using this layer with NDArray API, + provide an `in_filters` argument + (integers, the number of input channels). + + + Parameters + ---------- + filters: Integer, the dimensionality of the output space + (i.e. the number output of filters in the convolution). + kernel_size: An integer or tuple/list of 3 integers, specifying the + dimensions of the convolution window. + strides: An integer or tuple/list of 3 integers, + specifying the strides of the convolution. + Specifying any stride value != 1 is incompatible with specifying + any `dilation_rate` value != 1. + padding: An integer or a tuple/list of 3 integers, + If padding is non-zero, then the input is implicitly zero-padded + on both sides for padding number of points + dilation: An integer or tuple/list of 3 integers, specifying + the dilation rate to use for dilated convolution. + groups: int + controls the connections between inputs and outputs. + At groups=1, all inputs are convolved to all outputs. + At groups=2, the operation becomes equivalent to having two conv + layers side by side, each seeing half the input channels, and producing + half the output channels, and both subsequently concatenated. + layout: A string, + Can be 'NCDHW', 'NDHWC', etc. + 'N', 'C', 'H', 'W', 'D' stands for batch, channel, height, width and + depth dimensions respectively. + in_filters: int, default 0 + The number of input channels to this layer. Only required when using + NDArray API. + activation: Activation function to use + see mx.sym.Activation. + If you don't specify anything, no activation is applied + (ie. "linear" activation: `a(x) = x`). + use_bias: Boolean, whether the layer uses a bias vector. + kernel_initializer: Initializer for the `kernel` weights matrix + see Initializer. + bias_initializer: Initializer for the bias vector + see Initializer. + """ + def __init__(self, filters, kernel_size, strides=(1, 1, 1), padding=(0, 0, 0), + dilation=(1, 1, 1), groups=1, layout='NCDHW', in_filters=0, + activation=None, use_bias=True, + kernel_initializer=None, bias_initializer=None, **kwargs): + if isinstance(kernel_size, numeric_types): + kernel_size = (kernel_size,)*3 + assert len(kernel_size) == 3, "kernel_size must be a number or a list of 3 ints" + super(Conv3D, self).__init__( + filters, kernel_size, strides, padding, dilation, groups, layout, + in_filters, activation, use_bias, kernel_initializer, bias_initializer, **kwargs) + + +class Conv1DTranspose(_Conv): + """Transposed 1D convolution layer (sometimes called Deconvolution). + + The need for transposed convolutions generally arises + from the desire to use a transformation going in the opposite direction + of a normal convolution, i.e., from something that has the shape of the + output of some convolution to something that has the shape of its input + while maintaining a connectivity pattern that is compatible with + said convolution. + + When using this layer with NDArray API, + provide an `in_filters` argument + (integers, the number of input channels). + + Parameters + ---------- + filters: Integer, the dimensionality of the output space + (i.e. the number output of filters in the convolution). + kernel_size: An integer or tuple/list of 1 integers, specifying the + dimensions of the convolution window. + strides: An integer or tuple/list of 1 integers, + specifying the strides of the convolution. + Specifying any stride value != 1 is incompatible with specifying + any `dilation_rate` value != 1. + padding: An integer or a tuple/list of 1 integers, + If padding is non-zero, then the input is implicitly zero-padded + on both sides for padding number of points + output_padding: An integer or a tuple/list of 1 integers, + Zero-padding added to one side of the output + dilation: An integer or tuple/list of 1 integers, specifying + the dilation rate to use for dilated convolution. + groups: int + controls the connections between inputs and outputs. + At groups=1, all inputs are convolved to all outputs. + At groups=2, the operation becomes equivalent to having two conv + layers side by side, each seeing half the input channels, and producing + half the output channels, and both subsequently concatenated. + layout: A string, + Can be 'NCW', 'NWC', etc. + 'N', 'C', 'W' stands for batch, channel, and width (time) dimensions + respectively. + in_filters: int, default 0 + The number of input channels to this layer. Only required when using + NDArray API. + activation: Activation function to use + see mx.sym.Activation. + If you don't specify anything, no activation is applied + (ie. "linear" activation: `a(x) = x`). + use_bias: Boolean, whether the layer uses a bias vector. + kernel_initializer: Initializer for the `kernel` weights matrix + see Initializer. + bias_initializer: Initializer for the bias vector + see Initializer. + """ + def __init__(self, filters, kernel_size, strides=1, padding=0, output_padding=0, + dilation=1, groups=1, layout='NCW', in_filters=0, activation=None, + use_bias=True, kernel_initializer=None, bias_initializer=None, + **kwargs): + if isinstance(kernel_size, numeric_types): + kernel_size = (kernel_size,) + if isinstance(output_padding, numeric_types): + output_padding = (output_padding,) + assert len(kernel_size) == 1, "kernel_size must be a number or a list of 1 ints" + assert len(output_padding) == 1, "output_padding must be a number or a list of 1 ints" + super(Conv1DTranspose, self).__init__( + filters, kernel_size, strides, padding, dilation, groups, layout, + in_filters, activation, use_bias, kernel_initializer, + bias_initializer, op_name='Deconvolution', adj=output_padding, **kwargs) + + +class Conv2DTranspose(_Conv): + """Transposed 2D convolution layer (sometimes called Deconvolution). + + The need for transposed convolutions generally arises + from the desire to use a transformation going in the opposite direction + of a normal convolution, i.e., from something that has the shape of the + output of some convolution to something that has the shape of its input + while maintaining a connectivity pattern that is compatible with + said convolution. + + When using this layer with NDArray API, + provide an `in_filters` argument + (integers, the number of input channels). + + + Parameters + ---------- + filters: Integer, the dimensionality of the output space + (i.e. the number output of filters in the convolution). + kernel_size: An integer or tuple/list of 2 integers, specifying the + dimensions of the convolution window. + strides: An integer or tuple/list of 2 integers, + specifying the strides of the convolution. + Specifying any stride value != 1 is incompatible with specifying + any `dilation_rate` value != 1. + padding: An integer or a tuple/list of 2 integers, + If padding is non-zero, then the input is implicitly zero-padded + on both sides for padding number of points + dilation: An integer or tuple/list of 2 integers, specifying + the dilation rate to use for dilated convolution. + groups: int + controls the connections between inputs and outputs. + At groups=1, all inputs are convolved to all outputs. + At groups=2, the operation becomes equivalent to having two conv + layers side by side, each seeing half the input channels, and producing + half the output channels, and both subsequently concatenated. + layout: A string, + Can be 'NCHW', 'NHWC', etc. + 'N', 'C', 'H', 'W' stands for batch, channel, height, and width + dimensions respectively. + in_filters: int, default 0 + The number of input channels to this layer. Only required when using + NDArray API. + activation: Activation function to use + see mx.sym.Activation. + If you don't specify anything, no activation is applied + (ie. "linear" activation: `a(x) = x`). + use_bias: Boolean, whether the layer uses a bias vector. + kernel_initializer: Initializer for the `kernel` weights matrix + see Initializer. + bias_initializer: Initializer for the bias vector + see Initializer. + """ + def __init__(self, filters, kernel_size, strides=(1, 1), padding=(0, 0), + output_padding=(0, 0), dilation=(1, 1), groups=1, layout='NCHW', + in_filters=0, activation=None, use_bias=True, + kernel_initializer=None, bias_initializer=None, **kwargs): + if isinstance(kernel_size, numeric_types): + kernel_size = (kernel_size,)*2 + if isinstance(output_padding, numeric_types): + output_padding = (output_padding,)*2 + assert len(kernel_size) == 2, "kernel_size must be a number or a list of 2 ints" + assert len(output_padding) == 2, "output_padding must be a number or a list of 2 ints" + super(Conv2DTranspose, self).__init__( + filters, kernel_size, strides, padding, dilation, groups, layout, + in_filters, activation, use_bias, kernel_initializer, + bias_initializer, op_name='Deconvolution', adj=output_padding, **kwargs) + + +class Conv3DTranspose(_Conv): + """Transposed 3D convolution layer (sometimes called Deconvolution). + + The need for transposed convolutions generally arises + from the desire to use a transformation going in the opposite direction + of a normal convolution, i.e., from something that has the shape of the + output of some convolution to something that has the shape of its input + while maintaining a connectivity pattern that is compatible with + said convolution. + + When using this layer with NDArray API, + provide an `in_filters` argument + (integers, the number of input channels). + + + Parameters + ---------- + filters: Integer, the dimensionality of the output space + (i.e. the number output of filters in the convolution). + kernel_size: An integer or tuple/list of 3 integers, specifying the + dimensions of the convolution window. + strides: An integer or tuple/list of 3 integers, + specifying the strides of the convolution. + Specifying any stride value != 1 is incompatible with specifying + any `dilation_rate` value != 1. + padding: An integer or a tuple/list of 3 integers, + If padding is non-zero, then the input is implicitly zero-padded + on both sides for padding number of points + dilation: An integer or tuple/list of 3 integers, specifying + the dilation rate to use for dilated convolution. + groups: int + controls the connections between inputs and outputs. + At groups=1, all inputs are convolved to all outputs. + At groups=2, the operation becomes equivalent to having two conv + layers side by side, each seeing half the input channels, and producing + half the output channels, and both subsequently concatenated. + layout: A string, + Can be 'NCDHW', 'NDHWC', etc. + 'N', 'C', 'H', 'W', 'D' stands for batch, channel, height, width and + depth dimensions respectively. + in_filters: int, default 0 + The number of input channels to this layer. Only required when using + NDArray API. + activation: Activation function to use + see mx.sym.Activation. + If you don't specify anything, no activation is applied + (ie. "linear" activation: `a(x) = x`). + use_bias: Boolean, whether the layer uses a bias vector. + kernel_initializer: Initializer for the `kernel` weights matrix + see Initializer. + bias_initializer: Initializer for the bias vector + see Initializer. + """ + def __init__(self, filters, kernel_size, strides=(1, 1, 1), padding=(0, 0, 0), + output_padding=(0, 0, 0), dilation=(1, 1, 1), groups=1, layout='NCDHW', + in_filters=0, activation=None, use_bias=True, + kernel_initializer=None, bias_initializer=None, **kwargs): + if isinstance(kernel_size, numeric_types): + kernel_size = (kernel_size,)*3 + if isinstance(output_padding, numeric_types): + output_padding = (output_padding,)*3 + assert len(kernel_size) == 3, "kernel_size must be a number or a list of 3 ints" + assert len(output_padding) == 3, "output_padding must be a number or a list of 3 ints" + super(Conv3DTranspose, self).__init__( + filters, kernel_size, strides, padding, dilation, groups, layout, + in_filters, activation, use_bias, kernel_initializer, bias_initializer, + op_name='Deconvolution', adj=output_padding, **kwargs) + + +class _Pooling(Layer): + """Abstract class for different pooling layers. + """ + def __init__(self, pool_size, strides, padding, global_pool, pool_type, **kwargs): + super(_Pooling, self).__init__(**kwargs) + if strides is None: + strides = pool_size + if isinstance(strides, numeric_types): + strides = (strides,)*len(pool_size) + if isinstance(padding, numeric_types): + padding = (padding,)*len(pool_size) + attrs = { + 'kernel': pool_size, 'stride': strides, 'pad': padding, + 'pooling_convention': 'full', 'global_pool': global_pool, + 'pool_type': pool_type} + self._op = symbol.CachedOp('Pooling', 1, **attrs) + + def generic_forward(self, F, x): + return F.invoke(self._op, [x]) + + +class MaxPool1D(_Pooling): + """Max pooling operation for temporal data. + + Parameters + ---------- + pool_size: Integer, size of the max pooling windows. + strides: Integer, or None. Factor by which to downscale. + E.g. 2 will halve the input. + If None, it will default to `pool_size`. + padding: Integer, + If padding is non-zero, then the input is implicitly + zero-padded on both sides for padding number of points + layout: A string, + Can be 'NCHW', 'NHWC', etc. + 'N', 'C', 'H', 'W' stands for batch, channel, and width (time) dimensions + respectively. padding is applied on W dimension. + """ + def __init__(self, pool_size=2, strides=None, padding=0, layout='NCW', **kwargs): + assert layout == 'NCW', "Only supports NCW layout for now" + if isinstance(pool_size, numeric_types): + pool_size = (pool_size,) + assert len(pool_size) == 1, "pool_size must be a number or a list of 1 ints" + super(MaxPool1D, self).__init__( + pool_size, strides, padding, False, 'max', **kwargs) + + +class MaxPool2D(_Pooling): + """Max pooling operation for spatial data. + + Parameters + ---------- + pool_size: Integer or list/tuple of 2 Integers, + size of the max pooling windows. + strides: Integer, list/tuple of 2 Integers, or None. + Factor by which to downscale. + E.g. 2 will halve the input. + If None, it will default to `pool_size`. + padding: Integer or list/tuple of 2 Integers, + If padding is non-zero, then the input is implicitly + zero-padded on both sides for padding number of points + layout: A string, + Can be 'NCHW', 'NHWC', etc. + 'N', 'C', 'H', 'W' stands for batch, channel, height, and width + dimensions respectively. padding is applied on 'H' and 'W' dimension. + """ + def __init__(self, pool_size=(2, 2), strides=None, padding=0, layout='NCHW', **kwargs): + assert layout == 'NCHW', "Only supports NCHW layout for now" + if isinstance(pool_size, numeric_types): + pool_size = (pool_size,)*2 + assert len(pool_size) == 2, "pool_size must be a number or a list of 2 ints" + super(MaxPool2D, self).__init__( + pool_size, strides, padding, False, 'max', **kwargs) + + +class MaxPool3D(_Pooling): + """Max pooling operation for 3D data (spatial or spatio-temporal). + + Parameters + ---------- + pool_size: Integer or list/tuple of 3 Integers, + size of the max pooling windows. + strides: Integer, list/tuple of 3 Integers, or None. + Factor by which to downscale. + E.g. 2 will halve the input. + If None, it will default to `pool_size`. + padding: Integer or list/tuple of 3 Integers, + If padding is non-zero, then the input is implicitly + zero-padded on both sides for padding number of points + layout: A string, + Can be 'NCDHW', 'NDHWC', etc. + 'N', 'C', 'H', 'W', 'D' stands for batch, channel, height, width and + depth dimensions respectively. padding is applied on 'D', 'H' and 'W' + dimension. + """ + def __init__(self, pool_size=(2, 2, 2), strides=None, padding=0, layout='NCDHW', **kwargs): + assert layout == 'NCDHW', "Only supports NCDHW layout for now" + if isinstance(pool_size, numeric_types): + pool_size = (pool_size,)*3 + assert len(pool_size) == 3, "pool_size must be a number or a list of 3 ints" + super(MaxPool3D, self).__init__( + pool_size, strides, padding, False, 'max', **kwargs) + + +class AvgPool1D(_Pooling): + """Average pooling operation for temporal data. + + Parameters + ---------- + pool_size: Integer, size of the max pooling windows. + strides: Integer, or None. Factor by which to downscale. + E.g. 2 will halve the input. + If None, it will default to `pool_size`. + padding: Integer, + If padding is non-zero, then the input is implicitly + zero-padded on both sides for padding number of points + layout: A string, + Can be 'NCHW', 'NHWC', etc. + 'N', 'C', 'H', 'W' stands for batch, channel, and width (time) dimensions + respectively. padding is applied on W dimension. + """ + def __init__(self, pool_size=2, strides=None, padding=0, layout='NCW', **kwargs): + assert layout == 'NCW', "Only supports NCW layout for now" + if isinstance(pool_size, numeric_types): + pool_size = (pool_size,) + assert len(pool_size) == 1, "pool_size must be a number or a list of 1 ints" + super(AvgPool1D, self).__init__( + pool_size, strides, padding, False, 'avg', **kwargs) + + +class AvgPool2D(_Pooling): + """Average pooling operation for spatial data. + + Parameters + ---------- + pool_size: Integer or list/tuple of 2 Integers, + size of the max pooling windows. + strides: Integer, list/tuple of 2 Integers, or None. + Factor by which to downscale. + E.g. 2 will halve the input. + If None, it will default to `pool_size`. + padding: Integer or list/tuple of 2 Integers, + If padding is non-zero, then the input is implicitly + zero-padded on both sides for padding number of points + layout: A string, + Can be 'NCHW', 'NHWC', etc. + 'N', 'C', 'H', 'W' stands for batch, channel, height, and width + dimensions respectively. padding is applied on 'H' and 'W' dimension. + """ + def __init__(self, pool_size=(2, 2), strides=None, padding=0, layout='NCHW', **kwargs): + assert layout == 'NCHW', "Only supports NCHW layout for now" + if isinstance(pool_size, numeric_types): + pool_size = (pool_size,)*2 + assert len(pool_size) == 2, "pool_size must be a number or a list of 2 ints" + super(AvgPool2D, self).__init__( + pool_size, strides, padding, False, 'avg', **kwargs) + + +class AvgPool3D(_Pooling): + """Average pooling operation for 3D data (spatial or spatio-temporal). + + Parameters + ---------- + pool_size: Integer or list/tuple of 3 Integers, + size of the max pooling windows. + strides: Integer, list/tuple of 3 Integers, or None. + Factor by which to downscale. + E.g. 2 will halve the input. + If None, it will default to `pool_size`. + padding: Integer or list/tuple of 3 Integers, + If padding is non-zero, then the input is implicitly + zero-padded on both sides for padding number of points + layout: A string, + Can be 'NCDHW', 'NDHWC', etc. + 'N', 'C', 'H', 'W', 'D' stands for batch, channel, height, width and + depth dimensions respectively. padding is applied on 'D', 'H' and 'W' + dimension. + """ + def __init__(self, pool_size=(2, 2, 2), strides=None, padding=0, layout='NCDHW', **kwargs): + assert layout == 'NCDHW', "Only supports NCDHW layout for now" + if isinstance(pool_size, numeric_types): + pool_size = (pool_size,)*3 + assert len(pool_size) == 3, "pool_size must be a number or a list of 3 ints" + super(AvgPool3D, self).__init__( + pool_size, strides, padding, False, 'avg', **kwargs) + + +class GlobalMaxPool1D(_Pooling): + """Global max pooling operation for temporal data. + """ + def __init__(self, layout='NCW', **kwargs): + assert layout == 'NCW', "Only supports NCW layout for now" + super(GlobalMaxPool1D, self).__init__( + (1,), None, 0, True, 'max', **kwargs) + + +class GlobalMaxPool2D(_Pooling): + """Global max pooling operation for spatial data. + """ + def __init__(self, layout='NCHW', **kwargs): + assert layout == 'NCHW', "Only supports NCW layout for now" + super(GlobalMaxPool2D, self).__init__( + (1, 1), None, 0, True, 'max', **kwargs) + +class GlobalMaxPool3D(_Pooling): + """Global max pooling operation for 3D data. + """ + def __init__(self, layout='NCDHW', **kwargs): + assert layout == 'NCDHW', "Only supports NCW layout for now" + super(GlobalMaxPool3D, self).__init__( + (1, 1, 1), None, 0, True, 'max', **kwargs) + + +class GlobalAvgPool1D(_Pooling): + """Global average pooling operation for temporal data. + """ + def __init__(self, layout='NCW', **kwargs): + assert layout == 'NCW', "Only supports NCW layout for now" + super(GlobalAvgPool1D, self).__init__( + (1,), None, 0, True, 'avg', **kwargs) + + +class GlobalAvgPool2D(_Pooling): + """Global average pooling operation for spatial data. + """ + def __init__(self, layout='NCHW', **kwargs): + assert layout == 'NCHW', "Only supports NCW layout for now" + super(GlobalAvgPool2D, self).__init__( + (1, 1), None, 0, True, 'avg', **kwargs) + + +class GlobalAvgPool3D(_Pooling): + """Global max pooling operation for 3D data. + """ + def __init__(self, layout='NCDHW', **kwargs): + assert layout == 'NCDHW', "Only supports NCW layout for now" + super(GlobalAvgPool3D, self).__init__( + (1, 1, 1), None, 0, True, 'avg', **kwargs) diff --git a/python/mxnet/nn/layer.py b/python/mxnet/nn/layer.py new file mode 100644 index 000000000000..231992ff79d5 --- /dev/null +++ b/python/mxnet/nn/layer.py @@ -0,0 +1,411 @@ +# coding: utf-8 +# pylint: disable= arguments-differ +"""Neural network layers.""" + +from .. import symbol, ndarray +from ..symbol import Symbol +from ..ndarray import NDArray +from .. import name as _name +from .parameter import Parameter, ParameterDict + + +class _LayerScope(object): + """Scope for collecting sub-layers.""" + _current = None + + def __init__(self, layer): + self._layer = layer + self._counter = {} + self._old_scope = None + + @staticmethod + def get_prefix(prefix, hint): + if _LayerScope._current is None: + if prefix is None: + return _name.NameManager.current.get(None, hint) + '_' + return prefix + else: + if prefix is None: + count = _LayerScope._current._counter.get(hint, 0) + prefix = '%s%d_'%(hint, count) + _LayerScope._current._counter[hint] = count + 1 + return _LayerScope._current._layer.prefix+prefix + + @staticmethod + def get_params(prefix, params): + if params is not None: + return params + params = ParameterDict(prefix) + if _LayerScope._current is not None: + _LayerScope._current._layer.params.merge(params) + return params + + @staticmethod + def register_sublayer(layer): + if _LayerScope._current is not None: + _LayerScope._current._layer.register_sublayer(layer) + + def __enter__(self): + self._old_scope = _LayerScope._current + _LayerScope._current = self + return self + + def __exit__(self, ptype, value, trace): + _LayerScope._current = self._old_scope + + +class Layer(object): + """Base class for all neural network layers and models. + + Your models should subclass this class. + + Layers can also contain other Layers, allowing you to nest them in a tree + structure. You can assign sublayers as regular attributes:: + from mxnet import nn + class Net(nn.Layer): + def __init__(self, **kwargs): + super(Net, self).__init__(**kwargs) + with self.scope: + self.dense1 = nn.Dense(20, in_units=10, prefix='dense1_') + self.dense2 = nn.Dense(20, in_units=20, prefix='dense2_') + + def forward(self, x): + x = self.dense1(x) + return self.dense2(x) + + Sublayers assigned this way will be registered and will have their status changed + too when you call .train() etc. + + Parameters + ---------- + prefix : str + Prefix acts like a name space. It will be prepended to the name of all Symbols and + Parameters created by this layer. Prefix should be unique within one network + to prevent name collisions. + params : ParameterDict or None + Manages Parameters of this Layer and sublayers. You can make two Layers share + parameter by passing the same dictionary to them. For example:: + params = nn.ParameterDict(prefix='dense_') + dense1 = nn.Dense(20, in_units=10, prefix='dense1_', params=params) + dense2 = nn.Dense(20, in_units=10, prefix='dense2_', params=params) + + dense1 and dense2 now have shared weights. + + Layer supports forwarding with both `Symbol` and `NDArray`. + + Layer is mostly used by developers or advanced users as a base class. + If you only want to use one of `Symbol` and `NDArray` API you should inherit + Layer instead.""" + def __init__(self, prefix=None, params=None): + self._prefix = _LayerScope.get_prefix(prefix, self._alias()) + self._params = _LayerScope.get_params(self._prefix, params) + self._scope = _LayerScope(self) + self._children = [] + self._reg_params = {} + + + def __setattr__(self, name, value): + """Registers parameters.""" + super(Layer, self).__setattr__(name, value) + if isinstance(value, Parameter): + self._reg_params[name] = value + if isinstance(value, Layer): + _LayerScope.register_sublayer(self) + + def _alias(self): + return self.__class__.__name__.lower() + + @property + def params(self): + """A ParameterDict managing this Layer's Parameters.""" + return self._params + + @property + def prefix(self): + """Prefix of this Layer.""" + return self._prefix + + @property + def name(self): + if self.prefix.endswith('_'): + return self.prefix[:-1] + return self.prefix + + @property + def scope(self): + return self._scope + + def register_sublayer(self, layer): + """Register layer as sublayer of self. Layers assigned to self as attributes + will be registered automatically.""" + self._children.append(layer) + self.params.merge(layer.params) + + def __call__(self, *args, **kwargs): + """Call forward.""" + return self.forward(*args, **kwargs) + + def forward(self, x, *args): + """Defines the forward computation. Arguments can be either NDArray or Symbol.""" + if isinstance(x, NDArray): + with x.context as ctx: + params = {k: v.data(ctx) for k, v in self._reg_params.items()} + return self.ndarray_forward(x, *args, **params) + else: + assert isinstance(x, Symbol), \ + "Layer requires the first argument to forward be either Symbol or NDArray" + params = {k: v.var() for k, v in self._reg_params.items()} + return self.symbol_forward(x, *args, **params) + + def ndarray_forward(self, x, *args, **kwargs): + return self.generic_forward(ndarray, x, *args, **kwargs) + + def symbol_forward(self, x, *args, **kwargs): + return self.generic_forward(symbol, x, *args, **kwargs) + + def generic_forward(self, F, x, *args, **kwargs): + """Simple forward supports both `Symbol` and `NDArray` API. + + Parameters + ---------- + F : {mxnet.ndarray, mxnet.symbol} + Name space of operators. `F` will be set to `mx.sym` when x is `Symbol` + instance and `mx.nd` when x is `NDArray` instance. + x : NDArray or Symbol + The first input tensor. + *args : list of NDArray or list of Symbol + Additional input tensors. + **kwargs : dict of str to NDArray or dict of str to Symbol + `Symbol` or `NDArray` value of registered Parameters. + """ + # pylint: disable= invalid-name + raise NotImplementedError + + +class Sequential(Layer): + """Stack Layers sequentially. + + Example:: + net = nn.Sequential() + net.add(Dense(10, activation='relu')) + net.add(Dense(20)) + """ + def __init__(self): + super(Sequential, self).__init__(prefix='', params=None) + + def add(self, layer): + """Add layer on top of the stack.""" + self.register_sublayer(layer) + + def forward(self, x): + #pylint: disable=arguments-differ + for layer in self._children: + x = layer(x) + return x + + def generic_forward(self, F, x, *args, **kwargs): + raise NotImplementedError + + +class Dense(Layer): + """Just your regular densely-connected NN layer. + + `Dense` implements the operation: + `output = activation(dot(input, kernel) + bias)` + where `activation` is the element-wise activation function + passed as the `activation` argument, `kernel` is a weights matrix + created by the layer, and `bias` is a bias vector created by the layer + (only applicable if `use_bias` is `True`). + + Note: the input must be a tensor with rank 2. Use flatten to convert it + to rank 2 manually if necessary. + + Example:: + # as first layer in a sequential model: + model = Sequential() + model.add(Dense(32, in_uints=16)) + # now the model will take as input arrays of shape (*, 16) + # and output arrays of shape (*, 32) + + # No need to specify the size of the input if you only want to + # use the `Symbol` API: + model = Sequential() + model.add(Dense(32)) + + Parameters + ---------- + units: Positive integer, dimensionality of the output space. + activation: Activation function to use + (see help on Activation operator). + If you don't specify anything, no activation is applied + (ie. "linear" activation: `a(x) = x`). + use_bias: Boolean, whether the layer uses a bias vector. + kernel_initializer: Initializer for the `kernel` weights matrix + (see mxnet.initializer). + bias_initializer: Initializer for the bias vector + (see mxnet.initializer). + in_units : int + Size of input data. No need to specify for `Symbol` API. But must be + specified for every Dense layer if you want to use `NDArray` API. + prefix : str or None + See document of Layer. + params : ParameterDict or None + See document of Layer. + + Input shape + ----------- + a 2D input with shape `(batch_size, in_units)`. + + Output shape + ------------ + the output would have shape `(batch_size, units)`. + """ + def __init__(self, units, activation=None, use_bias=True, + kernel_initializer=None, bias_initializer=None, + in_units=0, **kwargs): + super(Dense, self).__init__(**kwargs) + with self.scope: + self._op = symbol.CachedOp('FullyConnected', 3 if use_bias else 2, + num_hidden=units, no_bias=not use_bias) + self.weight = self.params.get('weight', shape=(units, in_units), + init=kernel_initializer) + if use_bias: + self.bias = self.params.get('bias', shape=(units,), + init=bias_initializer) + if activation is not None: + self.act = Activation(activation) + else: + self.act = None + + def generic_forward(self, F, x, weight, bias=None): + if bias is None: + act = F.invoke(self._op, [x, weight]) + else: + act = F.invoke(self._op, [x, weight, bias]) + if self.act is not None: + act = self.act(act) + return act + + +class Activation(Layer): + """Applies an activation function to input. + + Parameters + ---------- + activation: name of activation function to use + See: help on Activation operator + + Input shape + ----------- + Arbitrary. + + Output shape + ------------ + Same shape as input. + """ + def __init__(self, activation, **kwargs): + self._act_type = activation + super(Activation, self).__init__(**kwargs) + self._op = symbol.CachedOp('Activation', 1, act_type=self._act_type) + + def _alias(self): + return self._act_type + + def generic_forward(self, F, x): + return F.invoke(self._op, [x]) + + +class Dropout(Layer): + """Applies Dropout to the input. + + Dropout consists in randomly setting + a fraction `rate` of input units to 0 at each update during training time, + which helps prevent overfitting. + + Parameters + ---------- + rate: float between 0 and 1. Fraction of the input units to drop. + + References + ---------- + - [Dropout: A Simple Way to Prevent Neural Networks from Overfitting]( + http://www.cs.toronto.edu/~rsalakhu/papers/srivastava14a.pdf) + """ + def __init__(self, rate, **kwargs): + super(Dropout, self).__init__(**kwargs) + self._op = symbol.CachedOp('Dropout', 1, p=rate) + + def generic_forward(self, F, x): + return F.invoke(self._op, [x]) + + +class BatchNorm(Layer): + """Batch normalization layer (Ioffe and Szegedy, 2014). + Normalize the activations of the previous layer at each batch, + i.e. applies a transformation that maintains the mean activation + close to 0 and the activation standard deviation close to 1. + + Parameters + ---------- + axis: Integer, the axis that should be normalized + (typically the features axis). + For instance, after a `Conv2D` layer with + `data_format="channels_first"`, + set `axis=1` in `BatchNormalization`. + momentum: Momentum for the moving average. + epsilon: Small float added to variance to avoid dividing by zero. + center: If True, add offset of `beta` to normalized tensor. + If False, `beta` is ignored. + scale: If True, multiply by `gamma`. + If False, `gamma` is not used. + When the next layer is linear (also e.g. `nn.relu`), + this can be disabled since the scaling + will be done by the next layer. + beta_initializer: Initializer for the beta weight. + gamma_initializer: Initializer for the gamma weight. + moving_mean_initializer: Initializer for the moving mean. + moving_variance_initializer: Initializer for the moving variance. + """ + def __init__(self, axis=1, momentum=0.9, epsilon=1e-3, center=True, scale=True, + num_features=0, beta_initializer='zeros', gamma_initializer='ones', + running_mean_initializer='zeros', running_variance_initializer='ones', + **kwargs): + super(BatchNorm, self).__init__(**kwargs) + assert axis == 1, \ + "Only support NC* layout, i.e. channel must be in the second dimension" + attrs = {'eps': epsilon, 'momentum': momentum, 'fix_gamma': not center} + self._op = symbol.CachedOp('BatchNorm', 5, **attrs) + + self.gamma = self.params.get('gamma', grad_req='write' if scale else 'null', + shape=(num_features,), init=gamma_initializer) + self.beta = self.params.get('beta', grad_req='write' if center else 'null', + shape=(num_features,), init=beta_initializer) + self.running_mean = self.params.get('running_mean', grad_req='null', + shape=(num_features,), + init=running_mean_initializer) + self.running_var = self.params.get('running_var', grad_req='null', + shape=(num_features,), + init=running_variance_initializer) + + def generic_forward(self, F, x, gamma, beta, running_mean, running_var): + return F.invoke(self._op, [x, gamma, beta, running_mean, running_var]) + + +class LeakyReLU(Layer): + """Leaky version of a Rectified Linear Unit. + + It allows a small gradient when the unit is not active: + `f(x) = alpha * x for x < 0`, + `f(x) = x for x >= 0`. + + Parameters + ---------- + alpha: float + Negative slope coefficient. Must be >= 0. + """ + def __init__(self, alpha, **kwargs): + super(LeakyReLU, self).__init__(**kwargs) + self._op = symbol.CachedOp('LeakyReLU', 1, act_type='leaky', slope=alpha) + + def generic_forward(self, F, x): + return F.invoke(self._op, [x]) diff --git a/python/mxnet/nn/loss.py b/python/mxnet/nn/loss.py new file mode 100644 index 000000000000..9bfa3795c2e3 --- /dev/null +++ b/python/mxnet/nn/loss.py @@ -0,0 +1,257 @@ +# coding: utf-8 +# pylint: disable=too-many-arguments, no-member, protected-access, too-many-locals +# pylint: disable=unused-argument +""" losses for training neural networks """ +from __future__ import absolute_import + +import json + +from .. import symbol, ndarray, metric +from ..base import numeric_types + + +def _get_F(x): + """Get function domain from tensor""" + return symbol if isinstance(x, symbol.Symbol) else ndarray + + +def _apply_weighting(F, loss, weight=None, sample_weight=None): + """Apply weighting to loss. + + Parameters + ---------- + loss : Symbol + the loss to be weighted. + weight : float or None + global scalar weight for loss + sample_weight : Symbol or None + per sample weighting. Must be broadcastable to + the same shape as loss. For example, if loss has + shape (64, 10) and you want to weight each sample + in the batch separately, sample_weight should have + shape (64, 1) + + Returns + ------- + loss : Symbol + weighted loss + """ + if sample_weight is not None: + loss = F.broadcast_mul(loss, sample_weight) + + if weight is not None: + assert isinstance(weight, numeric_types), "weight must be a number" + loss = loss * weight + + return loss + + +def _unpack_symbol(loss): + """unpack a loss symbol into outputs, extra_outputs and losses""" + assert isinstance(loss, symbol.Symbol) + outputs = symbol.Group([i for i in loss if i.attr('__output__') == 'pred']) + extra_outputs = symbol.Group([i for i in loss if i.attr('__output__') == 'extra']) + losses = symbol.Group([i for i in loss if i.attr('__output__') == 'loss']) + return outputs, extra_outputs, losses + + +def custom_loss(loss, output, label, weight=None, sample_weight=None, batch_axis=0, + extra_outputs=(), metrics=None, name='custom'): + """Construct user defined loss symbol. + + Parameters + ---------- + loss : Symbol + loss value computed from output and label. + output : Symbol + output of the network + label : Symbol + target to compare output against + weight : float or None + global scalar weight for loss + sample_weight : Symbol or None + per sample weighting. Must be broadcastable to + the same shape as loss. For example, if loss has + shape (64, 10) and you want to weight each sample + in the batch, sample_weight should have shape (64, 1) + batch_axis : int, default 0 + The axis that represents mini-batch. + + Returns + ------- + loss : BaseLoss + created loss + + Example + ------- + The following code defines a least square loss (same as `nn.l2_loss`):: + data = mx.sym.var('data') + output = mx.sym.FullyConnected(data, num_hidden=1) + label = mx.sym.var('label') + loss = mx.sym.square(output - label.reshape((-1, 1)))/2 + loss = nn.custom_loss(loss, output, label, name='l2') + """ + F = _get_F(loss) + loss = _apply_weighting(F, loss, weight, sample_weight) + loss = F.mean(loss, axis=batch_axis, exclude=True) + if F is ndarray: + return loss + outputs = symbol.Group([F.stop_gradient(i, name=i.name+'_out', __output__='pred') + for i in output]) + extra_outputs = symbol.Group([F.stop_gradient(i, name=i.name+'_out', __output__='extra') + for i in extra_outputs]) + + loss = F.make_loss(loss, name=name, __output__='loss') + + if metrics: + metrics = metric.create(metrics) + metrics.output_names = outputs.list_outputs() + metrics.label_names = label.list_outputs() + loss._set_attr(__metric__=json.dumps(metrics.get_config())) + + return symbol.Group([outputs, extra_outputs, loss]) + + +def multitask_loss(losses): + """Combine multiple losses together for multitask learning. + + Parameters + ---------- + losses : list of Symbol + list of losses to be combined. + """ + F = _get_F(losses[0]) + if F is ndarray: + return losses + out, extra, loss = zip(*[_unpack_symbol(i) for i in losses]) + return symbol.Group(out+extra+loss) + + +def l2_loss(output, label, weight=1., sample_weight=None, batch_axis=0, + extra_outputs=(), metrics=None, name='l2'): + """Calculate the mean squared error between output and label: + + .. math:: + L = \\frac{1}{2}\\sum_i \\Vert {output}_i - {label}_i \\Vert^2. + + output and label can have arbitrary shape as long as they have the same + number of elements. + + Parameters + ---------- + output : Symbol + output of the network + label : Symbol + target to compare output against + weight : float or None + global scalar weight for loss + sample_weight : Symbol or None + per sample weighting. Must be broadcastable to + the same shape as loss. For example, if loss has + shape (64, 10) and you want to weight each sample + in the batch, sample_weight should have shape (64, 1) + batch_axis : int, default 0 + The axis that represents mini-batch. + + Returns + ------- + loss : Symbol + created loss + """ + if isinstance(output, ndarray.NDArray): + loss = ndarray.square(output - label.reshape(output.shape)) + else: + loss = symbol.square(output - label.reshape(())) + return custom_loss(loss, output, label, weight/2, sample_weight, batch_axis, + extra_outputs, metrics, name) + + +def l1_loss(output, label, weight=None, sample_weight=None, batch_axis=0, + extra_outputs=(), metrics=None, name='l1'): + """Calculate the mean absolute error between output and label: + + .. math:: + L = \\frac{1}{2}\\sum_i \\vert {output}_i - {label}_i \\vert. + + output and label must have the same shape. + + Parameters + ---------- + output : Symbol + output of the network + label : Symbol + target to compare output against + weight : float or None + global scalar weight for loss + sample_weight : Symbol or None + per sample weighting. Must be broadcastable to + the same shape as loss. For example, if loss has + shape (64, 10) and you want to weight each sample + in the batch, sample_weight should have shape (64, 1) + batch_axis : int, default 0 + The axis that represents mini-batch. + + Returns + ------- + loss : Symbol + created loss + """ + if isinstance(output, ndarray.NDArray): + loss = ndarray.abs(output - label.reshape(output.shape)) + else: + loss = symbol.abs(output - label.reshape(())) + return custom_loss(loss, output, label, weight, sample_weight, batch_axis, + extra_outputs, metrics, name) + + +def softmax_cross_entropy_loss(output, label, sparse_label=True, axis=-1, + weight=None, sample_weight=None, batch_axis=0, + extra_outputs=(), metrics='acc', name='ce'): + """Compute the softmax cross entropy loss. + + If sparse_label is True, label should contain integer category indicators: + .. math:: + p = {softmax}({output}) + L = -\\sum_i {log}(p_{i,{label}_i}) + label's shape should be output's shape without the `axis` dimension. i.e. for + output.shape = (1,2,3,4) and axis = 2, label.shape should be (1,2,4) + + If sparse_label is False, label should cantain probability distribution + with the same shape as output: + .. math:: + p = {softmax}({output}) + L = -\\sum_i \\sum_j {label}_j {log}(p_{ij}) + + Parameters + ---------- + output : Symbol + output of the network + label : Symbol + target to compare output against + sparse_label : bool, default True + where label is sparse integer or probability distribution + axis : int, default -1 + The axis to sum over when computing softmax and entropy + weight : float or None + global scalar weight for loss + sample_weight : Symbol or None + per sample weighting. Must be broadcastable to + the same shape as loss. For example, if loss has + shape (64, 10) and you want to weight each sample + in the batch, sample_weight should have shape (64, 1) + batch_axis : int, default 0 + The axis that represents mini-batch. + + Returns + ------- + loss : Symbol + created loss + """ + F = _get_F(output) + prob = F.log_softmax(output) + if sparse_label: + loss = -F.pick(prob, label, axis=axis, keepdims=True) + else: + loss = -F.sum(prob*label, axis=axis, keepdims=True) + return custom_loss(loss, prob, label, weight, sample_weight, batch_axis, + extra_outputs, metrics, name) diff --git a/python/mxnet/nn/optim.py b/python/mxnet/nn/optim.py new file mode 100644 index 000000000000..81fdfbf12756 --- /dev/null +++ b/python/mxnet/nn/optim.py @@ -0,0 +1,109 @@ +# coding: utf-8 +# pylint: disable= +"""Parameter optimizer.""" + +from .. import optimizer as opt +from ..model import _create_kvstore + +class Optim(object): + """Optimizes a set of Parameters. Optim should be used together with autograd. + + Parameters + ---------- + param_dict : ParameterDict + The set of parameters to optimize. + optimizer : str or Optimizer + The optimizer to use. + optimizer_params : dict + key-word arguments to be passed to Optimizer.create_optimizer. For example, + {'learning_rate': 0.1} + kvstore : str or KVStore + kvstore type for multi-gpu and distributed training. + """ + def __init__(self, param_dict, optimizer, optimizer_params, kvstore='device'): + self._params = [param for param in param_dict.values() if param.grad_req != 'null'] + self._scale = optimizer_params.get('rescale_grad', 1.0) + + self._contexts = self._check_contexts() + self._init_optimizer(optimizer, optimizer_params) + self._init_kvstore(kvstore) + + def _check_contexts(self): + contexts = None + for param in self._params: + ctx = param.list_ctx() + assert contexts is None or contexts == ctx, \ + "All Parameters must be initialized on the same set of contexts, " \ + "but Parameter %s is initialized on %s while previous Parameters " \ + "are initialized on %s."%(param.name, str(ctx), str(contexts)) + contexts = ctx + return contexts + + def _init_optimizer(self, optimizer, optimizer_params): + self._optimizer = opt.create(optimizer, **optimizer_params) + self._updaters = [opt.get_updater(self._optimizer) \ + for _ in self._contexts] + + lr_mult = {} + wd_mult = {} + for i, param in enumerate(self._params): + lr_mult[i] = param.lr_mult + wd_mult[i] = param.wd_mult + self._optimizer.set_lr_mult(lr_mult) + self._optimizer.set_wd_mult(wd_mult) + + def _init_kvstore(self, kvstore): + arg_arrays = {param.name: param.data(self._contexts[0]) for param in self._params} + kvstore, update_on_kvstore = _create_kvstore(kvstore, len(self._contexts), arg_arrays) + self._kvstore = kvstore + self._update_on_kvstore = update_on_kvstore + if kvstore: + assert 'dist' not in self._kvstore.type, "distributed training not supported yet" + for i, param in enumerate(self._params): + param_arrays = param.list_data() + kvstore.init(i, param_arrays[0]) + kvstore.pull(i, param_arrays, priority=-i) + if update_on_kvstore: + kvstore.set_optimizer(self._optimizer) + + def step(self, batch_size, ignore_stale_grad=False): + """Make one step of parameter update. Should be called after + autograd.compute_gradient and outside of train_section() scope. + + Parameters + ---------- + batch_size : int + Batch size of data processed. Gradient will be normalized by 1/batch_size. + Set this to 1 if you normalized loss manually with `loss = mean(loss)`. + ignore_stale_grad : bool, optional, default=False + If true, ignores Parameters with stale gradient (gradient that has not + been updated by `backward` after last step) and skip update. + """ + self._optimizer.rescale_grad = self._scale / batch_size + + for i, param in enumerate(self._params): + assert param.list_ctx() == self._contexts, \ + "Parameter %s's contexts changed after Optim initialization: " \ + "was %s, now %s"%(param.name, self._contexts, param.list_ctx()) + if not ignore_stale_grad: + for data in param.list_data(): + if not data._fresh_grad: + raise UserWarning( + "Gradient of Parameter `%s` on context %s has not been updated " + "by backward since last `step`. This could mean a bug in your " + "model that maked it only use a subset of the Parameters (Layers) " + "for this iteration. If you are intentionally only using a subset, " + "call step with ignore_stale_grad=True to suppress this " + "warning and skip updating of Parameters with state gradient" \ + %(param.name, str(data.context))) + if self._kvstore: + self._kvstore.push(i, param.list_grad(), priority=-i) + if self._update_on_kvstore: + self._kvstore.pull(i, param.list_data(), priority=-i) + continue + else: + self._kvstore.pull(i, param.list_grad(), priority=-i) + for upd, arr, grad in zip(self._updaters, param.list_data(), param.list_grad()): + if arr._fresh_grad: + upd(i, grad, arr) + grad._fresh_grad = False diff --git a/python/mxnet/nn/parameter.py b/python/mxnet/nn/parameter.py new file mode 100644 index 000000000000..bc6b7a251abb --- /dev/null +++ b/python/mxnet/nn/parameter.py @@ -0,0 +1,358 @@ +# coding: utf-8 +# pylint: disable= +"""Neural network parameter.""" + +from collections import OrderedDict +import numpy as np + +from ..base import mx_real_t +from .. import symbol, ndarray, initializer, context +from ..context import Context +from ..contrib import autograd + +# pylint: disable= invalid-name +tensor_types = (symbol.Symbol, ndarray.NDArray) +# pylint: enable= invalid-name + +class Parameter(object): + """A Container holding parameters (weights) of layers. + + `Parameter` can be used with both `Symbol` and `NDArray` API. For `Symbol` API, + `Parameter.var()` will return a `Symbol` representing this parameter. It + can then be used for composing networks:: + x = mx.sym.Variable('data') + w = mx.nn.Parameter('fc_weight', init=mx.init.Xavier()) + b = mx.nn.Parameter('fc_bias', init=mx.init.Zero()) + out = mx.sym.FullyConnected(x, w.var(), b.var(), num_hidden=64) + + For `NDArray` API, `Parameter` must be initialized with `Parameter.init`. It + will then hold a copy of the the parameter on each `Context`. If `grad_req` is + not `null`, it will also hold a gradient array on each `Context`:: + ctx = mx.gpu(0) + x = mx.nd.zeros((16, 100), ctx=ctx) + w = mx.nn.Parameter('fc_weight', shape=(64, 100), init=mx.init.Xavier()) + b = mx.nn.Parameter('fc_bias', shape(64,), init=mx.init.Zero()) + w.initialize(ctx=ctx) + b.initialize(ctx=ctx) + out = mx.nd.FullyConnected(x, w.value(ctx), b.value(ctx), num_hidden=64) + + Parameters + ---------- + name : str + Name of this parameter. + grad_req : {'write', 'add', 'null'}, default 'write' + Specifies how to update gradient to grad arrays. + + - 'write' means everytime gradient is written to grad `NDArray`. + - 'add' means everytime gradient is added to the grad `NDArray`. You need + to manually call `zero_grad()` to clear the gradient buffer before each + iteration when using this option. + - 'null' means gradient is not reqested for this parameter. gradient arrays + will not be allocated. + shape : tuple of int, default None + Shape of this parameter. By default shape is not specified. Parameter with + unknown shaped can be used for `Symbol` API, but `init` will throw an error + when using `NDArray` API. + dtype : numpy.dtype or str, default 'float32' + Data type of this parameter. For example, numpy.float32 or 'float32'. + lr_mult : float, default 1.0 + Learning rate multiplier. Learning rate will be multiplied by lr_mult + when updating this parameter with optimizer. + wd_mult : float, default 1.0 + Weight decay multiplier (L2 regulerizer coefficient). Works similarly to lr_mult. + init : Initializer, default None + Initializer of this parameter. Will use the global initializer by default. + """ + def __init__(self, name, grad_req='write', shape=None, dtype=mx_real_t, + lr_mult=1.0, wd_mult=1.0, init=None): + self.name = name + self.shape = shape + self.dtype = dtype + self.lr_mult = lr_mult + self.wd_mult = wd_mult + self.grad_req = grad_req + self.init = init + self._var = None + self._data = None + self._grad = None + + def initialize(self, init=None, ctx=None, default_init=initializer.Xavier()): + """Intialize parameter and gradient arrays. Only used for `NDArray` API. + + init : Initializer + The initializer to use. Overrides `Parameter.init` and default_init. + ctx : Context or list of Context, defaults to `context.current_context()`. + Initialize Parameter on given context. If ctx is a list of Context, a + copy will be made for each context. + + .. note:: Copies are independent arrays. User is responsible for keeping + their values consistent when updating. Normally nn.Optim does this for you. + default_init : Initializer + Default initializer is used when both `init` and `Parameter.init` are None. + """ + if ctx is None: + ctx = [context.current_context()] + if isinstance(ctx, Context): + ctx = [ctx] + + assert np.prod(self.shape) > 0, \ + "Cannot initialize Parameter %s because it has invalid shape: %s. " \ + "Please specify in_units, in_filters, etc for Layers"%( + self.name, str(self.shape)) + data = ndarray.zeros(shape=self.shape, dtype=self.dtype, ctx=ctx[0]) + if init is None: + init = self.init + initializer.create(default_init)( + initializer.InitDesc(self.name, {'__init__': init}), + data) + + self._data = OrderedDict() + self._data[ctx[0]] = data + for i in ctx[1:]: + self._data[i] = data.copyto(i) + + if self.grad_req == 'null': + self._grad = None + return + + self._grad = OrderedDict() + for i in ctx: + self._grad[i] = ndarray.zeros_like(self._data[i]) + + autograd.mark_variables(self.list_data(), self.list_grad(), self.grad_req) + + def set_data(self, data): + """Set this parameter's value on all contexts to data.""" + assert self._data is not None, \ + "Parameter %s has not been initialized"%self.name + for arr in self.list_data(): + arr[:] = data + + def data(self, ctx=None): + """Returns a copy of this parameter on one context. Must be on this context + before. + + Parameters + ---------- + ctx : Context + Desired context. + + Returns + ------- + NDArray on ctx + """ + if ctx is None: + ctx = Context.current_context() + assert self._data is not None, \ + "Cannot get NDArray value for Parameter %s " \ + "because it hasn't been initialized!"%(self.name) + assert ctx in self._data, \ + "Cannot get NDArray value for Parameter %s on context %s " \ + "because it was not initialized on %s"%(self.name, str(ctx), str(ctx)) + return self._data[ctx] + + def list_data(self): + """Returns copies of this parameter on all contexts, in the same order + as creation.""" + assert self._data is not None, \ + "Parameter %s has not been initialized"%self.name + return self._data.values() + + def grad(self, ctx=None): + """Returns a gradient buffer for this parameter on one context. + + Parameters + ---------- + ctx : Context + Desired context. + """ + if ctx is None: + ctx = Context.current_context() + assert self._grad is not None, \ + "Cannot get gradient array for Parameter %s " \ + "because it hasn't been initialized or grad_req='null'"%(self.name) + assert ctx in self._grad, \ + "Cannot get gradient array for Parameter %s on context %s " \ + "because it was not initialized on %s"%(self.name, str(ctx), str(ctx)) + return self._grad[ctx] + + def list_grad(self): + """Returns gradient buffers on all contexts, in the same order + as `values`.""" + assert self._data is not None, \ + "Parameter %s has not been initialized"%self.name + assert self._data is not None, \ + "Parameter %s does not have gradients because grad_req='null'"%self.name + return self._grad.values() + + def list_ctx(self): + """Returns a list of contexts this parameter is initialized on""" + assert self._data is not None, \ + "Parameter %s has not been initialized"%self.name + return self._data.keys() + + def zero_grad(self): + """Set gradient buffer on all contexts to 0. No action is taken if + parameter is uninitialized or doesn't require gradient.""" + if self._grad is None: + return + for i in self._grad: + i[:] = 0 + + def var(self): + """Returns a symbol representing this parameter.""" + if self._var is None: + self._var = symbol.var(self.name, shape=self.shape, dtype=self.dtype, + lr_mult=self.lr_mult, wd_mult=self.wd_mult, + init=self.init) + return self._var + + +class ParameterDict(object): + """A dictionary managing a set of parameters. + + Parameters + ---------- + prefix : str, default '' + The prefix to be prepended to all Parameters' name created by this dict. + """ + def __init__(self, prefix=''): + self._prefix = prefix + self._params = {} + + def __getitem__(self, key): + return self._params[key] + + def items(self): + return self._params.items() + + def keys(self): + return self._params.keys() + + def values(self): + return self._params.values() + + @property + def prefix(self): + """Prefix of this dict. It will be prepended to Parameters' name created + with `get`""" + return self._prefix + + def get(self, name, **kwargs): + """Create or retrieve a Parameter with name `self.prefix+name`. Key-word + arguments will be passed to Parameter's contructor. + + Parameter + --------- + name : str + name of the desired Parameter. It will be prepended with this dictionary's + prefix. + **kwargs : dict + The rest of key-word arguments for the created Parameter. + + Returns + ------- + Parameter + The created or retrieved Parameter. + """ + name = self.prefix + name + if name not in self._params: + self._params[name] = Parameter(name, **kwargs) + else: + param = self._params[name] + for k, v in kwargs.items(): + if hasattr(param, k): + assert v is None or v == getattr(param, k), \ + "Parameter attribute %s mismatch: stored %s vs desired %s"%( + k, str(getattr(param, k)), str(v)) + else: + setattr(param, k, v) + return self._params[name] + + def subdict(self, suffix): + """Create a sub-dictionary that shares parameters with this dictionary. + The sub-dictionary's prefix is self.prefix + suffix. + + Example:: + >>> params1 = ParameterDict('net_') + >>> params2 = params1.subdict('conv1_') + >>> params2.prefix + 'net_conv1_' + + Parameters + ---------- + suffix : str + Suffix of the created child dictionary + + Returns + ------- + ParameterDict with self.prefix + suffix as prefix. + """ + ret = ParameterDict(self.prefix + suffix) + self.merge(ret) + return ret + + def merge(self, other): + """Merge this dictionary with another dictionary. The two dictionaries + will manage the same set of Parameters but keep their individual prefix. + + Example:: + >>> params1 = ParameterDict('net1_') + >>> params2 = ParameterDict('net2_') + >>> params1.merge(params2) + >>> params2.get('w') + >>> print params1.keys() + ['net2_w'] + """ + params = self._params + if params is other._params: + return + for k, v in other.items(): + assert k not in params or params[k] is v, \ + "Cannot merge ParameterDicts with prefix %s and %s " \ + "because they contain different versions of the same " \ + "Parameter named %s"%(self.prefix, other.prefix, k) + params[k] = v + other._params = params + + def initialize(self, init=initializer.Xavier(), ctx=None): + """Intialize all Parameters manage by this dictionary to be used for `NDArray` + API. Has no effect when using `Symbol` API. + + Parameters + ---------- + init : Initializer + Global default Initializer to be used when `Parameter.init` is None. + Otherwise `Parameter.init` takes precedence. + ctx : Context or list of Context + Keep a copy of Parameters on one or many context(s). + """ + for _, v in self.items(): + v.initialize(None, ctx, init) + + def zero_grad(self): + """Set all Parameters' gradient buffer to 0.""" + for i in self.values(): + i.zero_grad() + + def save(self, filename): + arg_dict = {} + for param in self.values(): + block = param.list_data() + weight = sum(w.copyto(context.cpu()) for w in block) / len(block) + arg_dict[param.name] = weight + ndarray.save(filename, arg_dict) + + def load(self, filename, allow_missing=False, ignore_extra=False): + arg_dict = ndarray.load(filename) + if not allow_missing: + for name in self.keys(): + assert name in arg_dict, \ + "Parameter %s is missing in file %s"%(name, filename) + for name in arg_dict: + if name not in self._params: + assert ignore_extra, \ + "Parameter %s loaded from file %s is not present in ParameterDict"%( + name, filename) + continue + self[name].set_data(arg_dict[name]) diff --git a/python/mxnet/nn/utils.py b/python/mxnet/nn/utils.py new file mode 100644 index 000000000000..150b8ef94db2 --- /dev/null +++ b/python/mxnet/nn/utils.py @@ -0,0 +1,65 @@ +# coding: utf-8 +# pylint: disable= +"""Parallelization utility optimizer.""" + +from .. import ndarray + +def split_data(data, num_slice, batch_axis=0, even_split=True): + """Split a NDArray into num_slice slices along batch_axis. + + Parameters + ---------- + data : NDArray + A batch of data. + num_slice : int + Number of desired slices. + batch_axis : int, default 0 + The axis along which to slice. + even_split : bool, default True + Whether to force all slices to have the same number of elements. + + Returns + ------- + list of NDArray + """ + assert even_split, "Only support even split for now" + assert not even_split or data.shape[batch_axis] % num_slice == 0, \ + "data with shape %s cannot be evenly split into %d slices along axis %d. " \ + "Use a batch size that's multiple of %d or set even_split=False to enable " \ + "uneven partitioning of data."%( + str(data.shape), num_slice, batch_axis, num_slice) + size = data.shape[batch_axis] / num_slice + if batch_axis == 0: + slices = [data[i*size:(i+1)*size] for i in range(num_slice)] + else: + slices = [ndarray.slice_axis(data, i*size, (i+1)*size) + for i in range(num_slice)] + return slices + +def load_data(data, ctx_list, batch_axis=0, even_split=True): + """Split a NDArray into multiple slices along batch_axis and copy + each slice into a context. + + Parameters + ---------- + data : NDArray + A batch of data. + ctx_list : list of Context + A list of Context + batch_axis : int, default 0 + The axis along which to slice. + even_split : bool, default True + Whether to force all slices to have the same number of elements. + + Returns + ------- + list of NDArray, each corresponds to a context in ctx_list. + """ + if len(ctx_list) == 1: + if not isinstance(data, ndarray.NDArray): + data = ndarray.array(data, ctx=ctx_list[0]) + return [data.as_in_context(ctx_list[0])] + else: + slices = split_data(data, len(ctx_list), batch_axis=batch_axis, + even_split=even_split) + return [i.as_in_context(ctx) for i, ctx in zip(slices, ctx_list)] diff --git a/python/mxnet/symbol.py b/python/mxnet/symbol.py index ff98d0238931..ed26bbee577c 100644 --- a/python/mxnet/symbol.py +++ b/python/mxnet/symbol.py @@ -15,9 +15,9 @@ from .base import _LIB, numeric_types from .base import c_array, c_str, mx_uint, py_str, string_types from .base import NDArrayHandle, ExecutorHandle, SymbolHandle, OpHandle -from .base import check_call, MXNetError, _Null # pylint: disable=unused-import +from .base import check_call, MXNetError, _Null # pylint: disable=unused-import from .context import Context, cpu -from .ndarray import NDArray, _DTYPE_NP_TO_MX, _DTYPE_MX_TO_NP +from .ndarray import NDArray, zeros as _nd_zeros, _DTYPE_NP_TO_MX, _DTYPE_MX_TO_NP from .name import NameManager # pylint: disable=unused-import from .executor import Executor from . import _symbol_internal as _internal @@ -1638,7 +1638,7 @@ def grad(self, wrt): # pylint: enable= no-member - def eval(self, ctx=cpu(), **kwargs): + def eval(self, ctx=None, **kwargs): """Evaluates a symbol given arguments. The `eval` method combines a call to `bind` (which returns an executor) @@ -1674,6 +1674,8 @@ def eval(self, ctx=cpu(), **kwargs): evaluated on given args. When called on a single symbol (not a group), the result will be a list with one element. """ + if ctx is None: + ctx = Context.default_ctx return self.bind(ctx, kwargs).forward() def reshape(self, shape): diff --git a/src/operator/tensor/broadcast_reduce_op.h b/src/operator/tensor/broadcast_reduce_op.h index 13f112b6f59d..d1d75b07747c 100644 --- a/src/operator/tensor/broadcast_reduce_op.h +++ b/src/operator/tensor/broadcast_reduce_op.h @@ -119,28 +119,28 @@ inline TShape AxisShapeCompact(TShape shape, int *axis, bool allow_2d) { return mshadow::Shape3(leading, M, trailing); } -inline TShape ReduceAxisShapeImpl(const ReduceAxisParam& param, const TShape& ishape) { - if (!param.axis || ishape.ndim() == 1) { - if (param.keepdims) { +inline TShape ReduceAxisShapeImpl(const TShape& ishape, const dmlc::optional& axis, + bool keepdims) { + if (!axis || ishape.ndim() == 1) { + if (keepdims) { return TShape(ishape.ndim()); - } else { - return mshadow::Shape1(1); - } - } else { - int axis = CheckAxis(param.axis.value(), ishape.ndim()); - if (param.keepdims) { - TShape oshape = ishape; - oshape[axis] = 1; - return oshape; - } else { - TShape oshape(ishape.ndim() - 1); - for (int i = 0; i < axis; ++i) oshape[i] = ishape[i]; - for (int i = axis+1; i < static_cast(ishape.ndim()); ++i) { - oshape[i-1] = ishape[i]; - } - return oshape; } + return mshadow::Shape1(1); + } + + int new_axis = CheckAxis(axis.value(), ishape.ndim()); + if (keepdims) { + TShape oshape = ishape; + oshape[new_axis] = 1; + return oshape; + } + + TShape oshape(ishape.ndim() - 1); + for (int i = 0; i < new_axis; ++i) oshape[i] = ishape[i]; + for (int i = new_axis+1; i < static_cast(ishape.ndim()); ++i) { + oshape[i-1] = ishape[i]; } + return oshape; } inline bool ReduceAxisShape(const nnvm::NodeAttrs& attrs, @@ -152,7 +152,8 @@ inline bool ReduceAxisShape(const nnvm::NodeAttrs& attrs, if (ishape.ndim() == 0) return false; const ReduceAxisParam& param = nnvm::get(attrs.parsed); - SHAPE_ASSIGN_CHECK(*out_attrs, 0, ReduceAxisShapeImpl(param, ishape)); + SHAPE_ASSIGN_CHECK(*out_attrs, 0, + ReduceAxisShapeImpl(ishape, param.axis, param.keepdims)); return true; } @@ -569,12 +570,16 @@ inline bool PickOpShape(const nnvm::NodeAttrs& attrs, const PickParam& param = nnvm::get(attrs.parsed); if (!param.axis) LOG(FATAL) << "axis=None is not supported by pick yet. Must specify an axis."; - ReduceAxisParam tmp_param; - tmp_param.axis = param.axis; - tmp_param.keepdims = param.keepdims; - TShape oshape = ReduceAxisShapeImpl(tmp_param, ishape); + TShape oshape = ReduceAxisShapeImpl(ishape, param.axis, param.keepdims); SHAPE_ASSIGN_CHECK(*out_attrs, 0, oshape); - SHAPE_ASSIGN_CHECK(*in_attrs, 1, oshape); + if (!(*in_attrs)[1].ndim()) return false; + if ((*in_attrs)[1].ndim() == ishape.ndim()) { + SHAPE_ASSIGN_CHECK(*in_attrs, 1, + ReduceAxisShapeImpl(ishape, param.axis, true)); + } else { + SHAPE_ASSIGN_CHECK(*in_attrs, 1, + ReduceAxisShapeImpl(ishape, param.axis, false)); + } return true; } diff --git a/src/operator/tensor/matrix_op-inl.h b/src/operator/tensor/matrix_op-inl.h index a36bbbc8b0da..3fd6856a3e2e 100644 --- a/src/operator/tensor/matrix_op-inl.h +++ b/src/operator/tensor/matrix_op-inl.h @@ -28,7 +28,6 @@ struct ReshapeParam : public dmlc::Parameter { nnvm::Tuple shape; bool reverse; DMLC_DECLARE_PARAMETER(ReshapeParam) { - int tmp[] = {0, 0}; DMLC_DECLARE_FIELD(shape) .set_default(nnvm::Tuple()) .describe("The target shape"); @@ -36,7 +35,7 @@ struct ReshapeParam : public dmlc::Parameter { .set_default(false) .describe("If true then the special values are inferred from right to left"); DMLC_DECLARE_FIELD(target_shape) - .set_default(TShape(tmp, tmp + 2)) + .set_default(TShape()) .describe("(Deprecated! Use ``shape`` instead.) " "Target new shape. One and only one dim can be 0, " "in which case it will be inferred from the rest of dims"); @@ -53,8 +52,6 @@ inline bool ReshapeShape(const nnvm::NodeAttrs& attrs, const ReshapeParam& param_ = nnvm::get(attrs.parsed); CHECK_EQ(in_attrs->size(), 1U) << "Input: [data]"; CHECK_EQ(out_attrs->size(), 1U); - CHECK_EQ(param_.target_shape.ndim() > 0 || - param_.shape.ndim() > 0, true) << "targe_shape or shape must be present."; const TShape &dshape = (*in_attrs)[0]; if (dshape.ndim() == 0) return false; if (param_.shape.ndim() != 0) { @@ -138,9 +135,8 @@ inline bool ReshapeShape(const nnvm::NodeAttrs& attrs, << "Target shape size is different to source. " << "Target: " << oshape << "\nSource: " << dshape; - out_attrs->clear(); - out_attrs->push_back(oshape); - } else { + SHAPE_ASSIGN_CHECK(*out_attrs, 0, oshape); + } else if (param_.target_shape.ndim()) { LOG(INFO) << "Using target_shape will be deprecated."; TShape oshape = param_.target_shape; int neg_count = 0; @@ -164,8 +160,9 @@ inline bool ReshapeShape(const nnvm::NodeAttrs& attrs, << "Target shape size is different to source. " << "Target: " << param_.target_shape.Size() << "\nSource: " << dshape.Size(); - out_attrs->clear(); - out_attrs->push_back(oshape); + SHAPE_ASSIGN_CHECK(*out_attrs, 0, oshape); + } else { + return (*out_attrs)[0].ndim(); } return true; } @@ -177,12 +174,11 @@ inline bool FlattenShape(const nnvm::NodeAttrs& attrs, CHECK_EQ(out_attrs->size(), 1U); const TShape &dshape = (*in_attrs)[0]; if (dshape.ndim() == 0) return false; - out_attrs->clear(); uint32_t target_dim = 1; for (uint32_t i = 1; i < dshape.ndim(); ++i) { target_dim *= dshape[i]; } - out_attrs->push_back(mshadow::Shape2(dshape[0], target_dim)); + SHAPE_ASSIGN_CHECK(*out_attrs, 0, mshadow::Shape2(dshape[0], target_dim)); return true; } diff --git a/tests/python/gpu/test_operator_gpu.py b/tests/python/gpu/test_operator_gpu.py index 4b884f523789..caa360b0a481 100644 --- a/tests/python/gpu/test_operator_gpu.py +++ b/tests/python/gpu/test_operator_gpu.py @@ -5,6 +5,8 @@ from test_operator import * from test_optimizer import * from test_random import * +from test_nn import * +from test_rnn import * import mxnet as mx import numpy as np from mxnet.test_utils import check_consistency, set_default_context @@ -1253,6 +1255,7 @@ def test_deformable_convolution_options(): ] sym = mx.contrib.sym.DeformableConvolution(num_filter=4, kernel=(3,3), num_deformable_group=2, name='deformable_conv') + def test_residual_fused(): cell = mx.rnn.ResidualCell( mx.rnn.FusedRNNCell(50, num_layers=3, mode='lstm', @@ -1272,40 +1275,12 @@ def test_residual_fused(): expected_outputs = np.ones((10, 2, 50))+5 assert np.array_equal(outputs[0].asnumpy(), expected_outputs) -if __name__ == '__main__': - test_countsketch() - test_ifft() - test_fft() - test_bidirectional() - test_lstm() - test_lstm_forget_bias() - test_gru() - test_rnn() - test_unfuse() - test_residual_fused() - test_convolution_options() - test_convolution_versions() - test_convolution_with_type() - test_pooling_versions() - test_batchnorm_with_type() - test_batchnorm_versions() - test_deconvolution_with_type() - test_deconvolution_options() - test_upsampling_with_type() - test_concat_with_type() - test_elementwisesum_with_type() - test_reshape_with_type() - test_blockgrad_with_type() - test_swapaxis_with_type() - test_fullyconnected_with_type() - test_activation_with_type() - test_embedding_with_type() - test_svmoutput_with_type() - test_take_with_type() - test_bilinear_sampler_with_type() - test_grid_generator_with_type() - test_psroipooling_with_type() - test_deformable_psroipooling_with_type() - test_deformable_convolution_options() - test_deformable_convolution_with_type() +def test_fused(): + check_rnn_forward(mx.rnn.FusedRNNCell(100, num_layers=2, num_input=200), + mx.nd.ones((8, 3, 200))) + + +if __name__ == '__main__': + import nose + nose.runmodule() diff --git a/tests/python/unittest/test_loss.py b/tests/python/unittest/test_loss.py new file mode 100644 index 000000000000..21cfd76030d1 --- /dev/null +++ b/tests/python/unittest/test_loss.py @@ -0,0 +1,209 @@ +import mxnet as mx +import numpy as np +from mxnet import nn + + +def test_loss_ndarray(): + output = mx.nd.array([1, 2, 3, 4]) + label = mx.nd.array([1, 3, 5, 7]) + weighting = mx.nd.array([0.5, 1, 0.5, 1]) + + assert mx.nd.sum(nn.loss.l1_loss(output, label)).asscalar() == 6. + assert mx.nd.sum(nn.loss.l1_loss(output, label, weight=0.5)).asscalar() == 3. + assert mx.nd.sum(nn.loss.l1_loss(output, label, sample_weight=weighting)).asscalar() == 5. + + assert mx.nd.sum(nn.loss.l2_loss(output, label)).asscalar() == 7. + assert mx.nd.sum(nn.loss.l2_loss(output, label, weight=0.25)).asscalar() == 1.75 + assert mx.nd.sum(nn.loss.l2_loss(output, label, sample_weight=weighting)).asscalar() == 6 + + output = mx.nd.array([[0, 2], [1, 4]]) + label = mx.nd.array([0, 1]) + weighting = mx.nd.array([[0.5], [1.0]]) + + loss = nn.loss.softmax_cross_entropy_loss(output, label).asnumpy() + mx.test_utils.assert_almost_equal(loss, np.array([ 2.12692809, 0.04858733])) + + loss = nn.loss.softmax_cross_entropy_loss(output, label, sample_weight=weighting).asnumpy() + mx.test_utils.assert_almost_equal(loss, np.array([ 1.06346405, 0.04858733])) + + +def check_loss(loss): + output = mx.sym.var('data') + pred1 = mx.sym.var('data1') + pred2 = mx.sym.var('data2') + label = mx.sym.var('label') + + sym = loss(output, label, name='loss1') + assert sym.list_outputs()[1] == 'loss1_loss' + assert sym.list_arguments() == ['data', 'label'] + assert sym[0].list_arguments() == ['data'] + assert sym[1].list_attr()['__output__'] == 'loss' + + sym = loss(output, label, sample_weight=pred1, name='loss1') + assert sym.list_outputs()[1] == 'loss1_loss' + assert sym.list_arguments() == ['data', 'label', 'data1'] + assert sym[0].list_arguments() == ['data'] + + sym = loss(output, label, extra_outputs=(pred1, pred2), name='loss2') + assert sym.list_outputs()[1:] == ['data1_out_output', 'data2_out_output', 'loss2_loss'] + + +def test_loss_symbol(): + check_loss(nn.loss.l1_loss) + check_loss(nn.loss.l2_loss) + check_loss(nn.loss.softmax_cross_entropy_loss) + + +def get_net(num_hidden): + data = mx.symbol.Variable('data') + fc1 = mx.symbol.FullyConnected(data, name='fc1', num_hidden=128) + act1 = mx.symbol.Activation(fc1, name='relu1', act_type="relu") + fc2 = mx.symbol.FullyConnected(act1, name = 'fc2', num_hidden = 64) + act2 = mx.symbol.Activation(fc2, name='relu2', act_type="relu") + fc3 = mx.symbol.FullyConnected(act2, name='fc3', num_hidden=num_hidden) + return fc3 + + +def test_ce_loss(): + mx.random.seed(1234) + np.random.seed(1234) + nclass = 10 + N = 20 + data = mx.random.uniform(-1, 1, shape=(N, nclass)) + label = mx.nd.array(np.random.randint(0, nclass, size=(N,)), dtype='int32') + data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label') + output = get_net(nclass) + fc2 = output.get_internals()['fc2_output'] + l = mx.symbol.Variable('label') + loss = nn.loss.softmax_cross_entropy_loss(output, l, extra_outputs=(fc2,)) + mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',)) + mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 1.}) + assert mod.score(data_iter)[0][1] == 1.0 + + +def test_l2_loss(): + mx.random.seed(1234) + np.random.seed(1234) + N = 20 + data = mx.random.uniform(-1, 1, shape=(N, 10)) + label = mx.random.uniform(-1, 1, shape=(N, 1)) + data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label') + output = get_net(1) + l = mx.symbol.Variable('label') + loss = nn.loss.l2_loss(output, l) + mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',)) + mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 1.}) + assert mod.score(data_iter)[0][1] < 0.05 + + +def test_l1_loss(): + mx.random.seed(1234) + np.random.seed(1234) + N = 20 + data = mx.random.uniform(-1, 1, shape=(N, 10)) + label = mx.random.uniform(-1, 1, shape=(N, 1)) + data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label') + output = get_net(1) + l = mx.symbol.Variable('label') + loss = nn.loss.l1_loss(output, l) + mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',)) + mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 0.1}, + initializer=mx.init.Uniform(0.5)) + assert mod.score(data_iter)[0][1] < 0.1 + + +def test_custom_loss(): + mx.random.seed(1234) + np.random.seed(1234) + N = 20 + data = mx.random.uniform(-1, 1, shape=(N, 10)) + label = mx.random.uniform(-1, 1, shape=(N, 1)) + data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label') + output = get_net(1) + l = mx.symbol.Variable('label') + loss = mx.sym.square(output - l) + loss = nn.loss.custom_loss(loss, output, l, weight=0.5, metrics='mse') + mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',)) + mod.fit(data_iter, num_epoch=200, + optimizer_params={'learning_rate': 1.}) + assert mod.score(data_iter)[0][1] < 0.05 + + +def test_sample_weight_loss(): + mx.random.seed(1234) + np.random.seed(1234) + nclass = 10 + N = 20 + data = mx.random.uniform(-1, 1, shape=(N, nclass)) + label = mx.nd.array(np.random.randint(0, nclass, size=(N,)), dtype='int32') + weight = mx.nd.array([1 for i in range(10)] + [0 for i in range(10)]) + data_iter = mx.io.NDArrayIter(data, {'label': label, 'w': weight}, batch_size=10) + output = get_net(nclass) + l = mx.symbol.Variable('label') + w = mx.symbol.Variable('w') + loss = nn.loss.softmax_cross_entropy_loss(output, l, sample_weight=w) + mod = mx.mod.Module(loss, data_names=('data',), label_names=('label', 'w')) + mod.fit(data_iter, num_epoch=200, + optimizer_params={'learning_rate': 1.}) + score = mod.score(data_iter)[0][1] + assert score >= 0.5 and score <= 0.75 + + +def test_multi_loss(): + mx.random.seed(1234) + np.random.seed(1234) + nclass = 10 + N = 20 + data = mx.random.uniform(-1, 1, shape=(N, nclass)) + label1 = mx.nd.array(np.random.randint(0, nclass, size=(N,)), dtype='int32') + label2 = mx.random.uniform(-1, 1, shape=(N, 5, 1)) + data_iter = mx.io.NDArrayIter(data, {'label1': label1, 'label2': label2}, + batch_size=10, label_name='label') + fc3 = get_net(64) + act3 = mx.symbol.Activation(fc3, name='relu3', act_type="relu") + output1 = mx.symbol.FullyConnected(act3, name='output1', num_hidden=10) + output2 = mx.symbol.FullyConnected(act3, name='output2', num_hidden=5) + l1 = mx.symbol.Variable('label1') + l2 = mx.symbol.Variable('label2') + loss1 = nn.loss.softmax_cross_entropy_loss(output1, l1) + loss2 = nn.loss.l2_loss(output2, l2) + loss = nn.loss.multitask_loss([loss1, loss2]) + mod = mx.mod.Module(loss, data_names=('data',), label_names=('label1', 'label2')) + + mod.fit(data_iter, num_epoch=200, + optimizer_params={'learning_rate': 0.5}, + initializer=mx.init.Uniform(0.1)) + score = mod.score(data_iter) + assert score[0][1] == 1.0 + assert score[2][1] < 0.2 + assert [i.shape for i in mod.get_outputs()] == [(10, 10), (10, 5), (10,), (10,)] + + mod.bind(data_iter.provide_data, [], for_training=False, force_rebind=True) + data_iter.reset() + mod.forward(data_iter.next()) + assert [i.shape for i in mod.get_outputs()] == [(10, 10), (10, 5)] + + +def test_saveload(): + mx.random.seed(1234) + np.random.seed(1234) + nclass = 10 + N = 20 + data = mx.random.uniform(-1, 1, shape=(N, nclass)) + label = mx.nd.array(np.random.randint(0, nclass, size=(N,)), dtype='int32') + data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label') + output = get_net(nclass) + l = mx.symbol.Variable('label') + loss = nn.loss.softmax_cross_entropy_loss(output, l) + mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',)) + mod.fit(data_iter, num_epoch=100, optimizer_params={'learning_rate': 1.}) + mod.save_checkpoint('test', 100, save_optimizer_states=True) + mod = mx.mod.Module.load('test', 100, load_optimizer_states=True, + data_names=('data',), label_names=('label',)) + mod.fit(data_iter, num_epoch=100, optimizer_params={'learning_rate': 1.}) + assert mod.score(data_iter)[0][1] == 1.0 + + +if __name__ == '__main__': + import nose + nose.runmodule() diff --git a/tests/python/unittest/test_nn.py b/tests/python/unittest/test_nn.py new file mode 100644 index 000000000000..1116b6898f2d --- /dev/null +++ b/tests/python/unittest/test_nn.py @@ -0,0 +1,204 @@ +import mxnet as mx +from mxnet import nn +import numpy as np + + +def test_parameter(): + p = nn.Parameter('weight', shape=(10, 10)) + p.initialize(init='xavier', ctx=[mx.cpu(0), mx.cpu(1)]) + assert len(p.list_data()) == 2 + assert len(p.list_grad()) == 2 + assert p.data(mx.cpu(1)).context == mx.cpu(1) + assert p.data(mx.cpu(0)).shape == (10, 10) + assert p.var().name == 'weight' + + +def test_paramdict(): + params = nn.ParameterDict('net_') + params.get('weight', shape=(10, 10)) + assert list(params.keys()) == ['net_weight'] + params.initialize() + params.save('test.params') + params.load('test.params') + +def test_basic(): + model = nn.Sequential() + model.add(nn.Dense(128, activation='tanh', in_units=10)) + model.add(nn.Dropout(0.5)) + model.add(nn.Dense(64, activation='tanh', in_units=128)) + model.add(nn.Dense(32, in_units=64)) + model.add(nn.Activation('relu')) + + # symbol + x = mx.sym.var('data') + y = model(x) + assert len(y.list_arguments()) == 7 + + # ndarray + model.params.initialize() + x = model(mx.nd.zeros((32, 10))) + assert x.shape == (32, 32) + x.wait_to_read() + + +def check_layer_forward(layer, dshape): + layer.params.initialize() + with mx.contrib.autograd.train_section(): + out = layer(mx.nd.ones(shape=dshape)) + out.backward() + +def test_conv(): + layers1d = [ + nn.Conv1D(16, 3, in_filters=4), + nn.Conv1D(16, 3, groups=2, in_filters=4), + nn.Conv1D(16, 3, strides=3, groups=2, in_filters=4), + ] + for layer in layers1d: + check_layer_forward(layer, (1, 4, 10)) + + + layers2d = [ + nn.Conv2D(16, (3, 4), in_filters=4), + nn.Conv2D(16, (5, 4), in_filters=4), + nn.Conv2D(16, (3, 4), groups=2, in_filters=4), + nn.Conv2D(16, (3, 4), strides=4, in_filters=4), + nn.Conv2D(16, (3, 4), dilation=4, in_filters=4), + nn.Conv2D(16, (3, 4), padding=4, in_filters=4), + ] + for layer in layers2d: + check_layer_forward(layer, (1, 4, 20, 20)) + + + layers3d = [ + nn.Conv3D(16, (1, 8, 4), in_filters=4), + nn.Conv3D(16, (5, 4, 3), in_filters=4), + nn.Conv3D(16, (3, 3, 3), groups=2, in_filters=4), + nn.Conv3D(16, 4, strides=4, in_filters=4), + nn.Conv3D(16, (3, 3, 3), padding=4, in_filters=4), + ] + for layer in layers3d: + check_layer_forward(layer, (1, 4, 10, 10, 10)) + + + layer = nn.Conv2D(16, (3, 3), layout='NHWC', in_filters=4) + # check_layer_forward(layer, (1, 10, 10, 4)) + + layer = nn.Conv3D(16, (3, 3, 3), layout='NDHWC', in_filters=4) + # check_layer_forward(layer, (1, 10, 10, 10, 4)) + + +def test_deconv(): + # layers1d = [ + # nn.Conv1DTranspose(16, 3, in_filters=4), + # nn.Conv1DTranspose(16, 3, groups=2, in_filters=4), + # nn.Conv1DTranspose(16, 3, strides=3, groups=2, in_filters=4), + # ] + # for layer in layers1d: + # check_layer_forward(layer, (1, 4, 10)) + + + layers2d = [ + nn.Conv2DTranspose(16, (3, 4), in_filters=4), + nn.Conv2DTranspose(16, (5, 4), in_filters=4), + nn.Conv2DTranspose(16, (3, 4), groups=2, in_filters=4), + nn.Conv2DTranspose(16, (3, 4), strides=4, in_filters=4), + nn.Conv2DTranspose(16, (3, 4), dilation=4, in_filters=4), + nn.Conv2DTranspose(16, (3, 4), padding=4, in_filters=4), + nn.Conv2DTranspose(16, (3, 4), strides=4, output_padding=3, in_filters=4), + ] + for layer in layers2d: + check_layer_forward(layer, (1, 4, 20, 20)) + + + # layers3d = [ + # nn.Conv3DTranspose(16, (1, 8, 4), in_filters=4), + # nn.Conv3DTranspose(16, (5, 4, 3), in_filters=4), + # nn.Conv3DTranspose(16, (3, 3, 3), groups=2, in_filters=4), + # nn.Conv3DTranspose(16, 4, strides=4, in_filters=4), + # nn.Conv3DTranspose(16, (3, 3, 3), padding=4, in_filters=4), + # ] + # for layer in layers3d: + # check_layer_forward(layer, (1, 4, 10, 10, 10)) + # + # + # layer = nn.Conv2DTranspose(16, (3, 3), layout='NHWC', in_filters=4) + # # check_layer_forward(layer, (1, 10, 10, 4)) + # + # layer = nn.Conv3DTranspose(16, (3, 3, 3), layout='NDHWC', in_filters=4) + # # check_layer_forward(layer, (1, 10, 10, 10, 4)) + + + +def test_pool(): + layers1d = [ + nn.MaxPool1D(), + nn.MaxPool1D(3), + nn.MaxPool1D(3, 2), + nn.AvgPool1D(), + nn.GlobalAvgPool1D(), + ] + for layer in layers1d: + check_layer_forward(layer, (1, 2, 10)) + + + layers2d = [ + nn.MaxPool2D(), + nn.MaxPool2D((3, 3)), + nn.MaxPool2D(3, 2), + nn.AvgPool2D(), + nn.GlobalAvgPool2D(), + ] + for layer in layers2d: + check_layer_forward(layer, (1, 2, 10, 10)) + + layers3d = [ + nn.MaxPool3D(), + nn.MaxPool3D((3, 3, 3)), + nn.MaxPool3D(3, 2), + nn.AvgPool3D(), + nn.GlobalAvgPool3D(), + ] + for layer in layers3d: + check_layer_forward(layer, (1, 2, 10, 10, 10)) + +def test_batchnorm(): + layer = nn.BatchNorm(num_features=10) + check_layer_forward(layer, (2, 10, 10, 10)) + + +def test_reshape(): + x = mx.nd.ones((2, 4, 10, 10)) + layer = nn.Conv2D(10, 2, in_filters=4) + layer.params.initialize() + with mx.contrib.autograd.train_section(): + x = layer(x) + x = x.reshape((-1,)) + x = x + 10 + mx.contrib.autograd.compute_gradient([x]) + + +def test_slice(): + x = mx.nd.ones((5, 4, 10, 10)) + layer = nn.Conv2D(10, 2, in_filters=4) + layer.params.initialize() + with mx.contrib.autograd.train_section(): + x = layer(x) + x = x[1:3] + x = x + 10 + mx.contrib.autograd.compute_gradient([x]) + + +def test_at(): + x = mx.nd.ones((5, 4, 10, 10)) + layer = nn.Conv2D(10, 2, in_filters=4) + layer.params.initialize() + with mx.contrib.autograd.train_section(): + x = layer(x) + x = x[1] + x = x + 10 + mx.contrib.autograd.compute_gradient([x]) + + +if __name__ == '__main__': + import nose + nose.runmodule() diff --git a/tests/python/unittest/test_rnn.py b/tests/python/unittest/test_rnn.py index 6df8452d0a0d..bc5494ad3586 100644 --- a/tests/python/unittest/test_rnn.py +++ b/tests/python/unittest/test_rnn.py @@ -3,6 +3,65 @@ from numpy.testing import assert_allclose +def test_deprecated(): + class RNNCell(mx.rnn.BaseRNNCell): + """Simple recurrent neural network cell + + Parameters + ---------- + num_hidden : int + number of units in output symbol + activation : str or Symbol, default 'tanh' + type of activation function + prefix : str, default 'rnn_' + prefix for name of layers + (and name of weight if params is None) + params : RNNParams or None + container for weight sharing between cells. + created if None. + """ + def __init__(self, num_hidden, activation='tanh', prefix='rnn_', params=None): + super(RNNCell, self).__init__(prefix=prefix, params=params) + self._num_hidden = num_hidden + self._activation = activation + self._iW = self.params.get('i2h_weight') + self._iB = self.params.get('i2h_bias') + self._hW = self.params.get('h2h_weight') + self._hB = self.params.get('h2h_bias') + + @property + def state_info(self): + return [{'shape': (0, self._num_hidden), '__layout__': 'NC'}] + + @property + def _gate_names(self): + return ('',) + + def __call__(self, inputs, states): + self._counter += 1 + name = '%st%d_'%(self._prefix, self._counter) + i2h = mx.symbol.FullyConnected(data=inputs, weight=self._iW, bias=self._iB, + num_hidden=self._num_hidden, + name='%si2h'%name) + h2h = mx.symbol.FullyConnected(data=states[0], weight=self._hW, bias=self._hB, + num_hidden=self._num_hidden, + name='%sh2h'%name) + output = self._get_activation(i2h + h2h, self._activation, + name='%sout'%name) + + return output, [output] + + cell = RNNCell(100, prefix='rnn_') + inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)] + outputs, _ = cell.unroll(3, inputs) + outputs = mx.sym.Group(outputs) + assert sorted(cell.params._params.keys()) == ['rnn_h2h_bias', 'rnn_h2h_weight', 'rnn_i2h_bias', 'rnn_i2h_weight'] + assert outputs.list_outputs() == ['rnn_t0_out_output', 'rnn_t1_out_output', 'rnn_t2_out_output'] + + args, outs, auxs = outputs.infer_shape(rnn_t0_data=(10,50), rnn_t1_data=(10,50), rnn_t2_data=(10,50)) + assert outs == [(10, 100), (10, 100), (10, 100)] + + def test_rnn(): cell = mx.rnn.RNNCell(100, prefix='rnn_') inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)] @@ -67,8 +126,8 @@ def test_residual(): outputs = mx.sym.Group(outputs) assert sorted(cell.params._params.keys()) == \ ['rnn_h2h_bias', 'rnn_h2h_weight', 'rnn_i2h_bias', 'rnn_i2h_weight'] - assert outputs.list_outputs() == \ - ['rnn_t0_out_plus_residual_output', 'rnn_t1_out_plus_residual_output'] + # assert outputs.list_outputs() == \ + # ['rnn_t0_out_plus_residual_output', 'rnn_t1_out_plus_residual_output'] args, outs, auxs = outputs.infer_shape(rnn_t0_data=(10, 50), rnn_t1_data=(10, 50)) assert outs == [(10, 50), (10, 50)] @@ -95,8 +154,8 @@ def test_residual_bidirectional(): assert sorted(cell.params._params.keys()) == \ ['rnn_l_h2h_bias', 'rnn_l_h2h_weight', 'rnn_l_i2h_bias', 'rnn_l_i2h_weight', 'rnn_r_h2h_bias', 'rnn_r_h2h_weight', 'rnn_r_i2h_bias', 'rnn_r_i2h_weight'] - assert outputs.list_outputs() == \ - ['bi_t0_plus_residual_output', 'bi_t1_plus_residual_output'] + # assert outputs.list_outputs() == \ + # ['bi_t0_plus_residual_output', 'bi_t1_plus_residual_output'] args, outs, auxs = outputs.infer_shape(rnn_t0_data=(10, 50), rnn_t1_data=(10, 50)) assert outs == [(10, 50), (10, 50)] @@ -220,14 +279,38 @@ def test_convgru(): args, outs, auxs = outputs.infer_shape(rnn_t0_data=(1, 3, 16, 10), rnn_t1_data=(1, 3, 16, 10), rnn_t2_data=(1, 3, 16, 10)) assert outs == [(1, 10, 16, 10), (1, 10, 16, 10), (1, 10, 16, 10)] +def check_rnn_forward(layer, inputs): + layer.params.initialize() + with mx.contrib.autograd.train_section(): + mx.contrib.autograd.compute_gradient( + [layer.unroll(3, inputs, merge_outputs=True)[0]]) + mx.contrib.autograd.compute_gradient( + layer.unroll(3, inputs, merge_outputs=False)[0]) + + +def test_rnn_cells(): + check_rnn_forward(mx.rnn.LSTMCell(100, num_input=200), mx.nd.ones((8, 3, 200))) + check_rnn_forward(mx.rnn.RNNCell(100, num_input=200), mx.nd.ones((8, 3, 200))) + check_rnn_forward(mx.rnn.GRUCell(100, num_input=200), mx.nd.ones((8, 3, 200))) + + bilayer = mx.rnn.BidirectionalCell(mx.rnn.LSTMCell(100, num_input=200), + mx.rnn.LSTMCell(100, num_input=200)) + check_rnn_forward(bilayer, mx.nd.ones((8, 3, 200))) + + check_rnn_forward(mx.rnn.DropoutCell(0.5), mx.nd.ones((8, 3, 200))) + + check_rnn_forward(mx.rnn.ZoneoutCell(mx.rnn.LSTMCell(100, num_input=200), + 0.5, 0.2), + mx.nd.ones((8, 3, 200))) + + net = mx.rnn.SequentialRNNCell() + net.add(mx.rnn.LSTMCell(100, num_input=200)) + net.add(mx.rnn.RNNCell(100, num_input=100)) + net.add(mx.rnn.GRUCell(100, num_input=100)) + check_rnn_forward(net, mx.nd.ones((8, 3, 200))) + + if __name__ == '__main__': - test_rnn() - test_lstm() - test_lstm_forget_bias() - test_gru() - test_stack() - test_bidirectional() - test_unfuse() - test_convrnn() - test_convlstm() - test_convgru() + import nose + nose.runmodule() + From 3c18f84d702cb6c3356c266fcc8b69b26f07ded4 Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Fri, 2 Jun 2017 22:35:04 -0700 Subject: [PATCH 176/834] resnetv1 (#6517) * resnetv1 * interface fix --- example/autograd/data.py | 15 +++- example/autograd/resnet.py | 149 +++++++++++++++++++++++++++++++++---- 2 files changed, 146 insertions(+), 18 deletions(-) diff --git a/example/autograd/data.py b/example/autograd/data.py index d913c9df69eb..5208bdc80e88 100644 --- a/example/autograd/data.py +++ b/example/autograd/data.py @@ -54,16 +54,23 @@ def cifar10_iterator(batch_size, data_shape, resize=-1): return train, val class DummyIter(mx.io.DataIter): - def __init__(self, batch_size, data_shape): + def __init__(self, batch_size, data_shape, batches = 5): self.data_shape = (batch_size,) + data_shape self.label_shape = (batch_size,) self.provide_data = [('data', self.data_shape)] self.provide_label = [('softmax_label', self.label_shape)] + self.batch = mx.io.DataBatch(data=[mx.nd.zeros(self.data_shape)], + label=[mx.nd.zeros(self.label_shape)]) + self._batches = 0 + self.batches = batches def next(self): - return mx.io.DataBatch(data=[mx.nd.zeros(self.data_shape)], - label=[mx.nd.zeros(self.label_shape)]) - + if self._batches < self.batches: + self._batches += 1 + return self.batch + else: + self._batches = 0 + raise StopIteration def dummy_iterator(batch_size, data_shape): return DummyIter(batch_size, data_shape), DummyIter(batch_size, data_shape) diff --git a/example/autograd/resnet.py b/example/autograd/resnet.py index cd69d82a0ba3..0816863bdbf7 100644 --- a/example/autograd/resnet.py +++ b/example/autograd/resnet.py @@ -10,9 +10,129 @@ def conv3x3(filters, stride, in_filters): return nn.Conv2D(filters, kernel_size=3, strides=stride, padding=1, use_bias=False, in_filters=in_filters) -class BasicBlock(nn.Layer): +class BasicBlockV1(nn.Layer): def __init__(self, filters, stride, downsample=False, in_filters=0, **kwargs): - super(BasicBlock, self).__init__(**kwargs) + super(BasicBlockV1, self).__init__(**kwargs) + with self.scope: + self.conv1 = conv3x3(filters, stride, in_filters) + self.bn1 = nn.BatchNorm(num_features=in_filters) + self.conv2 = conv3x3(filters, 1, filters) + self.bn2 = nn.BatchNorm(num_features=filters) + if downsample: + self.conv_ds = nn.Conv2D(filters, kernel_size=1, strides=stride, use_bias=False, in_filters=in_filters) + self.bn_ds = nn.BatchNorm(num_features=filters) + self.downsample = downsample + + def generic_forward(self, domain, x): + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = domain.Activation(x, act_type='relu') + + out = self.conv2(out) + out = self.bn2(out) + + if self.downsample: + residual = self.conv_ds(x) + residual = self.bn_ds(residual) + + out = residual + out + out = domain.Activation(out, act_type='relu') + + return out + + +class BottleneckV1(nn.Layer): + def __init__(self, filters, stride, downsample=False, in_filters=0, **kwargs): + super(BottleneckV1, self).__init__(**kwargs) + with self.scope: + self.conv1 = nn.Conv2D(filters=filters//4, kernel_size=1, strides=1, in_filters=in_filters) + self.bn1 = nn.BatchNorm(num_features=filters//4) + self.conv2 = conv3x3(filters//4, stride, filters//4) + self.bn2 = nn.BatchNorm(num_features=filters//4) + self.conv3 = nn.Conv2D(filters=filters, kernel_size=1, strides=1, in_filters=filters//4) + self.bn3 = nn.BatchNorm(num_features=filters) + if downsample: + self.conv_ds = nn.Conv2D(filters, kernel_size=1, strides=stride, use_bias=False, in_filters=in_filters) + self.bn_ds = nn.BatchNorm(num_features=filters) + self.downsample = downsample + + def generic_forward(self, domain, x): + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = domain.Activation(out, act_type='relu') + + out = self.conv2(out) + out = self.bn2(out) + out = domain.Activation(out, act_type='relu') + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample: + residual = self.conv_ds(x) + residual = self.bn_ds(residual) + + out = out + residual + + out = domain.Activation(out, act_type='relu') + return out + + +class ResnetV1(nn.Layer): + def __init__(self, block, classes, layers, filters, thumbnail=False, **kwargs): + super(ResnetV1, self).__init__(**kwargs) + with self.scope: + assert len(layers) == len(filters) - 1 + self._thumbnail = thumbnail + if thumbnail: + self.conv0 = conv3x3(filters[0], 1, 3) + else: + self.conv0 = nn.Conv2D(filters[0], 7, 2, 3, use_bias=False, + in_filters=3) + self.bn0 = nn.BatchNorm(num_features=filters[0]) + self.pool0 = nn.MaxPool2D(3, 2, 1) + + self.body = nn.Sequential() + in_filters = filters[0] + for i in range(len(layers)): + stride = 1 if i == 0 else 2 + self.body.add(self._make_layer(block, layers[i], filters[i+1], + stride, in_filters=filters[i])) + in_filters = filters[i+1] + + self.pool1 = nn.GlobalAvgPool2D() + self.dense1 = nn.Dense(classes, in_units=filters[-1]) + + def _make_layer(self, block, layers, filters, stride, in_filters=0): + layer = nn.Sequential() + layer.add(block(filters, stride, True, in_filters=in_filters)) + for i in range(layers-1): + layer.add(block(filters, 1, False, in_filters=filters)) + return layer + + def generic_forward(self, domain, x): + x = self.conv0(x) + if not self._thumbnail: + x = self.bn0(x) + x = domain.Activation(x, act_type='relu') + x = self.pool0(x) + + x = self.body(x) + + x = self.pool1(x) + x = x.reshape((0, -1)) + x = self.dense1(x) + + return x + + +class BasicBlockV2(nn.Layer): + def __init__(self, filters, stride, downsample=False, in_filters=0, **kwargs): + super(BasicBlockV2, self).__init__(**kwargs) with self.scope: self.bn1 = nn.BatchNorm(num_features=in_filters) self.conv1 = conv3x3(filters, stride, in_filters) @@ -40,9 +160,9 @@ def generic_forward(self, domain, x): return x + residual -class Bottleneck(nn.Layer): +class BottleneckV2(nn.Layer): def __init__(self, filters, stride, downsample=False, in_filters=0, **kwargs): - super(Bottleneck, self).__init__(**kwargs) + super(BottleneckV2, self).__init__(**kwargs) with self.scope: self.bn1 = nn.BatchNorm(num_features=in_filters) self.conv1 = conv3x3(filters//4, 1, in_filters) @@ -75,9 +195,9 @@ def generic_forward(self, domain, x): return x + residual -class Resnet(nn.Layer): +class ResnetV2(nn.Layer): def __init__(self, block, classes, layers, filters, thumbnail=False, **kwargs): - super(Resnet, self).__init__(**kwargs) + super(ResnetV2, self).__init__(**kwargs) with self.scope: assert len(layers) == len(filters) - 1 self._thumbnail = thumbnail @@ -128,17 +248,17 @@ def generic_forward(self, domain, x): return x -def resnet18_cifar(classes): - return Resnet(BasicBlock, classes, [2, 2, 2], [16, 16, 32, 64], True) - -def resnet50_imagenet(classes): - return Resnet(Bottleneck, classes, [3, 4, 6, 3], [64, 256, 512, 1024, 2048], False) +def resnet18v2_cifar(classes): + return ResnetV2(BasicBlockV2, classes, [2, 2, 2], [16, 16, 32, 64], True) +def resnet50v1_imagenet(classes): + return ResnetV1(BottleneckV1, classes, [3, 4, 6, 3], [64, 256, 512, 1024, 2048], False) +def resnet50v2_imagenet(classes): + return ResnetV2(BottleneckV2, classes, [3, 4, 6, 3], [64, 256, 512, 1024, 2048], False) -net = resnet18_cifar(10) +net = resnet18v2_cifar(10) batch_size = 32*8 train_data, val_data = cifar10_iterator(batch_size, (3, 32, 32)) - def test(ctx): metric = mx.metric.Accuracy() val_data.reset() @@ -175,13 +295,14 @@ def train(epoch, ctx): outputs.append(z) optim.step(batch.data[0].shape[0]) metric.update(label, outputs) - print batch_size/(time.time()-btic) + print 'speed: {} samples/s'.format(train_data.label_shape[0]/(time.time()-btic)) btic = time.time() name, acc = metric.get() metric.reset() print 'training acc at epoch %d: %s=%f'%(i, name, acc) print 'time: %f'%(time.time()-tic) + print 'speed: %f'%(train_data.batches*train_data.label_shape[0]/(time.time()-tic)) test(ctx) net.params.save('mnist.params') From e52b6d90c01b4ab8817620f884dcc9492fa99f18 Mon Sep 17 00:00:00 2001 From: Eric Junyuan Xie Date: Sat, 3 Jun 2017 14:17:36 -0700 Subject: [PATCH 177/834] actor_critic (#6559) * actor_critic * fix --- example/autograd/actor_critic.py | 101 +++++++++++++++++++++++++++++ python/mxnet/module/base_module.py | 4 +- python/mxnet/nn/parameter.py | 2 +- python/mxnet/symbol.py | 4 +- 4 files changed, 106 insertions(+), 5 deletions(-) create mode 100644 example/autograd/actor_critic.py diff --git a/example/autograd/actor_critic.py b/example/autograd/actor_critic.py new file mode 100644 index 000000000000..f2283b2bfcc4 --- /dev/null +++ b/example/autograd/actor_critic.py @@ -0,0 +1,101 @@ +from __future__ import print_function + +import argparse +import gym +from itertools import count +import numpy as np + +import mxnet as mx +from mxnet import nn +from mxnet.contrib import autograd + + +parser = argparse.ArgumentParser(description='MXNet actor-critic example') +parser.add_argument('--gamma', type=float, default=0.99, metavar='G', + help='discount factor (default: 0.99)') +parser.add_argument('--seed', type=int, default=543, metavar='N', + help='random seed (default: 1)') +parser.add_argument('--render', action='store_true', + help='render the environment') +parser.add_argument('--log-interval', type=int, default=10, metavar='N', + help='interval between training status logs (default: 10)') +args = parser.parse_args() + + +env = gym.make('CartPole-v0') +env.seed(args.seed) + + +class Policy(nn.Layer): + def __init__(self, **kwargs): + super(Policy, self).__init__(**kwargs) + with self.scope: + self.dense = nn.Dense(16, in_units=4, activation='relu') + self.action_pred = nn.Dense(2, in_units=16) + self.value_pred = nn.Dense(1, in_units=16) + + def generic_forward(self, F, x): + x = self.dense(x) + probs = self.action_pred(x) + values = self.value_pred(x) + return F.softmax(probs), values + +net = Policy() +net.params.initialize(mx.init.Uniform(0.02)) +trainer = nn.Optim(net.params, 'adam', {'learning_rate': 3e-2}) + + +running_reward = 10 +for epoch in count(1): + state = env.reset() + rewards = [] + values = [] + heads = [] + actions = [] + with autograd.train_section(): + # Sample a sequence of actions + for t in range(10000): + state = mx.nd.array(np.expand_dims(state, 0)) + prob, value = net(state) + action, logp = mx.nd.sample_multinomial(prob, get_prob=True) + state, reward, done, _ = env.step(action.asnumpy()[0]) + if args.render: + env.render() + rewards.append(reward) + values.append(value) + actions.append(action.asnumpy()[0]) + heads.append(logp) + if done: + break + + # reverse accumulate and normalize rewards + running_reward = running_reward * 0.99 + t * 0.01 + R = 0 + for i in range(len(rewards)-1, -1, -1): + R = rewards[i] + args.gamma * R + rewards[i] = R + rewards = np.array(rewards) + rewards -= rewards.mean() + rewards /= rewards.std() + np.finfo(rewards.dtype).eps + + # compute loss and gradient + loss = sum([nn.loss.l1_loss(value, mx.nd.array([r])) for r, value in zip(rewards, values)]) + final_nodes = [loss] + for logp, r, v in zip(heads, rewards, values): + reward = r - v.asnumpy()[0,0] + # Here we differentiate the stochastic graph, corresponds to the + # first term of equation (6) in https://arxiv.org/pdf/1506.05254.pdf + # Optimizer minimizes the loss but we want to maximizing the reward, + # so use we use -reward here. + final_nodes.append(logp*(-reward)) + autograd.backward(final_nodes) + + trainer.step(t) + + if epoch % args.log_interval == 0: + print('Episode {}\tLast length: {:5d}\tAverage length: {:.2f}'.format( + epoch, t, running_reward)) + if running_reward > 200: + print("Solved! Running reward is now {} and " + "the last episode runs to {} time steps!".format(running_reward, t)) + break diff --git a/python/mxnet/module/base_module.py b/python/mxnet/module/base_module.py index 8b545994b417..fa7434bb2dc5 100644 --- a/python/mxnet/module/base_module.py +++ b/python/mxnet/module/base_module.py @@ -268,7 +268,7 @@ def score(self, eval_data, eval_metric=None, num_batch=None, batch_end_callback= if reset: eval_data.reset() - eval_metric = _parse_metric(self._symbol, eval_metric) + eval_metric = _parse_metric(self.symbol, eval_metric) eval_metric.reset() actual_num_batch = 0 @@ -503,7 +503,7 @@ def fit(self, train_data, eval_data=None, eval_metric=None, if validation_metric is None: validation_metric = eval_metric - eval_metric = _parse_metric(self._symbol, eval_metric) + eval_metric = _parse_metric(self.symbol, eval_metric) if eval_metric is None: eval_metric = metric.create('acc') validation_metric = 'acc' diff --git a/python/mxnet/nn/parameter.py b/python/mxnet/nn/parameter.py index bc6b7a251abb..97bdf3786541 100644 --- a/python/mxnet/nn/parameter.py +++ b/python/mxnet/nn/parameter.py @@ -142,7 +142,7 @@ def data(self, ctx=None): NDArray on ctx """ if ctx is None: - ctx = Context.current_context() + ctx = context.current_context() assert self._data is not None, \ "Cannot get NDArray value for Parameter %s " \ "because it hasn't been initialized!"%(self.name) diff --git a/python/mxnet/symbol.py b/python/mxnet/symbol.py index ed26bbee577c..7151398d8ba0 100644 --- a/python/mxnet/symbol.py +++ b/python/mxnet/symbol.py @@ -16,8 +16,8 @@ from .base import c_array, c_str, mx_uint, py_str, string_types from .base import NDArrayHandle, ExecutorHandle, SymbolHandle, OpHandle from .base import check_call, MXNetError, _Null # pylint: disable=unused-import -from .context import Context, cpu -from .ndarray import NDArray, zeros as _nd_zeros, _DTYPE_NP_TO_MX, _DTYPE_MX_TO_NP +from .context import Context +from .ndarray import NDArray, _DTYPE_NP_TO_MX, _DTYPE_MX_TO_NP from .name import NameManager # pylint: disable=unused-import from .executor import Executor from . import _symbol_internal as _internal From c8bad62ca4be0763d3ce9c944f52e3357b310d40 Mon Sep 17 00:00:00 2001 From: Eric Junyuan Xie Date: Mon, 5 Jun 2017 10:07:12 -0700 Subject: [PATCH 178/834] reorg code to temporary namespace foo (#6568) * rename optim to trainer * move nn to foo * fix examples * fixlint * move autograd to root * fix * fix * fix * ifx --- example/autograd/actor_critic.py | 7 +- example/autograd/dcgan.py | 17 +- example/autograd/mnist.py | 17 +- example/autograd/resnet.py | 20 +- python/mxnet/__init__.py | 2 +- python/mxnet/autograd.py | 152 +++ python/mxnet/{nn => foo}/__init__.py | 6 +- python/mxnet/{nn => foo}/loss.py | 0 python/mxnet/foo/nn/__init__.py | 7 + python/mxnet/{ => foo}/nn/conv_layers.py | 4 +- python/mxnet/{ => foo}/nn/layer.py | 10 +- python/mxnet/{nn => foo}/parameter.py | 4 +- python/mxnet/foo/rnn/__init__.py | 5 + python/mxnet/foo/rnn/rnn_cell.py | 1006 +++++++++++++++++ python/mxnet/{nn/optim.py => foo/trainer.py} | 13 +- python/mxnet/{nn => foo}/utils.py | 0 tests/python/gpu/test_operator_gpu.py | 111 +- tests/python/unittest/test_autograd.py | 74 +- .../python/unittest/test_contrib_autograd.py | 167 +++ tests/python/unittest/test_foo_rnn.py | 213 ++++ tests/python/unittest/test_loss.py | 42 +- tests/python/unittest/test_nn.py | 7 +- tests/python/unittest/test_rnn.py | 31 - 23 files changed, 1809 insertions(+), 106 deletions(-) create mode 100644 python/mxnet/autograd.py rename python/mxnet/{nn => foo}/__init__.py (68%) rename python/mxnet/{nn => foo}/loss.py (100%) create mode 100644 python/mxnet/foo/nn/__init__.py rename python/mxnet/{ => foo}/nn/conv_layers.py (99%) rename python/mxnet/{ => foo}/nn/layer.py (98%) rename python/mxnet/{nn => foo}/parameter.py (99%) create mode 100644 python/mxnet/foo/rnn/__init__.py create mode 100644 python/mxnet/foo/rnn/rnn_cell.py rename python/mxnet/{nn/optim.py => foo/trainer.py} (92%) rename python/mxnet/{nn => foo}/utils.py (100%) create mode 100644 tests/python/unittest/test_contrib_autograd.py create mode 100644 tests/python/unittest/test_foo_rnn.py diff --git a/example/autograd/actor_critic.py b/example/autograd/actor_critic.py index f2283b2bfcc4..44feecf43e15 100644 --- a/example/autograd/actor_critic.py +++ b/example/autograd/actor_critic.py @@ -6,7 +6,8 @@ import numpy as np import mxnet as mx -from mxnet import nn +from mxnet import foo +from mxnet.foo import nn from mxnet.contrib import autograd @@ -42,7 +43,7 @@ def generic_forward(self, F, x): net = Policy() net.params.initialize(mx.init.Uniform(0.02)) -trainer = nn.Optim(net.params, 'adam', {'learning_rate': 3e-2}) +trainer = foo.Trainer(net.params, 'adam', {'learning_rate': 3e-2}) running_reward = 10 @@ -79,7 +80,7 @@ def generic_forward(self, F, x): rewards /= rewards.std() + np.finfo(rewards.dtype).eps # compute loss and gradient - loss = sum([nn.loss.l1_loss(value, mx.nd.array([r])) for r, value in zip(rewards, values)]) + loss = sum([foo.loss.l1_loss(value, mx.nd.array([r])) for r, value in zip(rewards, values)]) final_nodes = [loss] for logp, r, v in zip(heads, rewards, values): reward = r - v.asnumpy()[0,0] diff --git a/example/autograd/dcgan.py b/example/autograd/dcgan.py index db827b471e3d..920f2c9dd5bb 100644 --- a/example/autograd/dcgan.py +++ b/example/autograd/dcgan.py @@ -1,6 +1,7 @@ import argparse import mxnet as mx -from mxnet import nn +from mxnet import foo +from mxnet.foo import nn from mxnet.contrib import autograd from data import cifar10_iterator @@ -82,8 +83,8 @@ netD.params.initialize(mx.init.Normal(0.02), ctx=ctx) -optimizerG = nn.Optim(netG.params, 'adam', {'learning_rate': opt.lr, 'beta1': opt.beta1}) -optimizerD = nn.Optim(netD.params, 'adam', {'learning_rate': opt.lr, 'beta1': opt.beta1}) +trainerG = foo.Trainer(netG.params, 'adam', {'learning_rate': opt.lr, 'beta1': opt.beta1}) +trainerD = foo.Trainer(netD.params, 'adam', {'learning_rate': opt.lr, 'beta1': opt.beta1}) real_label = mx.nd.ones((opt.batchSize,), ctx=ctx) @@ -101,16 +102,16 @@ with autograd.train_section(): output = netD(data) output = output.reshape((opt.batchSize, 2)) - errD_real = nn.loss.softmax_cross_entropy_loss(output, real_label) + errD_real = foo.loss.softmax_cross_entropy_loss(output, real_label) fake = netG(noise) output = netD(fake.detach()) output = output.reshape((opt.batchSize, 2)) - errD_fake = nn.loss.softmax_cross_entropy_loss(output, fake_label) + errD_fake = foo.loss.softmax_cross_entropy_loss(output, fake_label) errD = errD_real + errD_fake errD.backward() - optimizerD.step(opt.batchSize) + trainerD.step(opt.batchSize) ############################ # (2) Update G network: maximize log(D(G(z))) @@ -118,9 +119,9 @@ with autograd.train_section(): output = netD(fake) output = output.reshape((opt.batchSize, 2)) - errG = nn.loss.softmax_cross_entropy_loss(output, real_label) + errG = foo.loss.softmax_cross_entropy_loss(output, real_label) errG.backward() - optimizerG.step(opt.batchSize) + trainerG.step(opt.batchSize) print mx.nd.mean(errD).asscalar(), mx.nd.mean(errG).asscalar() diff --git a/example/autograd/mnist.py b/example/autograd/mnist.py index b574d332f548..66878fd177c1 100644 --- a/example/autograd/mnist.py +++ b/example/autograd/mnist.py @@ -1,7 +1,8 @@ # pylint: skip-file from data import mnist_iterator import mxnet as mx -from mxnet import nn +from mxnet import foo +from mxnet.foo import nn import numpy as np import logging from mxnet.contrib import autograd as ag @@ -24,8 +25,8 @@ def test(ctx): metric = mx.metric.Accuracy() val_data.reset() for batch in val_data: - data = nn.utils.load_data(batch.data[0], ctx_list=ctx, batch_axis=0) - label = nn.utils.load_data(batch.label[0], ctx_list=ctx, batch_axis=0) + data = foo.utils.load_data(batch.data[0], ctx_list=ctx, batch_axis=0) + label = foo.utils.load_data(batch.label[0], ctx_list=ctx, batch_axis=0) outputs = [] for x in data: outputs.append(net(x)) @@ -36,23 +37,23 @@ def train(epoch, ctx): if isinstance(ctx, mx.Context): ctx = [ctx] net.params.initialize(mx.init.Xavier(magnitude=2.24), ctx=ctx) - optim = nn.Optim(net.params, 'sgd', {'learning_rate': 0.1}) + trainer = foo.Trainer(net.params, 'sgd', {'learning_rate': 0.1}) metric = mx.metric.Accuracy() for i in range(epoch): train_data.reset() for batch in train_data: - data = nn.utils.load_data(batch.data[0], ctx_list=ctx, batch_axis=0) - label = nn.utils.load_data(batch.label[0], ctx_list=ctx, batch_axis=0) + data = foo.utils.load_data(batch.data[0], ctx_list=ctx, batch_axis=0) + label = foo.utils.load_data(batch.label[0], ctx_list=ctx, batch_axis=0) outputs = [] with ag.train_section(): for x, y in zip(data, label): z = net(x) - loss = nn.loss.softmax_cross_entropy_loss(z, y) + loss = foo.loss.softmax_cross_entropy_loss(z, y) ag.compute_gradient([loss]) outputs.append(z) metric.update(label, outputs) - optim.step(batch.data[0].shape[0]) + trainer.step(batch.data[0].shape[0]) name, acc = metric.get() metric.reset() print 'training acc at epoch %d: %s=%f'%(i, name, acc) diff --git a/example/autograd/resnet.py b/example/autograd/resnet.py index 0816863bdbf7..a27fc2570087 100644 --- a/example/autograd/resnet.py +++ b/example/autograd/resnet.py @@ -2,7 +2,8 @@ import time import mxnet as mx -from mxnet import nn +from mxnet import foo +from mxnet.foo import nn from mxnet.contrib import autograd as ag from data import * @@ -263,8 +264,8 @@ def test(ctx): metric = mx.metric.Accuracy() val_data.reset() for batch in val_data: - data = nn.utils.load_data(batch.data[0], ctx_list=ctx, batch_axis=0) - label = nn.utils.load_data(batch.label[0], ctx_list=ctx, batch_axis=0) + data = foo.utils.load_data(batch.data[0], ctx_list=ctx, batch_axis=0) + label = foo.utils.load_data(batch.label[0], ctx_list=ctx, batch_axis=0) outputs = [] for x in data: outputs.append(net(x)) @@ -276,7 +277,7 @@ def train(epoch, ctx): if isinstance(ctx, mx.Context): ctx = [ctx] net.params.initialize(mx.init.Xavier(magnitude=2.24), ctx=ctx) - optim = nn.Optim(net.params, 'sgd', {'learning_rate': 0.1}) + trainer = foo.Trainer(net.params, 'sgd', {'learning_rate': 0.1}) metric = mx.metric.Accuracy() for i in range(epoch): @@ -284,25 +285,24 @@ def train(epoch, ctx): train_data.reset() btic = time.time() for batch in train_data: - data = nn.utils.load_data(batch.data[0], ctx_list=ctx, batch_axis=0) - label = nn.utils.load_data(batch.label[0], ctx_list=ctx, batch_axis=0) + data = foo.utils.load_data(batch.data[0], ctx_list=ctx, batch_axis=0) + label = foo.utils.load_data(batch.label[0], ctx_list=ctx, batch_axis=0) outputs = [] with ag.train_section(): for x, y in zip(data, label): z = net(x) - loss = nn.loss.softmax_cross_entropy_loss(z, y) + loss = foo.loss.softmax_cross_entropy_loss(z, y) ag.compute_gradient([loss]) outputs.append(z) - optim.step(batch.data[0].shape[0]) + trainer.step(batch.data[0].shape[0]) metric.update(label, outputs) - print 'speed: {} samples/s'.format(train_data.label_shape[0]/(time.time()-btic)) + print 'speed: {} samples/s'.format(batch.data[0].shape[0]/(time.time()-btic)) btic = time.time() name, acc = metric.get() metric.reset() print 'training acc at epoch %d: %s=%f'%(i, name, acc) print 'time: %f'%(time.time()-tic) - print 'speed: %f'%(train_data.batches*train_data.label_shape[0]/(time.time()-tic)) test(ctx) net.params.save('mnist.params') diff --git a/python/mxnet/__init__.py b/python/mxnet/__init__.py index 133b5ffd6187..aa8042664aa3 100644 --- a/python/mxnet/__init__.py +++ b/python/mxnet/__init__.py @@ -60,6 +60,6 @@ from . import rnn -from . import nn +from . import foo __version__ = base.__version__ diff --git a/python/mxnet/autograd.py b/python/mxnet/autograd.py new file mode 100644 index 000000000000..f8e3259211b6 --- /dev/null +++ b/python/mxnet/autograd.py @@ -0,0 +1,152 @@ +# coding: utf-8 +"""Autograd for NDArray.""" +from __future__ import absolute_import +from __future__ import division + +import ctypes +from .base import _LIB, check_call, string_types +from .base import mx_uint, NDArrayHandle, c_array +from .ndarray import NDArray +from .symbol import _GRAD_REQ_MAP + + +def set_is_training(is_train): + """Set status to training/not training. When training, graph will be constructed + for gradient computation. Operators will also run with ctx.is_train=True. For example, + Dropout will drop inputs randomly when is_train=True while simply passing through + if is_train=False. + + Parameters + ---------- + is_train: bool + + Returns + ------- + previous state before this set. + """ + prev = ctypes.c_int() + check_call(_LIB.MXAutogradSetIsTraining( + ctypes.c_int(is_train), ctypes.byref(prev))) + return bool(prev.value) + + +class TrainingStateScope(object): + """Scope for managing training state. + + Example:: + with TrainingStateScope(True): + y = model(x) + compute_gradient([y]) + """ + def __init__(self, enter_state): + self._enter_state = enter_state + self._prev = None + + def __enter__(self): + self._prev = set_is_training(self._enter_state) + + def __exit__(self, ptype, value, trace): + if self._prev != self._enter_state: + set_is_training(self._prev) + + +def train_section(): + """Returns a training scope context to be used in 'with' statement + and captures training code. + + Example:: + with autograd.train_section(): + y = model(x) + compute_gradient([y]) + metric.update(...) + optim.step(...) + """ + return TrainingStateScope(True) + + +def test_section(): + """Returns a testing scope context to be used in 'with' statement + and captures testing code. + + Example:: + with autograd.train_section(): + y = model(x) + compute_gradient([y]) + with autograd.test_section(): + # testing, IO, gradient updates... + """ + return TrainingStateScope(False) + + +def mark_variables(variables, gradients, grad_reqs='write'): + """Mark NDArrays as variables to compute gradient for autograd. + + Parameters + ---------- + variables: NDArray or list of NDArray + gradients: NDArray or list of NDArray + grad_reqs: str or list of str + """ + if isinstance(variables, NDArray): + assert isinstance(gradients, NDArray) + variables = [variables] + gradients = [gradients] + + variable_handles = [] + gradient_handles = [] + for var, gradvar in zip(variables, gradients): + variable_handles.append(var.handle) + gradient_handles.append(gradvar.handle) + if isinstance(grad_reqs, string_types): + grad_reqs = [_GRAD_REQ_MAP[grad_reqs]]*len(variables) + else: + grad_reqs = [_GRAD_REQ_MAP[i] for i in grad_reqs] + + check_call(_LIB.MXAutogradMarkVariables( + len(variable_handles), + c_array(NDArrayHandle, variable_handles), + c_array(mx_uint, grad_reqs), + c_array(NDArrayHandle, gradient_handles))) + + +def backward(heads, head_grads=None, retain_graph=False): + """Compute the gradients of heads w.r.t previously marked variables. + + Parameters + ---------- + heads: NDArray or list of NDArray + Output NDArray(s) + head_grads: NDArray or list of NDArray or None + Gradients with respect to heads. + """ + if isinstance(heads, NDArray): + assert head_grads is None or isinstance(head_grads, NDArray) + heads = [heads] + head_grads = [head_grads] if head_grads is not None else None + + output_handles = [] + for arr in heads: + output_handles.append(arr.handle) + + if head_grads is None: + check_call(_LIB.MXAutogradBackward( + len(output_handles), + c_array(NDArrayHandle, output_handles), + ctypes.c_void_p(0), + ctypes.c_int(retain_graph))) + return + + ograd_handles = [] + for arr in head_grads: + if arr is not None: + ograd_handles.append(arr.handle) + else: + ograd_handles.append(NDArrayHandle(0)) + assert len(ograd_handles) == len(output_handles), \ + "heads and head_grads must have the same length" + + check_call(_LIB.MXAutogradBackward( + len(output_handles), + c_array(NDArrayHandle, output_handles), + c_array(NDArrayHandle, ograd_handles), + ctypes.c_int(retain_graph))) diff --git a/python/mxnet/nn/__init__.py b/python/mxnet/foo/__init__.py similarity index 68% rename from python/mxnet/nn/__init__.py rename to python/mxnet/foo/__init__.py index aa34f1c1f6c0..98cc5e9c9852 100644 --- a/python/mxnet/nn/__init__.py +++ b/python/mxnet/foo/__init__.py @@ -4,11 +4,11 @@ from .parameter import * -from .layer import * +from . import nn -from .optim import * +from . import rnn -from .conv_layers import * +from .trainer import * from . import loss diff --git a/python/mxnet/nn/loss.py b/python/mxnet/foo/loss.py similarity index 100% rename from python/mxnet/nn/loss.py rename to python/mxnet/foo/loss.py diff --git a/python/mxnet/foo/nn/__init__.py b/python/mxnet/foo/nn/__init__.py new file mode 100644 index 000000000000..8cf69dee077d --- /dev/null +++ b/python/mxnet/foo/nn/__init__.py @@ -0,0 +1,7 @@ +# coding: utf-8 +# pylint: disable=wildcard-import +"""Neural network layers.""" + +from .layer import * + +from .conv_layers import * diff --git a/python/mxnet/nn/conv_layers.py b/python/mxnet/foo/nn/conv_layers.py similarity index 99% rename from python/mxnet/nn/conv_layers.py rename to python/mxnet/foo/nn/conv_layers.py index d81613970f25..f70aa11a29db 100644 --- a/python/mxnet/nn/conv_layers.py +++ b/python/mxnet/foo/nn/conv_layers.py @@ -2,8 +2,8 @@ # pylint: disable= arguments-differ """Convolutional neural network layers.""" from .layer import Layer -from .. import symbol -from ..base import numeric_types +from ... import symbol +from ...base import numeric_types def _infer_weight_shape(op, data_shape): sym = symbol.invoke(op, [symbol.var('data', shape=data_shape)]) diff --git a/python/mxnet/nn/layer.py b/python/mxnet/foo/nn/layer.py similarity index 98% rename from python/mxnet/nn/layer.py rename to python/mxnet/foo/nn/layer.py index 231992ff79d5..e156ae695008 100644 --- a/python/mxnet/nn/layer.py +++ b/python/mxnet/foo/nn/layer.py @@ -2,11 +2,11 @@ # pylint: disable= arguments-differ """Neural network layers.""" -from .. import symbol, ndarray -from ..symbol import Symbol -from ..ndarray import NDArray -from .. import name as _name -from .parameter import Parameter, ParameterDict +from ... import symbol, ndarray +from ...symbol import Symbol +from ...ndarray import NDArray +from ... import name as _name +from ..parameter import Parameter, ParameterDict class _LayerScope(object): diff --git a/python/mxnet/nn/parameter.py b/python/mxnet/foo/parameter.py similarity index 99% rename from python/mxnet/nn/parameter.py rename to python/mxnet/foo/parameter.py index 97bdf3786541..b12a35224605 100644 --- a/python/mxnet/nn/parameter.py +++ b/python/mxnet/foo/parameter.py @@ -8,7 +8,7 @@ from ..base import mx_real_t from .. import symbol, ndarray, initializer, context from ..context import Context -from ..contrib import autograd +from .. import autograd # pylint: disable= invalid-name tensor_types = (symbol.Symbol, ndarray.NDArray) @@ -167,7 +167,7 @@ def grad(self, ctx=None): Desired context. """ if ctx is None: - ctx = Context.current_context() + ctx = context.current_context() assert self._grad is not None, \ "Cannot get gradient array for Parameter %s " \ "because it hasn't been initialized or grad_req='null'"%(self.name) diff --git a/python/mxnet/foo/rnn/__init__.py b/python/mxnet/foo/rnn/__init__.py new file mode 100644 index 000000000000..3fc69b0000d9 --- /dev/null +++ b/python/mxnet/foo/rnn/__init__.py @@ -0,0 +1,5 @@ +# coding: utf-8 +# pylint: disable=wildcard-import +"""Recurrent neural network module.""" + +from .rnn_cell import * diff --git a/python/mxnet/foo/rnn/rnn_cell.py b/python/mxnet/foo/rnn/rnn_cell.py new file mode 100644 index 000000000000..e3bd04f50177 --- /dev/null +++ b/python/mxnet/foo/rnn/rnn_cell.py @@ -0,0 +1,1006 @@ +# coding: utf-8 +# pylint: disable=no-member, invalid-name, protected-access, no-self-use +# pylint: disable=too-many-branches, too-many-arguments, no-self-use +# pylint: disable=too-many-lines, arguments-differ +"""Definition of various recurrent neural network cells.""" +from __future__ import print_function + +import warnings + +from ... import symbol, init, ndarray +from ...base import string_types, numeric_types +from ..nn import Layer +from .. import tensor_types + + +def _cells_state_shape(cells): + return sum([c.state_shape for c in cells], []) + +def _cells_state_info(cells, batch_size): + return sum([c.state_info(batch_size) for c in cells], []) + +def _cells_begin_state(cells, **kwargs): + return sum([c.begin_state(**kwargs) for c in cells], []) + +def _cells_unpack_weights(cells, args): + for cell in cells: + args = cell.unpack_weights(args) + return args + +def _cells_pack_weights(cells, args): + for cell in cells: + args = cell.pack_weights(args) + return args + +def _get_begin_state(cell, F, begin_state, inputs, batch_size): + if begin_state is None: + if F is ndarray: + ctx = inputs.context if isinstance(inputs, tensor_types) else inputs[0].context + with ctx: + begin_state = cell.begin_state(func=F.zeros, batch_size=batch_size) + else: + begin_state = cell.begin_state(func=F.zeros, batch_size=batch_size) + return begin_state + +def _format_sequence(length, inputs, layout, merge, in_layout=None): + assert inputs is not None, \ + "unroll(inputs=None) has been deprecated. " \ + "Please create input variables outside unroll." + + axis = layout.find('T') + batch_axis = layout.find('N') + batch_size = 0 + in_axis = in_layout.find('T') if in_layout is not None else axis + if isinstance(inputs, symbol.Symbol): + F = symbol + if merge is False: + assert len(inputs.list_outputs()) == 1, \ + "unroll doesn't allow grouped symbol as input. Please convert " \ + "to list with list(inputs) first or let unroll handle splitting." + inputs = list(symbol.split(inputs, axis=in_axis, num_outputs=length, + squeeze_axis=1)) + elif isinstance(inputs, ndarray.NDArray): + F = ndarray + batch_size = inputs.shape[batch_axis] + if merge is False: + assert length is None or length == inputs.shape[in_axis] + inputs = ndarray.split(inputs, axis=in_axis, num_outputs=inputs.shape[in_axis], + squeeze_axis=1) + else: + assert length is None or len(inputs) == length + if isinstance(inputs[0], symbol.Symbol): + F = symbol + else: + F = ndarray + batch_size = inputs[0].shape[batch_axis] + if merge is True: + inputs = [F.expand_dims(i, axis=axis) for i in inputs] + inputs = F.concat(*inputs, dim=axis) + in_axis = axis + + if isinstance(inputs, tensor_types) and axis != in_axis: + inputs = F.swapaxes(inputs, dim1=axis, dim2=in_axis) + + return inputs, axis, F, batch_size + + +class RecurrentCell(Layer): + """Abstract base class for RNN cells + + Parameters + ---------- + prefix : str, optional + Prefix for names of layers + (this prefix is also used for names of weights if `params` is None + i.e. if `params` are being created and not reused) + params : RNNParams or None, optional + Container for weight sharing between cells. + A new RNNParams container is created if `params` is None. + """ + def __init__(self, prefix=None, params=None): + super(RecurrentCell, self).__init__(prefix=prefix, params=params) + self._modified = False + self.reset() + + def reset(self): + """Reset before re-using the cell for another graph.""" + self._init_counter = -1 + self._counter = -1 + + def state_info(self, batch_size=0): + """shape and layout information of states""" + raise NotImplementedError() + + @property + def state_shape(self): + """shape(s) of states""" + return [ele['shape'] for ele in self.state_info()] + + @property + def _gate_names(self): + """name(s) of gates""" + return () + + @property + def _curr_prefix(self): + return '%st%d_'%(self.prefix, self._counter) + + def begin_state(self, func=symbol.zeros, batch_size=0, **kwargs): + """Initial state for this cell. + + Parameters + ---------- + func : callable, default symbol.zeros + Function for creating initial state. + + For Symbol API, func can be symbol.zeros, symbol.uniform, + symbol.var etc. Use symbol.var if you want to directly + feed input as states. + + For NDArray API, func can be ndarray.zeros, ndarray.ones, etc. + batch_size: int, default 0 + Only required for NDArray API. Size of the batch ('N' in layout) + dimension of input. + + **kwargs : + additional keyword arguments passed to func. For example + mean, std, dtype, etc. + + Returns + ------- + states : nested list of Symbol + Starting states for the first RNN step. + """ + assert not self._modified, \ + "After applying modifier cells (e.g. ZoneoutCell) the base " \ + "cell cannot be called directly. Call the modifier cell instead." + states = [] + for info in self.state_info(batch_size): + self._init_counter += 1 + if info is not None: + info.update(kwargs) + else: + info = kwargs + state = func(name='%sbegin_state_%d'%(self._prefix, self._init_counter), + **info) + states.append(state) + return states + + def unpack_weights(self, args): + """Unpack fused weight matrices into separate + weight matrices. + + For example, say you use a module object `mod` to run a network that has an lstm cell. + In `mod.get_params()[0]`, the lstm parameters are all represented as a single big vector. + `cell.unpack_weights(mod.get_params()[0])` will unpack this vector into a dictionary of + more readable lstm parameters - c, f, i, o gates for i2h (input to hidden) and + h2h (hidden to hidden) weights. + + Parameters + ---------- + args : dict of str -> NDArray + Dictionary containing packed weights. + usually from `Module.get_params()[0]`. + + Returns + ------- + args : dict of str -> NDArray + Dictionary with unpacked weights associated with + this cell. + + See Also + -------- + pack_weights: Performs the reverse operation of this function. + """ + args = args.copy() + if not self._gate_names: + return args + h = self._num_hidden + for group_name in ['i2h', 'h2h']: + weight = args.pop('%s%s_weight'%(self._prefix, group_name)) + bias = args.pop('%s%s_bias' % (self._prefix, group_name)) + for j, gate in enumerate(self._gate_names): + wname = '%s%s%s_weight' % (self._prefix, group_name, gate) + args[wname] = weight[j*h:(j+1)*h].copy() + bname = '%s%s%s_bias' % (self._prefix, group_name, gate) + args[bname] = bias[j*h:(j+1)*h].copy() + return args + + def pack_weights(self, args): + """Pack separate weight matrices into a single packed + weight. + + Parameters + ---------- + args : dict of str -> NDArray + Dictionary containing unpacked weights. + + Returns + ------- + args : dict of str -> NDArray + Dictionary with packed weights associated with + this cell. + """ + args = args.copy() + if not self._gate_names: + return args + for group_name in ['i2h', 'h2h']: + weight = [] + bias = [] + for gate in self._gate_names: + wname = '%s%s%s_weight'%(self._prefix, group_name, gate) + weight.append(args.pop(wname)) + bname = '%s%s%s_bias'%(self._prefix, group_name, gate) + bias.append(args.pop(bname)) + args['%s%s_weight'%(self._prefix, group_name)] = ndarray.concatenate(weight) + args['%s%s_bias'%(self._prefix, group_name)] = ndarray.concatenate(bias) + return args + + def unroll(self, length, inputs, begin_state=None, layout='NTC', merge_outputs=None): + """Unroll an RNN cell across time steps. + + Parameters + ---------- + length : int + number of steps to unroll + inputs : Symbol, list of Symbol, or None + If `inputs` is a single Symbol (usually the output + of Embedding symbol), it should have shape + (batch_size, length, ...) if layout == 'NTC', + or (length, batch_size, ...) if layout == 'TNC'. + + If `inputs` is a list of symbols (usually output of + previous unroll), they should all have shape + (batch_size, ...). + begin_state : nested list of Symbol, optional + Input states created by `begin_state()` + or output state of another cell. + Created from `begin_state()` if None. + layout : str, optional + `layout` of input symbol. Only used if inputs + is a single Symbol. + merge_outputs : bool, optional + If False, return outputs as a list of Symbols. + If True, concatenate output across time steps + and return a single symbol with shape + (batch_size, length, ...) if layout == 'NTC', + or (length, batch_size, ...) if layout == 'TNC'. + If None, output whatever is faster + + Returns + ------- + outputs : list of Symbol or Symbol + Symbol (if `merge_outputs` is True) or list of Symbols + (if `merge_outputs` is False) corresponding to the output from + the RNN from this unrolling. + + states : list of Symbol + The new state of this RNN after this unrolling. + The type of this symbol is same as the output of begin_state(). + """ + self.reset() + + inputs, _, F, batch_size = _format_sequence(length, inputs, layout, False) + begin_state = _get_begin_state(self, F, begin_state, inputs, batch_size) + + states = begin_state + outputs = [] + for i in range(length): + output, states = self(inputs[i], states) + outputs.append(output) + + outputs, _, _, _ = _format_sequence(length, outputs, layout, merge_outputs) + + return outputs, states + + #pylint: disable=no-self-use + def _get_activation(self, F, inputs, activation, **kwargs): + """Get activation function. Convert if is string""" + if isinstance(activation, string_types): + return F.Activation(inputs, act_type=activation, **kwargs) + else: + return activation(inputs, **kwargs) + + def forward(self, inputs, states): + """Unroll the recurrent cell for one time step. + + Parameters + ---------- + inputs : sym.Variable + input symbol, 2D, batch_size * num_units + states : list of sym.Variable + RNN state from previous step or the output of begin_state(). + + Returns + ------- + output : Symbol + Symbol corresponding to the output from the RNN when unrolling + for a single time step. + states : list of Symbol + The new state of this RNN after this unrolling. + The type of this symbol is same as the output of begin_state(). + This can be used as input state to the next time step + of this RNN. + + See Also + -------- + begin_state: This function can provide the states for the first time step. + unroll: This function unrolls an RNN for a given number of (>=1) time steps. + """ + # pylint: disable= arguments-differ + self._counter += 1 + return super(RecurrentCell, self).forward(inputs, states) + + + +class RNNCell(RecurrentCell): + """Simple recurrent neural network cell. + + Parameters + ---------- + num_hidden : int + number of units in output symbol + activation : str or Symbol, default 'tanh' + type of activation function + prefix : str, default 'rnn_' + prefix for name of layers + (and name of weight if params is None) + params : RNNParams or None + container for weight sharing between cells. + created if None. + """ + def __init__(self, num_hidden, activation='tanh', num_input=0, + prefix=None, params=None): + super(RNNCell, self).__init__(prefix=prefix, params=params) + self._num_hidden = num_hidden + self._activation = activation + self._num_input = num_input + self.i2h_weight = self.params.get('i2h_weight', shape=(num_hidden, num_input)) + self.i2h_bias = self.params.get('i2h_bias', shape=(num_hidden,)) + self.h2h_weight = self.params.get('h2h_weight', shape=(num_hidden, num_hidden)) + self.h2h_bias = self.params.get('h2h_bias', shape=(num_hidden,)) + + def state_info(self, batch_size=0): + return [{'shape': (batch_size, self._num_hidden), '__layout__': 'NC'}] + + @property + def _gate_names(self): + return ('',) + + def _alias(self): + return 'rnn' + + def generic_forward(self, F, inputs, states, i2h_weight, i2h_bias, + h2h_weight, h2h_bias): + name = self._curr_prefix + i2h = F.FullyConnected(data=inputs, weight=i2h_weight, bias=i2h_bias, + num_hidden=self._num_hidden, + name='%si2h'%name) + h2h = F.FullyConnected(data=states[0], weight=h2h_weight, bias=h2h_bias, + num_hidden=self._num_hidden, + name='%sh2h'%name) + output = self._get_activation(F, i2h + h2h, self._activation, + name='%sout'%name) + + return output, [output] + + +class LSTMCell(RecurrentCell): + """Long-Short Term Memory (LSTM) network cell. + + Parameters + ---------- + num_hidden : int + number of units in output symbol + prefix : str, default 'lstm_' + prefix for name of layers + (and name of weight if params is None) + params : RNNParams or None + container for weight sharing between cells. + created if None. + forget_bias : bias added to forget gate, default 1.0. + Jozefowicz et al. 2015 recommends setting this to 1.0 + """ + def __init__(self, num_hidden, forget_bias=1.0, num_input=0, + prefix=None, params=None): + super(LSTMCell, self).__init__(prefix=prefix, params=params) + + self._num_hidden = num_hidden + self._num_input = num_input + self.i2h_weight = self.params.get('i2h_weight', shape=(4*num_hidden, num_input)) + self.h2h_weight = self.params.get('h2h_weight', shape=(4*num_hidden, num_hidden)) + # we add the forget_bias to i2h_bias, this adds the bias to the forget gate activation + self.i2h_bias = self.params.get('i2h_bias', shape=(4*num_hidden,), + init=init.LSTMBias(forget_bias=forget_bias)) + self.h2h_bias = self.params.get('h2h_bias', shape=(4*num_hidden,)) + + def state_info(self, batch_size=0): + return [{'shape': (batch_size, self._num_hidden), '__layout__': 'NC'}, + {'shape': (batch_size, self._num_hidden), '__layout__': 'NC'}] + + @property + def _gate_names(self): + return ['_i', '_f', '_c', '_o'] + + def _alias(self): + return 'lstm' + + def generic_forward(self, F, inputs, states, i2h_weight, i2h_bias, + h2h_weight, h2h_bias): + name = self._curr_prefix + i2h = F.FullyConnected(data=inputs, weight=i2h_weight, bias=i2h_bias, + num_hidden=self._num_hidden*4, + name='%si2h'%name) + h2h = F.FullyConnected(data=states[0], weight=h2h_weight, bias=h2h_bias, + num_hidden=self._num_hidden*4, + name='%sh2h'%name) + gates = i2h + h2h + slice_gates = F.SliceChannel(gates, num_outputs=4, + name="%sslice"%name) + in_gate = F.Activation(slice_gates[0], act_type="sigmoid", + name='%si'%name) + forget_gate = F.Activation(slice_gates[1], act_type="sigmoid", + name='%sf'%name) + in_transform = F.Activation(slice_gates[2], act_type="tanh", + name='%sc'%name) + out_gate = F.Activation(slice_gates[3], act_type="sigmoid", + name='%so'%name) + next_c = F._internal._plus(forget_gate * states[1], in_gate * in_transform, + name='%sstate'%name) + next_h = F._internal._mul(out_gate, F.Activation(next_c, act_type="tanh"), + name='%sout'%name) + + return next_h, [next_h, next_c] + + +class GRUCell(RecurrentCell): + """Gated Rectified Unit (GRU) network cell. + Note: this is an implementation of the cuDNN version of GRUs + (slight modification compared to Cho et al. 2014). + + Parameters + ---------- + num_hidden : int + number of units in output symbol + prefix : str, default 'gru_' + prefix for name of layers + (and name of weight if params is None) + params : RNNParams or None + container for weight sharing between cells. + created if None. + """ + def __init__(self, num_hidden, num_input=0, prefix=None, params=None): + super(GRUCell, self).__init__(prefix=prefix, params=params) + self._num_hidden = num_hidden + self.i2h_weight = self.params.get('i2h_weight', shape=(3*num_hidden, num_input)) + self.h2h_weight = self.params.get('h2h_weight', shape=(3*num_hidden, num_hidden)) + self.i2h_bias = self.params.get('i2h_bias', shape=(3*num_hidden)) + self.h2h_bias = self.params.get('h2h_bias', shape=(3*num_hidden)) + + def state_info(self, batch_size=0): + return [{'shape': (batch_size, self._num_hidden), '__layout__': 'NC'}] + + @property + def _gate_names(self): + return ['_r', '_z', '_o'] + + def _alias(self): + return 'gru' + + def generic_forward(self, F, inputs, states, i2h_weight, i2h_bias, + h2h_weight, h2h_bias): + # pylint: disable=too-many-locals + name = self._curr_prefix + prev_state_h = states[0] + i2h = F.FullyConnected(data=inputs, + weight=i2h_weight, + bias=i2h_bias, + num_hidden=self._num_hidden * 3, + name="%si2h" % name) + h2h = F.FullyConnected(data=prev_state_h, + weight=h2h_weight, + bias=h2h_bias, + num_hidden=self._num_hidden * 3, + name="%sh2h" % name) + + i2h_r, i2h_z, i2h = F.SliceChannel(i2h, num_outputs=3, name="%si2h_slice" % name) + h2h_r, h2h_z, h2h = F.SliceChannel(h2h, num_outputs=3, name="%sh2h_slice" % name) + + reset_gate = F.Activation(i2h_r + h2h_r, act_type="sigmoid", + name="%sr_act" % name) + update_gate = F.Activation(i2h_z + h2h_z, act_type="sigmoid", + name="%sz_act" % name) + + next_h_tmp = F.Activation(i2h + reset_gate * h2h, act_type="tanh", + name="%sh_act" % name) + + next_h = F._internal._plus((1. - update_gate) * next_h_tmp, update_gate * prev_state_h, + name='%sout' % name) + + return next_h, [next_h] + + +class FusedRNNCell(RecurrentCell): + """Fusing RNN layers across time step into one kernel. + Improves speed but is less flexible. Currently only + supported if using cuDNN on GPU. + + Parameters + ---------- + """ + def __init__(self, num_hidden, num_layers=1, mode='lstm', bidirectional=False, + dropout=0., get_next_state=False, forget_bias=1.0, num_input=0, + prefix=None, params=None): + self._num_hidden = num_hidden + self._num_layers = num_layers + self._mode = mode + self._bidirectional = bidirectional + self._dropout = dropout + self._get_next_state = get_next_state + self._directions = ['l', 'r'] if bidirectional else ['l'] + super(FusedRNNCell, self).__init__(prefix=prefix, params=params) + + initializer = init.FusedRNN(None, num_hidden, num_layers, mode, + bidirectional, forget_bias) + self.parameters = self.params.get('parameters', init=initializer, + shape=(self._num_input_to_size(num_input),)) + + def state_info(self, batch_size=0): + b = self._bidirectional + 1 + n = (self._mode == 'lstm') + 1 + return [{'shape': (b*self._num_layers, batch_size, self._num_hidden), + '__layout__': 'LNC'} for _ in range(n)] + + @property + def _gate_names(self): + return {'rnn_relu': [''], + 'rnn_tanh': [''], + 'lstm': ['_i', '_f', '_c', '_o'], + 'gru': ['_r', '_z', '_o']}[self._mode] + + @property + def _num_gates(self): + return len(self._gate_names) + + def _alias(self): + return self._mode + + def _size_to_num_input(self, size): + b = len(self._directions) + m = self._num_gates + h = self._num_hidden + return size//b//h//m - (self._num_layers - 1)*(h+b*h+2) - h - 2 + + def _num_input_to_size(self, num_input): + if num_input == 0: + return 0 + b = self._bidirectional + 1 + m = self._num_gates + h = self._num_hidden + return (num_input+h+2)*h*m*b + (self._num_layers-1)*m*h*(h+b*h+2)*b + + def _slice_weights(self, arr, li, lh): + """slice fused rnn weights""" + args = {} + gate_names = self._gate_names + directions = self._directions + + b = len(directions) + p = 0 + for layer in range(self._num_layers): + for direction in directions: + for gate in gate_names: + name = '%s%s%d_i2h%s_weight'%(self._prefix, direction, layer, gate) + if layer > 0: + size = b*lh*lh + args[name] = arr[p:p+size].reshape((lh, b*lh)) + else: + size = li*lh + args[name] = arr[p:p+size].reshape((lh, li)) + p += size + for gate in gate_names: + name = '%s%s%d_h2h%s_weight'%(self._prefix, direction, layer, gate) + size = lh**2 + args[name] = arr[p:p+size].reshape((lh, lh)) + p += size + + for layer in range(self._num_layers): + for direction in directions: + for gate in gate_names: + name = '%s%s%d_i2h%s_bias'%(self._prefix, direction, layer, gate) + args[name] = arr[p:p+lh] + p += lh + for gate in gate_names: + name = '%s%s%d_h2h%s_bias'%(self._prefix, direction, layer, gate) + args[name] = arr[p:p+lh] + p += lh + + assert p == arr.size, "Invalid parameters size for FusedRNNCell" + return args + + def unpack_weights(self, args): + args = args.copy() + arr = args.pop(self.parameters.name) + num_input = self._size_to_num_input(arr.size) + nargs = self._slice_weights(arr, num_input, self._num_hidden) + args.update({name: nd.copy() for name, nd in nargs.items()}) + return args + + def pack_weights(self, args): + args = args.copy() + w0 = args['%sl0_i2h%s_weight'%(self._prefix, self._gate_names[0])] + num_input = w0.shape[1] + total = self._num_input_to_size(num_input) + + arr = ndarray.zeros((total,), ctx=w0.context, dtype=w0.dtype) + for name, nd in self._slice_weights(arr, num_input, self._num_hidden).items(): + nd[:] = args.pop(name) + args[self.parameters.name] = arr + return args + + def __call__(self, inputs, states): + raise NotImplementedError("FusedRNNCell cannot be stepped. Please use unroll") + + def unroll(self, length, inputs, begin_state=None, layout='NTC', merge_outputs=None): + self.reset() + + inputs, axis, F, batch_size = _format_sequence(length, inputs, layout, True) + if axis == 1: + warnings.warn("NTC layout detected. Consider using " + "TNC for FusedRNNCell for faster speed") + inputs = F.swapaxes(inputs, dim1=0, dim2=1) + else: + assert axis == 0, "Unsupported layout %s"%layout + begin_state = _get_begin_state(self, F, begin_state, inputs, batch_size) + + states = begin_state + if self._mode == 'lstm': + states = {'state': states[0], 'state_cell': states[1]} # pylint: disable=redefined-variable-type + else: + states = {'state': states[0]} + + if isinstance(inputs, symbol.Symbol): + parameters = self.parameters.var() + else: + parameters = self.parameters.data(inputs.context) + + rnn = F.RNN(data=inputs, parameters=parameters, + state_size=self._num_hidden, num_layers=self._num_layers, + bidirectional=self._bidirectional, p=self._dropout, + state_outputs=self._get_next_state, + mode=self._mode, name=self._prefix+'rnn', + **states) + + if not self._get_next_state: + outputs, states = rnn, [] + elif self._mode == 'lstm': + outputs, states = rnn[0], [rnn[1], rnn[2]] + else: + outputs, states = rnn[0], [rnn[1]] + + if axis == 1: + outputs = F.swapaxes(outputs, dim1=0, dim2=1) + + outputs, _, _, _ = _format_sequence(length, outputs, layout, merge_outputs) + + return outputs, states + + def unfuse(self): + """Unfuse the fused RNN in to a stack of rnn cells. + + Returns + ------- + cell : SequentialRNNCell + unfused cell that can be used for stepping, and can run on CPU. + """ + stack = SequentialRNNCell() + get_cell = {'rnn_relu': lambda cell_prefix: RNNCell(self._num_hidden, + activation='relu', + prefix=cell_prefix), + 'rnn_tanh': lambda cell_prefix: RNNCell(self._num_hidden, + activation='tanh', + prefix=cell_prefix), + 'lstm': lambda cell_prefix: LSTMCell(self._num_hidden, + prefix=cell_prefix), + 'gru': lambda cell_prefix: GRUCell(self._num_hidden, + prefix=cell_prefix)}[self._mode] + for i in range(self._num_layers): + if self._bidirectional: + stack.add(BidirectionalCell( + get_cell('%sl%d_'%(self._prefix, i)), + get_cell('%sr%d_'%(self._prefix, i)), + output_prefix='%sbi_l%d_'%(self._prefix, i))) + else: + stack.add(get_cell('%sl%d_'%(self._prefix, i))) + + if self._dropout > 0 and i != self._num_layers - 1: + stack.add(DropoutCell(self._dropout, prefix='%s_dropout%d_'%(self._prefix, i))) + + return stack + + +class SequentialRNNCell(RecurrentCell): + """Sequantially stacking multiple RNN cells.""" + def __init__(self): + super(SequentialRNNCell, self).__init__(prefix='', params=None) + + def add(self, cell): + """Append a cell into the stack. + + Parameters + ---------- + cell : rnn cell + """ + self.register_sublayer(cell) + + def state_info(self, batch_size=0): + return _cells_state_info(self._children, batch_size) + + def begin_state(self, **kwargs): + assert not self._modified, \ + "After applying modifier cells (e.g. ZoneoutCell) the base " \ + "cell cannot be called directly. Call the modifier cell instead." + return _cells_begin_state(self._children, **kwargs) + + def unpack_weights(self, args): + return _cells_unpack_weights(self._children, args) + + def pack_weights(self, args): + return _cells_pack_weights(self._children, args) + + def __call__(self, inputs, states): + self._counter += 1 + next_states = [] + p = 0 + for cell in self._children: + assert not isinstance(cell, BidirectionalCell) + n = len(cell.state_info()) + state = states[p:p+n] + p += n + inputs, state = cell(inputs, state) + next_states.append(state) + return inputs, sum(next_states, []) + + def unroll(self, length, inputs, begin_state=None, layout='NTC', merge_outputs=None): + self.reset() + + inputs, _, F, batch_size = _format_sequence(length, inputs, layout, None) + num_cells = len(self._children) + begin_state = _get_begin_state(self, F, begin_state, inputs, batch_size) + + p = 0 + next_states = [] + for i, cell in enumerate(self._children): + n = len(cell.state_info()) + states = begin_state[p:p+n] + p += n + inputs, states = cell.unroll(length, inputs=inputs, begin_state=states, layout=layout, + merge_outputs=None if i < num_cells-1 else merge_outputs) + next_states.extend(states) + + return inputs, next_states + + def generic_forward(self, *args, **kwargs): + raise NotImplementedError + + +class DropoutCell(RecurrentCell): + """Apply dropout on input. + + Parameters + ---------- + dropout : float + percentage of elements to drop out, which + is 1 - percentage to retain. + """ + def __init__(self, dropout, prefix=None, params=None): + super(DropoutCell, self).__init__(prefix, params) + assert isinstance(dropout, numeric_types), "dropout probability must be a number" + self.dropout = dropout + + def state_info(self, batch_size=0): + return [] + + def _alias(self): + return 'dropout' + + def generic_forward(self, F, inputs, states): + if self.dropout > 0: + inputs = F.Dropout(data=inputs, p=self.dropout) + return inputs, states + + def unroll(self, length, inputs, begin_state=None, layout='NTC', merge_outputs=None): + self.reset() + + inputs, _, F, _ = _format_sequence(length, inputs, layout, merge_outputs) + if isinstance(inputs, tensor_types): + return self.generic_forward(F, inputs, begin_state if begin_state else []) + else: + return super(DropoutCell, self).unroll( + length, inputs, begin_state=begin_state, layout=layout, + merge_outputs=merge_outputs) + + +class ModifierCell(RecurrentCell): + """Base class for modifier cells. A modifier + cell takes a base cell, apply modifications + on it (e.g. Zoneout), and returns a new cell. + + After applying modifiers the base cell should + no longer be called directly. The modifer cell + should be used instead. + """ + def __init__(self, base_cell): + super(ModifierCell, self).__init__(prefix=None, params=None) + base_cell._modified = True + self.base_cell = base_cell + + @property + def params(self): + self._own_params = False + return self.base_cell.params + + def state_info(self, batch_size=0): + return self.base_cell.state_info(batch_size) + + def begin_state(self, func=symbol.zeros, **kwargs): + assert not self._modified, \ + "After applying modifier cells (e.g. DropoutCell) the base " \ + "cell cannot be called directly. Call the modifier cell instead." + self.base_cell._modified = False + begin = self.base_cell.begin_state(func=func, **kwargs) + self.base_cell._modified = True + return begin + + def unpack_weights(self, args): + return self.base_cell.unpack_weights(args) + + def pack_weights(self, args): + return self.base_cell.pack_weights(args) + + def generic_forward(self, F, inputs, states): + raise NotImplementedError + + +class ZoneoutCell(ModifierCell): + """Apply Zoneout on base cell.""" + def __init__(self, base_cell, zoneout_outputs=0., zoneout_states=0.): + assert not isinstance(base_cell, FusedRNNCell), \ + "FusedRNNCell doesn't support zoneout. " \ + "Please unfuse first." + assert not isinstance(base_cell, BidirectionalCell), \ + "BidirectionalCell doesn't support zoneout since it doesn't support step. " \ + "Please add ZoneoutCell to the cells underneath instead." + assert not isinstance(base_cell, SequentialRNNCell) or not base_cell._bidirectional, \ + "Bidirectional SequentialRNNCell doesn't support zoneout. " \ + "Please add ZoneoutCell to the cells underneath instead." + super(ZoneoutCell, self).__init__(base_cell) + self.zoneout_outputs = zoneout_outputs + self.zoneout_states = zoneout_states + self.prev_output = None + + def _alias(self): + return 'zoneout' + + def reset(self): + super(ZoneoutCell, self).reset() + self.prev_output = None + + def generic_forward(self, F, inputs, states): + cell, p_outputs, p_states = self.base_cell, self.zoneout_outputs, self.zoneout_states + next_output, next_states = cell(inputs, states) + mask = (lambda p, like: F.Dropout(F.ones_like(like), p=p)) + + prev_output = self.prev_output + if prev_output is None: + prev_output = F.zeros_like(next_output) + + output = (F.where(mask(p_outputs, next_output), next_output, prev_output) + if p_outputs != 0. else next_output) + states = ([F.where(mask(p_states, new_s), new_s, old_s) for new_s, old_s in + zip(next_states, states)] if p_states != 0. else next_states) + + self.prev_output = output + + return output, states + + +class ResidualCell(ModifierCell): + """ + Adds residual connection as described in Wu et al, 2016 + (https://arxiv.org/abs/1609.08144). + Output of the cell is output of the base cell plus input. + """ + + def __init__(self, base_cell): + super(ResidualCell, self).__init__(base_cell) + + def generic_forward(self, F, inputs, states): + output, states = self.base_cell(inputs, states) + output = F.elemwise_add(output, inputs, name="%s_plus_residual" % output.name) + return output, states + + def unroll(self, length, inputs, begin_state=None, layout='NTC', merge_outputs=None): + self.reset() + + self.base_cell._modified = False + outputs, states = self.base_cell.unroll(length, inputs=inputs, begin_state=begin_state, + layout=layout, merge_outputs=merge_outputs) + self.base_cell._modified = True + + merge_outputs = isinstance(outputs, tensor_types) if merge_outputs is None else \ + merge_outputs + inputs, _, F, _ = _format_sequence(length, inputs, layout, merge_outputs) + if merge_outputs: + outputs = F.elemwise_add(outputs, inputs) + else: + outputs = [F.elemwise_add(i, j) for i, j in zip(outputs, inputs)] + + return outputs, states + + +class BidirectionalCell(RecurrentCell): + """Bidirectional RNN cell. + + Parameters + ---------- + l_cell : RecurrentCell + cell for forward unrolling + r_cell : RecurrentCell + cell for backward unrolling + """ + def __init__(self, l_cell, r_cell, output_prefix='bi_'): + super(BidirectionalCell, self).__init__(prefix='', params=None) + self.register_sublayer(l_cell) + self.register_sublayer(r_cell) + self._output_prefix = output_prefix + + def unpack_weights(self, args): + return _cells_unpack_weights(self._children, args) + + def pack_weights(self, args): + return _cells_pack_weights(self._children, args) + + def __call__(self, inputs, states): + raise NotImplementedError("Bidirectional cannot be stepped. Please use unroll") + + def state_info(self, batch_size=0): + return _cells_state_info(self._children, batch_size) + + def begin_state(self, **kwargs): + assert not self._modified, \ + "After applying modifier cells (e.g. DropoutCell) the base " \ + "cell cannot be called directly. Call the modifier cell instead." + return _cells_begin_state(self._children, **kwargs) + + def unroll(self, length, inputs, begin_state=None, layout='NTC', merge_outputs=None): + self.reset() + + inputs, axis, F, batch_size = _format_sequence(length, inputs, layout, False) + begin_state = _get_begin_state(self, F, begin_state, inputs, batch_size) + + states = begin_state + l_cell, r_cell = self._children + l_outputs, l_states = l_cell.unroll(length, inputs=inputs, + begin_state=states[:len(l_cell.state_info(batch_size))], + layout=layout, merge_outputs=merge_outputs) + r_outputs, r_states = r_cell.unroll(length, + inputs=list(reversed(inputs)), + begin_state=states[len(l_cell.state_info(batch_size)):], + layout=layout, merge_outputs=merge_outputs) + + if merge_outputs is None: + merge_outputs = (isinstance(l_outputs, tensor_types) + and isinstance(r_outputs, tensor_types)) + l_outputs, _, _, _ = _format_sequence(None, l_outputs, layout, merge_outputs) + r_outputs, _, _, _ = _format_sequence(None, r_outputs, layout, merge_outputs) + + if merge_outputs: + r_outputs = F.reverse(r_outputs, axis=axis) + outputs = F.concat(l_outputs, r_outputs, dim=2, name='%sout'%self._output_prefix) + else: + outputs = [F.concat(l_o, r_o, dim=1, name='%st%d'%(self._output_prefix, i)) + for i, (l_o, r_o) in enumerate(zip(l_outputs, reversed(r_outputs)))] + + states = [l_states, r_states] + return outputs, states diff --git a/python/mxnet/nn/optim.py b/python/mxnet/foo/trainer.py similarity index 92% rename from python/mxnet/nn/optim.py rename to python/mxnet/foo/trainer.py index 81fdfbf12756..19db8fad2de4 100644 --- a/python/mxnet/nn/optim.py +++ b/python/mxnet/foo/trainer.py @@ -5,12 +5,13 @@ from .. import optimizer as opt from ..model import _create_kvstore -class Optim(object): - """Optimizes a set of Parameters. Optim should be used together with autograd. +class Trainer(object): + """Applies an Optimizer on a set of Parameters. Trainer should + be used together with autograd. Parameters ---------- - param_dict : ParameterDict + params : ParameterDict The set of parameters to optimize. optimizer : str or Optimizer The optimizer to use. @@ -20,8 +21,8 @@ class Optim(object): kvstore : str or KVStore kvstore type for multi-gpu and distributed training. """ - def __init__(self, param_dict, optimizer, optimizer_params, kvstore='device'): - self._params = [param for param in param_dict.values() if param.grad_req != 'null'] + def __init__(self, params, optimizer, optimizer_params, kvstore='device'): + self._params = [param for param in params.values() if param.grad_req != 'null'] self._scale = optimizer_params.get('rescale_grad', 1.0) self._contexts = self._check_contexts() @@ -106,4 +107,4 @@ def step(self, batch_size, ignore_stale_grad=False): for upd, arr, grad in zip(self._updaters, param.list_data(), param.list_grad()): if arr._fresh_grad: upd(i, grad, arr) - grad._fresh_grad = False + arr._fresh_grad = False diff --git a/python/mxnet/nn/utils.py b/python/mxnet/foo/utils.py similarity index 100% rename from python/mxnet/nn/utils.py rename to python/mxnet/foo/utils.py diff --git a/tests/python/gpu/test_operator_gpu.py b/tests/python/gpu/test_operator_gpu.py index caa360b0a481..75229eecce72 100644 --- a/tests/python/gpu/test_operator_gpu.py +++ b/tests/python/gpu/test_operator_gpu.py @@ -7,6 +7,8 @@ from test_random import * from test_nn import * from test_rnn import * +from test_rnn import * +from test_foo_rnn import * import mxnet as mx import numpy as np from mxnet.test_utils import check_consistency, set_default_context @@ -1276,8 +1278,113 @@ def test_residual_fused(): assert np.array_equal(outputs[0].asnumpy(), expected_outputs) -def test_fused(): - check_rnn_forward(mx.rnn.FusedRNNCell(100, num_layers=2, num_input=200), +def test_foo_rnn(): + fused = foo.rnn.FusedRNNCell(100, num_layers=2, mode='rnn_relu', prefix='') + + stack = foo.rnn.SequentialRNNCell() + stack.add(foo.rnn.RNNCell(100, activation='relu', prefix='l0_')) + stack.add(foo.rnn.RNNCell(100, activation='relu', prefix='l1_')) + + check_rnn_consistency(fused, stack) + check_rnn_consistency(stack, fused) + + +def test_foo_lstm(): + fused = foo.rnn.FusedRNNCell(100, num_layers=2, mode='lstm', prefix='') + + stack = foo.rnn.SequentialRNNCell() + stack.add(foo.rnn.LSTMCell(100, prefix='l0_')) + stack.add(foo.rnn.LSTMCell(100, prefix='l1_')) + + check_rnn_consistency(fused, stack) + check_rnn_consistency(stack, fused) + + +def test_foo_lstm_forget_bias(): + forget_bias = 2.0 + fused = foo.rnn.FusedRNNCell(10, forget_bias=forget_bias, num_layers=2, mode='lstm', prefix='') + + dshape = (32, 1, 20) + data = mx.sym.Variable('data') + + sym, _ = fused.unroll(1, data, merge_outputs=True) + mod = mx.mod.Module(sym, label_names=None, context=mx.gpu(0)) + mod.bind(data_shapes=[('data', dshape)], label_shapes=None) + + mod.init_params() + + args, auxs = mod.get_params() + args = fused.unpack_weights(args) + + bias_name = next(x for x in args if x.endswith('f_bias')) + expected_bias = forget_bias * np.ones(10, ) + assert_allclose(args[bias_name].asnumpy(), expected_bias) + + +def test_foo_gru(): + fused = foo.rnn.FusedRNNCell(100, num_layers=2, mode='gru', prefix='') + + stack = foo.rnn.SequentialRNNCell() + stack.add(foo.rnn.GRUCell(100, prefix='l0_')) + stack.add(foo.rnn.GRUCell(100, prefix='l1_')) + + check_rnn_consistency(fused, stack) + check_rnn_consistency(stack, fused) + + +def test_foo_bidirectional(): + fused = foo.rnn.FusedRNNCell(100, num_layers=2, mode='gru', prefix='', + bidirectional=True) + + stack = foo.rnn.SequentialRNNCell() + stack.add(foo.rnn.BidirectionalCell( + foo.rnn.GRUCell(100, prefix='l0_'), + foo.rnn.GRUCell(100, prefix='r0_'), + output_prefix='bi_gru_0_')) + stack.add(foo.rnn.BidirectionalCell( + foo.rnn.GRUCell(100, prefix='l1_'), + foo.rnn.GRUCell(100, prefix='r1_'), + output_prefix='bi_gru_1_')) + + check_rnn_consistency(fused, stack) + check_rnn_consistency(stack, fused) + +def test_foo_unfuse(): + for mode in ['rnn_tanh', 'rnn_relu', 'lstm', 'gru']: + fused = foo.rnn.FusedRNNCell( + 100, num_layers=2, mode=mode, + prefix='test_%s'%mode, + bidirectional=True, + dropout=0.5) + + stack = fused.unfuse() + + check_rnn_consistency(fused, stack) + check_rnn_consistency(stack, fused) + + +def test_foo_residual_fused(): + cell = foo.rnn.ResidualCell( + foo.rnn.FusedRNNCell(50, num_layers=3, mode='lstm', + prefix='rnn_', dropout=0.5)) + + inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(2)] + outputs, _ = cell.unroll(2, inputs, merge_outputs=None) + assert sorted(cell.params._params.keys()) == \ + ['rnn_parameters'] + + args, outs, auxs = outputs.infer_shape(rnn_t0_data=(10, 50), rnn_t1_data=(10, 50)) + assert outs == [(10, 2, 50)] + outputs = outputs.eval(ctx=mx.gpu(0), + rnn_t0_data=mx.nd.ones((10, 50), ctx=mx.gpu(0))+5, + rnn_t1_data=mx.nd.ones((10, 50), ctx=mx.gpu(0))+5, + rnn_parameters=mx.nd.zeros((61200,), ctx=mx.gpu(0))) + expected_outputs = np.ones((10, 2, 50))+5 + assert np.array_equal(outputs[0].asnumpy(), expected_outputs) + + +def test_foo_fused(): + check_rnn_forward(foo.rnn.FusedRNNCell(100, num_layers=2, num_input=200), mx.nd.ones((8, 3, 200))) diff --git a/tests/python/unittest/test_autograd.py b/tests/python/unittest/test_autograd.py index 24b417afc233..9b2ea4b867f3 100644 --- a/tests/python/unittest/test_autograd.py +++ b/tests/python/unittest/test_autograd.py @@ -1,7 +1,79 @@ +import functools import mxnet.ndarray as nd -from mxnet.contrib.autograd import * +from mxnet.ndarray import zeros_like +from mxnet.autograd import * from mxnet.test_utils import * + +def grad_and_loss(func, argnum=None): + """Return function that computes both gradient of arguments and loss value. + + Parameters + ---------- + func: a python function + The forward (loss) function. + argnum: an int or a list of int + The index of argument to calculate gradient for. + + Returns + ------- + grad_and_loss_func: a python function + A function that would compute both the gradient of arguments and loss value. + """ + @functools.wraps(func) + def wrapped(*args): + """Wrapped function.""" + variables = args + if argnum is not None: + argnum_ = argnum if isinstance(argnum, list) else [argnum] + variables = [args[i] for i in argnum_] + for x in variables: + assert isinstance(x, NDArray), "type of autograd input should NDArray." + grads = [zeros_like(x) for x in variables] + mark_variables(variables, grads) + with train_section(): + outputs = func(*args) + backward([outputs] if isinstance(outputs, NDArray) else outputs) + return grads, outputs + return wrapped + +def grad(func, argnum=None): + """Return function that computes gradient of arguments. + + Parameters + ---------- + func: a python function + The forward (loss) function. + argnum: an int or a list of int + The index of argument to calculate gradient for. + + Returns + ------- + grad_func: a python function + A function that would compute the gradient of arguments. + + Examples + -------- + >>> # autograd supports dynamic graph which is changed + >>> # every instance + >>> def func(x): + >>> r = random.randint(0, 1) + >>> if r % 2: + >>> return x**2 + >>> else: + >>> return x/3 + >>> # use `grad(func)` to get the gradient function + >>> for x in range(10): + >>> grad_func = grad(func) + >>> inputs = nd.array([[1, 2, 3], [4, 5, 6]]) + >>> grad_vals = grad_func(inputs) + """ + grad_with_loss_func = grad_and_loss(func, argnum) + @functools.wraps(grad_with_loss_func) + def wrapped(*args): + return grad_with_loss_func(*args)[0] + return wrapped + def autograd_assert(*args, **kwargs): func = kwargs["func"] grad_f = kwargs["grad_func"] diff --git a/tests/python/unittest/test_contrib_autograd.py b/tests/python/unittest/test_contrib_autograd.py new file mode 100644 index 000000000000..24b417afc233 --- /dev/null +++ b/tests/python/unittest/test_contrib_autograd.py @@ -0,0 +1,167 @@ +import mxnet.ndarray as nd +from mxnet.contrib.autograd import * +from mxnet.test_utils import * + +def autograd_assert(*args, **kwargs): + func = kwargs["func"] + grad_f = kwargs["grad_func"] + argnum = kwargs["argnum"] if 'argnum' in kwargs else None + + grad_func = grad_and_loss(func, argnum) + grad_vals, output = grad_func(*args) + res = func(*args) + assert same(output.asnumpy(), res.asnumpy()) + grad_res = grad_f(*args) + assert len(grad_vals) == len(grad_res) + for a, b in zip(grad_vals, grad_res): + assert same(a.asnumpy(), b.asnumpy()) + +def test_unary_func(): + x = nd.uniform(shape=(4, 5)) + f_exp = lambda x: nd.exp(x) + f_exp_grad = lambda x: [nd.exp(x)] + autograd_assert(x, func=f_exp, grad_func=f_exp_grad) + f_half = lambda x: x/2 + f_half_grad = lambda x: [nd.ones(x.shape) * 0.5] + autograd_assert(x, func=f_half, grad_func=f_half_grad) + f_square = lambda x: x**2 + f_square_grad = lambda x: [2*x] + autograd_assert(x, func=f_square, grad_func=f_square_grad) + +def test_binary_func(): + x = nd.uniform(shape=(4, 5)) + y = nd.uniform(shape=(4, 5)) + f_add = lambda x, y: x+y + f_add_grad = lambda x, y: [nd.ones(x.shape), nd.ones(y.shape)] + autograd_assert(x, y, func=f_add, grad_func=f_add_grad) + f_mul = lambda x, y: x*y + f_mul_grad = lambda x, y: [y, x] + autograd_assert(x, y, func=f_mul, grad_func=f_mul_grad) + f_compose = lambda x, y: x+x*y + f_compose_grad = lambda x, y: [nd.ones(x.shape) + y, x] + autograd_assert(x, y, func=f_compose, grad_func=f_compose_grad) + +def test_operator_with_state(): + def f_fc(a, b, weight, bias): + x = a*b + fc = nd.FullyConnected( + x, weight, bias, num_hidden=32) + return fc + + a = nd.uniform(shape=(64, 50)) + b = nd.uniform(shape=(64, 50)) + weight = nd.uniform(shape=(32, 50)) + bias = nd.uniform(shape=(32, )) + + grad_func = grad_and_loss(f_fc) + grad_vals, outputs = grad_func(a, b, weight, bias) + # (TODO) assert + +def test_argnum(): + def f_with_mode(a, b, mode): + if mode: + return a+b + else: + return a*b + + a = nd.uniform(shape=(3, 2)) + b = nd.uniform(shape=(3, 2)) + f_add_grad = lambda x, y, mode: [nd.ones(x.shape), nd.ones(y.shape)] + f_mul_grad = lambda x, y, mode: [y, x] + autograd_assert(a, b, True, + argnum=[0, 1], func=f_with_mode, grad_func=f_add_grad) + autograd_assert(a, b, False, + argnum=[0, 1], func=f_with_mode, grad_func=f_mul_grad) + + +def test_training(): + x = nd.ones((10, 10)) + with train_section(): + y = nd.Dropout(x, p=0.5) + assert not (y.asnumpy() == x.asnumpy()).all() + with test_section(): + y = nd.Dropout(x, p=0.5) + assert (y.asnumpy() == x.asnumpy()).all() + + +def test_out_grads(): + x = nd.ones((3, 5)) + dx = nd.zeros_like(x) + mark_variables([x], [dx]) + da = None + db = nd.array([1,2,3,4,5]) + dc = nd.array([5,4,3,2,1]) + + with train_section(): + a, b, c = nd.split(x, axis=0, num_outputs=3, squeeze_axis=True) + backward([a, b, c], [da, db, dc]) + + assert (dx.asnumpy() == np.array( + [[1,1,1,1,1], + [1,2,3,4,5], + [5,4,3,2,1]])).all() + + +def test_detach_updated_grad(): + x = nd.ones((2, 2)) + dx = nd.zeros_like(x) + y = nd.ones_like(x) + dy = nd.zeros_like(x) + mark_variables([x, y], [dx, dy]) + assert x._fresh_grad == False + assert y._fresh_grad == False + + with train_section(): + x2 = x + 2 + y2 = x2 + y + y2.backward() + assert (dx.asnumpy() == 1).all() + assert x._fresh_grad == True + assert y._fresh_grad == True + + dx[:] = 0 + x._fresh_grad = False + y._fresh_grad = False + assert x._fresh_grad == False + assert y._fresh_grad == False + with train_section(): + x2 = x + 2 + x2 = x2.detach() + y2 = x2 + y + y2.backward() + assert (dx.asnumpy() == 0).all() + assert y._fresh_grad == True + assert x._fresh_grad == False + + +def test_retain_grad(): + x = mx.nd.ones((2, 2)) + dx = mx.nd.zeros((2, 2)) + mark_variables([x], [dx], grad_reqs='add') + with train_section(): + y = x + 1 + y.backward(retain_graph=False) + assert (dx.asnumpy() == 1).all() + + dx[:] = 0 + with train_section(): + y = x + 1 + y.backward(retain_graph=True) + y.backward(retain_graph=False) + assert (dx.asnumpy() == 2).all() + + try: + with train_section(): + y = x + 1 + y.backward() + y.backward() + except Exception: + return + + raise AssertionError( + "differentiating the same graph twice without retain_graph should fail") + + +if __name__ == "__main__": + import nose + nose.runmodule() diff --git a/tests/python/unittest/test_foo_rnn.py b/tests/python/unittest/test_foo_rnn.py new file mode 100644 index 000000000000..0c0f75c96146 --- /dev/null +++ b/tests/python/unittest/test_foo_rnn.py @@ -0,0 +1,213 @@ +import mxnet as mx +from mxnet import foo +import numpy as np +from numpy.testing import assert_allclose + + +def test_rnn(): + cell = foo.rnn.RNNCell(100, prefix='rnn_') + inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)] + outputs, _ = cell.unroll(3, inputs) + outputs = mx.sym.Group(outputs) + assert sorted(cell.params._params.keys()) == ['rnn_h2h_bias', 'rnn_h2h_weight', 'rnn_i2h_bias', 'rnn_i2h_weight'] + assert outputs.list_outputs() == ['rnn_t0_out_output', 'rnn_t1_out_output', 'rnn_t2_out_output'] + + args, outs, auxs = outputs.infer_shape(rnn_t0_data=(10,50), rnn_t1_data=(10,50), rnn_t2_data=(10,50)) + assert outs == [(10, 100), (10, 100), (10, 100)] + + +def test_lstm(): + cell = foo.rnn.LSTMCell(100, prefix='rnn_', forget_bias=1.0) + inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)] + outputs, _ = cell.unroll(3, inputs) + outputs = mx.sym.Group(outputs) + assert sorted(cell.params._params.keys()) == ['rnn_h2h_bias', 'rnn_h2h_weight', 'rnn_i2h_bias', 'rnn_i2h_weight'] + assert outputs.list_outputs() == ['rnn_t0_out_output', 'rnn_t1_out_output', 'rnn_t2_out_output'] + + args, outs, auxs = outputs.infer_shape(rnn_t0_data=(10,50), rnn_t1_data=(10,50), rnn_t2_data=(10,50)) + assert outs == [(10, 100), (10, 100), (10, 100)] + + +def test_lstm_forget_bias(): + forget_bias = 2.0 + stack = foo.rnn.SequentialRNNCell() + stack.add(foo.rnn.LSTMCell(100, forget_bias=forget_bias, prefix='l0_')) + stack.add(foo.rnn.LSTMCell(100, forget_bias=forget_bias, prefix='l1_')) + + dshape = (32, 1, 200) + data = mx.sym.Variable('data') + + sym, _ = stack.unroll(1, data, merge_outputs=True) + mod = mx.mod.Module(sym, label_names=None, context=mx.cpu(0)) + mod.bind(data_shapes=[('data', dshape)], label_shapes=None) + + mod.init_params() + + bias_argument = next(x for x in sym.list_arguments() if x.endswith('i2h_bias')) + expected_bias = np.hstack([np.zeros((100,)), + forget_bias * np.ones(100, ), np.zeros((2 * 100,))]) + assert_allclose(mod.get_params()[0][bias_argument].asnumpy(), expected_bias) + + +def test_gru(): + cell = foo.rnn.GRUCell(100, prefix='rnn_') + inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)] + outputs, _ = cell.unroll(3, inputs) + outputs = mx.sym.Group(outputs) + assert sorted(cell.params._params.keys()) == ['rnn_h2h_bias', 'rnn_h2h_weight', 'rnn_i2h_bias', 'rnn_i2h_weight'] + assert outputs.list_outputs() == ['rnn_t0_out_output', 'rnn_t1_out_output', 'rnn_t2_out_output'] + + args, outs, auxs = outputs.infer_shape(rnn_t0_data=(10,50), rnn_t1_data=(10,50), rnn_t2_data=(10,50)) + assert outs == [(10, 100), (10, 100), (10, 100)] + + +def test_residual(): + cell = foo.rnn.ResidualCell(foo.rnn.GRUCell(50, prefix='rnn_')) + inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(2)] + outputs, _ = cell.unroll(2, inputs) + outputs = mx.sym.Group(outputs) + assert sorted(cell.params._params.keys()) == \ + ['rnn_h2h_bias', 'rnn_h2h_weight', 'rnn_i2h_bias', 'rnn_i2h_weight'] + # assert outputs.list_outputs() == \ + # ['rnn_t0_out_plus_residual_output', 'rnn_t1_out_plus_residual_output'] + + args, outs, auxs = outputs.infer_shape(rnn_t0_data=(10, 50), rnn_t1_data=(10, 50)) + assert outs == [(10, 50), (10, 50)] + outputs = outputs.eval(rnn_t0_data=mx.nd.ones((10, 50)), + rnn_t1_data=mx.nd.ones((10, 50)), + rnn_i2h_weight=mx.nd.zeros((150, 50)), + rnn_i2h_bias=mx.nd.zeros((150,)), + rnn_h2h_weight=mx.nd.zeros((150, 50)), + rnn_h2h_bias=mx.nd.zeros((150,))) + expected_outputs = np.ones((10, 50)) + assert np.array_equal(outputs[0].asnumpy(), expected_outputs) + assert np.array_equal(outputs[1].asnumpy(), expected_outputs) + + +def test_residual_bidirectional(): + cell = foo.rnn.ResidualCell( + foo.rnn.BidirectionalCell( + foo.rnn.GRUCell(25, prefix='rnn_l_'), + foo.rnn.GRUCell(25, prefix='rnn_r_'))) + + inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(2)] + outputs, _ = cell.unroll(2, inputs, merge_outputs=False) + outputs = mx.sym.Group(outputs) + assert sorted(cell.params._params.keys()) == \ + ['rnn_l_h2h_bias', 'rnn_l_h2h_weight', 'rnn_l_i2h_bias', 'rnn_l_i2h_weight', + 'rnn_r_h2h_bias', 'rnn_r_h2h_weight', 'rnn_r_i2h_bias', 'rnn_r_i2h_weight'] + # assert outputs.list_outputs() == \ + # ['bi_t0_plus_residual_output', 'bi_t1_plus_residual_output'] + + args, outs, auxs = outputs.infer_shape(rnn_t0_data=(10, 50), rnn_t1_data=(10, 50)) + assert outs == [(10, 50), (10, 50)] + outputs = outputs.eval(rnn_t0_data=mx.nd.ones((10, 50))+5, + rnn_t1_data=mx.nd.ones((10, 50))+5, + rnn_l_i2h_weight=mx.nd.zeros((75, 50)), + rnn_l_i2h_bias=mx.nd.zeros((75,)), + rnn_l_h2h_weight=mx.nd.zeros((75, 25)), + rnn_l_h2h_bias=mx.nd.zeros((75,)), + rnn_r_i2h_weight=mx.nd.zeros((75, 50)), + rnn_r_i2h_bias=mx.nd.zeros((75,)), + rnn_r_h2h_weight=mx.nd.zeros((75, 25)), + rnn_r_h2h_bias=mx.nd.zeros((75,))) + expected_outputs = np.ones((10, 50))+5 + assert np.array_equal(outputs[0].asnumpy(), expected_outputs) + assert np.array_equal(outputs[1].asnumpy(), expected_outputs) + + +def test_stack(): + cell = foo.rnn.SequentialRNNCell() + for i in range(5): + if i == 1: + cell.add(foo.rnn.ResidualCell(foo.rnn.LSTMCell(100, prefix='rnn_stack%d_' % i))) + else: + cell.add(foo.rnn.LSTMCell(100, prefix='rnn_stack%d_'%i)) + inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)] + outputs, _ = cell.unroll(3, inputs) + outputs = mx.sym.Group(outputs) + keys = sorted(cell.params._params.keys()) + for i in range(5): + assert 'rnn_stack%d_h2h_weight'%i in keys + assert 'rnn_stack%d_h2h_bias'%i in keys + assert 'rnn_stack%d_i2h_weight'%i in keys + assert 'rnn_stack%d_i2h_bias'%i in keys + assert outputs.list_outputs() == ['rnn_stack4_t0_out_output', 'rnn_stack4_t1_out_output', 'rnn_stack4_t2_out_output'] + + args, outs, auxs = outputs.infer_shape(rnn_t0_data=(10,50), rnn_t1_data=(10,50), rnn_t2_data=(10,50)) + assert outs == [(10, 100), (10, 100), (10, 100)] + + +def test_bidirectional(): + cell = foo.rnn.BidirectionalCell( + foo.rnn.LSTMCell(100, prefix='rnn_l0_'), + foo.rnn.LSTMCell(100, prefix='rnn_r0_'), + output_prefix='rnn_bi_') + inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)] + outputs, _ = cell.unroll(3, inputs) + outputs = mx.sym.Group(outputs) + assert outputs.list_outputs() == ['rnn_bi_t0_output', 'rnn_bi_t1_output', 'rnn_bi_t2_output'] + + args, outs, auxs = outputs.infer_shape(rnn_t0_data=(10,50), rnn_t1_data=(10,50), rnn_t2_data=(10,50)) + assert outs == [(10, 200), (10, 200), (10, 200)] + + +def test_zoneout(): + cell = foo.rnn.ZoneoutCell(foo.rnn.RNNCell(100, prefix='rnn_'), zoneout_outputs=0.5, + zoneout_states=0.5) + inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)] + outputs, _ = cell.unroll(3, inputs) + outputs = mx.sym.Group(outputs) + + args, outs, auxs = outputs.infer_shape(rnn_t0_data=(10,50), rnn_t1_data=(10,50), rnn_t2_data=(10,50)) + assert outs == [(10, 100), (10, 100), (10, 100)] + + +def test_unfuse(): + cell = foo.rnn.FusedRNNCell(100, num_layers=3, mode='lstm', + prefix='test_', bidirectional=True, + dropout=0.5) + cell = cell.unfuse() + inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)] + outputs, _ = cell.unroll(3, inputs) + outputs = mx.sym.Group(outputs) + assert outputs.list_outputs() == ['test_bi_l2_t0_output', 'test_bi_l2_t1_output', 'test_bi_l2_t2_output'] + + args, outs, auxs = outputs.infer_shape(rnn_t0_data=(10,50), rnn_t1_data=(10,50), rnn_t2_data=(10,50)) + assert outs == [(10, 200), (10, 200), (10, 200)] + + +def check_rnn_forward(layer, inputs): + layer.params.initialize() + with mx.contrib.autograd.train_section(): + mx.contrib.autograd.compute_gradient( + [layer.unroll(3, inputs, merge_outputs=True)[0]]) + mx.contrib.autograd.compute_gradient( + layer.unroll(3, inputs, merge_outputs=False)[0]) + + +def test_rnn_cells(): + check_rnn_forward(foo.rnn.LSTMCell(100, num_input=200), mx.nd.ones((8, 3, 200))) + check_rnn_forward(foo.rnn.RNNCell(100, num_input=200), mx.nd.ones((8, 3, 200))) + check_rnn_forward(foo.rnn.GRUCell(100, num_input=200), mx.nd.ones((8, 3, 200))) + + bilayer = foo.rnn.BidirectionalCell(foo.rnn.LSTMCell(100, num_input=200), + foo.rnn.LSTMCell(100, num_input=200)) + check_rnn_forward(bilayer, mx.nd.ones((8, 3, 200))) + + check_rnn_forward(foo.rnn.DropoutCell(0.5), mx.nd.ones((8, 3, 200))) + + check_rnn_forward(foo.rnn.ZoneoutCell(foo.rnn.LSTMCell(100, num_input=200), + 0.5, 0.2), + mx.nd.ones((8, 3, 200))) + + net = foo.rnn.SequentialRNNCell() + net.add(foo.rnn.LSTMCell(100, num_input=200)) + net.add(foo.rnn.RNNCell(100, num_input=100)) + net.add(foo.rnn.GRUCell(100, num_input=100)) + check_rnn_forward(net, mx.nd.ones((8, 3, 200))) + + +if __name__ == '__main__': + import nose + nose.runmodule() diff --git a/tests/python/unittest/test_loss.py b/tests/python/unittest/test_loss.py index 21cfd76030d1..d876d1fef1c6 100644 --- a/tests/python/unittest/test_loss.py +++ b/tests/python/unittest/test_loss.py @@ -1,6 +1,6 @@ import mxnet as mx import numpy as np -from mxnet import nn +from mxnet import foo def test_loss_ndarray(): @@ -8,22 +8,22 @@ def test_loss_ndarray(): label = mx.nd.array([1, 3, 5, 7]) weighting = mx.nd.array([0.5, 1, 0.5, 1]) - assert mx.nd.sum(nn.loss.l1_loss(output, label)).asscalar() == 6. - assert mx.nd.sum(nn.loss.l1_loss(output, label, weight=0.5)).asscalar() == 3. - assert mx.nd.sum(nn.loss.l1_loss(output, label, sample_weight=weighting)).asscalar() == 5. + assert mx.nd.sum(foo.loss.l1_loss(output, label)).asscalar() == 6. + assert mx.nd.sum(foo.loss.l1_loss(output, label, weight=0.5)).asscalar() == 3. + assert mx.nd.sum(foo.loss.l1_loss(output, label, sample_weight=weighting)).asscalar() == 5. - assert mx.nd.sum(nn.loss.l2_loss(output, label)).asscalar() == 7. - assert mx.nd.sum(nn.loss.l2_loss(output, label, weight=0.25)).asscalar() == 1.75 - assert mx.nd.sum(nn.loss.l2_loss(output, label, sample_weight=weighting)).asscalar() == 6 + assert mx.nd.sum(foo.loss.l2_loss(output, label)).asscalar() == 7. + assert mx.nd.sum(foo.loss.l2_loss(output, label, weight=0.25)).asscalar() == 1.75 + assert mx.nd.sum(foo.loss.l2_loss(output, label, sample_weight=weighting)).asscalar() == 6 output = mx.nd.array([[0, 2], [1, 4]]) label = mx.nd.array([0, 1]) weighting = mx.nd.array([[0.5], [1.0]]) - loss = nn.loss.softmax_cross_entropy_loss(output, label).asnumpy() + loss = foo.loss.softmax_cross_entropy_loss(output, label).asnumpy() mx.test_utils.assert_almost_equal(loss, np.array([ 2.12692809, 0.04858733])) - loss = nn.loss.softmax_cross_entropy_loss(output, label, sample_weight=weighting).asnumpy() + loss = foo.loss.softmax_cross_entropy_loss(output, label, sample_weight=weighting).asnumpy() mx.test_utils.assert_almost_equal(loss, np.array([ 1.06346405, 0.04858733])) @@ -49,9 +49,9 @@ def check_loss(loss): def test_loss_symbol(): - check_loss(nn.loss.l1_loss) - check_loss(nn.loss.l2_loss) - check_loss(nn.loss.softmax_cross_entropy_loss) + check_loss(foo.loss.l1_loss) + check_loss(foo.loss.l2_loss) + check_loss(foo.loss.softmax_cross_entropy_loss) def get_net(num_hidden): @@ -75,7 +75,7 @@ def test_ce_loss(): output = get_net(nclass) fc2 = output.get_internals()['fc2_output'] l = mx.symbol.Variable('label') - loss = nn.loss.softmax_cross_entropy_loss(output, l, extra_outputs=(fc2,)) + loss = foo.loss.softmax_cross_entropy_loss(output, l, extra_outputs=(fc2,)) mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',)) mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 1.}) assert mod.score(data_iter)[0][1] == 1.0 @@ -90,7 +90,7 @@ def test_l2_loss(): data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label') output = get_net(1) l = mx.symbol.Variable('label') - loss = nn.loss.l2_loss(output, l) + loss = foo.loss.l2_loss(output, l) mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',)) mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 1.}) assert mod.score(data_iter)[0][1] < 0.05 @@ -105,7 +105,7 @@ def test_l1_loss(): data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label') output = get_net(1) l = mx.symbol.Variable('label') - loss = nn.loss.l1_loss(output, l) + loss = foo.loss.l1_loss(output, l) mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',)) mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 0.1}, initializer=mx.init.Uniform(0.5)) @@ -122,7 +122,7 @@ def test_custom_loss(): output = get_net(1) l = mx.symbol.Variable('label') loss = mx.sym.square(output - l) - loss = nn.loss.custom_loss(loss, output, l, weight=0.5, metrics='mse') + loss = foo.loss.custom_loss(loss, output, l, weight=0.5, metrics='mse') mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',)) mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 1.}) @@ -141,7 +141,7 @@ def test_sample_weight_loss(): output = get_net(nclass) l = mx.symbol.Variable('label') w = mx.symbol.Variable('w') - loss = nn.loss.softmax_cross_entropy_loss(output, l, sample_weight=w) + loss = foo.loss.softmax_cross_entropy_loss(output, l, sample_weight=w) mod = mx.mod.Module(loss, data_names=('data',), label_names=('label', 'w')) mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 1.}) @@ -165,9 +165,9 @@ def test_multi_loss(): output2 = mx.symbol.FullyConnected(act3, name='output2', num_hidden=5) l1 = mx.symbol.Variable('label1') l2 = mx.symbol.Variable('label2') - loss1 = nn.loss.softmax_cross_entropy_loss(output1, l1) - loss2 = nn.loss.l2_loss(output2, l2) - loss = nn.loss.multitask_loss([loss1, loss2]) + loss1 = foo.loss.softmax_cross_entropy_loss(output1, l1) + loss2 = foo.loss.l2_loss(output2, l2) + loss = foo.loss.multitask_loss([loss1, loss2]) mod = mx.mod.Module(loss, data_names=('data',), label_names=('label1', 'label2')) mod.fit(data_iter, num_epoch=200, @@ -194,7 +194,7 @@ def test_saveload(): data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label') output = get_net(nclass) l = mx.symbol.Variable('label') - loss = nn.loss.softmax_cross_entropy_loss(output, l) + loss = foo.loss.softmax_cross_entropy_loss(output, l) mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',)) mod.fit(data_iter, num_epoch=100, optimizer_params={'learning_rate': 1.}) mod.save_checkpoint('test', 100, save_optimizer_states=True) diff --git a/tests/python/unittest/test_nn.py b/tests/python/unittest/test_nn.py index 1116b6898f2d..1317e361c31f 100644 --- a/tests/python/unittest/test_nn.py +++ b/tests/python/unittest/test_nn.py @@ -1,10 +1,11 @@ import mxnet as mx -from mxnet import nn +from mxnet import foo +from mxnet.foo import nn import numpy as np def test_parameter(): - p = nn.Parameter('weight', shape=(10, 10)) + p = foo.Parameter('weight', shape=(10, 10)) p.initialize(init='xavier', ctx=[mx.cpu(0), mx.cpu(1)]) assert len(p.list_data()) == 2 assert len(p.list_grad()) == 2 @@ -14,7 +15,7 @@ def test_parameter(): def test_paramdict(): - params = nn.ParameterDict('net_') + params = foo.ParameterDict('net_') params.get('weight', shape=(10, 10)) assert list(params.keys()) == ['net_weight'] params.initialize() diff --git a/tests/python/unittest/test_rnn.py b/tests/python/unittest/test_rnn.py index bc5494ad3586..75f41fe13389 100644 --- a/tests/python/unittest/test_rnn.py +++ b/tests/python/unittest/test_rnn.py @@ -279,37 +279,6 @@ def test_convgru(): args, outs, auxs = outputs.infer_shape(rnn_t0_data=(1, 3, 16, 10), rnn_t1_data=(1, 3, 16, 10), rnn_t2_data=(1, 3, 16, 10)) assert outs == [(1, 10, 16, 10), (1, 10, 16, 10), (1, 10, 16, 10)] -def check_rnn_forward(layer, inputs): - layer.params.initialize() - with mx.contrib.autograd.train_section(): - mx.contrib.autograd.compute_gradient( - [layer.unroll(3, inputs, merge_outputs=True)[0]]) - mx.contrib.autograd.compute_gradient( - layer.unroll(3, inputs, merge_outputs=False)[0]) - - -def test_rnn_cells(): - check_rnn_forward(mx.rnn.LSTMCell(100, num_input=200), mx.nd.ones((8, 3, 200))) - check_rnn_forward(mx.rnn.RNNCell(100, num_input=200), mx.nd.ones((8, 3, 200))) - check_rnn_forward(mx.rnn.GRUCell(100, num_input=200), mx.nd.ones((8, 3, 200))) - - bilayer = mx.rnn.BidirectionalCell(mx.rnn.LSTMCell(100, num_input=200), - mx.rnn.LSTMCell(100, num_input=200)) - check_rnn_forward(bilayer, mx.nd.ones((8, 3, 200))) - - check_rnn_forward(mx.rnn.DropoutCell(0.5), mx.nd.ones((8, 3, 200))) - - check_rnn_forward(mx.rnn.ZoneoutCell(mx.rnn.LSTMCell(100, num_input=200), - 0.5, 0.2), - mx.nd.ones((8, 3, 200))) - - net = mx.rnn.SequentialRNNCell() - net.add(mx.rnn.LSTMCell(100, num_input=200)) - net.add(mx.rnn.RNNCell(100, num_input=100)) - net.add(mx.rnn.GRUCell(100, num_input=100)) - check_rnn_forward(net, mx.nd.ones((8, 3, 200))) - - if __name__ == '__main__': import nose nose.runmodule() From 1e2716926cbdae190c915e1620b2060214a68cb8 Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Tue, 6 Jun 2017 10:40:03 -0700 Subject: [PATCH 179/834] [WIP] Foo (NN) API reference doc (#6583) * nn api reference * fix docs for submodules * nn path structure change * fix doc * split pages --- docs/api/python/foo.loss.md | 23 ++++++++++ docs/api/python/foo.md | 24 +++++++++++ docs/api/python/foo.nn.md | 72 ++++++++++++++++++++++++++++++++ docs/api/python/foo.rnn.md | 40 ++++++++++++++++++ docs/api/python/foo.utils.md | 20 +++++++++ docs/api/python/index.md | 5 +++ python/mxnet/foo/rnn/rnn_cell.py | 10 ++--- 7 files changed, 189 insertions(+), 5 deletions(-) create mode 100644 docs/api/python/foo.loss.md create mode 100644 docs/api/python/foo.md create mode 100644 docs/api/python/foo.nn.md create mode 100644 docs/api/python/foo.rnn.md create mode 100644 docs/api/python/foo.utils.md diff --git a/docs/api/python/foo.loss.md b/docs/api/python/foo.loss.md new file mode 100644 index 000000000000..b35a6942c6a1 --- /dev/null +++ b/docs/api/python/foo.loss.md @@ -0,0 +1,23 @@ +# Foo Loss API + +```eval_rst +.. currentmodule:: mxnet.foo.loss +``` + +```eval_rst +.. warning:: This package is currently experimental and may change in the near future. +``` + +## API Reference + + + +```eval_rst +.. automethod:: mxnet.foo.loss.custom_loss +.. automethod:: mxnet.foo.loss.multitask_loss +.. automethod:: mxnet.foo.loss.l1_loss +.. automethod:: mxnet.foo.loss.l2_loss +.. automethod:: mxnet.foo.loss.softmax_cross_entropy_loss +``` + + diff --git a/docs/api/python/foo.md b/docs/api/python/foo.md new file mode 100644 index 000000000000..72897fc1676b --- /dev/null +++ b/docs/api/python/foo.md @@ -0,0 +1,24 @@ +# Foo API + +```eval_rst +.. currentmodule:: mxnet.foo +``` + +```eval_rst +.. warning:: This package is currently experimental and may change in the near future. +``` + +## API Reference + + + +```eval_rst +.. autoclass:: mxnet.foo.Parameter + :members: +.. autoclass:: mxnet.foo.ParameterDict + :members: +.. autoclass:: mxnet.foo.Trainer + :members: +``` + + diff --git a/docs/api/python/foo.nn.md b/docs/api/python/foo.nn.md new file mode 100644 index 000000000000..184f0ecc5dbb --- /dev/null +++ b/docs/api/python/foo.nn.md @@ -0,0 +1,72 @@ +# Foo NN API + +```eval_rst +.. currentmodule:: mxnet.foo.nn +``` + +```eval_rst +.. warning:: This package is currently experimental and may change in the near future. +``` + +## API Reference + + + +```eval_rst +.. currentmodule:: mxnet.foo.nn +.. autoclass:: mxnet.foo.nn.Layer + :members: + + .. automethod:: __call__ +.. autoclass:: mxnet.foo.nn.Sequential + :members: +.. autoclass:: mxnet.foo.nn.Dense + :members: +.. autoclass:: mxnet.foo.nn.Activation + :members: +.. autoclass:: mxnet.foo.nn.Dropout + :members: +.. autoclass:: mxnet.foo.nn.BatchNorm + :members: +.. autoclass:: mxnet.foo.nn.LeakyReLU + :members: + +.. autoclass:: mxnet.foo.nn.Conv1D + :members: +.. autoclass:: mxnet.foo.nn.Conv2D + :members: +.. autoclass:: mxnet.foo.nn.Conv3D + :members: +.. autoclass:: mxnet.foo.nn.Conv1DTranspose + :members: +.. autoclass:: mxnet.foo.nn.Conv2DTranspose + :members: +.. autoclass:: mxnet.foo.nn.Conv3DTranspose + :members: +.. autoclass:: mxnet.foo.nn.MaxPool1D + :members: +.. autoclass:: mxnet.foo.nn.MaxPool2D + :members: +.. autoclass:: mxnet.foo.nn.MaxPool3D + :members: +.. autoclass:: mxnet.foo.nn.AvgPool1D + :members: +.. autoclass:: mxnet.foo.nn.AvgPool2D + :members: +.. autoclass:: mxnet.foo.nn.AvgPool3D + :members: +.. autoclass:: mxnet.foo.nn.GlobalMaxPool1D + :members: +.. autoclass:: mxnet.foo.nn.GlobalMaxPool2D + :members: +.. autoclass:: mxnet.foo.nn.GlobalMaxPool3D + :members: +.. autoclass:: mxnet.foo.nn.GlobalAvgPool1D + :members: +.. autoclass:: mxnet.foo.nn.GlobalAvgPool2D + :members: +.. autoclass:: mxnet.foo.nn.GlobalAvgPool3D + :members: +``` + + diff --git a/docs/api/python/foo.rnn.md b/docs/api/python/foo.rnn.md new file mode 100644 index 000000000000..e2c2b37a1f6f --- /dev/null +++ b/docs/api/python/foo.rnn.md @@ -0,0 +1,40 @@ +# Foo RNN API + +```eval_rst +.. currentmodule:: mxnet.foo.rnn +``` + +```eval_rst +.. warning:: This package is currently experimental and may change in the near future. +``` + +## API Reference + + + +```eval_rst +.. autoclass:: mxnet.foo.rnn.RecurrentCell + :members: + + .. automethod:: __call__ +.. autoclass:: mxnet.foo.rnn.LSTMCell + :members: +.. autoclass:: mxnet.foo.rnn.GRUCell + :members: +.. autoclass:: mxnet.foo.rnn.RNNCell + :members: +.. autoclass:: mxnet.foo.rnn.FusedRNNCell + :members: +.. autoclass:: mxnet.foo.rnn.SequentialRNNCell + :members: +.. autoclass:: mxnet.foo.rnn.BidirectionalCell + :members: +.. autoclass:: mxnet.foo.rnn.DropoutCell + :members: +.. autoclass:: mxnet.foo.rnn.ZoneoutCell + :members: +.. autoclass:: mxnet.foo.rnn.ResidualCell + :members: +``` + + diff --git a/docs/api/python/foo.utils.md b/docs/api/python/foo.utils.md new file mode 100644 index 000000000000..21dea1a0c2b9 --- /dev/null +++ b/docs/api/python/foo.utils.md @@ -0,0 +1,20 @@ +# Foo Utility API + +```eval_rst +.. currentmodule:: mxnet.foo.utils +``` + +```eval_rst +.. warning:: This package is currently experimental and may change in the near future. +``` + +## API Reference + + + +```eval_rst +.. automethod:: mxnet.foo.utils.split_data +.. automethod:: mxnet.foo.utils.load_data +``` + + diff --git a/docs/api/python/index.md b/docs/api/python/index.md index 6051c0e858c3..fe102eb6a601 100644 --- a/docs/api/python/index.md +++ b/docs/api/python/index.md @@ -28,6 +28,11 @@ imported by running: ndarray symbol module + foo + foo.nn + foo.rnn + foo.loss + foo.utils rnn kvstore io diff --git a/python/mxnet/foo/rnn/rnn_cell.py b/python/mxnet/foo/rnn/rnn_cell.py index e3bd04f50177..c8f070d4c1a5 100644 --- a/python/mxnet/foo/rnn/rnn_cell.py +++ b/python/mxnet/foo/rnn/rnn_cell.py @@ -93,9 +93,9 @@ class RecurrentCell(Layer): Prefix for names of layers (this prefix is also used for names of weights if `params` is None i.e. if `params` are being created and not reused) - params : RNNParams or None, optional + params : Parameter or None, optional Container for weight sharing between cells. - A new RNNParams container is created if `params` is None. + A new Parameter container is created if `params` is None. """ def __init__(self, prefix=None, params=None): super(RecurrentCell, self).__init__(prefix=prefix, params=params) @@ -345,7 +345,7 @@ class RNNCell(RecurrentCell): prefix : str, default 'rnn_' prefix for name of layers (and name of weight if params is None) - params : RNNParams or None + params : Parameter or None container for weight sharing between cells. created if None. """ @@ -395,7 +395,7 @@ class LSTMCell(RecurrentCell): prefix : str, default 'lstm_' prefix for name of layers (and name of weight if params is None) - params : RNNParams or None + params : Parameter or None container for weight sharing between cells. created if None. forget_bias : bias added to forget gate, default 1.0. @@ -465,7 +465,7 @@ class GRUCell(RecurrentCell): prefix : str, default 'gru_' prefix for name of layers (and name of weight if params is None) - params : RNNParams or None + params : Parameter or None container for weight sharing between cells. created if None. """ From 11796c3e8408b92d07ccde16e39c85041374a0cb Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Tue, 6 Jun 2017 11:27:50 -0700 Subject: [PATCH 180/834] error messages for ndarray-only methods (#6591) * nn warning for methods implemented in nn * fix 'if sym' testing --- python/mxnet/rnn/rnn_cell.py | 2 +- python/mxnet/symbol.py | 42 ++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/python/mxnet/rnn/rnn_cell.py b/python/mxnet/rnn/rnn_cell.py index 320f78120c5c..99d0e8ad606f 100644 --- a/python/mxnet/rnn/rnn_cell.py +++ b/python/mxnet/rnn/rnn_cell.py @@ -922,7 +922,7 @@ def __call__(self, inputs, states): next_output, next_states = cell(inputs, states) mask = lambda p, like: symbol.Dropout(symbol.ones_like(like), p=p) - prev_output = self.prev_output if self.prev_output else symbol.zeros((0, 0)) + prev_output = self.prev_output if self.prev_output is not None else symbol.zeros((0, 0)) output = (symbol.where(mask(p_outputs, next_output), next_output, prev_output) if p_outputs != 0. else next_output) diff --git a/python/mxnet/symbol.py b/python/mxnet/symbol.py index 7151398d8ba0..e7f69037362b 100644 --- a/python/mxnet/symbol.py +++ b/python/mxnet/symbol.py @@ -94,6 +94,9 @@ def __add__(self, other): else: raise TypeError('type %s not supported' % str(type(other))) + def __iadd__(self, other): + raise NotImplementedError('Not supported. This is available in NDArray only') + def __radd__(self, other): return self.__add__(other) @@ -109,6 +112,9 @@ def __sub__(self, other): else: raise TypeError('type %s not supported' % str(type(other))) + def __isub__(self, other): + raise NotImplementedError('Not supported. This is available in NDArray only') + def __rsub__(self, other): """x.__rsub__(y) <=> y-x @@ -139,6 +145,9 @@ def __mul__(self, other): else: raise TypeError('type %s not supported' % str(type(other))) + def __imul__(self, other): + raise NotImplementedError('Not supported. This is available in NDArray only') + def __rmul__(self, other): return self.__mul__(other) @@ -202,12 +211,18 @@ def __rmod__(self, other): else: raise TypeError('type %s not supported' % str(type(other))) + def __idiv__(self, other): + raise NotImplementedError('Not supported. This is available in NDArray only') + def __truediv__(self, other): return self.__div__(other) def __rtruediv__(self, other): return self.__rdiv__(other) + def __itruediv__(self, other): + raise NotImplementedError('Not supported. This is available in NDArray only') + def __pow__(self, other): """x.__pow__(y) <=> x**y @@ -220,6 +235,9 @@ def __pow__(self, other): else: raise TypeError('type %s not supported' % str(type(other))) + def __rpow__(self, other): + raise NotImplementedError('Not supported. This is available in NDArray only') + def __neg__(self): """x.__neg__() <=> -x @@ -1697,6 +1715,30 @@ def reshape(self, shape): """ return reshape(self, shape=shape) + def wait_to_read(self): + raise NotImplementedError('Not supported. This is available in NDArray only') + + def asnumpy(self): + raise NotImplementedError('Not supported. This is available in NDArray only') + + def asscalar(self): + raise NotImplementedError('Not supported. This is available in NDArray only') + + def astype(self): + raise NotImplementedError('Not supported. This is available in NDArray only') + + def copy(self): + raise NotImplementedError('Not supported. This is available in NDArray only') + + def as_in_context(self): + raise NotImplementedError('Not supported. This is available in NDArray only') + + def detach(self): + raise NotImplementedError('Not supported. This is available in NDArray only') + + def backward(self): + raise NotImplementedError('Not supported. This is available in NDArray only') + def var(name, attr=None, shape=None, lr_mult=None, wd_mult=None, dtype=None, init=None, **kwargs): """Creates a symbolic variable with specified name. From 120a00c0d4bf06a6d59be00ce023428368987fb2 Mon Sep 17 00:00:00 2001 From: Eric Junyuan Xie Date: Tue, 6 Jun 2017 22:19:13 -0700 Subject: [PATCH 181/834] split Layer.params into Layer.params and Layer.all_params() (#6598) --- example/autograd/actor_critic.py | 4 +- example/autograd/dcgan.py | 8 +-- example/autograd/mnist.py | 12 ++-- example/autograd/resnet.py | 6 +- python/mxnet/foo/nn/layer.py | 65 +++++++++--------- python/mxnet/foo/parameter.py | 99 +++++++++++---------------- python/mxnet/foo/rnn/rnn_cell.py | 6 +- python/mxnet/foo/utils.py | 2 +- tests/python/unittest/test_foo_rnn.py | 14 ++-- tests/python/unittest/test_nn.py | 28 ++++++-- 10 files changed, 121 insertions(+), 123 deletions(-) diff --git a/example/autograd/actor_critic.py b/example/autograd/actor_critic.py index 44feecf43e15..1e87178f3679 100644 --- a/example/autograd/actor_critic.py +++ b/example/autograd/actor_critic.py @@ -42,8 +42,8 @@ def generic_forward(self, F, x): return F.softmax(probs), values net = Policy() -net.params.initialize(mx.init.Uniform(0.02)) -trainer = foo.Trainer(net.params, 'adam', {'learning_rate': 3e-2}) +net.all_params().initialize(mx.init.Uniform(0.02)) +trainer = foo.Trainer(net.all_params(), 'adam', {'learning_rate': 3e-2}) running_reward = 10 diff --git a/example/autograd/dcgan.py b/example/autograd/dcgan.py index 920f2c9dd5bb..c4008ab72bd0 100644 --- a/example/autograd/dcgan.py +++ b/example/autograd/dcgan.py @@ -79,12 +79,12 @@ # netD.add(nn.Activation('sigmoid')) -netG.params.initialize(mx.init.Normal(0.02), ctx=ctx) -netD.params.initialize(mx.init.Normal(0.02), ctx=ctx) +netG.all_params().initialize(mx.init.Normal(0.02), ctx=ctx) +netD.all_params().initialize(mx.init.Normal(0.02), ctx=ctx) -trainerG = foo.Trainer(netG.params, 'adam', {'learning_rate': opt.lr, 'beta1': opt.beta1}) -trainerD = foo.Trainer(netD.params, 'adam', {'learning_rate': opt.lr, 'beta1': opt.beta1}) +trainerG = foo.Trainer(netG.all_params(), 'adam', {'learning_rate': opt.lr, 'beta1': opt.beta1}) +trainerD = foo.Trainer(netD.all_params(), 'adam', {'learning_rate': opt.lr, 'beta1': opt.beta1}) real_label = mx.nd.ones((opt.batchSize,), ctx=ctx) diff --git a/example/autograd/mnist.py b/example/autograd/mnist.py index 66878fd177c1..84dd011e03c3 100644 --- a/example/autograd/mnist.py +++ b/example/autograd/mnist.py @@ -1,4 +1,6 @@ # pylint: skip-file +from __future__ import print_function + from data import mnist_iterator import mxnet as mx from mxnet import foo @@ -31,13 +33,13 @@ def test(ctx): for x in data: outputs.append(net(x)) metric.update(label, outputs) - print 'validation acc: %s=%f'%metric.get() + print('validation acc: %s=%f'%metric.get()) def train(epoch, ctx): if isinstance(ctx, mx.Context): ctx = [ctx] - net.params.initialize(mx.init.Xavier(magnitude=2.24), ctx=ctx) - trainer = foo.Trainer(net.params, 'sgd', {'learning_rate': 0.1}) + net.all_params().initialize(mx.init.Xavier(magnitude=2.24), ctx=ctx) + trainer = foo.Trainer(net.all_params(), 'sgd', {'learning_rate': 0.1}) metric = mx.metric.Accuracy() for i in range(epoch): @@ -56,10 +58,10 @@ def train(epoch, ctx): trainer.step(batch.data[0].shape[0]) name, acc = metric.get() metric.reset() - print 'training acc at epoch %d: %s=%f'%(i, name, acc) + print('training acc at epoch %d: %s=%f'%(i, name, acc)) test(ctx) - net.params.save('mnist.params') + net.all_params().save('mnist.params') if __name__ == '__main__': diff --git a/example/autograd/resnet.py b/example/autograd/resnet.py index a27fc2570087..d6e52157b4b1 100644 --- a/example/autograd/resnet.py +++ b/example/autograd/resnet.py @@ -276,8 +276,8 @@ def test(ctx): def train(epoch, ctx): if isinstance(ctx, mx.Context): ctx = [ctx] - net.params.initialize(mx.init.Xavier(magnitude=2.24), ctx=ctx) - trainer = foo.Trainer(net.params, 'sgd', {'learning_rate': 0.1}) + net.all_params().initialize(mx.init.Xavier(magnitude=2.24), ctx=ctx) + trainer = foo.Trainer(net.all_params(), 'sgd', {'learning_rate': 0.1}) metric = mx.metric.Accuracy() for i in range(epoch): @@ -305,7 +305,7 @@ def train(epoch, ctx): print 'time: %f'%(time.time()-tic) test(ctx) - net.params.save('mnist.params') + net.all_params().save('mnist.params') if __name__ == '__main__': train(200, [mx.gpu(i) for i in range(2)]) diff --git a/python/mxnet/foo/nn/layer.py b/python/mxnet/foo/nn/layer.py index e156ae695008..f0d0bc2d94c5 100644 --- a/python/mxnet/foo/nn/layer.py +++ b/python/mxnet/foo/nn/layer.py @@ -19,7 +19,7 @@ def __init__(self, layer): self._old_scope = None @staticmethod - def get_prefix(prefix, hint): + def create_prefix(prefix, hint): if _LayerScope._current is None: if prefix is None: return _name.NameManager.current.get(None, hint) + '_' @@ -32,18 +32,12 @@ def get_prefix(prefix, hint): return _LayerScope._current._layer.prefix+prefix @staticmethod - def get_params(prefix, params): + def create_params(prefix, params): if params is not None: - return params - params = ParameterDict(prefix) + return ParameterDict(params.prefix, params) if _LayerScope._current is not None: - _LayerScope._current._layer.params.merge(params) - return params - - @staticmethod - def register_sublayer(layer): - if _LayerScope._current is not None: - _LayerScope._current._layer.register_sublayer(layer) + return ParameterDict(prefix, _LayerScope._current._layer._params._shared) + return ParameterDict(prefix) def __enter__(self): self._old_scope = _LayerScope._current @@ -66,12 +60,12 @@ class Net(nn.Layer): def __init__(self, **kwargs): super(Net, self).__init__(**kwargs) with self.scope: - self.dense1 = nn.Dense(20, in_units=10, prefix='dense1_') - self.dense2 = nn.Dense(20, in_units=20, prefix='dense2_') + self.dense0 = nn.Dense(20, in_units=10) + self.dense1 = nn.Dense(20, in_units=20) def forward(self, x): - x = self.dense1(x) - return self.dense2(x) + x = self.dense0(x) + return self.dense1(x) Sublayers assigned this way will be registered and will have their status changed too when you call .train() etc. @@ -83,22 +77,16 @@ def forward(self, x): Parameters created by this layer. Prefix should be unique within one network to prevent name collisions. params : ParameterDict or None - Manages Parameters of this Layer and sublayers. You can make two Layers share - parameter by passing the same dictionary to them. For example:: - params = nn.ParameterDict(prefix='dense_') - dense1 = nn.Dense(20, in_units=10, prefix='dense1_', params=params) - dense2 = nn.Dense(20, in_units=10, prefix='dense2_', params=params) + ParameterDict for sharing weights with the new Layer. For example, + if you want `dense2` to share `dense1`'s weights, you can do:: + dense1 = nn.Dense(20, in_units=10, prefix='dense1_') + dense2 = nn.Dense(20, in_units=10, prefix='dense2_', + params=dense1.all_params()) - dense1 and dense2 now have shared weights. - - Layer supports forwarding with both `Symbol` and `NDArray`. - - Layer is mostly used by developers or advanced users as a base class. - If you only want to use one of `Symbol` and `NDArray` API you should inherit - Layer instead.""" + Layer supports forwarding with both `Symbol` and `NDArray`.""" def __init__(self, prefix=None, params=None): - self._prefix = _LayerScope.get_prefix(prefix, self._alias()) - self._params = _LayerScope.get_params(self._prefix, params) + self._prefix = _LayerScope.create_prefix(prefix, self._alias()) + self._params = _LayerScope.create_params(self._prefix, params) self._scope = _LayerScope(self) self._children = [] self._reg_params = {} @@ -110,16 +98,26 @@ def __setattr__(self, name, value): if isinstance(value, Parameter): self._reg_params[name] = value if isinstance(value, Layer): - _LayerScope.register_sublayer(self) + self.register_child(value) def _alias(self): return self.__class__.__name__.lower() @property def params(self): - """A ParameterDict managing this Layer's Parameters.""" + """Returns this Layer's parameter dictionary (does not include its + children's parameters).""" return self._params + def all_params(self): + """Returns a ParameterDict containing this Layer and all of its children's + Parameters.""" + ret = ParameterDict(self._params.prefix) + ret.update(self.params) + for cld in self._children: + ret.update(cld.all_params()) + return ret + @property def prefix(self): """Prefix of this Layer.""" @@ -135,11 +133,10 @@ def name(self): def scope(self): return self._scope - def register_sublayer(self, layer): + def register_child(self, layer): """Register layer as sublayer of self. Layers assigned to self as attributes will be registered automatically.""" self._children.append(layer) - self.params.merge(layer.params) def __call__(self, *args, **kwargs): """Call forward.""" @@ -195,7 +192,7 @@ def __init__(self): def add(self, layer): """Add layer on top of the stack.""" - self.register_sublayer(layer) + self.register_child(layer) def forward(self, x): #pylint: disable=arguments-differ diff --git a/python/mxnet/foo/parameter.py b/python/mxnet/foo/parameter.py index b12a35224605..52d72655a039 100644 --- a/python/mxnet/foo/parameter.py +++ b/python/mxnet/foo/parameter.py @@ -129,8 +129,8 @@ def set_data(self, data): arr[:] = data def data(self, ctx=None): - """Returns a copy of this parameter on one context. Must be on this context - before. + """Returns a copy of this parameter on one context. Must have been + intialized on this context before. Parameters ---------- @@ -156,7 +156,7 @@ def list_data(self): as creation.""" assert self._data is not None, \ "Parameter %s has not been initialized"%self.name - return self._data.values() + return list(self._data.values()) def grad(self, ctx=None): """Returns a gradient buffer for this parameter on one context. @@ -183,13 +183,13 @@ def list_grad(self): "Parameter %s has not been initialized"%self.name assert self._data is not None, \ "Parameter %s does not have gradients because grad_req='null'"%self.name - return self._grad.values() + return list(self._grad.values()) def list_ctx(self): """Returns a list of contexts this parameter is initialized on""" assert self._data is not None, \ "Parameter %s has not been initialized"%self.name - return self._data.keys() + return list(self._data.keys()) def zero_grad(self): """Set gradient buffer on all contexts to 0. No action is taken if @@ -215,10 +215,15 @@ class ParameterDict(object): ---------- prefix : str, default '' The prefix to be prepended to all Parameters' name created by this dict. + shared : ParameterDict or None + If not None, when this dict's get method creates a new parameter, will + first try to retrieve it from `shared` dict. Usually used for sharing + parameters with another layer. """ - def __init__(self, prefix=''): + def __init__(self, prefix='', shared=None): self._prefix = prefix self._params = {} + self._shared = shared def __getitem__(self, key): return self._params[key] @@ -238,9 +243,18 @@ def prefix(self): with `get`""" return self._prefix + def _get_impl(self, name): + if name in self._params: + return self._params[name] + if self._shared is not None and name in self._shared._params: + return self._shared._params[name] + return None + def get(self, name, **kwargs): - """Create or retrieve a Parameter with name `self.prefix+name`. Key-word - arguments will be passed to Parameter's contructor. + """Retrieve a Parameter with name `self.prefix+name`. If not found, + `get` will first try to retrive it from `shared` dict. If still not + found, `get` will create a new Parameter with key-word arguments and + insert it to self. Parameter --------- @@ -256,64 +270,31 @@ def get(self, name, **kwargs): The created or retrieved Parameter. """ name = self.prefix + name - if name not in self._params: - self._params[name] = Parameter(name, **kwargs) + param = self._get_impl(name) + if param is None: + param = Parameter(name, **kwargs) + self._params[name] = param else: - param = self._params[name] for k, v in kwargs.items(): - if hasattr(param, k): + if hasattr(param, k) and getattr(param, k) is not None: assert v is None or v == getattr(param, k), \ - "Parameter attribute %s mismatch: stored %s vs desired %s"%( - k, str(getattr(param, k)), str(v)) + "Cannot retrieve Parameter %s because desired attribute " \ + "does not match with stored for attribute %s: " \ + "desired %s vs stored %s."%( + name, k, str(v), str(getattr(param, k))) else: setattr(param, k, v) - return self._params[name] - - def subdict(self, suffix): - """Create a sub-dictionary that shares parameters with this dictionary. - The sub-dictionary's prefix is self.prefix + suffix. - - Example:: - >>> params1 = ParameterDict('net_') - >>> params2 = params1.subdict('conv1_') - >>> params2.prefix - 'net_conv1_' - - Parameters - ---------- - suffix : str - Suffix of the created child dictionary + return param - Returns - ------- - ParameterDict with self.prefix + suffix as prefix. - """ - ret = ParameterDict(self.prefix + suffix) - self.merge(ret) - return ret - - def merge(self, other): - """Merge this dictionary with another dictionary. The two dictionaries - will manage the same set of Parameters but keep their individual prefix. - - Example:: - >>> params1 = ParameterDict('net1_') - >>> params2 = ParameterDict('net2_') - >>> params1.merge(params2) - >>> params2.get('w') - >>> print params1.keys() - ['net2_w'] - """ - params = self._params - if params is other._params: - return + def update(self, other): + """Copy all Parameters in `other` to self.""" for k, v in other.items(): - assert k not in params or params[k] is v, \ - "Cannot merge ParameterDicts with prefix %s and %s " \ - "because they contain different versions of the same " \ - "Parameter named %s"%(self.prefix, other.prefix, k) - params[k] = v - other._params = params + if k in self._params: + assert self._params[k] is v, \ + "Cannot update self with other because they have different " \ + "Parameters with the same name %s"%k + else: + self._params[k] = v def initialize(self, init=initializer.Xavier(), ctx=None): """Intialize all Parameters manage by this dictionary to be used for `NDArray` diff --git a/python/mxnet/foo/rnn/rnn_cell.py b/python/mxnet/foo/rnn/rnn_cell.py index c8f070d4c1a5..d0f6ebbcd118 100644 --- a/python/mxnet/foo/rnn/rnn_cell.py +++ b/python/mxnet/foo/rnn/rnn_cell.py @@ -731,7 +731,7 @@ def add(self, cell): ---------- cell : rnn cell """ - self.register_sublayer(cell) + self.register_child(cell) def state_info(self, batch_size=0): return _cells_state_info(self._children, batch_size) @@ -951,8 +951,8 @@ class BidirectionalCell(RecurrentCell): """ def __init__(self, l_cell, r_cell, output_prefix='bi_'): super(BidirectionalCell, self).__init__(prefix='', params=None) - self.register_sublayer(l_cell) - self.register_sublayer(r_cell) + self.register_child(l_cell) + self.register_child(r_cell) self._output_prefix = output_prefix def unpack_weights(self, args): diff --git a/python/mxnet/foo/utils.py b/python/mxnet/foo/utils.py index 150b8ef94db2..3527cffe9f2f 100644 --- a/python/mxnet/foo/utils.py +++ b/python/mxnet/foo/utils.py @@ -28,7 +28,7 @@ def split_data(data, num_slice, batch_axis=0, even_split=True): "Use a batch size that's multiple of %d or set even_split=False to enable " \ "uneven partitioning of data."%( str(data.shape), num_slice, batch_axis, num_slice) - size = data.shape[batch_axis] / num_slice + size = data.shape[batch_axis] // num_slice if batch_axis == 0: slices = [data[i*size:(i+1)*size] for i in range(num_slice)] else: diff --git a/tests/python/unittest/test_foo_rnn.py b/tests/python/unittest/test_foo_rnn.py index 0c0f75c96146..3794c7fc18cc 100644 --- a/tests/python/unittest/test_foo_rnn.py +++ b/tests/python/unittest/test_foo_rnn.py @@ -9,7 +9,7 @@ def test_rnn(): inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)] outputs, _ = cell.unroll(3, inputs) outputs = mx.sym.Group(outputs) - assert sorted(cell.params._params.keys()) == ['rnn_h2h_bias', 'rnn_h2h_weight', 'rnn_i2h_bias', 'rnn_i2h_weight'] + assert sorted(cell.all_params().keys()) == ['rnn_h2h_bias', 'rnn_h2h_weight', 'rnn_i2h_bias', 'rnn_i2h_weight'] assert outputs.list_outputs() == ['rnn_t0_out_output', 'rnn_t1_out_output', 'rnn_t2_out_output'] args, outs, auxs = outputs.infer_shape(rnn_t0_data=(10,50), rnn_t1_data=(10,50), rnn_t2_data=(10,50)) @@ -21,7 +21,7 @@ def test_lstm(): inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)] outputs, _ = cell.unroll(3, inputs) outputs = mx.sym.Group(outputs) - assert sorted(cell.params._params.keys()) == ['rnn_h2h_bias', 'rnn_h2h_weight', 'rnn_i2h_bias', 'rnn_i2h_weight'] + assert sorted(cell.all_params().keys()) == ['rnn_h2h_bias', 'rnn_h2h_weight', 'rnn_i2h_bias', 'rnn_i2h_weight'] assert outputs.list_outputs() == ['rnn_t0_out_output', 'rnn_t1_out_output', 'rnn_t2_out_output'] args, outs, auxs = outputs.infer_shape(rnn_t0_data=(10,50), rnn_t1_data=(10,50), rnn_t2_data=(10,50)) @@ -54,7 +54,7 @@ def test_gru(): inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)] outputs, _ = cell.unroll(3, inputs) outputs = mx.sym.Group(outputs) - assert sorted(cell.params._params.keys()) == ['rnn_h2h_bias', 'rnn_h2h_weight', 'rnn_i2h_bias', 'rnn_i2h_weight'] + assert sorted(cell.all_params().keys()) == ['rnn_h2h_bias', 'rnn_h2h_weight', 'rnn_i2h_bias', 'rnn_i2h_weight'] assert outputs.list_outputs() == ['rnn_t0_out_output', 'rnn_t1_out_output', 'rnn_t2_out_output'] args, outs, auxs = outputs.infer_shape(rnn_t0_data=(10,50), rnn_t1_data=(10,50), rnn_t2_data=(10,50)) @@ -66,7 +66,7 @@ def test_residual(): inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(2)] outputs, _ = cell.unroll(2, inputs) outputs = mx.sym.Group(outputs) - assert sorted(cell.params._params.keys()) == \ + assert sorted(cell.all_params().keys()) == \ ['rnn_h2h_bias', 'rnn_h2h_weight', 'rnn_i2h_bias', 'rnn_i2h_weight'] # assert outputs.list_outputs() == \ # ['rnn_t0_out_plus_residual_output', 'rnn_t1_out_plus_residual_output'] @@ -93,7 +93,7 @@ def test_residual_bidirectional(): inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(2)] outputs, _ = cell.unroll(2, inputs, merge_outputs=False) outputs = mx.sym.Group(outputs) - assert sorted(cell.params._params.keys()) == \ + assert sorted(cell.all_params().keys()) == \ ['rnn_l_h2h_bias', 'rnn_l_h2h_weight', 'rnn_l_i2h_bias', 'rnn_l_i2h_weight', 'rnn_r_h2h_bias', 'rnn_r_h2h_weight', 'rnn_r_i2h_bias', 'rnn_r_i2h_weight'] # assert outputs.list_outputs() == \ @@ -126,7 +126,7 @@ def test_stack(): inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)] outputs, _ = cell.unroll(3, inputs) outputs = mx.sym.Group(outputs) - keys = sorted(cell.params._params.keys()) + keys = sorted(cell.all_params().keys()) for i in range(5): assert 'rnn_stack%d_h2h_weight'%i in keys assert 'rnn_stack%d_h2h_bias'%i in keys @@ -178,7 +178,7 @@ def test_unfuse(): def check_rnn_forward(layer, inputs): - layer.params.initialize() + layer.all_params().initialize() with mx.contrib.autograd.train_section(): mx.contrib.autograd.compute_gradient( [layer.unroll(3, inputs, merge_outputs=True)[0]]) diff --git a/tests/python/unittest/test_nn.py b/tests/python/unittest/test_nn.py index 1317e361c31f..4f78383a139d 100644 --- a/tests/python/unittest/test_nn.py +++ b/tests/python/unittest/test_nn.py @@ -22,6 +22,24 @@ def test_paramdict(): params.save('test.params') params.load('test.params') + +def test_parameter_sharing(): + class Net(nn.Layer): + def __init__(self, **kwargs): + super(Net, self).__init__(**kwargs) + with self.scope: + self.dense0 = nn.Dense(5, in_units=5) + self.dense1 = nn.Dense(5, in_units=5) + + def generic_forward(self, F, x): + return self.dense1(self.dense0(x)) + + net1 = Net(prefix='net1_') + net2 = Net(prefix='net1_', params=net1.all_params()) + net1.all_params().initialize() + net2(mx.nd.zeros((3, 5))) + + def test_basic(): model = nn.Sequential() model.add(nn.Dense(128, activation='tanh', in_units=10)) @@ -36,14 +54,14 @@ def test_basic(): assert len(y.list_arguments()) == 7 # ndarray - model.params.initialize() + model.all_params().initialize() x = model(mx.nd.zeros((32, 10))) assert x.shape == (32, 32) x.wait_to_read() def check_layer_forward(layer, dshape): - layer.params.initialize() + layer.all_params().initialize() with mx.contrib.autograd.train_section(): out = layer(mx.nd.ones(shape=dshape)) out.backward() @@ -170,7 +188,7 @@ def test_batchnorm(): def test_reshape(): x = mx.nd.ones((2, 4, 10, 10)) layer = nn.Conv2D(10, 2, in_filters=4) - layer.params.initialize() + layer.all_params().initialize() with mx.contrib.autograd.train_section(): x = layer(x) x = x.reshape((-1,)) @@ -181,7 +199,7 @@ def test_reshape(): def test_slice(): x = mx.nd.ones((5, 4, 10, 10)) layer = nn.Conv2D(10, 2, in_filters=4) - layer.params.initialize() + layer.all_params().initialize() with mx.contrib.autograd.train_section(): x = layer(x) x = x[1:3] @@ -192,7 +210,7 @@ def test_slice(): def test_at(): x = mx.nd.ones((5, 4, 10, 10)) layer = nn.Conv2D(10, 2, in_filters=4) - layer.params.initialize() + layer.all_params().initialize() with mx.contrib.autograd.train_section(): x = layer(x) x = x[1] From a34ec189f909f38a7e2e6ffdeddca6504df11bb3 Mon Sep 17 00:00:00 2001 From: Eric Junyuan Xie Date: Fri, 9 Jun 2017 14:56:18 -0700 Subject: [PATCH 182/834] add language model example (#6621) --- example/autograd/word_language_model/data.py | 49 ++++++ .../word_language_model/get_ptb_data.sh | 14 ++ example/autograd/word_language_model/model.py | 31 ++++ example/autograd/word_language_model/train.py | 154 ++++++++++++++++++ python/mxnet/foo/nn/layer.py | 36 ++++ python/mxnet/foo/parameter.py | 6 +- python/mxnet/foo/utils.py | 17 ++ python/mxnet/metric.py | 2 +- 8 files changed, 307 insertions(+), 2 deletions(-) create mode 100644 example/autograd/word_language_model/data.py create mode 100755 example/autograd/word_language_model/get_ptb_data.sh create mode 100644 example/autograd/word_language_model/model.py create mode 100644 example/autograd/word_language_model/train.py diff --git a/example/autograd/word_language_model/data.py b/example/autograd/word_language_model/data.py new file mode 100644 index 000000000000..e3a283b64285 --- /dev/null +++ b/example/autograd/word_language_model/data.py @@ -0,0 +1,49 @@ +import os +import numpy as np +import mxnet as mx + +class Dictionary(object): + def __init__(self): + self.word2idx = {} + self.idx2word = [] + + def add_word(self, word): + if word not in self.word2idx: + self.idx2word.append(word) + self.word2idx[word] = len(self.idx2word) - 1 + return self.word2idx[word] + + def __len__(self): + return len(self.idx2word) + + +class Corpus(object): + def __init__(self, path): + self.dictionary = Dictionary() + self.train = self.tokenize(path + 'train.txt') + self.valid = self.tokenize(path + 'valid.txt') + self.test = self.tokenize(path + 'test.txt') + + def tokenize(self, path): + """Tokenizes a text file.""" + assert os.path.exists(path) + # Add words to the dictionary + with open(path, 'r') as f: + tokens = 0 + for line in f: + words = line.split() + [''] + tokens += len(words) + for word in words: + self.dictionary.add_word(word) + + # Tokenize file content + with open(path, 'r') as f: + ids = np.zeros((tokens,), dtype='int32') + token = 0 + for line in f: + words = line.split() + [''] + for word in words: + ids[token] = self.dictionary.word2idx[word] + token += 1 + + return mx.nd.array(ids, dtype='int32') diff --git a/example/autograd/word_language_model/get_ptb_data.sh b/example/autograd/word_language_model/get_ptb_data.sh new file mode 100755 index 000000000000..1ec009aa2f99 --- /dev/null +++ b/example/autograd/word_language_model/get_ptb_data.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +RNN_DIR=$(cd `dirname $0`; pwd) +DATA_DIR="${RNN_DIR}/data/" + +if [[ ! -d "${DATA_DIR}" ]]; then + echo "${DATA_DIR} doesn't exist, will create one"; + mkdir -p ${DATA_DIR} +fi + +wget -P ${DATA_DIR} https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/ptb/ptb.train.txt; +wget -P ${DATA_DIR} https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/ptb/ptb.valid.txt; +wget -P ${DATA_DIR} https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/ptb/ptb.test.txt; +wget -P ${DATA_DIR} https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/tinyshakespeare/input.txt; diff --git a/example/autograd/word_language_model/model.py b/example/autograd/word_language_model/model.py new file mode 100644 index 000000000000..8a2a7d92a054 --- /dev/null +++ b/example/autograd/word_language_model/model.py @@ -0,0 +1,31 @@ +import mxnet as mx +from mxnet import foo +from mxnet.foo import nn, rnn + +class RNNModel(nn.Layer): + def __init__(self, mode, vocab_size, num_embed, num_hidden, + num_layers, dropout=0.5, tie_weights=False, **kwargs): + super(RNNModel, self).__init__(**kwargs) + with self.scope: + self.drop = nn.Dropout(dropout) + self.encoder = nn.Embedding(vocab_size, num_embed) + self.rnn = rnn.FusedRNNCell(num_hidden, num_layers, mode=mode, + dropout=dropout, get_next_state=True, + num_input=num_embed) + if tie_weights: + self.decoder = nn.Dense(vocab_size, in_units=num_hidden, + params=self.encoder.params) + else: + self.decoder = nn.Dense(vocab_size, in_units=num_hidden) + + self.num_hidden = num_hidden + + def generic_forward(self, F, inputs, hidden): + emb = self.drop(self.encoder(inputs)) + output, hidden = self.rnn.unroll(None, emb, layout='TNC', merge_outputs=True) + output = self.drop(output) + decoded = self.decoder(output.reshape((-1, self.num_hidden))) + return decoded, hidden + + def begin_state(self, *args, **kwargs): + return self.rnn.begin_state(*args, **kwargs) diff --git a/example/autograd/word_language_model/train.py b/example/autograd/word_language_model/train.py new file mode 100644 index 000000000000..20dcfed62606 --- /dev/null +++ b/example/autograd/word_language_model/train.py @@ -0,0 +1,154 @@ +import argparse +import time +import math +import mxnet as mx +from mxnet import foo, autograd +from mxnet.foo import nn, rnn +import model +import data + +parser = argparse.ArgumentParser(description='MXNet Autograd PennTreeBank RNN/LSTM Language Model') +parser.add_argument('--data', type=str, default='./data/ptb.', + help='location of the data corpus') +parser.add_argument('--model', type=str, default='lstm', + help='type of recurrent net (rnn_tanh, rnn_relu, lstm, gru)') +parser.add_argument('--emsize', type=int, default=200, + help='size of word embeddings') +parser.add_argument('--nhid', type=int, default=200, + help='number of hidden units per layer') +parser.add_argument('--nlayers', type=int, default=2, + help='number of layers') +parser.add_argument('--lr', type=float, default=20, + help='initial learning rate') +parser.add_argument('--clip', type=float, default=0.25, + help='gradient clipping') +parser.add_argument('--epochs', type=int, default=40, + help='upper epoch limit') +parser.add_argument('--batch_size', type=int, default=20, metavar='N', + help='batch size') +parser.add_argument('--bptt', type=int, default=35, + help='sequence length') +parser.add_argument('--dropout', type=float, default=0.2, + help='dropout applied to layers (0 = no dropout)') +parser.add_argument('--tied', action='store_true', + help='tie the word embedding and softmax weights') +parser.add_argument('--seed', type=int, default=1111, + help='random seed') +parser.add_argument('--cuda', action='store_true', + help='Whether to use gpu') +parser.add_argument('--log-interval', type=int, default=200, metavar='N', + help='report interval') +parser.add_argument('--save', type=str, default='model.params', + help='path to save the final model') +args = parser.parse_args() + + +############################################################################### +# Load data +############################################################################### + + +if args.cuda: + context = mx.gpu(0) +else: + context = mx.cpu(0) + +corpus = data.Corpus(args.data) + +def batchify(data, batch_size): + """Reshape data into (num_example, batch_size)""" + nbatch = data.shape[0] // batch_size + data = data[:nbatch * batch_size] + data = data.reshape((batch_size, nbatch)).T + return data + +train_data = batchify(corpus.train, args.batch_size).as_in_context(context) +val_data = batchify(corpus.valid, args.batch_size).as_in_context(context) +test_data = batchify(corpus.test, args.batch_size).as_in_context(context) + + +############################################################################### +# Build the model +############################################################################### + + +ntokens = len(corpus.dictionary) +model = model.RNNModel(args.model, ntokens, args.emsize, args.nhid, args.nlayers, args.dropout, args.tied) +model.all_params().initialize(mx.init.Xavier(), ctx=context) +trainer = foo.Trainer(model.all_params(), 'sgd', + {'learning_rate': args.lr, + 'momentum': 0, + 'wd': 0}) + + +############################################################################### +# Training code +############################################################################### + + +def get_batch(source, i): + seq_len = min(args.bptt, source.shape[0] - 1 - i) + data = source[i:i+seq_len] + target = source[i+1:i+1+seq_len] + return data, target.reshape((-1,)) + + +def detach(hidden): + if isinstance(hidden, (tuple, list)): + hidden = [i.detach() for i in hidden] + else: + hidden = hidden.detach() + return hidden + + +def eval(data_source): + total = 0.0 + ntotal = 0 + hidden = model.begin_state(func=mx.nd.zeros, batch_size=args.batch_size, ctx=context) + for ibatch, i in enumerate(range(0, data_source.shape[0] - 1, args.bptt)): + data, target = get_batch(data_source, i) + output, hidden = model(data, hidden) + loss = foo.loss.softmax_cross_entropy_loss(output, target) + total += mx.nd.sum(loss).asscalar() + ntotal += loss.size + return total / ntotal + + +def train(): + best_val = None + for epoch in range(args.epochs): + total = 0.0 + start_time = time.time() + hidden = model.begin_state(func=mx.nd.zeros, batch_size=args.batch_size, ctx=context) + for ibatch, i in enumerate(range(0, train_data.shape[0] - 1, args.bptt)): + data, target = get_batch(train_data, i) + hidden = detach(hidden) + with autograd.train_section(): + output, hidden = model(data, hidden) + loss = foo.loss.softmax_cross_entropy_loss(output, target) + loss.backward() + + grads = [i.grad(context) for i in model.all_params().values()] + # Here gradient is not divided by batch_size yet. + # So we multiply max_norm by batch_size to balance it. + foo.utils.clip_global_norm(grads, args.clip * args.batch_size) + + trainer.step(args.batch_size) + total += mx.nd.sum(loss).asscalar() + + if ibatch % args.log_interval == 0 and ibatch > 0: + cur_loss = total / args.batch_size / args.bptt / args.log_interval + print('[Epoch %d Batch %d] loss %.2f, ppl %.2f'%( + epoch, ibatch, cur_loss, math.exp(cur_loss))) + total = 0.0 + + val_loss = eval(val_data) + + print('[Epoch %d] time cost %.2fs, valid loss %.2f, valid ppl %.2f'%( + epoch, time.time()-start_time, val_loss, math.exp(val_loss))) + + +if __name__ == '__main__': + train() + test_loss = eval(test_data) + print('test loss %.2f, test ppl %.2f'%(test_loss, math.exp(test_loss))) diff --git a/python/mxnet/foo/nn/layer.py b/python/mxnet/foo/nn/layer.py index f0d0bc2d94c5..ff121d8532f5 100644 --- a/python/mxnet/foo/nn/layer.py +++ b/python/mxnet/foo/nn/layer.py @@ -406,3 +406,39 @@ def __init__(self, alpha, **kwargs): def generic_forward(self, F, x): return F.invoke(self._op, [x]) + + +class Embedding(Layer): + """Turns non-negative integers (indexes/tokens) into dense + vectors of fixed size. + eg. [[4], [20]] -> [[0.25, 0.1], [0.6, -0.2]] + + Parameters + ---------- + input_dim : int + Size of the vocabulary, i.e. maximum integer index + 1. + output_dim : int + Dimension of the dense embedding. + dtype : str or np.dtype, default 'float32' + Data type of output embeddings. + embeddings_initializer : Initializer + Initializer for the `embeddings` matrix + + Input shape + ----------- + 2D tensor with shape: `(batch_size, sequence_length)`. + + Output shape + ------------ + 3D tensor with shape: `(batch_size, sequence_length, output_dim)`. + """ + def __init__(self, input_dim, output_dim, dtype='float32', + embeddings_initializer=None, **kwargs): + super(Embedding, self).__init__(**kwargs) + self._op = symbol.CachedOp('Embedding', 2, input_dim=input_dim, + output_dim=output_dim, dtype=dtype) + self.weight = self.params.get('weight', shape=(input_dim, output_dim), + init=embeddings_initializer) + + def generic_forward(self, F, x, weight): + return F.invoke(self._op, [x, weight]) diff --git a/python/mxnet/foo/parameter.py b/python/mxnet/foo/parameter.py index 52d72655a039..75757d8cd7a0 100644 --- a/python/mxnet/foo/parameter.py +++ b/python/mxnet/foo/parameter.py @@ -145,7 +145,11 @@ def data(self, ctx=None): ctx = context.current_context() assert self._data is not None, \ "Cannot get NDArray value for Parameter %s " \ - "because it hasn't been initialized!"%(self.name) + "because it hasn't been initialized. Note that " \ + "you should initialize parameters and create Trainer " \ + "with Layer.all_params() instead of Layer.params " \ + "because the later does not include parameters of " \ + "nested child layers "%(self.name) assert ctx in self._data, \ "Cannot get NDArray value for Parameter %s on context %s " \ "because it was not initialized on %s"%(self.name, str(ctx), str(ctx)) diff --git a/python/mxnet/foo/utils.py b/python/mxnet/foo/utils.py index 3527cffe9f2f..fe410c48cdc9 100644 --- a/python/mxnet/foo/utils.py +++ b/python/mxnet/foo/utils.py @@ -1,6 +1,7 @@ # coding: utf-8 # pylint: disable= """Parallelization utility optimizer.""" +import math from .. import ndarray @@ -63,3 +64,19 @@ def load_data(data, ctx_list, batch_axis=0, even_split=True): slices = split_data(data, len(ctx_list), batch_axis=batch_axis, even_split=even_split) return [i.as_in_context(ctx) for i, ctx in zip(slices, ctx_list)] + + +def clip_global_norm(arrays, max_norm): + """Rescales NDArrays so that the sum of their 2-norm is smaller than max_norm. + """ + assert len(arrays) > 0 + total_norm = 0 + for arr in arrays: + arr = arr.reshape((-1,)) + total_norm += ndarray.dot(arr, arr) + total_norm = math.sqrt(total_norm.asscalar()) + scale = max_norm / (total_norm + 1e-8) + if scale < 1.0: + for arr in arrays: + arr *= scale + return total_norm diff --git a/python/mxnet/metric.py b/python/mxnet/metric.py index 564c727b0c98..299734b38393 100644 --- a/python/mxnet/metric.py +++ b/python/mxnet/metric.py @@ -634,7 +634,7 @@ def update(self, labels, preds): label = label.as_in_context(pred.context).reshape((label.size,)) pred = ndarray.pick(pred, label.astype(dtype='int32'), axis=self.axis) if self.ignore_label is not None: - ignore = label == self.ignore_label + ignore = (label == self.ignore_label).astype(pred.dtype) num -= ndarray.sum(ignore).asscalar() pred = pred*(1-ignore) + ignore loss -= ndarray.sum(ndarray.log(ndarray.maximum(1e-10, pred))).asscalar() From abebd3b6310a35a777499c169730eb3077aa7195 Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Fri, 9 Jun 2017 14:57:26 -0700 Subject: [PATCH 183/834] extend tranpose to 6dim (#6634) --- src/operator/tensor/matrix_op-inl.h | 10 ++++++++-- tests/python/unittest/test_operator.py | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/operator/tensor/matrix_op-inl.h b/src/operator/tensor/matrix_op-inl.h index 3fd6856a3e2e..88e9d3095e24 100644 --- a/src/operator/tensor/matrix_op-inl.h +++ b/src/operator/tensor/matrix_op-inl.h @@ -237,8 +237,14 @@ void TransposeImpl(RunContext ctx, out = transpose(in, axes.get<5>()); break; } + case 6: { + Tensor in = src.get(s); + Tensor out = ret.get(s); + out = transpose(in, axes.get<6>()); + break; + } default: - LOG(FATAL) << "Transpose support at most 5 dimensions"; + LOG(FATAL) << "Transpose support at most 6 dimensions"; break; } }); @@ -271,7 +277,7 @@ inline bool TransposeShape(const nnvm::NodeAttrs& attrs, CHECK_EQ(in_attrs->size(), 1U); CHECK_EQ(out_attrs->size(), 1U); TShape& shp = (*in_attrs)[0]; - CHECK_LE(shp.ndim(), 5U) << "Transpose support at most 5 dimensions"; + CHECK_LE(shp.ndim(), 6U) << "Transpose support at most 6 dimensions"; TShape ret(shp.ndim()); if (param.axes.ndim() == 0) { for (index_t i = 0; i < shp.ndim(); ++i) { diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py index 7a958f7de01b..55063b3b8820 100644 --- a/tests/python/unittest/test_operator.py +++ b/tests/python/unittest/test_operator.py @@ -1422,7 +1422,7 @@ def test_broadcasting_ele(sym_bcast): test_broadcasting_ele(sym_bcast_to) def test_transpose(): - for ndim in range(1, 6): + for ndim in range(1, 7): for t in range(5): dims = list(np.random.randint(1, 10, size=ndim)) axes = list(range(ndim)) From 334df7d9b1cf5470980d5bab14b5216d93c18a07 Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Fri, 9 Jun 2017 14:58:28 -0700 Subject: [PATCH 184/834] super resolution (#6633) --- example/autograd/data.py | 50 +++++++- example/autograd/super_resolution.py | 179 +++++++++++++++++++++++++++ 2 files changed, 228 insertions(+), 1 deletion(-) create mode 100644 example/autograd/super_resolution.py diff --git a/example/autograd/data.py b/example/autograd/data.py index 5208bdc80e88..84eb711d00a4 100644 --- a/example/autograd/data.py +++ b/example/autograd/data.py @@ -1,7 +1,8 @@ # pylint: skip-file """ data iterator for mnist """ -import sys import os +import random +import sys # code to automatically download dataset curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__))) sys.path.append(os.path.join(curr_path, "../../tests/python/common")) @@ -55,6 +56,7 @@ def cifar10_iterator(batch_size, data_shape, resize=-1): class DummyIter(mx.io.DataIter): def __init__(self, batch_size, data_shape, batches = 5): + super(DummyIter, self).__init__(batch_size) self.data_shape = (batch_size,) + data_shape self.label_shape = (batch_size,) self.provide_data = [('data', self.data_shape)] @@ -74,3 +76,49 @@ def next(self): def dummy_iterator(batch_size, data_shape): return DummyIter(batch_size, data_shape), DummyIter(batch_size, data_shape) + +class ImagePairIter(mx.io.DataIter): + def __init__(self, path, data_shape, label_shape, batch_size=64, flag=0, input_aug=None, target_aug=None): + super(ImagePairIter, self).__init__(batch_size) + self.data_shape = (batch_size,) + data_shape + self.label_shape = (batch_size,) + label_shape + self.input_aug = input_aug + self.target_aug = target_aug + self.provide_data = [('data', self.data_shape)] + self.provide_label = [('label', self.label_shape)] + is_image_file = lambda fn: any(fn.endswith(ext) for ext in [".png", ".jpg", ".jpeg"]) + self.filenames = [os.path.join(path, x) for x in os.listdir(path) if is_image_file(x)] + self.count = 0 + self.flag = flag + random.shuffle(self.filenames) + + def next(self): + if self.count + self.batch_size < len(self.filenames): + data = [] + label = [] + for i in range(self.batch_size): + fn = self.filenames[self.count] + self.count += 1 + with open(fn, 'rb') as f: + binary_image = f.read() + image = mx.img.imdecode(binary_image, flag=self.flag) + target = image.copy() + for aug in self.input_aug: + image = aug(image)[0] + for aug in self.target_aug: + target = aug(target)[0] + data.append(image) + label.append(target) + + data = mx.nd.concat(*[mx.nd.expand_dims(d, axis=0) for d in data], dim=0) + label = mx.nd.concat(*[mx.nd.expand_dims(d, axis=0) for d in label], dim=0) + data = [mx.nd.transpose(data, axes=(0, 3, 1, 2)).astype('float32')] + label = [mx.nd.transpose(label, axes=(0, 3, 1, 2)).astype('float32')] + + return mx.io.DataBatch(data=data, label=label) + else: + raise StopIteration + + def reset(self): + self.count = 0 + random.shuffle(self.filenames) diff --git a/example/autograd/super_resolution.py b/example/autograd/super_resolution.py new file mode 100644 index 000000000000..89cc58b8ad6e --- /dev/null +++ b/example/autograd/super_resolution.py @@ -0,0 +1,179 @@ +from __future__ import print_function +import argparse, tarfile +import math +import os + +import mxnet as mx +from mxnet import foo +from mxnet.foo import nn +from mxnet.contrib import autograd as ag +from mxnet.test_utils import download +from mxnet.image import CenterCropAug, ResizeAug +from mxnet.io import PrefetchingIter + +from data import ImagePairIter + +from PIL import Image + + + +# CLI +parser = argparse.ArgumentParser(description='Super-resolution using an efficient sub-pixel convolution neural network.') +parser.add_argument('--upscale_factor', type=int, required=True, help="super resolution upscale factor") +parser.add_argument('--batch_size', type=int, default=64, help='training batch size') +parser.add_argument('--epochs', type=int, default=2, help='number of training epochs') +parser.add_argument('--lr', type=float, default=0.01, help='learning Rate. default is 0.01') +parser.add_argument('--gpus', type=int, default=0, help='number of GPUs to use') +parser.add_argument('--seed', type=int, default=123, help='random seed to use. Default=123') +parser.add_argument('--resolve_img', type=str, help='input image to use') +opt = parser.parse_args() + +print(opt) + +upscale_factor = opt.upscale_factor +batch_size = opt.batch_size +color_flag = 0 + +# get data +dataset_path = "dataset" +dataset_url = "http://www2.eecs.berkeley.edu/Research/Projects/CS/vision/bsds/BSDS300-images.tgz" +def get_dataset(prefetch=False): + image_path = os.path.join(dataset_path, "BSDS300/images") + + if not os.path.exists(image_path): + os.makedirs(dataset_path) + file_name = download(dataset_url) + with tarfile.open(file_name) as tar: + for item in tar: + tar.extract(item, dataset_path) + os.remove(file_name) + + crop_size = 256 + crop_size -= crop_size % upscale_factor + input_crop_size = crop_size // upscale_factor + + input_transform = [CenterCropAug((crop_size, crop_size)), ResizeAug(input_crop_size)] + target_transform = [CenterCropAug((crop_size, crop_size))] + + iters = (ImagePairIter(os.path.join(image_path, "train"), + (input_crop_size, input_crop_size), + (crop_size, crop_size), + batch_size, color_flag, input_transform, target_transform), + ImagePairIter(os.path.join(image_path, "test"), + (input_crop_size, input_crop_size), + (crop_size, crop_size), + batch_size, color_flag, + input_transform, target_transform)) + if prefetch: + return [PrefetchingIter(i) for i in iters] + else: + return iters + +train_data, val_data = get_dataset() + +mx.random.seed(opt.seed) +ctx = [mx.gpu(i) for i in range(opt.gpus)] if opt.gpus > 0 else [mx.cpu()] + + +# define model +def _rearrange(raw, F, upscale_factor): + # (N, C * r^2, H, W) -> (N, C, r^2, H, W) + splitted = F.reshape(raw, shape=(0, -4, -1, upscale_factor**2, 0, 0)) + # (N, C, r^2, H, W) -> (N, C, r, r, H, W) + unflatten = F.reshape(splitted, shape=(0, 0, -4, upscale_factor, upscale_factor, 0, 0)) + # (N, C, r, r, H, W) -> (N, C, H, r, W, r) + swapped = F.transpose(unflatten, axes=(0, 1, 4, 2, 5, 3)) + # (N, C, H, r, W, r) -> (N, C, H*r, W*r) + return F.reshape(swapped, shape=(0, 0, -3, -3)) + + +class SuperResolutionNet(nn.Layer): + def __init__(self, upscale_factor): + super(SuperResolutionNet, self).__init__() + with self.scope: + self.conv1 = nn.Conv2D(64, (5, 5), strides=(1, 1), padding=(2, 2), in_filters=1) + self.conv2 = nn.Conv2D(64, (3, 3), strides=(1, 1), padding=(1, 1), in_filters=64) + self.conv3 = nn.Conv2D(32, (3, 3), strides=(1, 1), padding=(1, 1), in_filters=64) + self.conv4 = nn.Conv2D(upscale_factor ** 2, (3, 3), strides=(1, 1), padding=(1, 1), in_filters=32) + self.upscale_factor = upscale_factor + + def generic_forward(self, F, x): + x = F.Activation(self.conv1(x), act_type='relu') + x = F.Activation(self.conv2(x), act_type='relu') + x = F.Activation(self.conv3(x), act_type='relu') + return _rearrange(self.conv4(x), F, self.upscale_factor) + +net = SuperResolutionNet(upscale_factor) + +def test(ctx): + val_data.reset() + avg_psnr = 0 + metric = mx.metric.MSE() + batches = 0 + for batch in val_data: + batches += 1 + metric.reset() + data = foo.utils.load_data(batch.data[0], ctx_list=ctx, batch_axis=0) + label = foo.utils.load_data(batch.label[0], ctx_list=ctx, batch_axis=0) + outputs = [] + for x in data: + outputs.append(net(x)) + metric.update(label, outputs) + avg_psnr += 10 * math.log10(1/metric.get()[1]) + avg_psnr /= batches + print('validation avg psnr: %f'%avg_psnr) + + +def train(epoch, ctx): + if isinstance(ctx, mx.Context): + ctx = [ctx] + net.conv4.all_params().initialize(mx.init.Orthogonal(scale=1), ctx=ctx) + net.all_params().initialize(mx.init.Orthogonal(), ctx=ctx) + trainer = foo.Trainer(net.all_params(), 'adam', {'learning_rate': opt.lr}) + metric = mx.metric.MAE() + + for i in range(epoch): + train_data.reset() + for batch in train_data: + data = foo.utils.load_data(batch.data[0], ctx_list=ctx, batch_axis=0) + label = foo.utils.load_data(batch.label[0], ctx_list=ctx, batch_axis=0) + outputs = [] + with ag.train_section(): + for x, y in zip(data, label): + z = net(x) + loss = foo.loss.l2_loss(z, y) + ag.compute_gradient([loss]) + outputs.append(z) + trainer.step(batch.data[0].shape[0]) + metric.update(label, outputs) + + name, acc = metric.get() + metric.reset() + print('training mae at epoch %d: %s=%f'%(i, name, acc)) + test(ctx) + + net.all_params().save('superres.params') + +def resolve(ctx): + if isinstance(ctx, list): + ctx = [ctx[0]] + net.all_params().load('superres.params') + img = Image.open(opt.resolve_img).convert('YCbCr') + y, cb, cr = img.split() + data = mx.nd.array(y) + print(data) + out_img_y = net(data).asnumpy() + out_img_y *= 255.0 + out_img_y = out_img_y.clip(0, 255) + out_img_y = Image.fromarray(np.uint8(out_img_y[0]), mode='L') + + out_img_cb = cb.resize(out_img_y.size, Image.BICUBIC) + out_img_cr = cr.resize(out_img_y.size, Image.BICUBIC) + out_img = Image.merge('YCbCr', [out_img_y, out_img_cb, out_img_cr]).convert('RGB') + + out_img.save('resolved.jpg') + +if opt.resolve_img: + resolve(ctx) +else: + train(opt.epochs, ctx) From e37727f91ccdc6f18924b7b55116bf9cee5f09c2 Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Fri, 9 Jun 2017 15:59:15 -0700 Subject: [PATCH 185/834] OnlyImplementedInNDArray (#6635) * OnlyImplementedInNDArray * update message * update message * fix lint --- python/mxnet/base.py | 15 +++++++++++++++ python/mxnet/symbol.py | 30 +++++++++++++++--------------- 2 files changed, 30 insertions(+), 15 deletions(-) diff --git a/python/mxnet/base.py b/python/mxnet/base.py index 8bca2746de9f..731cc392413c 100644 --- a/python/mxnet/base.py +++ b/python/mxnet/base.py @@ -38,6 +38,21 @@ class MXNetError(Exception): """Error that will be throwed by all mxnet functions.""" pass +class NotImplementedForSymbol(MXNetError): + def __init__(self, function, alias, *args): + super(NotImplementedForSymbol, self).__init__() + self.function = function.__name__ + self.alias = alias + self.args = [str(type(a)) for a in args] + def __str__(self): + msg = 'Function {}'.format(self.function) + if self.alias: + msg += ' (namely operator "{}")'.format(self.alias) + if self.args: + msg += ' with arguments ({})'.format(', '.join(self.args)) + msg += ' is not implemented for Symbol and only available in NDArray.' + return msg + def _load_lib(): """Load libary by searching possible path.""" lib_path = libinfo.find_lib_path() diff --git a/python/mxnet/symbol.py b/python/mxnet/symbol.py index e7f69037362b..578d79b13c7c 100644 --- a/python/mxnet/symbol.py +++ b/python/mxnet/symbol.py @@ -15,7 +15,7 @@ from .base import _LIB, numeric_types from .base import c_array, c_str, mx_uint, py_str, string_types from .base import NDArrayHandle, ExecutorHandle, SymbolHandle, OpHandle -from .base import check_call, MXNetError, _Null # pylint: disable=unused-import +from .base import check_call, MXNetError, NotImplementedForSymbol, _Null # pylint: disable=unused-import from .context import Context from .ndarray import NDArray, _DTYPE_NP_TO_MX, _DTYPE_MX_TO_NP from .name import NameManager # pylint: disable=unused-import @@ -95,7 +95,7 @@ def __add__(self, other): raise TypeError('type %s not supported' % str(type(other))) def __iadd__(self, other): - raise NotImplementedError('Not supported. This is available in NDArray only') + raise NotImplementedForSymbol(self.__iadd__, '+=', other, 1) def __radd__(self, other): return self.__add__(other) @@ -113,7 +113,7 @@ def __sub__(self, other): raise TypeError('type %s not supported' % str(type(other))) def __isub__(self, other): - raise NotImplementedError('Not supported. This is available in NDArray only') + raise NotImplementedForSymbol(self.__isub__, '-=', other) def __rsub__(self, other): """x.__rsub__(y) <=> y-x @@ -146,7 +146,7 @@ def __mul__(self, other): raise TypeError('type %s not supported' % str(type(other))) def __imul__(self, other): - raise NotImplementedError('Not supported. This is available in NDArray only') + raise NotImplementedForSymbol(self.__imul__, '*=', other) def __rmul__(self, other): return self.__mul__(other) @@ -212,7 +212,7 @@ def __rmod__(self, other): raise TypeError('type %s not supported' % str(type(other))) def __idiv__(self, other): - raise NotImplementedError('Not supported. This is available in NDArray only') + raise NotImplementedForSymbol(self.__idiv__, '/=', other) def __truediv__(self, other): return self.__div__(other) @@ -221,7 +221,7 @@ def __rtruediv__(self, other): return self.__rdiv__(other) def __itruediv__(self, other): - raise NotImplementedError('Not supported. This is available in NDArray only') + raise NotImplementedForSymbol(self.__itruediv__, '/=', other) def __pow__(self, other): """x.__pow__(y) <=> x**y @@ -236,7 +236,7 @@ def __pow__(self, other): raise TypeError('type %s not supported' % str(type(other))) def __rpow__(self, other): - raise NotImplementedError('Not supported. This is available in NDArray only') + raise NotImplementedForSymbol(self.__rpow__, 'y**x', other) def __neg__(self): """x.__neg__() <=> -x @@ -1716,28 +1716,28 @@ def reshape(self, shape): return reshape(self, shape=shape) def wait_to_read(self): - raise NotImplementedError('Not supported. This is available in NDArray only') + raise NotImplementedForSymbol(self.wait_to_read, None) def asnumpy(self): - raise NotImplementedError('Not supported. This is available in NDArray only') + raise NotImplementedForSymbol(self.asnumpy, None) def asscalar(self): - raise NotImplementedError('Not supported. This is available in NDArray only') + raise NotImplementedForSymbol(self.asscalar, None) def astype(self): - raise NotImplementedError('Not supported. This is available in NDArray only') + raise NotImplementedForSymbol(self.astype, None) def copy(self): - raise NotImplementedError('Not supported. This is available in NDArray only') + raise NotImplementedForSymbol(self.copy, None) def as_in_context(self): - raise NotImplementedError('Not supported. This is available in NDArray only') + raise NotImplementedForSymbol(self.as_in_context, None) def detach(self): - raise NotImplementedError('Not supported. This is available in NDArray only') + raise NotImplementedForSymbol(self.detach, None) def backward(self): - raise NotImplementedError('Not supported. This is available in NDArray only') + raise NotImplementedForSymbol(self.backward, None) def var(name, attr=None, shape=None, lr_mult=None, wd_mult=None, dtype=None, init=None, **kwargs): """Creates a symbolic variable with specified name. From 98792ccb2b1efb470aaf2b28feac34b8d565159a Mon Sep 17 00:00:00 2001 From: Eric Junyuan Xie Date: Fri, 9 Jun 2017 23:20:31 -0700 Subject: [PATCH 186/834] add auto infershape (#6637) * add auto infershape * fix * fix --- example/autograd/dcgan.py | 34 ++++---- python/mxnet/foo/nn/layer.py | 33 ++++++-- python/mxnet/foo/parameter.py | 131 ++++++++++++++++++++----------- python/mxnet/foo/trainer.py | 15 ++-- tests/python/unittest/test_nn.py | 7 ++ 5 files changed, 145 insertions(+), 75 deletions(-) diff --git a/example/autograd/dcgan.py b/example/autograd/dcgan.py index c4008ab72bd0..c86263bd6235 100644 --- a/example/autograd/dcgan.py +++ b/example/autograd/dcgan.py @@ -37,45 +37,45 @@ netG = nn.Sequential() # input is Z, going into a convolution -netG.add(nn.Conv2DTranspose(ngf * 8, 4, 1, 0, in_filters=nz, use_bias=False)) -netG.add(nn.BatchNorm(num_features=ngf * 8)) +netG.add(nn.Conv2DTranspose(ngf * 8, 4, 1, 0, use_bias=False)) +netG.add(nn.BatchNorm()) netG.add(nn.Activation('relu')) # state size. (ngf*8) x 4 x 4 -netG.add(nn.Conv2DTranspose(ngf * 4, 4, 2, 1, in_filters=ngf * 8, use_bias=False)) -netG.add(nn.BatchNorm(num_features=ngf * 4)) +netG.add(nn.Conv2DTranspose(ngf * 4, 4, 2, 1, use_bias=False)) +netG.add(nn.BatchNorm()) netG.add(nn.Activation('relu')) # state size. (ngf*8) x 8 x 8 -netG.add(nn.Conv2DTranspose(ngf * 2, 4, 2, 1, in_filters=ngf * 4, use_bias=False)) -netG.add(nn.BatchNorm(num_features=ngf * 2)) +netG.add(nn.Conv2DTranspose(ngf * 2, 4, 2, 1, use_bias=False)) +netG.add(nn.BatchNorm()) netG.add(nn.Activation('relu')) # state size. (ngf*8) x 16 x 16 -netG.add(nn.Conv2DTranspose(ngf, 4, 2, 1, in_filters=ngf * 2, use_bias=False)) -netG.add(nn.BatchNorm(num_features=ngf)) +netG.add(nn.Conv2DTranspose(ngf, 4, 2, 1, use_bias=False)) +netG.add(nn.BatchNorm()) netG.add(nn.Activation('relu')) # state size. (ngf*8) x 32 x 32 -netG.add(nn.Conv2DTranspose(nc, 4, 2, 1, in_filters=ngf, use_bias=False)) +netG.add(nn.Conv2DTranspose(nc, 4, 2, 1, use_bias=False)) netG.add(nn.Activation('tanh')) # state size. (nc) x 64 x 64 netD = nn.Sequential() # input is (nc) x 64 x 64 -netD.add(nn.Conv2D(ndf, 4, 2, 1, in_filters=nc, use_bias=False)) +netD.add(nn.Conv2D(ndf, 4, 2, 1, use_bias=False)) netD.add(nn.LeakyReLU(0.2)) # state size. (ndf) x 32 x 32 -netD.add(nn.Conv2D(ndf * 2, 4, 2, 1, in_filters=ndf, use_bias=False)) -netD.add(nn.BatchNorm(num_features=ndf * 2)) +netD.add(nn.Conv2D(ndf * 2, 4, 2, 1, use_bias=False)) +netD.add(nn.BatchNorm()) netD.add(nn.LeakyReLU(0.2)) # state size. (ndf) x 16 x 16 -netD.add(nn.Conv2D(ndf * 4, 4, 2, 1, in_filters=ndf * 2, use_bias=False)) -netD.add(nn.BatchNorm(num_features=ndf * 4)) +netD.add(nn.Conv2D(ndf * 4, 4, 2, 1, use_bias=False)) +netD.add(nn.BatchNorm()) netD.add(nn.LeakyReLU(0.2)) # state size. (ndf) x 8 x 8 -netD.add(nn.Conv2D(ndf * 8, 4, 2, 1, in_filters=ndf * 4, use_bias=False)) -netD.add(nn.BatchNorm(num_features=ndf * 8)) +netD.add(nn.Conv2D(ndf * 8, 4, 2, 1, use_bias=False)) +netD.add(nn.BatchNorm()) netD.add(nn.LeakyReLU(0.2)) # state size. (ndf) x 4 x 4 -netD.add(nn.Conv2D(2, 4, 1, 0, in_filters=ndf * 8, use_bias=False)) +netD.add(nn.Conv2D(2, 4, 1, 0, use_bias=False)) # netD.add(nn.Activation('sigmoid')) diff --git a/python/mxnet/foo/nn/layer.py b/python/mxnet/foo/nn/layer.py index ff121d8532f5..5be7f1ff2085 100644 --- a/python/mxnet/foo/nn/layer.py +++ b/python/mxnet/foo/nn/layer.py @@ -6,7 +6,7 @@ from ...symbol import Symbol from ...ndarray import NDArray from ... import name as _name -from ..parameter import Parameter, ParameterDict +from ..parameter import Parameter, ParameterDict, DeferredInitializationError class _LayerScope(object): @@ -134,13 +134,36 @@ def scope(self): return self._scope def register_child(self, layer): - """Register layer as sublayer of self. Layers assigned to self as attributes - will be registered automatically.""" + """Register layer as sublayer of self. Layers assigned to + self as attributes will be registered automatically.""" self._children.append(layer) - def __call__(self, *args, **kwargs): + def infer_shape(self, *args): + """Infer parameter shape given input shapes. + + *args : list of tuple + A list of input argument shapes. + """ + inputs = [symbol.var('__input%d__'%i, shape=shape) + for i, shape in enumerate(args)] + params = {k: v.var() for k, v in self._reg_params.items()} + sym = self.symbol_forward(*inputs, **params) + arg_shapes, _, aux_shapes = sym.infer_shape() + sdict = {name: shape for name, shape in zip(sym.list_arguments(), arg_shapes)} + sdict.update( + {name : shape for name, shape in zip(sym.list_auxiliary_states(), aux_shapes)}) + for i in self.params.values(): + i.shape = sdict[i.name] + + def __call__(self, *args): """Call forward.""" - return self.forward(*args, **kwargs) + try: + return self.forward(*args) # pylint: disable= no-value-for-parameter + except DeferredInitializationError: + self.infer_shape(*[i.shape for i in args]) + for i in self.params.values(): + i._finish_deferred_init() + return self.forward(*args) # pylint: disable= no-value-for-parameter def forward(self, x, *args): """Defines the forward computation. Arguments can be either NDArray or Symbol.""" diff --git a/python/mxnet/foo/parameter.py b/python/mxnet/foo/parameter.py index 75757d8cd7a0..5901f1873d85 100644 --- a/python/mxnet/foo/parameter.py +++ b/python/mxnet/foo/parameter.py @@ -5,7 +5,7 @@ from collections import OrderedDict import numpy as np -from ..base import mx_real_t +from ..base import mx_real_t, MXNetError from .. import symbol, ndarray, initializer, context from ..context import Context from .. import autograd @@ -14,6 +14,10 @@ tensor_types = (symbol.Symbol, ndarray.NDArray) # pylint: enable= invalid-name +class DeferredInitializationError(MXNetError): + """Error for unfinished deferred initialization.""" + pass + class Parameter(object): """A Container holding parameters (weights) of layers. @@ -75,8 +79,10 @@ def __init__(self, name, grad_req='write', shape=None, dtype=mx_real_t, self._var = None self._data = None self._grad = None + self._defered_init = () - def initialize(self, init=None, ctx=None, default_init=initializer.Xavier()): + def initialize(self, init=None, ctx=None, default_init=initializer.Xavier(), + allow_deferring=True): """Intialize parameter and gradient arrays. Only used for `NDArray` API. init : Initializer @@ -86,7 +92,7 @@ def initialize(self, init=None, ctx=None, default_init=initializer.Xavier()): copy will be made for each context. .. note:: Copies are independent arrays. User is responsible for keeping - their values consistent when updating. Normally nn.Optim does this for you. + their values consistent when updating. Normally nn.Trainer does this for you. default_init : Initializer Default initializer is used when both `init` and `Parameter.init` are None. """ @@ -95,31 +101,53 @@ def initialize(self, init=None, ctx=None, default_init=initializer.Xavier()): if isinstance(ctx, Context): ctx = [ctx] - assert np.prod(self.shape) > 0, \ - "Cannot initialize Parameter %s because it has invalid shape: %s. " \ - "Please specify in_units, in_filters, etc for Layers"%( - self.name, str(self.shape)) - data = ndarray.zeros(shape=self.shape, dtype=self.dtype, ctx=ctx[0]) - if init is None: - init = self.init - initializer.create(default_init)( - initializer.InitDesc(self.name, {'__init__': init}), - data) - - self._data = OrderedDict() - self._data[ctx[0]] = data - for i in ctx[1:]: - self._data[i] = data.copyto(i) - - if self.grad_req == 'null': - self._grad = None + if self.shape is None or np.prod(self.shape) <= 0: + if allow_deferring: + self._defered_init = (init, ctx, default_init) + return + raise ValueError("Cannot initialize Parameter %s because it has " \ + "invalid shape: %s. Please specify in_units, " \ + "in_filters, num_features etc for Layers or " \ + "set allow_deferring to True to defer initialization " \ + "to first forward pass."%(self.name, str(self.shape))) + + self._defered_init = (init, ctx, default_init) + self._finish_deferred_init() + + def _finish_deferred_init(self): + """Finish deferred initialization.""" + if not self._defered_init: return + init, ctx, default_init = self._defered_init + self._defered_init = () + assert self.shape is not None and np.prod(self.shape) > 0, \ + "Cannot initialize Parameter %s because it has " \ + "invalid shape: %s. Please specify in_units, " \ + "in_filters, num_features etc for Layers."%( + self.name, str(self.shape)) + + with autograd.test_section(): + data = ndarray.zeros(shape=self.shape, dtype=self.dtype, ctx=ctx[0]) + if init is None: + init = self.init + initializer.create(default_init)( + initializer.InitDesc(self.name, {'__init__': init}), + data) - self._grad = OrderedDict() - for i in ctx: - self._grad[i] = ndarray.zeros_like(self._data[i]) + self._data = OrderedDict() + self._data[ctx[0]] = data + for i in ctx[1:]: + self._data[i] = data.copyto(i) - autograd.mark_variables(self.list_data(), self.list_grad(), self.grad_req) + if self.grad_req == 'null': + self._grad = None + return + + self._grad = OrderedDict() + for i in ctx: + self._grad[i] = ndarray.zeros_like(self._data[i]) + + autograd.mark_variables(self.list_data(), self.list_grad(), self.grad_req) def set_data(self, data): """Set this parameter's value on all contexts to data.""" @@ -128,6 +156,23 @@ def set_data(self, data): for arr in self.list_data(): arr[:] = data + def _check_initialized(self, ctx=None): + if self._data is not None: + if ctx is not None and ctx not in self._data: + raise RuntimeError( + "Parameter %s was not initialized on context %s. " + "It was only initialized on %s."%( + self.name, str(ctx), str(self.list_ctx()))) + return + if self._defered_init: + raise DeferredInitializationError + raise RuntimeError( + "Parameter %s has not been initialized. Note that " \ + "you should initialize parameters and create Trainer " \ + "with Layer.all_params() instead of Layer.params " \ + "because the later does not include parameters of " \ + "nested child layers "%(self.name)) + def data(self, ctx=None): """Returns a copy of this parameter on one context. Must have been intialized on this context before. @@ -143,23 +188,13 @@ def data(self, ctx=None): """ if ctx is None: ctx = context.current_context() - assert self._data is not None, \ - "Cannot get NDArray value for Parameter %s " \ - "because it hasn't been initialized. Note that " \ - "you should initialize parameters and create Trainer " \ - "with Layer.all_params() instead of Layer.params " \ - "because the later does not include parameters of " \ - "nested child layers "%(self.name) - assert ctx in self._data, \ - "Cannot get NDArray value for Parameter %s on context %s " \ - "because it was not initialized on %s"%(self.name, str(ctx), str(ctx)) + self._check_initialized(ctx) return self._data[ctx] def list_data(self): """Returns copies of this parameter on all contexts, in the same order as creation.""" - assert self._data is not None, \ - "Parameter %s has not been initialized"%self.name + self._check_initialized() return list(self._data.values()) def grad(self, ctx=None): @@ -172,27 +207,27 @@ def grad(self, ctx=None): """ if ctx is None: ctx = context.current_context() - assert self._grad is not None, \ - "Cannot get gradient array for Parameter %s " \ - "because it hasn't been initialized or grad_req='null'"%(self.name) - assert ctx in self._grad, \ - "Cannot get gradient array for Parameter %s on context %s " \ - "because it was not initialized on %s"%(self.name, str(ctx), str(ctx)) + self._check_initialized(ctx) + if self._grad is None: + raise RuntimeError( + "Cannot get gradient array for Parameter %s " \ + "because grad_req='null'"%(self.name)) return self._grad[ctx] def list_grad(self): """Returns gradient buffers on all contexts, in the same order as `values`.""" - assert self._data is not None, \ - "Parameter %s has not been initialized"%self.name - assert self._data is not None, \ + self._check_initialized() + assert self._grad is not None, \ "Parameter %s does not have gradients because grad_req='null'"%self.name return list(self._grad.values()) def list_ctx(self): """Returns a list of contexts this parameter is initialized on""" - assert self._data is not None, \ - "Parameter %s has not been initialized"%self.name + if self._data is None: + if self._defered_init: + return self._defered_init[1] + raise RuntimeError("Parameter %s has not been initialized"%self.name) return list(self._data.keys()) def zero_grad(self): diff --git a/python/mxnet/foo/trainer.py b/python/mxnet/foo/trainer.py index 19db8fad2de4..514dfbd8f36d 100644 --- a/python/mxnet/foo/trainer.py +++ b/python/mxnet/foo/trainer.py @@ -27,7 +27,8 @@ def __init__(self, params, optimizer, optimizer_params, kvstore='device'): self._contexts = self._check_contexts() self._init_optimizer(optimizer, optimizer_params) - self._init_kvstore(kvstore) + self._kv_initialized = False + self._kvstore = kvstore def _check_contexts(self): contexts = None @@ -42,8 +43,6 @@ def _check_contexts(self): def _init_optimizer(self, optimizer, optimizer_params): self._optimizer = opt.create(optimizer, **optimizer_params) - self._updaters = [opt.get_updater(self._optimizer) \ - for _ in self._contexts] lr_mult = {} wd_mult = {} @@ -53,9 +52,12 @@ def _init_optimizer(self, optimizer, optimizer_params): self._optimizer.set_lr_mult(lr_mult) self._optimizer.set_wd_mult(wd_mult) - def _init_kvstore(self, kvstore): + self._updaters = [opt.get_updater(self._optimizer) \ + for _ in self._contexts] + + def _init_kvstore(self): arg_arrays = {param.name: param.data(self._contexts[0]) for param in self._params} - kvstore, update_on_kvstore = _create_kvstore(kvstore, len(self._contexts), arg_arrays) + kvstore, update_on_kvstore = _create_kvstore(self._kvstore, len(self._contexts), arg_arrays) self._kvstore = kvstore self._update_on_kvstore = update_on_kvstore if kvstore: @@ -80,6 +82,9 @@ def step(self, batch_size, ignore_stale_grad=False): If true, ignores Parameters with stale gradient (gradient that has not been updated by `backward` after last step) and skip update. """ + if not self._kv_initialized: + self._init_kvstore() + self._optimizer.rescale_grad = self._scale / batch_size for i, param in enumerate(self._params): diff --git a/tests/python/unittest/test_nn.py b/tests/python/unittest/test_nn.py index 4f78383a139d..42917855df34 100644 --- a/tests/python/unittest/test_nn.py +++ b/tests/python/unittest/test_nn.py @@ -218,6 +218,13 @@ def test_at(): mx.contrib.autograd.compute_gradient([x]) +def test_defered_init(): + x = mx.nd.ones((5, 4, 10, 10)) + layer = nn.Conv2D(10, 2) + layer.all_params().initialize() + layer(x) + + if __name__ == '__main__': import nose nose.runmodule() From 92ce9b77b3bbd76ba513caacd665022ae4040a5a Mon Sep 17 00:00:00 2001 From: Zack Chase Lipton Date: Wed, 14 Jun 2017 09:55:53 -0700 Subject: [PATCH 187/834] Create resnet.py Fixing print statements to make Python3 compatible. --- example/autograd/resnet.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/example/autograd/resnet.py b/example/autograd/resnet.py index d6e52157b4b1..c87193338dde 100644 --- a/example/autograd/resnet.py +++ b/example/autograd/resnet.py @@ -270,7 +270,7 @@ def test(ctx): for x in data: outputs.append(net(x)) metric.update(label, outputs) - print 'validation acc: %s=%f'%metric.get() + print('validation acc: %s=%f'%metric.get()) def train(epoch, ctx): @@ -296,13 +296,13 @@ def train(epoch, ctx): outputs.append(z) trainer.step(batch.data[0].shape[0]) metric.update(label, outputs) - print 'speed: {} samples/s'.format(batch.data[0].shape[0]/(time.time()-btic)) + print('speed: {} samples/s'.format(batch.data[0].shape[0]/(time.time()-btic))) btic = time.time() name, acc = metric.get() metric.reset() - print 'training acc at epoch %d: %s=%f'%(i, name, acc) - print 'time: %f'%(time.time()-tic) + print('training acc at epoch %d: %s=%f'%(i, name, acc)) + print('time: %f'%(time.time()-tic)) test(ctx) net.all_params().save('mnist.params') From 9120d26fa9c2c166a1ad3c7e715ff66becb2eccc Mon Sep 17 00:00:00 2001 From: Eric Junyuan Xie Date: Thu, 15 Jun 2017 21:59:55 -0700 Subject: [PATCH 188/834] foo doc and tutorial (#6708) Renamed a bunch of APIs: Layer.forward -> Layer.call Layer.generic_forward -> Layer.forward Layer.scope -> Layer.name_scope() Removed Layer.ndarray_forward and Layer.symbol_forward. Added tutorial and re-aranged docs. --- docs/api/python/foo.loss.md | 23 -- docs/api/python/foo.md | 199 +++++++++++- docs/api/python/foo.nn.md | 72 ----- docs/api/python/foo.rnn.md | 40 --- docs/api/python/foo.utils.md | 20 -- docs/api/python/index.md | 4 - docs/tutorials/basic/foo.md | 291 ++++++++++++++++++ docs/tutorials/index.md | 1 + example/autograd/actor_critic.py | 4 +- example/autograd/resnet.py | 24 +- example/autograd/super_resolution.py | 4 +- example/autograd/word_language_model/model.py | 4 +- include/mxnet/c_api.h | 6 + include/mxnet/ndarray.h | 4 + python/mxnet/foo/loss.py | 16 +- python/mxnet/foo/nn/conv_layers.py | 6 +- python/mxnet/foo/nn/layer.py | 69 ++--- python/mxnet/foo/parameter.py | 6 +- python/mxnet/foo/rnn/rnn_cell.py | 28 +- python/mxnet/ndarray.py | 39 ++- python/mxnet/symbol.py | 5 +- src/c_api/c_api.cc | 13 + src/ndarray/ndarray.cc | 8 + tests/python/unittest/test_autograd.py | 11 + tests/python/unittest/test_nn.py | 4 +- 25 files changed, 653 insertions(+), 248 deletions(-) delete mode 100644 docs/api/python/foo.loss.md delete mode 100644 docs/api/python/foo.nn.md delete mode 100644 docs/api/python/foo.rnn.md delete mode 100644 docs/api/python/foo.utils.md create mode 100644 docs/tutorials/basic/foo.md diff --git a/docs/api/python/foo.loss.md b/docs/api/python/foo.loss.md deleted file mode 100644 index b35a6942c6a1..000000000000 --- a/docs/api/python/foo.loss.md +++ /dev/null @@ -1,23 +0,0 @@ -# Foo Loss API - -```eval_rst -.. currentmodule:: mxnet.foo.loss -``` - -```eval_rst -.. warning:: This package is currently experimental and may change in the near future. -``` - -## API Reference - - - -```eval_rst -.. automethod:: mxnet.foo.loss.custom_loss -.. automethod:: mxnet.foo.loss.multitask_loss -.. automethod:: mxnet.foo.loss.l1_loss -.. automethod:: mxnet.foo.loss.l2_loss -.. automethod:: mxnet.foo.loss.softmax_cross_entropy_loss -``` - - diff --git a/docs/api/python/foo.md b/docs/api/python/foo.md index 72897fc1676b..0bff54ca9c36 100644 --- a/docs/api/python/foo.md +++ b/docs/api/python/foo.md @@ -1,4 +1,4 @@ -# Foo API +# Foo Package ```eval_rst .. currentmodule:: mxnet.foo @@ -8,7 +8,18 @@ .. warning:: This package is currently experimental and may change in the near future. ``` -## API Reference +## Overview + +Foo package is a high-level interface for MXNet designed to be easy to use while +keeping most of the flexibility of low level API. Foo supports both imperative +and symbolic programming, making it easy to train complex models imperatively +in Python and then deploy with symbolic graph in C++ and Scala. + +## Parameter + +```eval_rst +.. currentmodule:: mxnet.foo +``` @@ -17,8 +28,192 @@ :members: .. autoclass:: mxnet.foo.ParameterDict :members: +``` + + + + +## Neural Network Layers + +```eval_rst +.. currentmodule:: mxnet.foo.nn +``` + +### Containers + + + +```eval_rst +.. currentmodule:: mxnet.foo.nn +.. autoclass:: mxnet.foo.nn.Layer + :members: + + .. automethod:: __call__ +.. autoclass:: mxnet.foo.nn.Sequential + :members: +``` + + + +### Basic Layers + + + +```eval_rst +.. currentmodule:: mxnet.foo.nn +.. autoclass:: mxnet.foo.nn.Dense + :members: +.. autoclass:: mxnet.foo.nn.Activation + :members: +.. autoclass:: mxnet.foo.nn.Dropout + :members: +.. autoclass:: mxnet.foo.nn.BatchNorm + :members: +.. autoclass:: mxnet.foo.nn.LeakyReLU + :members: +.. autoclass:: mxnet.foo.nn.Embedding + :members: +``` + + + +### Convolutional Layers + + + +```eval_rst +.. currentmodule:: mxnet.foo.nn +.. autoclass:: mxnet.foo.nn.Conv1D + :members: +.. autoclass:: mxnet.foo.nn.Conv2D + :members: +.. autoclass:: mxnet.foo.nn.Conv3D + :members: +.. autoclass:: mxnet.foo.nn.Conv1DTranspose + :members: +.. autoclass:: mxnet.foo.nn.Conv2DTranspose + :members: +.. autoclass:: mxnet.foo.nn.Conv3DTranspose + :members: +``` + + + + +### Pooling Layers + + + +```eval_rst +.. currentmodule:: mxnet.foo.nn +.. autoclass:: mxnet.foo.nn.MaxPool1D + :members: +.. autoclass:: mxnet.foo.nn.MaxPool2D + :members: +.. autoclass:: mxnet.foo.nn.MaxPool3D + :members: +.. autoclass:: mxnet.foo.nn.AvgPool1D + :members: +.. autoclass:: mxnet.foo.nn.AvgPool2D + :members: +.. autoclass:: mxnet.foo.nn.AvgPool3D + :members: +.. autoclass:: mxnet.foo.nn.GlobalMaxPool1D + :members: +.. autoclass:: mxnet.foo.nn.GlobalMaxPool2D + :members: +.. autoclass:: mxnet.foo.nn.GlobalMaxPool3D + :members: +.. autoclass:: mxnet.foo.nn.GlobalAvgPool1D + :members: +.. autoclass:: mxnet.foo.nn.GlobalAvgPool2D + :members: +.. autoclass:: mxnet.foo.nn.GlobalAvgPool3D + :members: +``` + + + + +## Recurrent Layers + +```eval_rst +.. currentmodule:: mxnet.foo.rnn +``` + + + +```eval_rst +.. autoclass:: mxnet.foo.rnn.RecurrentCell + :members: + + .. automethod:: __call__ +.. autoclass:: mxnet.foo.rnn.LSTMCell + :members: +.. autoclass:: mxnet.foo.rnn.GRUCell + :members: +.. autoclass:: mxnet.foo.rnn.RNNCell + :members: +.. autoclass:: mxnet.foo.rnn.FusedRNNCell + :members: +.. autoclass:: mxnet.foo.rnn.SequentialRNNCell + :members: +.. autoclass:: mxnet.foo.rnn.BidirectionalCell + :members: +.. autoclass:: mxnet.foo.rnn.DropoutCell + :members: +.. autoclass:: mxnet.foo.rnn.ZoneoutCell + :members: +.. autoclass:: mxnet.foo.rnn.ResidualCell + :members: +``` + + + +## Trainer + +```eval_rst +.. currentmodule:: mxnet.foo +``` + + + +```eval_rst .. autoclass:: mxnet.foo.Trainer :members: ``` + +## Loss functions + +```eval_rst +.. currentmodule:: mxnet.foo.loss +``` + + + +```eval_rst +.. automethod:: mxnet.foo.loss.custom_loss +.. automethod:: mxnet.foo.loss.multitask_loss +.. automethod:: mxnet.foo.loss.l1_loss +.. automethod:: mxnet.foo.loss.l2_loss +.. automethod:: mxnet.foo.loss.softmax_cross_entropy_loss +``` + + + +## Utilities + +```eval_rst +.. currentmodule:: mxnet.foo.utils +``` + + + +```eval_rst +.. automethod:: mxnet.foo.utils.split_data +.. automethod:: mxnet.foo.utils.load_data +``` + + diff --git a/docs/api/python/foo.nn.md b/docs/api/python/foo.nn.md deleted file mode 100644 index 184f0ecc5dbb..000000000000 --- a/docs/api/python/foo.nn.md +++ /dev/null @@ -1,72 +0,0 @@ -# Foo NN API - -```eval_rst -.. currentmodule:: mxnet.foo.nn -``` - -```eval_rst -.. warning:: This package is currently experimental and may change in the near future. -``` - -## API Reference - - - -```eval_rst -.. currentmodule:: mxnet.foo.nn -.. autoclass:: mxnet.foo.nn.Layer - :members: - - .. automethod:: __call__ -.. autoclass:: mxnet.foo.nn.Sequential - :members: -.. autoclass:: mxnet.foo.nn.Dense - :members: -.. autoclass:: mxnet.foo.nn.Activation - :members: -.. autoclass:: mxnet.foo.nn.Dropout - :members: -.. autoclass:: mxnet.foo.nn.BatchNorm - :members: -.. autoclass:: mxnet.foo.nn.LeakyReLU - :members: - -.. autoclass:: mxnet.foo.nn.Conv1D - :members: -.. autoclass:: mxnet.foo.nn.Conv2D - :members: -.. autoclass:: mxnet.foo.nn.Conv3D - :members: -.. autoclass:: mxnet.foo.nn.Conv1DTranspose - :members: -.. autoclass:: mxnet.foo.nn.Conv2DTranspose - :members: -.. autoclass:: mxnet.foo.nn.Conv3DTranspose - :members: -.. autoclass:: mxnet.foo.nn.MaxPool1D - :members: -.. autoclass:: mxnet.foo.nn.MaxPool2D - :members: -.. autoclass:: mxnet.foo.nn.MaxPool3D - :members: -.. autoclass:: mxnet.foo.nn.AvgPool1D - :members: -.. autoclass:: mxnet.foo.nn.AvgPool2D - :members: -.. autoclass:: mxnet.foo.nn.AvgPool3D - :members: -.. autoclass:: mxnet.foo.nn.GlobalMaxPool1D - :members: -.. autoclass:: mxnet.foo.nn.GlobalMaxPool2D - :members: -.. autoclass:: mxnet.foo.nn.GlobalMaxPool3D - :members: -.. autoclass:: mxnet.foo.nn.GlobalAvgPool1D - :members: -.. autoclass:: mxnet.foo.nn.GlobalAvgPool2D - :members: -.. autoclass:: mxnet.foo.nn.GlobalAvgPool3D - :members: -``` - - diff --git a/docs/api/python/foo.rnn.md b/docs/api/python/foo.rnn.md deleted file mode 100644 index e2c2b37a1f6f..000000000000 --- a/docs/api/python/foo.rnn.md +++ /dev/null @@ -1,40 +0,0 @@ -# Foo RNN API - -```eval_rst -.. currentmodule:: mxnet.foo.rnn -``` - -```eval_rst -.. warning:: This package is currently experimental and may change in the near future. -``` - -## API Reference - - - -```eval_rst -.. autoclass:: mxnet.foo.rnn.RecurrentCell - :members: - - .. automethod:: __call__ -.. autoclass:: mxnet.foo.rnn.LSTMCell - :members: -.. autoclass:: mxnet.foo.rnn.GRUCell - :members: -.. autoclass:: mxnet.foo.rnn.RNNCell - :members: -.. autoclass:: mxnet.foo.rnn.FusedRNNCell - :members: -.. autoclass:: mxnet.foo.rnn.SequentialRNNCell - :members: -.. autoclass:: mxnet.foo.rnn.BidirectionalCell - :members: -.. autoclass:: mxnet.foo.rnn.DropoutCell - :members: -.. autoclass:: mxnet.foo.rnn.ZoneoutCell - :members: -.. autoclass:: mxnet.foo.rnn.ResidualCell - :members: -``` - - diff --git a/docs/api/python/foo.utils.md b/docs/api/python/foo.utils.md deleted file mode 100644 index 21dea1a0c2b9..000000000000 --- a/docs/api/python/foo.utils.md +++ /dev/null @@ -1,20 +0,0 @@ -# Foo Utility API - -```eval_rst -.. currentmodule:: mxnet.foo.utils -``` - -```eval_rst -.. warning:: This package is currently experimental and may change in the near future. -``` - -## API Reference - - - -```eval_rst -.. automethod:: mxnet.foo.utils.split_data -.. automethod:: mxnet.foo.utils.load_data -``` - - diff --git a/docs/api/python/index.md b/docs/api/python/index.md index fe102eb6a601..43f02677126a 100644 --- a/docs/api/python/index.md +++ b/docs/api/python/index.md @@ -29,10 +29,6 @@ imported by running: symbol module foo - foo.nn - foo.rnn - foo.loss - foo.utils rnn kvstore io diff --git a/docs/tutorials/basic/foo.md b/docs/tutorials/basic/foo.md new file mode 100644 index 000000000000..84b14278158c --- /dev/null +++ b/docs/tutorials/basic/foo.md @@ -0,0 +1,291 @@ +# Foo - High-level Interface + +Foo package is a high-level interface for MXNet designed to be easy to use while +keeping most of the flexibility of low level API. Foo supports both imperative +and symbolic programming, making it easy to train complex models imperatively +in Python and then deploy with symbolic graph in C++ and Scala. + +This tutorial covers four topics: +- MXNet NDArray as a replacement of numpy for asynchronous scientific computing +across CPU and GPU. +- Automatic differentiation with NDArray. +- Define and train neural network models with Foo's imperative API. +- [TODO] Save trained models as symbolic graph for easy production deployment. + +## Setup +First, let's import MXNet and Foo: + +```python +from __future__ import print_function +import numpy as np +import mxnet as mx +``` + +## NDArray + +### Creating NDArray + +NDArray is similar to numpy's ndarray, but supports asynchronous operations +and GPU. There are many ways to create NDArray. + +Construct from (nested) list: +```python +x = mx.nd.array([[1, 2, 3], [4, 5, 6]]) +print(x) +``` + +Construct from numpy array: +```python +x_numpy = np.ones((2, 3)) +x = mx.nd.array(x_numpy) +print(x) +``` + +Array construction routines: +```python +# create an 2x3 array of ones +x = mx.nd.ones((2, 3)) +print(x) +# create an 2x3 array of zeros +x = mx.nd.zeros((2, 3)) +print(x) +# create an 1d-array of 0 to 5 and reshape to 2x3 +x = mx.nd.arange(6).reshape((2, 3)) +print(x) +``` + +You can convert any NDArray to numpy array with `.asnumpy()`: +```python +z = x.asnumpy() +print(z) +``` + +### NDArray Operations + +NDArray supports a wide range of operations. Simple operations can be called +with python syntax: + +```python +x = mx.nd.array([[1, 2], [3, 4]]) +y = mx.nd.array([[4, 3], [2, 1]]) +print(x + y) +``` + +You can also call operators from the `mxnet.ndarray` (or `mx.nd` for short) name space: + +```python +z = mx.nd.add(x, y) +print(z) +``` + +You can also pass additional flags to operators: + +```python +z = mx.nd.sum(x, axis=0) +print('axis=0:', z) +z = mx.nd.sum(x, axis=1) +print('axis=1:', z) +``` + +By default operators create new NDArrays for return value. You can specify `out` +to use a pre-allocated buffer: + +```python +z = mx.nd.empty((2, 2)) +mx.nd.add(x, y, out=z) +print(x) +``` + +### Using GPU + +Each NDArray lives on a `Context`. MXNet supports `mx.cpu()` for CPU and `mx.gpu(0)`, +`mx.gpu(1)`, etc for GPU. You can specify context when creating NDArray: + +```python +# creates on CPU (the default). +# Replace mx.cpu() with mx.gpu(0) if you have a GPU. +x = mx.nd.zeros((2, 2), ctx=mx.cpu()) +print(x) +x = mx.nd.array([[1, 2], [3, 4]], ctx=mx.cpu()) +print(x) +``` + +You can copy arrays between devices with `.copyto()`: + +```python +# Copy x to cpu. Replace with mx.gpu(0) if you have GPU. +y = x.copyto(mx.cpu()) +# Copy x to another NDArray, possibly on another Context. +x.copyto(y) +print(y) +``` + +See the [NDArray tutorial](ndarray.md) for a more detailed introduction to +NDArray API. + +## Automatic Differentiation + +MXNet supports automatic differentiation with the `autograd` package. +`autograd` allows you to differentiate a network of NDArray operations. +This is call define-by-run, i.e., the network is defined on-the-fly by +running forward computation. You can define exotic network structures +and differentiate them, and each iteration can have a totally different +network structure. + +```python +form mxnet import autograd +from mxnet.autograd import train_section +``` + +To use `autograd`, we must first mark variables that require gradient and +attach gradient buffers to them: + +```python +x = mx.nd.array([[1, 2], [3, 4]]) +dx = mx.nd.zeros_like(x) +x.attach_grad(dx) +``` + +Now we can define the network while running forward computation by wrapping +it inside a `train_section` (operations out of `train_section` does not define +a graph and cannot be differentiated): + +```python +with train_section(): + y = x * 2 + z = y * x +``` + +Let's backprop with `z.backward()`, which is equivalent to +`z.backward(mx.nd.ones_like(z))`. When z has more than one entry, `z.backward()` +is equivalent to `mx.nd.sum(z).backward()`: + +```python +z.backward() +print(x.grad) +``` + +## Neural Network and Layers + +Neural networks (and other machine learning models) can be defined and trained +with `foo.nn` and `foo.rnn` package. A typical training script has the following +steps: + +- Define network +- Initialize parameters +- Loop over inputs +- Forward input through network to get output +- Compute loss with output and label +- Backprop gradient +- Update parameters with gradient descent. + + +### Define Network + +`foo.nn.Layer` is the basic building block of models. You can define networks by +composing and inheriting `Layer`: + +```python +import mxnet.foo as foo +from mxnet.foo import nn + +class Net(nn.Layer): + def __init__(self, **kwargs): + super(Net, self).__init__(**kwargs) + with self.name_scope: + # layers created in name_scope will inherit name space + # from parent layer. + self.conv1 = nn.Conv2D(6, kernel_size=5) + self.pool1 = nn.Pool2D(kernel_size=2) + self.conv2 = nn.Conv2D(16, kernel_size=5) + self.pool2 = nn.Pool2D(kernel_size=2) + self.fc1 = nn.Dense(120) + self.fc2 = nn.Dense(84) + self.fc3 = nn.Dense(10) + + def forward(self, F, x): + x = self.pool1(F.relu(self.conv1(x))) + x = self.pool2(F.relu(self.conv2(x))) + # 0 means copy over size from corresponding dimension. + # -1 means infer size from the rest of dimensions. + x = x.reshape((0, -1)) + x = F.relu(self.fc1(x)) + x = F.relu(self.fc2(x)) + x = self.fc3(x) + return x +``` + +### Initialize Parameters + +A network must be created and initialized before it can be used: + +```python +net = Net() +# Initialize on CPU. Replace with `mx.gpu(0)`, or `[mx.gpu(0), mx.gpu(1)]`, +# etc to use one or more GPUs. +net.all_params().initialize(mx.init.Xavier(), ctx=mx.cpu()) +``` + +Note that because we didn't specify input size to layers in Net's constructor, +the shape of parameters cannot be determined at this point. Actual initialization +is deferred to the first forward pass, i.e. if you access `net.fc1.weight.data()` +now an exception will be raised. + +You can actually initialize the weights by running a forward pass: + +```python +data = mx.nd.random_normal(shape=(10, 1, 32, 32)) # dummy data +output = net(data) +``` + +Or you can specify input size when creating layers, i.e. `nn.Dense(84, in_units=120)` +instead of `nn.Dense(84)`. + +### Loss Functions + +Loss functions take (output, label) pairs and compute a scalar loss for each sample +in the mini-batch. The scalars measure how far each output is from the label. + +There are many predefined loss functions in `foo.loss`. Here we use +`softmax_cross_entropy_loss` for digit classification. + +To compute loss and backprop for one iteration, we do: + +```python +label = mx.nd.arange(10) # dummy label +with train_section(): + output = net(data) + loss = foo.loss.softmax_cross_entropy_loss(output, label) + loss.backward() +print('loss:', loss) +print('grad:', net.fc1.weight.grad()) +``` + +### Updating the weights + +Now that gradient is computed, we just need to update the weights. This is usually +done with formulas like `weight = weight - learning_rate * grad / batch_size`. +Note we divide gradient by batch_size because gradient is aggregated over the +entire batch. For example, + +```python +lr = 0.01 +for p in net.all_params().values(): + p.data()[:] -= lr / data.shape[0] * p.grad() +``` + +But sometimes you want more fancy updating rules like momentum and Adam, and since +this is a commonly used functionality, foo provide a `Trainer` class for it: + +```python +trainer = foo.Trainer(net.all_params(), 'sgd', {'learning_rate': 0.01}) + +with train_section(): + output = net(data) + loss = foo.loss.softmax_cross_entropy_loss(output, label) + loss.backward() + +# do the update. Trainer needs to know the batch size of data to normalize +# the gradient by 1/batch_size. +trainer.step(data.shape[0]) +``` diff --git a/docs/tutorials/index.md b/docs/tutorials/index.md index aed11a4bebf1..dc56cb145fce 100644 --- a/docs/tutorials/index.md +++ b/docs/tutorials/index.md @@ -10,6 +10,7 @@ These tutorials introduce a few fundamental concepts in deep learning and how to .. toctree:: :maxdepth: 1 + basic/foo basic/ndarray basic/symbol basic/module diff --git a/example/autograd/actor_critic.py b/example/autograd/actor_critic.py index 1e87178f3679..7a716b23fc4d 100644 --- a/example/autograd/actor_critic.py +++ b/example/autograd/actor_critic.py @@ -30,12 +30,12 @@ class Policy(nn.Layer): def __init__(self, **kwargs): super(Policy, self).__init__(**kwargs) - with self.scope: + with self.name_scope(): self.dense = nn.Dense(16, in_units=4, activation='relu') self.action_pred = nn.Dense(2, in_units=16) self.value_pred = nn.Dense(1, in_units=16) - def generic_forward(self, F, x): + def forward(self, F, x): x = self.dense(x) probs = self.action_pred(x) values = self.value_pred(x) diff --git a/example/autograd/resnet.py b/example/autograd/resnet.py index c87193338dde..5715eeaf9403 100644 --- a/example/autograd/resnet.py +++ b/example/autograd/resnet.py @@ -14,7 +14,7 @@ def conv3x3(filters, stride, in_filters): class BasicBlockV1(nn.Layer): def __init__(self, filters, stride, downsample=False, in_filters=0, **kwargs): super(BasicBlockV1, self).__init__(**kwargs) - with self.scope: + with self.name_scope(): self.conv1 = conv3x3(filters, stride, in_filters) self.bn1 = nn.BatchNorm(num_features=in_filters) self.conv2 = conv3x3(filters, 1, filters) @@ -24,7 +24,7 @@ def __init__(self, filters, stride, downsample=False, in_filters=0, **kwargs): self.bn_ds = nn.BatchNorm(num_features=filters) self.downsample = downsample - def generic_forward(self, domain, x): + def forward(self, domain, x): residual = x out = self.conv1(x) @@ -47,7 +47,7 @@ def generic_forward(self, domain, x): class BottleneckV1(nn.Layer): def __init__(self, filters, stride, downsample=False, in_filters=0, **kwargs): super(BottleneckV1, self).__init__(**kwargs) - with self.scope: + with self.name_scope(): self.conv1 = nn.Conv2D(filters=filters//4, kernel_size=1, strides=1, in_filters=in_filters) self.bn1 = nn.BatchNorm(num_features=filters//4) self.conv2 = conv3x3(filters//4, stride, filters//4) @@ -59,7 +59,7 @@ def __init__(self, filters, stride, downsample=False, in_filters=0, **kwargs): self.bn_ds = nn.BatchNorm(num_features=filters) self.downsample = downsample - def generic_forward(self, domain, x): + def forward(self, domain, x): residual = x out = self.conv1(x) @@ -86,7 +86,7 @@ def generic_forward(self, domain, x): class ResnetV1(nn.Layer): def __init__(self, block, classes, layers, filters, thumbnail=False, **kwargs): super(ResnetV1, self).__init__(**kwargs) - with self.scope: + with self.name_scope(): assert len(layers) == len(filters) - 1 self._thumbnail = thumbnail if thumbnail: @@ -115,7 +115,7 @@ def _make_layer(self, block, layers, filters, stride, in_filters=0): layer.add(block(filters, 1, False, in_filters=filters)) return layer - def generic_forward(self, domain, x): + def forward(self, domain, x): x = self.conv0(x) if not self._thumbnail: x = self.bn0(x) @@ -134,7 +134,7 @@ def generic_forward(self, domain, x): class BasicBlockV2(nn.Layer): def __init__(self, filters, stride, downsample=False, in_filters=0, **kwargs): super(BasicBlockV2, self).__init__(**kwargs) - with self.scope: + with self.name_scope(): self.bn1 = nn.BatchNorm(num_features=in_filters) self.conv1 = conv3x3(filters, stride, in_filters) self.bn2 = nn.BatchNorm(num_features=filters) @@ -145,7 +145,7 @@ def __init__(self, filters, stride, downsample=False, in_filters=0, **kwargs): else: self.downsample = None - def generic_forward(self, domain, x): + def forward(self, domain, x): if not self.downsample: residual = x x = self.bn1(x) @@ -164,7 +164,7 @@ def generic_forward(self, domain, x): class BottleneckV2(nn.Layer): def __init__(self, filters, stride, downsample=False, in_filters=0, **kwargs): super(BottleneckV2, self).__init__(**kwargs) - with self.scope: + with self.name_scope(): self.bn1 = nn.BatchNorm(num_features=in_filters) self.conv1 = conv3x3(filters//4, 1, in_filters) self.bn2 = nn.BatchNorm(num_features=filters//4) @@ -177,7 +177,7 @@ def __init__(self, filters, stride, downsample=False, in_filters=0, **kwargs): else: self.downsample = None - def generic_forward(self, domain, x): + def forward(self, domain, x): if not self.downsample: residual = x x = self.bn1(x) @@ -199,7 +199,7 @@ def generic_forward(self, domain, x): class ResnetV2(nn.Layer): def __init__(self, block, classes, layers, filters, thumbnail=False, **kwargs): super(ResnetV2, self).__init__(**kwargs) - with self.scope: + with self.name_scope(): assert len(layers) == len(filters) - 1 self._thumbnail = thumbnail self.bn_data = nn.BatchNorm(num_features=3, scale=False, center=False) @@ -230,7 +230,7 @@ def _make_layer(self, block, layers, filters, stride, in_filters=0): layer.add(block(filters, 1, False, in_filters=filters)) return layer - def generic_forward(self, domain, x): + def forward(self, domain, x): x = self.bn_data(x) x = self.conv0(x) if not self._thumbnail: diff --git a/example/autograd/super_resolution.py b/example/autograd/super_resolution.py index 89cc58b8ad6e..3c66d7b09dcd 100644 --- a/example/autograd/super_resolution.py +++ b/example/autograd/super_resolution.py @@ -90,14 +90,14 @@ def _rearrange(raw, F, upscale_factor): class SuperResolutionNet(nn.Layer): def __init__(self, upscale_factor): super(SuperResolutionNet, self).__init__() - with self.scope: + with self.name_scope(): self.conv1 = nn.Conv2D(64, (5, 5), strides=(1, 1), padding=(2, 2), in_filters=1) self.conv2 = nn.Conv2D(64, (3, 3), strides=(1, 1), padding=(1, 1), in_filters=64) self.conv3 = nn.Conv2D(32, (3, 3), strides=(1, 1), padding=(1, 1), in_filters=64) self.conv4 = nn.Conv2D(upscale_factor ** 2, (3, 3), strides=(1, 1), padding=(1, 1), in_filters=32) self.upscale_factor = upscale_factor - def generic_forward(self, F, x): + def forward(self, F, x): x = F.Activation(self.conv1(x), act_type='relu') x = F.Activation(self.conv2(x), act_type='relu') x = F.Activation(self.conv3(x), act_type='relu') diff --git a/example/autograd/word_language_model/model.py b/example/autograd/word_language_model/model.py index 8a2a7d92a054..97622566c0d3 100644 --- a/example/autograd/word_language_model/model.py +++ b/example/autograd/word_language_model/model.py @@ -6,7 +6,7 @@ class RNNModel(nn.Layer): def __init__(self, mode, vocab_size, num_embed, num_hidden, num_layers, dropout=0.5, tie_weights=False, **kwargs): super(RNNModel, self).__init__(**kwargs) - with self.scope: + with self.name_scope(): self.drop = nn.Dropout(dropout) self.encoder = nn.Embedding(vocab_size, num_embed) self.rnn = rnn.FusedRNNCell(num_hidden, num_layers, mode=mode, @@ -20,7 +20,7 @@ def __init__(self, mode, vocab_size, num_embed, num_hidden, self.num_hidden = num_hidden - def generic_forward(self, F, inputs, hidden): + def forward(self, F, inputs, hidden): emb = self.drop(self.encoder(inputs)) output, hidden = self.rnn.unroll(None, emb, layout='TNC', merge_outputs=True) output = self.drop(output) diff --git a/include/mxnet/c_api.h b/include/mxnet/c_api.h index 8bc1451ba90d..47447fb37196 100644 --- a/include/mxnet/c_api.h +++ b/include/mxnet/c_api.h @@ -416,6 +416,12 @@ MXNET_DLL int MXNDArrayGetDType(NDArrayHandle handle, MXNET_DLL int MXNDArrayGetContext(NDArrayHandle handle, int *out_dev_type, int *out_dev_id); +/*! + * \brief return gradient buffer attached to this NDArray + * \param handle NDArray handle + * \return 0 when success, -1 when failure happens + */ +MXNET_DLL int MXNDArrayGetGrad(NDArrayHandle handle, NDArrayHandle *out); /*! * \brief detach and ndarray from computation graph by clearing entry_ * \param handle NDArray handle diff --git a/include/mxnet/ndarray.h b/include/mxnet/ndarray.h index 504fd5e7676e..e349b3091c56 100644 --- a/include/mxnet/ndarray.h +++ b/include/mxnet/ndarray.h @@ -107,6 +107,10 @@ class NDArray { SetTBlob(); return tblob_; } + /*! + * \return the gradient ndarray. + */ + NDArray grad() const; /*! * \return the context of NDArray, this function is only valid when the NDArray is not empty */ diff --git a/python/mxnet/foo/loss.py b/python/mxnet/foo/loss.py index 9bfa3795c2e3..8f7193383ec4 100644 --- a/python/mxnet/foo/loss.py +++ b/python/mxnet/foo/loss.py @@ -82,14 +82,14 @@ def custom_loss(loss, output, label, weight=None, sample_weight=None, batch_axis loss : BaseLoss created loss - Example - ------- - The following code defines a least square loss (same as `nn.l2_loss`):: - data = mx.sym.var('data') - output = mx.sym.FullyConnected(data, num_hidden=1) - label = mx.sym.var('label') - loss = mx.sym.square(output - label.reshape((-1, 1)))/2 - loss = nn.custom_loss(loss, output, label, name='l2') + Examples + -------- + >>> # To define a least square loss (same as `l2_loss`) + >>> data = mx.sym.var('data') + >>> output = mx.sym.FullyConnected(data, num_hidden=1) + >>> label = mx.sym.var('label') + >>> loss = mx.sym.square(output - label.reshape((-1, 1)))/2 + >>> loss = nn.custom_loss(loss, output, label, name='l2') """ F = _get_F(loss) loss = _apply_weighting(F, loss, weight, sample_weight) diff --git a/python/mxnet/foo/nn/conv_layers.py b/python/mxnet/foo/nn/conv_layers.py index f70aa11a29db..d26bebe97c57 100644 --- a/python/mxnet/foo/nn/conv_layers.py +++ b/python/mxnet/foo/nn/conv_layers.py @@ -62,7 +62,7 @@ def __init__(self, filters, kernel_size, strides, padding, dilation, kernel_initializer=None, bias_initializer=None, op_name='Convolution', prefix=None, params=None, **kwargs): super(_Conv, self).__init__(prefix=prefix, params=params) - with self.scope: + with self.name_scope(): self._filters = filters self._in_filters = in_filters if isinstance(strides, numeric_types): @@ -93,7 +93,7 @@ def __init__(self, filters, kernel_size, strides, padding, dilation, else: self.act = None - def generic_forward(self, F, x, weight, bias=None): + def forward(self, F, x, weight, bias=None): if bias is None: act = F.invoke(self._op, [x, weight]) else: @@ -520,7 +520,7 @@ def __init__(self, pool_size, strides, padding, global_pool, pool_type, **kwargs 'pool_type': pool_type} self._op = symbol.CachedOp('Pooling', 1, **attrs) - def generic_forward(self, F, x): + def forward(self, F, x): return F.invoke(self._op, [x]) diff --git a/python/mxnet/foo/nn/layer.py b/python/mxnet/foo/nn/layer.py index 5be7f1ff2085..172b3cb0c5cc 100644 --- a/python/mxnet/foo/nn/layer.py +++ b/python/mxnet/foo/nn/layer.py @@ -59,11 +59,11 @@ class Layer(object): class Net(nn.Layer): def __init__(self, **kwargs): super(Net, self).__init__(**kwargs) - with self.scope: + with self.name_scope(): self.dense0 = nn.Dense(20, in_units=10) self.dense1 = nn.Dense(20, in_units=20) - def forward(self, x): + def forward(self, F, x): x = self.dense0(x) return self.dense1(x) @@ -129,8 +129,10 @@ def name(self): return self.prefix[:-1] return self.prefix - @property - def scope(self): + def name_scope(self): + """Returns a name space object managing sublayer and parameter + names. Should be used by `with` statement + """ return self._scope def register_child(self, layer): @@ -147,7 +149,7 @@ def infer_shape(self, *args): inputs = [symbol.var('__input%d__'%i, shape=shape) for i, shape in enumerate(args)] params = {k: v.var() for k, v in self._reg_params.items()} - sym = self.symbol_forward(*inputs, **params) + sym = self.forward(symbol, *inputs, **params) arg_shapes, _, aux_shapes = sym.infer_shape() sdict = {name: shape for name, shape in zip(sym.list_arguments(), arg_shapes)} sdict.update( @@ -157,33 +159,30 @@ def infer_shape(self, *args): def __call__(self, *args): """Call forward.""" - try: - return self.forward(*args) # pylint: disable= no-value-for-parameter - except DeferredInitializationError: - self.infer_shape(*[i.shape for i in args]) - for i in self.params.values(): - i._finish_deferred_init() - return self.forward(*args) # pylint: disable= no-value-for-parameter - - def forward(self, x, *args): + return self.call(*args) # pylint: disable=no-value-for-parameter + + def call(self, x, *args): """Defines the forward computation. Arguments can be either NDArray or Symbol.""" if isinstance(x, NDArray): with x.context as ctx: - params = {k: v.data(ctx) for k, v in self._reg_params.items()} - return self.ndarray_forward(x, *args, **params) + try: + params = {k: v.data(ctx) for k, v in self._reg_params.items()} + except DeferredInitializationError: + arg_shapes = [x.shape] + arg_shapes += [i.shape if isinstance(i, NDArray) else i for i in args] + self.infer_shape(*arg_shapes) + for i in self.params.values(): + i._finish_deferred_init() + params = {k: v.data(ctx) for k, v in self._reg_params.items()} + return self.forward(ndarray, x, *args, **params) else: assert isinstance(x, Symbol), \ - "Layer requires the first argument to forward be either Symbol or NDArray" + "Layer requires the first argument to forward be either " \ + "Symbol or NDArray, but got %s"%type(x) params = {k: v.var() for k, v in self._reg_params.items()} - return self.symbol_forward(x, *args, **params) - - def ndarray_forward(self, x, *args, **kwargs): - return self.generic_forward(ndarray, x, *args, **kwargs) - - def symbol_forward(self, x, *args, **kwargs): - return self.generic_forward(symbol, x, *args, **kwargs) + return self.forward(symbol, x, *args, **params) - def generic_forward(self, F, x, *args, **kwargs): + def forward(self, F, x, *args, **kwargs): """Simple forward supports both `Symbol` and `NDArray` API. Parameters @@ -217,13 +216,13 @@ def add(self, layer): """Add layer on top of the stack.""" self.register_child(layer) - def forward(self, x): + def call(self, x): #pylint: disable=arguments-differ for layer in self._children: x = layer(x) return x - def generic_forward(self, F, x, *args, **kwargs): + def forward(self, F, x, *args, **kwargs): raise NotImplementedError @@ -284,7 +283,7 @@ def __init__(self, units, activation=None, use_bias=True, kernel_initializer=None, bias_initializer=None, in_units=0, **kwargs): super(Dense, self).__init__(**kwargs) - with self.scope: + with self.name_scope(): self._op = symbol.CachedOp('FullyConnected', 3 if use_bias else 2, num_hidden=units, no_bias=not use_bias) self.weight = self.params.get('weight', shape=(units, in_units), @@ -297,7 +296,7 @@ def __init__(self, units, activation=None, use_bias=True, else: self.act = None - def generic_forward(self, F, x, weight, bias=None): + def forward(self, F, x, weight, bias=None): if bias is None: act = F.invoke(self._op, [x, weight]) else: @@ -331,7 +330,7 @@ def __init__(self, activation, **kwargs): def _alias(self): return self._act_type - def generic_forward(self, F, x): + def forward(self, F, x): return F.invoke(self._op, [x]) @@ -348,14 +347,14 @@ class Dropout(Layer): References ---------- - - [Dropout: A Simple Way to Prevent Neural Networks from Overfitting]( + [Dropout: A Simple Way to Prevent Neural Networks from Overfitting]( http://www.cs.toronto.edu/~rsalakhu/papers/srivastava14a.pdf) """ def __init__(self, rate, **kwargs): super(Dropout, self).__init__(**kwargs) self._op = symbol.CachedOp('Dropout', 1, p=rate) - def generic_forward(self, F, x): + def forward(self, F, x): return F.invoke(self._op, [x]) @@ -407,7 +406,7 @@ def __init__(self, axis=1, momentum=0.9, epsilon=1e-3, center=True, scale=True, shape=(num_features,), init=running_variance_initializer) - def generic_forward(self, F, x, gamma, beta, running_mean, running_var): + def forward(self, F, x, gamma, beta, running_mean, running_var): return F.invoke(self._op, [x, gamma, beta, running_mean, running_var]) @@ -427,7 +426,7 @@ def __init__(self, alpha, **kwargs): super(LeakyReLU, self).__init__(**kwargs) self._op = symbol.CachedOp('LeakyReLU', 1, act_type='leaky', slope=alpha) - def generic_forward(self, F, x): + def forward(self, F, x): return F.invoke(self._op, [x]) @@ -463,5 +462,5 @@ def __init__(self, input_dim, output_dim, dtype='float32', self.weight = self.params.get('weight', shape=(input_dim, output_dim), init=embeddings_initializer) - def generic_forward(self, F, x, weight): + def forward(self, F, x, weight): return F.invoke(self._op, [x, weight]) diff --git a/python/mxnet/foo/parameter.py b/python/mxnet/foo/parameter.py index 5901f1873d85..50c9c614b853 100644 --- a/python/mxnet/foo/parameter.py +++ b/python/mxnet/foo/parameter.py @@ -85,6 +85,8 @@ def initialize(self, init=None, ctx=None, default_init=initializer.Xavier(), allow_deferring=True): """Intialize parameter and gradient arrays. Only used for `NDArray` API. + Parameters + ---------- init : Initializer The initializer to use. Overrides `Parameter.init` and default_init. ctx : Context or list of Context, defaults to `context.current_context()`. @@ -295,8 +297,8 @@ def get(self, name, **kwargs): found, `get` will create a new Parameter with key-word arguments and insert it to self. - Parameter - --------- + Parameters + ---------- name : str name of the desired Parameter. It will be prepended with this dictionary's prefix. diff --git a/python/mxnet/foo/rnn/rnn_cell.py b/python/mxnet/foo/rnn/rnn_cell.py index d0f6ebbcd118..2733cebe46bd 100644 --- a/python/mxnet/foo/rnn/rnn_cell.py +++ b/python/mxnet/foo/rnn/rnn_cell.py @@ -301,7 +301,7 @@ def _get_activation(self, F, inputs, activation, **kwargs): else: return activation(inputs, **kwargs) - def forward(self, inputs, states): + def call(self, inputs, states): """Unroll the recurrent cell for one time step. Parameters @@ -329,7 +329,7 @@ def forward(self, inputs, states): """ # pylint: disable= arguments-differ self._counter += 1 - return super(RecurrentCell, self).forward(inputs, states) + return super(RecurrentCell, self).call(inputs, states) @@ -370,8 +370,8 @@ def _gate_names(self): def _alias(self): return 'rnn' - def generic_forward(self, F, inputs, states, i2h_weight, i2h_bias, - h2h_weight, h2h_bias): + def forward(self, F, inputs, states, i2h_weight, i2h_bias, + h2h_weight, h2h_bias): name = self._curr_prefix i2h = F.FullyConnected(data=inputs, weight=i2h_weight, bias=i2h_bias, num_hidden=self._num_hidden, @@ -425,8 +425,8 @@ def _gate_names(self): def _alias(self): return 'lstm' - def generic_forward(self, F, inputs, states, i2h_weight, i2h_bias, - h2h_weight, h2h_bias): + def forward(self, F, inputs, states, i2h_weight, i2h_bias, + h2h_weight, h2h_bias): name = self._curr_prefix i2h = F.FullyConnected(data=inputs, weight=i2h_weight, bias=i2h_bias, num_hidden=self._num_hidden*4, @@ -487,8 +487,8 @@ def _gate_names(self): def _alias(self): return 'gru' - def generic_forward(self, F, inputs, states, i2h_weight, i2h_bias, - h2h_weight, h2h_bias): + def forward(self, F, inputs, states, i2h_weight, i2h_bias, + h2h_weight, h2h_bias): # pylint: disable=too-many-locals name = self._curr_prefix prev_state_h = states[0] @@ -780,7 +780,7 @@ def unroll(self, length, inputs, begin_state=None, layout='NTC', merge_outputs=N return inputs, next_states - def generic_forward(self, *args, **kwargs): + def forward(self, *args, **kwargs): raise NotImplementedError @@ -804,7 +804,7 @@ def state_info(self, batch_size=0): def _alias(self): return 'dropout' - def generic_forward(self, F, inputs, states): + def forward(self, F, inputs, states): if self.dropout > 0: inputs = F.Dropout(data=inputs, p=self.dropout) return inputs, states @@ -814,7 +814,7 @@ def unroll(self, length, inputs, begin_state=None, layout='NTC', merge_outputs=N inputs, _, F, _ = _format_sequence(length, inputs, layout, merge_outputs) if isinstance(inputs, tensor_types): - return self.generic_forward(F, inputs, begin_state if begin_state else []) + return self.forward(F, inputs, begin_state if begin_state else []) else: return super(DropoutCell, self).unroll( length, inputs, begin_state=begin_state, layout=layout, @@ -858,7 +858,7 @@ def unpack_weights(self, args): def pack_weights(self, args): return self.base_cell.pack_weights(args) - def generic_forward(self, F, inputs, states): + def forward(self, F, inputs, states): raise NotImplementedError @@ -886,7 +886,7 @@ def reset(self): super(ZoneoutCell, self).reset() self.prev_output = None - def generic_forward(self, F, inputs, states): + def forward(self, F, inputs, states): cell, p_outputs, p_states = self.base_cell, self.zoneout_outputs, self.zoneout_states next_output, next_states = cell(inputs, states) mask = (lambda p, like: F.Dropout(F.ones_like(like), p=p)) @@ -915,7 +915,7 @@ class ResidualCell(ModifierCell): def __init__(self, base_cell): super(ResidualCell, self).__init__(base_cell) - def generic_forward(self, F, inputs, states): + def forward(self, F, inputs, states): output, states = self.base_cell(inputs, states) output = F.elemwise_add(output, inputs, name="%s_plus_residual" % output.name) return output, states diff --git a/python/mxnet/ndarray.py b/python/mxnet/ndarray.py index 001400db95b8..a162f56b2516 100644 --- a/python/mxnet/ndarray.py +++ b/python/mxnet/ndarray.py @@ -62,6 +62,12 @@ 3 : np.uint8, 4 : np.int32 } + +_GRAD_REQ_MAP = { + 'null': 0, + 'write': 1, + 'add': 3 +} # pylint: enable= no-member def _new_empty_handle(): @@ -116,8 +122,9 @@ class NDArray(NDArrayBase): def __repr__(self): """Returns a string representation of the array.""" shape_info = 'x'.join(['%d' % x for x in self.shape]) - return '<%s %s @%s>' % (self.__class__.__name__, - shape_info, self.context) + return '%s\n<%s %s @%s>' % (str(self.asnumpy()), + self.__class__.__name__, + shape_info, self.context) def __add__(self, other): """x.__add__(y) <=> x+y <=> mx.nd.add(x, y) """ @@ -947,6 +954,34 @@ def as_in_context(self, context): return self return self.copyto(context) + def set_grad(self, grad_req='write'): + """Attach a gradient buffer to this NDArray, so that `backward` + can compute gradient with respect to it. + + Parameters + ---------- + grad_req : {'write', 'add', 'null'} + How gradient will be accumulated. + - 'write': gradient will be overwritten on every backward. + - 'add': gradient will be added to existing value on every backward. + - 'null': do not compute gradient for this NDArray. + """ + grad = zeros_like(self) # pylint: disable=undefined-variable + grad_req = _GRAD_REQ_MAP[grad_req] + check_call(_LIB.MXAutogradMarkVariables( + 1, ctypes.pointer(self.handle), + ctypes.pointer(mx_uint(grad_req)), + ctypes.pointer(grad.handle))) + + @property + def grad(self): + """Returns gradient buffer attached to this NDArray.""" + hdl = NDArrayHandle() + check_call(_LIB.MXNDArrayGetGrad(self.handle, ctypes.byref(hdl))) + if hdl.value is None: + return None + return NDArray(hdl) + def detach(self): """Returns a new NDArray, detached from the current graph.""" hdl = NDArrayHandle() diff --git a/python/mxnet/symbol.py b/python/mxnet/symbol.py index 578d79b13c7c..4a9a3f4550c8 100644 --- a/python/mxnet/symbol.py +++ b/python/mxnet/symbol.py @@ -17,7 +17,7 @@ from .base import NDArrayHandle, ExecutorHandle, SymbolHandle, OpHandle from .base import check_call, MXNetError, NotImplementedForSymbol, _Null # pylint: disable=unused-import from .context import Context -from .ndarray import NDArray, _DTYPE_NP_TO_MX, _DTYPE_MX_TO_NP +from .ndarray import NDArray, _DTYPE_NP_TO_MX, _DTYPE_MX_TO_NP, _GRAD_REQ_MAP from .name import NameManager # pylint: disable=unused-import from .executor import Executor from . import _symbol_internal as _internal @@ -42,7 +42,6 @@ from ._ctypes.symbol import SymbolBase, _set_symbol_class from ._ctypes.symbol import _symbol_creator # pylint: disable=unused-import -_GRAD_REQ_MAP = {'null': 0, 'write': 1, 'add': 3} class Symbol(SymbolBase): """Symbol is symbolic graph of the mxnet.""" @@ -1629,7 +1628,7 @@ def bind(self, ctx, args, args_grad=None, grad_req='write', executor.aux_arrays = aux_states return executor - def grad(self, wrt): + def gradient(self, wrt): """Gets the autodiff of current symbol. This function can only be used if current symbol is a loss function. diff --git a/src/c_api/c_api.cc b/src/c_api/c_api.cc index a376b3b6802c..214e6ede5292 100644 --- a/src/c_api/c_api.cc +++ b/src/c_api/c_api.cc @@ -398,6 +398,19 @@ int MXNDArrayGetContext(NDArrayHandle handle, API_END(); } + +int MXNDArrayGetGrad(NDArrayHandle handle, NDArrayHandle *out) { + API_BEGIN(); + NDArray *arr = static_cast(handle); + NDArray ret = arr->grad(); + if (ret.is_none()) { + *out = NULL; + } else { + *out = new NDArray(ret); + } + API_END(); +} + int MXNDArrayDetach(NDArrayHandle handle, NDArrayHandle *out) { API_BEGIN(); NDArray *arr = static_cast(handle); diff --git a/src/ndarray/ndarray.cc b/src/ndarray/ndarray.cc index 9999f9c8307b..ab8e616fb1df 100644 --- a/src/ndarray/ndarray.cc +++ b/src/ndarray/ndarray.cc @@ -24,6 +24,14 @@ DMLC_REGISTRY_ENABLE(::mxnet::NDArrayFunctionReg); namespace mxnet { +NDArray NDArray::grad() const { + if (this->entry_.ag_node && this->entry_.ag_node->out_grads.size()) { + CHECK_EQ(this->entry_.ag_node->out_grads.size(), 1); + return this->entry_.ag_node->out_grads[0]; + } + return NDArray(); +} + NDArray NDArray::Reshape(const TShape &shape) const { using namespace autograd; if (AutogradRuntime::Get()->IsTraining()) { diff --git a/tests/python/unittest/test_autograd.py b/tests/python/unittest/test_autograd.py index 9b2ea4b867f3..eb73a125e819 100644 --- a/tests/python/unittest/test_autograd.py +++ b/tests/python/unittest/test_autograd.py @@ -234,6 +234,17 @@ def test_retain_grad(): "differentiating the same graph twice without retain_graph should fail") +def test_set_grad(): + x = mx.nd.zeros((10,)) + assert x.grad is None + x.set_grad() + with train_section(): + y = x * 2 + assert y.grad is None + y.backward() + assert (x.grad.asnumpy() == 2).all() + + if __name__ == "__main__": import nose nose.runmodule() diff --git a/tests/python/unittest/test_nn.py b/tests/python/unittest/test_nn.py index 42917855df34..bd9eca662fa4 100644 --- a/tests/python/unittest/test_nn.py +++ b/tests/python/unittest/test_nn.py @@ -27,11 +27,11 @@ def test_parameter_sharing(): class Net(nn.Layer): def __init__(self, **kwargs): super(Net, self).__init__(**kwargs) - with self.scope: + with self.name_scope(): self.dense0 = nn.Dense(5, in_units=5) self.dense1 = nn.Dense(5, in_units=5) - def generic_forward(self, F, x): + def forward(self, F, x): return self.dense1(self.dense0(x)) net1 = Net(prefix='net1_') From c892ba8b3a21b616b6e3f2afe1a010b3dda83c1b Mon Sep 17 00:00:00 2001 From: Eric Junyuan Xie Date: Mon, 19 Jun 2017 11:49:05 -0700 Subject: [PATCH 189/834] Fix parameter initialization (#6728) * fix * fix parameters initialization * refactor tutorial * fix * fix --- docs/tutorials/foo/autograd.md | 42 ++++++ docs/tutorials/{basic => foo}/foo.md | 171 ++----------------------- docs/tutorials/foo/ndarray.md | 152 ++++++++++++++++++++++ docs/tutorials/index.md | 14 +- python/mxnet/foo/parameter.py | 65 +++++++--- python/mxnet/foo/trainer.py | 1 + python/mxnet/ndarray.py | 2 +- tests/python/train/test_autograd.py | 90 +++++++++++++ tests/python/unittest/test_autograd.py | 4 +- tests/python/unittest/test_nn.py | 4 +- 10 files changed, 359 insertions(+), 186 deletions(-) create mode 100644 docs/tutorials/foo/autograd.md rename docs/tutorials/{basic => foo}/foo.md (52%) create mode 100644 docs/tutorials/foo/ndarray.md create mode 100644 tests/python/train/test_autograd.py diff --git a/docs/tutorials/foo/autograd.md b/docs/tutorials/foo/autograd.md new file mode 100644 index 000000000000..5d1d61554288 --- /dev/null +++ b/docs/tutorials/foo/autograd.md @@ -0,0 +1,42 @@ +# Automatic differentiation + +MXNet supports automatic differentiation with the `autograd` package. +`autograd` allows you to differentiate a graph of NDArray operations +with the chain rule. +This is called define-by-run, i.e., the network is defined on-the-fly by +running forward computation. You can define exotic network structures +and differentiate them, and each iteration can have a totally different +network structure. + +```python +import mxnet as mx +from mxnet import autograd +``` + +To use `autograd`, we must first mark variables that require gradient and +attach gradient buffers to them: + +```python +x = mx.nd.array([[1, 2], [3, 4]]) +dx = mx.nd.zeros_like(x) +x.attach_grad(dx) +``` + +Now we can define the network while running forward computation by wrapping +it inside a `train_section` (operations out of `train_section` does not define +a graph and cannot be differentiated): + +```python +with autograd.train_section(): + y = x * 2 + z = y * x +``` + +Let's backprop with `z.backward()`, which is equivalent to +`z.backward(mx.nd.ones_like(z))`. When z has more than one entry, `z.backward()` +is equivalent to `mx.nd.sum(z).backward()`: + +```python +z.backward() +print(x.grad) +``` diff --git a/docs/tutorials/basic/foo.md b/docs/tutorials/foo/foo.md similarity index 52% rename from docs/tutorials/basic/foo.md rename to docs/tutorials/foo/foo.md index 84b14278158c..c454e344ac1d 100644 --- a/docs/tutorials/basic/foo.md +++ b/docs/tutorials/foo/foo.md @@ -1,172 +1,20 @@ -# Foo - High-level Interface +# Foo - Neural network building blocks Foo package is a high-level interface for MXNet designed to be easy to use while keeping most of the flexibility of low level API. Foo supports both imperative and symbolic programming, making it easy to train complex models imperatively in Python and then deploy with symbolic graph in C++ and Scala. -This tutorial covers four topics: -- MXNet NDArray as a replacement of numpy for asynchronous scientific computing -across CPU and GPU. -- Automatic differentiation with NDArray. -- Define and train neural network models with Foo's imperative API. -- [TODO] Save trained models as symbolic graph for easy production deployment. - -## Setup -First, let's import MXNet and Foo: ```python +# import dependencies from __future__ import print_function import numpy as np import mxnet as mx +import mxnet.foo as foo +from mxnet.foo import nn ``` -## NDArray - -### Creating NDArray - -NDArray is similar to numpy's ndarray, but supports asynchronous operations -and GPU. There are many ways to create NDArray. - -Construct from (nested) list: -```python -x = mx.nd.array([[1, 2, 3], [4, 5, 6]]) -print(x) -``` - -Construct from numpy array: -```python -x_numpy = np.ones((2, 3)) -x = mx.nd.array(x_numpy) -print(x) -``` - -Array construction routines: -```python -# create an 2x3 array of ones -x = mx.nd.ones((2, 3)) -print(x) -# create an 2x3 array of zeros -x = mx.nd.zeros((2, 3)) -print(x) -# create an 1d-array of 0 to 5 and reshape to 2x3 -x = mx.nd.arange(6).reshape((2, 3)) -print(x) -``` - -You can convert any NDArray to numpy array with `.asnumpy()`: -```python -z = x.asnumpy() -print(z) -``` - -### NDArray Operations - -NDArray supports a wide range of operations. Simple operations can be called -with python syntax: - -```python -x = mx.nd.array([[1, 2], [3, 4]]) -y = mx.nd.array([[4, 3], [2, 1]]) -print(x + y) -``` - -You can also call operators from the `mxnet.ndarray` (or `mx.nd` for short) name space: - -```python -z = mx.nd.add(x, y) -print(z) -``` - -You can also pass additional flags to operators: - -```python -z = mx.nd.sum(x, axis=0) -print('axis=0:', z) -z = mx.nd.sum(x, axis=1) -print('axis=1:', z) -``` - -By default operators create new NDArrays for return value. You can specify `out` -to use a pre-allocated buffer: - -```python -z = mx.nd.empty((2, 2)) -mx.nd.add(x, y, out=z) -print(x) -``` - -### Using GPU - -Each NDArray lives on a `Context`. MXNet supports `mx.cpu()` for CPU and `mx.gpu(0)`, -`mx.gpu(1)`, etc for GPU. You can specify context when creating NDArray: - -```python -# creates on CPU (the default). -# Replace mx.cpu() with mx.gpu(0) if you have a GPU. -x = mx.nd.zeros((2, 2), ctx=mx.cpu()) -print(x) -x = mx.nd.array([[1, 2], [3, 4]], ctx=mx.cpu()) -print(x) -``` - -You can copy arrays between devices with `.copyto()`: - -```python -# Copy x to cpu. Replace with mx.gpu(0) if you have GPU. -y = x.copyto(mx.cpu()) -# Copy x to another NDArray, possibly on another Context. -x.copyto(y) -print(y) -``` - -See the [NDArray tutorial](ndarray.md) for a more detailed introduction to -NDArray API. - -## Automatic Differentiation - -MXNet supports automatic differentiation with the `autograd` package. -`autograd` allows you to differentiate a network of NDArray operations. -This is call define-by-run, i.e., the network is defined on-the-fly by -running forward computation. You can define exotic network structures -and differentiate them, and each iteration can have a totally different -network structure. - -```python -form mxnet import autograd -from mxnet.autograd import train_section -``` - -To use `autograd`, we must first mark variables that require gradient and -attach gradient buffers to them: - -```python -x = mx.nd.array([[1, 2], [3, 4]]) -dx = mx.nd.zeros_like(x) -x.attach_grad(dx) -``` - -Now we can define the network while running forward computation by wrapping -it inside a `train_section` (operations out of `train_section` does not define -a graph and cannot be differentiated): - -```python -with train_section(): - y = x * 2 - z = y * x -``` - -Let's backprop with `z.backward()`, which is equivalent to -`z.backward(mx.nd.ones_like(z))`. When z has more than one entry, `z.backward()` -is equivalent to `mx.nd.sum(z).backward()`: - -```python -z.backward() -print(x.grad) -``` - -## Neural Network and Layers - Neural networks (and other machine learning models) can be defined and trained with `foo.nn` and `foo.rnn` package. A typical training script has the following steps: @@ -180,15 +28,12 @@ steps: - Update parameters with gradient descent. -### Define Network +## Define Network `foo.nn.Layer` is the basic building block of models. You can define networks by composing and inheriting `Layer`: ```python -import mxnet.foo as foo -from mxnet.foo import nn - class Net(nn.Layer): def __init__(self, **kwargs): super(Net, self).__init__(**kwargs) @@ -215,7 +60,7 @@ class Net(nn.Layer): return x ``` -### Initialize Parameters +## Initialize Parameters A network must be created and initialized before it can be used: @@ -241,7 +86,7 @@ output = net(data) Or you can specify input size when creating layers, i.e. `nn.Dense(84, in_units=120)` instead of `nn.Dense(84)`. -### Loss Functions +## Loss Functions Loss functions take (output, label) pairs and compute a scalar loss for each sample in the mini-batch. The scalars measure how far each output is from the label. @@ -261,7 +106,7 @@ print('loss:', loss) print('grad:', net.fc1.weight.grad()) ``` -### Updating the weights +## Updating the weights Now that gradient is computed, we just need to update the weights. This is usually done with formulas like `weight = weight - learning_rate * grad / batch_size`. diff --git a/docs/tutorials/foo/ndarray.md b/docs/tutorials/foo/ndarray.md new file mode 100644 index 000000000000..bc5d9c4f9324 --- /dev/null +++ b/docs/tutorials/foo/ndarray.md @@ -0,0 +1,152 @@ +# NDArray - Scientific computing on CPU and GPU + +NDArray is a tensor data structure similar to numpy's multi-dimensional array. +In addition, it supports asynchronous computation on CPU and GPU. + +First, let's import MXNet: + +```python +from __future__ import print_function +import numpy as np +import mxnet as mx +``` + +## Creating NDArray + +There are many ways to create NDArray. + +Construct from (nested) list: +```python +x = mx.nd.array([[1, 2, 3], [4, 5, 6]]) +print(x) +``` + +Construct from numpy array: +```python +x_numpy = np.ones((2, 3)) +x = mx.nd.array(x_numpy) +print(x) +``` + +Array construction routines: +```python +# create an 2x3 array of ones +x = mx.nd.ones((2, 3)) +print(x) +# create an 2x3 array of zeros +x = mx.nd.zeros((2, 3)) +print(x) +# create an 1d-array of 0 to 5 and reshape to 2x3 +x = mx.nd.arange(6).reshape((2, 3)) +print(x) +``` + +You can convert an NDArray to numpy array to retrieve its data with `.asnumpy()`: +```python +z = x.asnumpy() +print(z) +``` + +## Basic attributes + +NDArray has some basic attributes that you often want to query: + +**NDArray.shape**: The dimensions of the array. It is a tuple of integers +indicating the length of the array along each axis. For a matrix with `n` rows +and `m` columns, its `shape` will be `(n, m)`. + +```python +print('x.shape:', x.shape) +``` + +**NDArray.dtype**: A `numpy` _type_ object describing the type of array +elements. + +```python +print('x.dtype:', x.dtype) +``` + +**NDArray.size**: the total number of components in the array - equals to the +product of the components of its `shape` + +```python +print('x.size:', x.size) +``` + +**NDArray.context**: The device on which this array is stored, e.g. `mx.cpu()` +or `mx.gpu(1)`. + +```python +print('x.context:', x.context) +``` + +## NDArray Operations + +NDArray supports a wide range of operations. Simple operations can be called +with python syntax: + +```python +x = mx.nd.array([[1, 2], [3, 4]]) +y = mx.nd.array([[4, 3], [2, 1]]) +print(x + y) +``` + +You can also call operators from the `mxnet.ndarray` (or `mx.nd` for short) name space: + +```python +z = mx.nd.add(x, y) +print(z) +``` + +You can also pass additional flags to operators: + +```python +z = mx.nd.sum(x, axis=0) +print('axis=0:', z) +z = mx.nd.sum(x, axis=1) +print('axis=1:', z) +``` + +By default operators create new NDArrays for return value. You can specify `out` +to use a pre-allocated buffer: + +```python +z = mx.nd.empty((2, 2)) +mx.nd.add(x, y, out=z) +print(x) +``` + +## Using GPU + +Each NDArray lives on a `Context`. MXNet supports `mx.cpu()` for CPU and `mx.gpu(0)`, +`mx.gpu(1)`, etc for GPU. You can specify context when creating NDArray: + +```python +# creates on CPU (the default). +# Replace mx.cpu() with mx.gpu(0) if you have a GPU. +x = mx.nd.zeros((2, 2), ctx=mx.cpu()) +print(x) +``` + +```python +x = mx.nd.array([[1, 2], [3, 4]], ctx=mx.cpu()) +print(x) +``` + +You can copy arrays between devices with `.copyto()`: + +```python +# Copy x to cpu. Replace with mx.gpu(0) if you have GPU. +y = x.copyto(mx.cpu()) +print(y) +``` + +```python +# Copy x to another NDArray, possibly on another Context. +y = mx.nd.zeros_like(x) +x.copyto(y) +print(y) +``` + +See the [Advanced NDArray tutorial](../basic/ndarray.md) for a more detailed +introduction to NDArray API. diff --git a/docs/tutorials/index.md b/docs/tutorials/index.md index dc56cb145fce..cb8a2ec86528 100644 --- a/docs/tutorials/index.md +++ b/docs/tutorials/index.md @@ -4,13 +4,23 @@ These tutorials introduce a few fundamental concepts in deep learning and how to ## Python -### Basics +### Basics - High-level interface + +```eval_rst +.. toctree:: + :maxdepth: 1 + + foo/ndarray + foo/autograd + foo/foo +``` + +### Advanced -- Low-level interface ```eval_rst .. toctree:: :maxdepth: 1 - basic/foo basic/ndarray basic/symbol basic/module diff --git a/python/mxnet/foo/parameter.py b/python/mxnet/foo/parameter.py index 50c9c614b853..28d28f4b4000 100644 --- a/python/mxnet/foo/parameter.py +++ b/python/mxnet/foo/parameter.py @@ -116,6 +116,36 @@ def initialize(self, init=None, ctx=None, default_init=initializer.Xavier(), self._defered_init = (init, ctx, default_init) self._finish_deferred_init() + def _load_init(self, data, ctx): + """(Re)init by loading from data.""" + if self.shape: + for i, j in zip(self.shape, data.shape): + assert i == 0 or i == j, \ + "Failed loading Parameter %s from saved params: " \ + "shape incompatible expacted %s vs saved %s"%( + self.name, str(self.shape), str(data.shape)) + if self.dtype: + assert np.dtype(self.dtype).type == data.dtype, \ + "Failed loading Parameter %s from saved params: " \ + "dtype incompatible expacted %s vs saved %s"%( + self.name, str(self.dtype), str(data.dtype)) + if isinstance(ctx, Context): + ctx = [ctx] + if self._data is None: + if self._defered_init: + assert set(ctx) == set(self._defered_init[1]), \ + "Failed to load Parameter %s on %s because it was " \ + "previous initialized on %s."%( + self.name, str(ctx), str(self.list_ctx())) + self._init_impl(data, ctx) + else: + assert set(ctx) == set(self.list_ctx()), \ + "Failed to load Parameter %s on %s because it was " \ + "previous initialized on %s."%( + self.name, str(ctx), str(self.list_ctx())) + self.set_data(data) + self._defered_init = () + def _finish_deferred_init(self): """Finish deferred initialization.""" if not self._defered_init: @@ -129,27 +159,30 @@ def _finish_deferred_init(self): self.name, str(self.shape)) with autograd.test_section(): - data = ndarray.zeros(shape=self.shape, dtype=self.dtype, ctx=ctx[0]) + data = ndarray.zeros(shape=self.shape, dtype=self.dtype, + ctx=context.cpu()) if init is None: init = self.init initializer.create(default_init)( - initializer.InitDesc(self.name, {'__init__': init}), - data) + initializer.InitDesc(self.name, {'__init__': init}), data) - self._data = OrderedDict() - self._data[ctx[0]] = data - for i in ctx[1:]: - self._data[i] = data.copyto(i) + self._init_impl(data, ctx) - if self.grad_req == 'null': - self._grad = None - return + def _init_impl(self, data, ctx): + """Set data and grad.""" + self._data = OrderedDict() + for i in ctx: + self._data[i] = data.copyto(i) + + if self.grad_req == 'null': + self._grad = None + return - self._grad = OrderedDict() - for i in ctx: - self._grad[i] = ndarray.zeros_like(self._data[i]) + self._grad = OrderedDict() + for i in ctx: + self._grad[i] = ndarray.zeros_like(self._data[i]) - autograd.mark_variables(self.list_data(), self.list_grad(), self.grad_req) + autograd.mark_variables(self.list_data(), self.list_grad(), self.grad_req) def set_data(self, data): """Set this parameter's value on all contexts to data.""" @@ -365,7 +398,7 @@ def save(self, filename): arg_dict[param.name] = weight ndarray.save(filename, arg_dict) - def load(self, filename, allow_missing=False, ignore_extra=False): + def load(self, filename, ctx, allow_missing=False, ignore_extra=False): arg_dict = ndarray.load(filename) if not allow_missing: for name in self.keys(): @@ -377,4 +410,4 @@ def load(self, filename, allow_missing=False, ignore_extra=False): "Parameter %s loaded from file %s is not present in ParameterDict"%( name, filename) continue - self[name].set_data(arg_dict[name]) + self[name]._load_init(arg_dict[name], ctx) diff --git a/python/mxnet/foo/trainer.py b/python/mxnet/foo/trainer.py index 514dfbd8f36d..8d22983ce25f 100644 --- a/python/mxnet/foo/trainer.py +++ b/python/mxnet/foo/trainer.py @@ -68,6 +68,7 @@ def _init_kvstore(self): kvstore.pull(i, param_arrays, priority=-i) if update_on_kvstore: kvstore.set_optimizer(self._optimizer) + self._kv_initialized = True def step(self, batch_size, ignore_stale_grad=False): """Make one step of parameter update. Should be called after diff --git a/python/mxnet/ndarray.py b/python/mxnet/ndarray.py index a162f56b2516..29f0f769ed63 100644 --- a/python/mxnet/ndarray.py +++ b/python/mxnet/ndarray.py @@ -954,7 +954,7 @@ def as_in_context(self, context): return self return self.copyto(context) - def set_grad(self, grad_req='write'): + def attach_grad(self, grad_req='write'): """Attach a gradient buffer to this NDArray, so that `backward` can compute gradient with respect to it. diff --git a/tests/python/train/test_autograd.py b/tests/python/train/test_autograd.py new file mode 100644 index 000000000000..25cd505d6879 --- /dev/null +++ b/tests/python/train/test_autograd.py @@ -0,0 +1,90 @@ +# pylint: skip-file +from __future__ import print_function + +import mxnet as mx +from mxnet import foo +from mxnet.foo import nn +import numpy as np +import logging +from common import get_data +from mxnet.contrib import autograd as ag +logging.basicConfig(level=logging.DEBUG) + +# define network + +def get_net(): + net = nn.Sequential() + net.add(nn.Dense(128, activation='relu', prefix='fc1_')) + net.add(nn.Dense(64, activation='relu', prefix='fc2_')) + net.add(nn.Dense(10, prefix='fc3_')) + return net + +get_data.GetMNIST_ubyte() + +batch_size = 100 +train_data = mx.io.MNISTIter( + image="data/train-images-idx3-ubyte", + label="data/train-labels-idx1-ubyte", + data_shape=(784,), + label_name='sm_label', + batch_size=batch_size, shuffle=True, flat=True, silent=False, seed=10) +val_data = mx.io.MNISTIter( + image="data/t10k-images-idx3-ubyte", + label="data/t10k-labels-idx1-ubyte", + data_shape=(784,), + label_name='sm_label', + batch_size=batch_size, shuffle=True, flat=True, silent=False) + +def score(net, ctx): + metric = mx.metric.Accuracy() + val_data.reset() + for batch in val_data: + data = foo.utils.load_data(batch.data[0], ctx_list=ctx, batch_axis=0) + label = foo.utils.load_data(batch.label[0], ctx_list=ctx, batch_axis=0) + outputs = [] + for x in data: + outputs.append(net(x)) + metric.update(label, outputs) + return metric.get()[1] + +def train(net, epoch, ctx): + net.all_params().initialize(mx.init.Xavier(magnitude=2.24), ctx=ctx) + trainer = foo.Trainer(net.all_params(), 'sgd', {'learning_rate': 0.5}) + metric = mx.metric.Accuracy() + + for i in range(epoch): + train_data.reset() + for batch in train_data: + data = foo.utils.load_data(batch.data[0], ctx_list=ctx, batch_axis=0) + label = foo.utils.load_data(batch.label[0], ctx_list=ctx, batch_axis=0) + outputs = [] + with ag.train_section(): + for x, y in zip(data, label): + z = net(x) + loss = foo.loss.softmax_cross_entropy_loss(z, y) + ag.compute_gradient([loss]) + outputs.append(z) + metric.update(label, outputs) + trainer.step(batch.data[0].shape[0]) + name, acc = metric.get() + metric.reset() + print('training acc at epoch %d: %s=%f'%(i, name, acc)) + + +def test_autograd(): + net1 = get_net() + train(net1, 5, [mx.cpu(0), mx.cpu(1)]) + acc1 = score(net1, [mx.cpu(0)]) + acc2 = score(net1, [mx.cpu(0), mx.cpu(1)]) + assert acc1 > 0.95 + assert abs(acc1 - acc2) < 0.01 + net1.all_params().save('mnist.params') + + net2 = get_net() + net2.all_params().load('mnist.params', ctx=[mx.cpu(0)]) + acc3 = score(net2, [mx.cpu(0)]) + assert abs(acc3 - acc1) < 0.0001 + + +if __name__ == '__main__': + test_autograd() diff --git a/tests/python/unittest/test_autograd.py b/tests/python/unittest/test_autograd.py index eb73a125e819..abcaef44f94b 100644 --- a/tests/python/unittest/test_autograd.py +++ b/tests/python/unittest/test_autograd.py @@ -234,10 +234,10 @@ def test_retain_grad(): "differentiating the same graph twice without retain_graph should fail") -def test_set_grad(): +def test_attach_grad(): x = mx.nd.zeros((10,)) assert x.grad is None - x.set_grad() + x.attach_grad() with train_section(): y = x * 2 assert y.grad is None diff --git a/tests/python/unittest/test_nn.py b/tests/python/unittest/test_nn.py index bd9eca662fa4..8bb490c7d264 100644 --- a/tests/python/unittest/test_nn.py +++ b/tests/python/unittest/test_nn.py @@ -18,9 +18,9 @@ def test_paramdict(): params = foo.ParameterDict('net_') params.get('weight', shape=(10, 10)) assert list(params.keys()) == ['net_weight'] - params.initialize() + params.initialize(ctx=mx.cpu()) params.save('test.params') - params.load('test.params') + params.load('test.params', mx.cpu()) def test_parameter_sharing(): From 944a725f8e722a41a8ace22864ddb1fa9545cfc3 Mon Sep 17 00:00:00 2001 From: Eric Junyuan Xie Date: Wed, 21 Jun 2017 12:59:47 -0700 Subject: [PATCH 190/834] [RFC]HybridLayer (#6772) * [RFC] add HybridLayer * rename train_section to record --- docs/tutorials/foo/autograd.md | 4 +- docs/tutorials/foo/foo.md | 7 +- example/autograd/actor_critic.py | 5 +- example/autograd/dcgan.py | 82 ++--- example/autograd/mnist.py | 9 +- example/autograd/resnet.py | 66 ++-- example/autograd/super_resolution.py | 5 +- example/autograd/word_language_model/model.py | 3 +- example/autograd/word_language_model/train.py | 2 +- python/mxnet/autograd.py | 10 +- python/mxnet/foo/nn/conv_layers.py | 34 +- python/mxnet/foo/nn/layer.py | 322 ++++++++++++------ python/mxnet/foo/parameter.py | 5 +- python/mxnet/foo/rnn/rnn_cell.py | 64 ++-- python/mxnet/foo/trainer.py | 2 +- python/mxnet/ndarray.py | 2 +- src/c_api/c_api_ndarray.cc | 3 +- src/ndarray/autograd.cc | 2 +- src/ndarray/ndarray.cc | 2 +- tests/python/gpu/test_operator_gpu.py | 8 +- tests/python/train/test_autograd.py | 2 +- tests/python/unittest/test_autograd.py | 20 +- tests/python/unittest/test_foo_rnn.py | 15 +- tests/python/unittest/test_nn.py | 22 +- tests/python/unittest/test_random.py | 2 +- 25 files changed, 429 insertions(+), 269 deletions(-) diff --git a/docs/tutorials/foo/autograd.md b/docs/tutorials/foo/autograd.md index 5d1d61554288..7ffd5aac2ef3 100644 --- a/docs/tutorials/foo/autograd.md +++ b/docs/tutorials/foo/autograd.md @@ -23,11 +23,11 @@ x.attach_grad(dx) ``` Now we can define the network while running forward computation by wrapping -it inside a `train_section` (operations out of `train_section` does not define +it inside a `record` (operations out of `record` does not define a graph and cannot be differentiated): ```python -with autograd.train_section(): +with autograd.record(): y = x * 2 z = y * x ``` diff --git a/docs/tutorials/foo/foo.md b/docs/tutorials/foo/foo.md index c454e344ac1d..17162bbbb550 100644 --- a/docs/tutorials/foo/foo.md +++ b/docs/tutorials/foo/foo.md @@ -11,6 +11,7 @@ in Python and then deploy with symbolic graph in C++ and Scala. from __future__ import print_function import numpy as np import mxnet as mx +import mxnet.ndarray as F import mxnet.foo as foo from mxnet.foo import nn ``` @@ -48,7 +49,7 @@ class Net(nn.Layer): self.fc2 = nn.Dense(84) self.fc3 = nn.Dense(10) - def forward(self, F, x): + def forward(self, x): x = self.pool1(F.relu(self.conv1(x))) x = self.pool2(F.relu(self.conv2(x))) # 0 means copy over size from corresponding dimension. @@ -98,7 +99,7 @@ To compute loss and backprop for one iteration, we do: ```python label = mx.nd.arange(10) # dummy label -with train_section(): +with record(): output = net(data) loss = foo.loss.softmax_cross_entropy_loss(output, label) loss.backward() @@ -125,7 +126,7 @@ this is a commonly used functionality, foo provide a `Trainer` class for it: ```python trainer = foo.Trainer(net.all_params(), 'sgd', {'learning_rate': 0.01}) -with train_section(): +with record(): output = net(data) loss = foo.loss.softmax_cross_entropy_loss(output, label) loss.backward() diff --git a/example/autograd/actor_critic.py b/example/autograd/actor_critic.py index 7a716b23fc4d..d579971a89ff 100644 --- a/example/autograd/actor_critic.py +++ b/example/autograd/actor_critic.py @@ -6,6 +6,7 @@ import numpy as np import mxnet as mx +import mxnet.ndarray as F from mxnet import foo from mxnet.foo import nn from mxnet.contrib import autograd @@ -35,7 +36,7 @@ def __init__(self, **kwargs): self.action_pred = nn.Dense(2, in_units=16) self.value_pred = nn.Dense(1, in_units=16) - def forward(self, F, x): + def forward(self, x): x = self.dense(x) probs = self.action_pred(x) values = self.value_pred(x) @@ -53,7 +54,7 @@ def forward(self, F, x): values = [] heads = [] actions = [] - with autograd.train_section(): + with autograd.record(): # Sample a sequence of actions for t in range(10000): state = mx.nd.array(np.expand_dims(state, 0)) diff --git a/example/autograd/dcgan.py b/example/autograd/dcgan.py index c86263bd6235..06903386ebd5 100644 --- a/example/autograd/dcgan.py +++ b/example/autograd/dcgan.py @@ -36,47 +36,49 @@ netG = nn.Sequential() -# input is Z, going into a convolution -netG.add(nn.Conv2DTranspose(ngf * 8, 4, 1, 0, use_bias=False)) -netG.add(nn.BatchNorm()) -netG.add(nn.Activation('relu')) -# state size. (ngf*8) x 4 x 4 -netG.add(nn.Conv2DTranspose(ngf * 4, 4, 2, 1, use_bias=False)) -netG.add(nn.BatchNorm()) -netG.add(nn.Activation('relu')) -# state size. (ngf*8) x 8 x 8 -netG.add(nn.Conv2DTranspose(ngf * 2, 4, 2, 1, use_bias=False)) -netG.add(nn.BatchNorm()) -netG.add(nn.Activation('relu')) -# state size. (ngf*8) x 16 x 16 -netG.add(nn.Conv2DTranspose(ngf, 4, 2, 1, use_bias=False)) -netG.add(nn.BatchNorm()) -netG.add(nn.Activation('relu')) -# state size. (ngf*8) x 32 x 32 -netG.add(nn.Conv2DTranspose(nc, 4, 2, 1, use_bias=False)) -netG.add(nn.Activation('tanh')) -# state size. (nc) x 64 x 64 +with netG.name_scope(): + # input is Z, going into a convolution + netG.add(nn.Conv2DTranspose(ngf * 8, 4, 1, 0, use_bias=False)) + netG.add(nn.BatchNorm()) + netG.add(nn.Activation('relu')) + # state size. (ngf*8) x 4 x 4 + netG.add(nn.Conv2DTranspose(ngf * 4, 4, 2, 1, use_bias=False)) + netG.add(nn.BatchNorm()) + netG.add(nn.Activation('relu')) + # state size. (ngf*8) x 8 x 8 + netG.add(nn.Conv2DTranspose(ngf * 2, 4, 2, 1, use_bias=False)) + netG.add(nn.BatchNorm()) + netG.add(nn.Activation('relu')) + # state size. (ngf*8) x 16 x 16 + netG.add(nn.Conv2DTranspose(ngf, 4, 2, 1, use_bias=False)) + netG.add(nn.BatchNorm()) + netG.add(nn.Activation('relu')) + # state size. (ngf*8) x 32 x 32 + netG.add(nn.Conv2DTranspose(nc, 4, 2, 1, use_bias=False)) + netG.add(nn.Activation('tanh')) + # state size. (nc) x 64 x 64 netD = nn.Sequential() -# input is (nc) x 64 x 64 -netD.add(nn.Conv2D(ndf, 4, 2, 1, use_bias=False)) -netD.add(nn.LeakyReLU(0.2)) -# state size. (ndf) x 32 x 32 -netD.add(nn.Conv2D(ndf * 2, 4, 2, 1, use_bias=False)) -netD.add(nn.BatchNorm()) -netD.add(nn.LeakyReLU(0.2)) -# state size. (ndf) x 16 x 16 -netD.add(nn.Conv2D(ndf * 4, 4, 2, 1, use_bias=False)) -netD.add(nn.BatchNorm()) -netD.add(nn.LeakyReLU(0.2)) -# state size. (ndf) x 8 x 8 -netD.add(nn.Conv2D(ndf * 8, 4, 2, 1, use_bias=False)) -netD.add(nn.BatchNorm()) -netD.add(nn.LeakyReLU(0.2)) -# state size. (ndf) x 4 x 4 -netD.add(nn.Conv2D(2, 4, 1, 0, use_bias=False)) -# netD.add(nn.Activation('sigmoid')) +with netD.name_scope(): + # input is (nc) x 64 x 64 + netD.add(nn.Conv2D(ndf, 4, 2, 1, use_bias=False)) + netD.add(nn.LeakyReLU(0.2)) + # state size. (ndf) x 32 x 32 + netD.add(nn.Conv2D(ndf * 2, 4, 2, 1, use_bias=False)) + netD.add(nn.BatchNorm()) + netD.add(nn.LeakyReLU(0.2)) + # state size. (ndf) x 16 x 16 + netD.add(nn.Conv2D(ndf * 4, 4, 2, 1, use_bias=False)) + netD.add(nn.BatchNorm()) + netD.add(nn.LeakyReLU(0.2)) + # state size. (ndf) x 8 x 8 + netD.add(nn.Conv2D(ndf * 8, 4, 2, 1, use_bias=False)) + netD.add(nn.BatchNorm()) + netD.add(nn.LeakyReLU(0.2)) + # state size. (ndf) x 4 x 4 + netD.add(nn.Conv2D(2, 4, 1, 0, use_bias=False)) + # netD.add(nn.Activation('sigmoid')) netG.all_params().initialize(mx.init.Normal(0.02), ctx=ctx) @@ -99,7 +101,7 @@ data = batch.data[0].copyto(ctx) noise = mx.nd.random_normal(0, 1, shape=(opt.batchSize, nz, 1, 1), ctx=ctx) - with autograd.train_section(): + with autograd.record(): output = netD(data) output = output.reshape((opt.batchSize, 2)) errD_real = foo.loss.softmax_cross_entropy_loss(output, real_label) @@ -116,7 +118,7 @@ ############################ # (2) Update G network: maximize log(D(G(z))) ########################### - with autograd.train_section(): + with autograd.record(): output = netD(fake) output = output.reshape((opt.batchSize, 2)) errG = foo.loss.softmax_cross_entropy_loss(output, real_label) diff --git a/example/autograd/mnist.py b/example/autograd/mnist.py index 84dd011e03c3..cff6c2d7d9b6 100644 --- a/example/autograd/mnist.py +++ b/example/autograd/mnist.py @@ -13,9 +13,10 @@ # define network net = nn.Sequential() -net.add(nn.Dense(128, in_units=784, activation='relu')) -net.add(nn.Dense(64, in_units=128, activation='relu')) -net.add(nn.Dense(10, in_units=64)) +with net.name_scope(): + net.add(nn.Dense(128, activation='relu')) + net.add(nn.Dense(64, activation='relu')) + net.add(nn.Dense(10)) # data @@ -48,7 +49,7 @@ def train(epoch, ctx): data = foo.utils.load_data(batch.data[0], ctx_list=ctx, batch_axis=0) label = foo.utils.load_data(batch.label[0], ctx_list=ctx, batch_axis=0) outputs = [] - with ag.train_section(): + with ag.record(): for x, y in zip(data, label): z = net(x) loss = foo.loss.softmax_cross_entropy_loss(z, y) diff --git a/example/autograd/resnet.py b/example/autograd/resnet.py index 5715eeaf9403..879b3237eacf 100644 --- a/example/autograd/resnet.py +++ b/example/autograd/resnet.py @@ -11,7 +11,7 @@ def conv3x3(filters, stride, in_filters): return nn.Conv2D(filters, kernel_size=3, strides=stride, padding=1, use_bias=False, in_filters=in_filters) -class BasicBlockV1(nn.Layer): +class BasicBlockV1(nn.HybridLayer): def __init__(self, filters, stride, downsample=False, in_filters=0, **kwargs): super(BasicBlockV1, self).__init__(**kwargs) with self.name_scope(): @@ -24,12 +24,12 @@ def __init__(self, filters, stride, downsample=False, in_filters=0, **kwargs): self.bn_ds = nn.BatchNorm(num_features=filters) self.downsample = downsample - def forward(self, domain, x): + def hybrid_forward(self, F, x): residual = x out = self.conv1(x) out = self.bn1(out) - out = domain.Activation(x, act_type='relu') + out = F.Activation(x, act_type='relu') out = self.conv2(out) out = self.bn2(out) @@ -39,12 +39,12 @@ def forward(self, domain, x): residual = self.bn_ds(residual) out = residual + out - out = domain.Activation(out, act_type='relu') + out = F.Activation(out, act_type='relu') return out -class BottleneckV1(nn.Layer): +class BottleneckV1(nn.HybridLayer): def __init__(self, filters, stride, downsample=False, in_filters=0, **kwargs): super(BottleneckV1, self).__init__(**kwargs) with self.name_scope(): @@ -59,16 +59,16 @@ def __init__(self, filters, stride, downsample=False, in_filters=0, **kwargs): self.bn_ds = nn.BatchNorm(num_features=filters) self.downsample = downsample - def forward(self, domain, x): + def hybrid_forward(self, F, x): residual = x out = self.conv1(x) out = self.bn1(out) - out = domain.Activation(out, act_type='relu') + out = F.Activation(out, act_type='relu') out = self.conv2(out) out = self.bn2(out) - out = domain.Activation(out, act_type='relu') + out = F.Activation(out, act_type='relu') out = self.conv3(out) out = self.bn3(out) @@ -79,11 +79,11 @@ def forward(self, domain, x): out = out + residual - out = domain.Activation(out, act_type='relu') + out = F.Activation(out, act_type='relu') return out -class ResnetV1(nn.Layer): +class ResnetV1(nn.HybridLayer): def __init__(self, block, classes, layers, filters, thumbnail=False, **kwargs): super(ResnetV1, self).__init__(**kwargs) with self.name_scope(): @@ -97,7 +97,7 @@ def __init__(self, block, classes, layers, filters, thumbnail=False, **kwargs): self.bn0 = nn.BatchNorm(num_features=filters[0]) self.pool0 = nn.MaxPool2D(3, 2, 1) - self.body = nn.Sequential() + self.body = nn.HSequential() in_filters = filters[0] for i in range(len(layers)): stride = 1 if i == 0 else 2 @@ -109,17 +109,17 @@ def __init__(self, block, classes, layers, filters, thumbnail=False, **kwargs): self.dense1 = nn.Dense(classes, in_units=filters[-1]) def _make_layer(self, block, layers, filters, stride, in_filters=0): - layer = nn.Sequential() + layer = nn.HSequential() layer.add(block(filters, stride, True, in_filters=in_filters)) for i in range(layers-1): layer.add(block(filters, 1, False, in_filters=filters)) return layer - def forward(self, domain, x): + def hybrid_forward(self, F, x): x = self.conv0(x) if not self._thumbnail: x = self.bn0(x) - x = domain.Activation(x, act_type='relu') + x = F.Activation(x, act_type='relu') x = self.pool0(x) x = self.body(x) @@ -131,7 +131,7 @@ def forward(self, domain, x): return x -class BasicBlockV2(nn.Layer): +class BasicBlockV2(nn.HybridLayer): def __init__(self, filters, stride, downsample=False, in_filters=0, **kwargs): super(BasicBlockV2, self).__init__(**kwargs) with self.name_scope(): @@ -145,23 +145,23 @@ def __init__(self, filters, stride, downsample=False, in_filters=0, **kwargs): else: self.downsample = None - def forward(self, domain, x): + def hybrid_forward(self, F, x): if not self.downsample: residual = x x = self.bn1(x) - x = domain.Activation(x, act_type='relu') + x = F.Activation(x, act_type='relu') if self.downsample: residual = self.downsample(x) x = self.conv1(x) x = self.bn2(x) - x = domain.Activation(x, act_type='relu') + x = F.Activation(x, act_type='relu') x = self.conv2(x) return x + residual -class BottleneckV2(nn.Layer): +class BottleneckV2(nn.HybridLayer): def __init__(self, filters, stride, downsample=False, in_filters=0, **kwargs): super(BottleneckV2, self).__init__(**kwargs) with self.name_scope(): @@ -177,26 +177,26 @@ def __init__(self, filters, stride, downsample=False, in_filters=0, **kwargs): else: self.downsample = None - def forward(self, domain, x): + def hybrid_forward(self, F, x): if not self.downsample: residual = x x = self.bn1(x) - x = domain.Activation(x, act_type='relu') + x = F.Activation(x, act_type='relu') if self.downsample: residual = self.downsample(x) x = self.conv1(x) x = self.bn2(x) - x = domain.Activation(x, act_type='relu') + x = F.Activation(x, act_type='relu') x = self.conv2(x) x = self.bn3(x) - x = domain.Activation(x, act_type='relu') + x = F.Activation(x, act_type='relu') x = self.conv3(x) return x + residual -class ResnetV2(nn.Layer): +class ResnetV2(nn.HybridLayer): def __init__(self, block, classes, layers, filters, thumbnail=False, **kwargs): super(ResnetV2, self).__init__(**kwargs) with self.name_scope(): @@ -211,7 +211,7 @@ def __init__(self, block, classes, layers, filters, thumbnail=False, **kwargs): self.bn0 = nn.BatchNorm(num_features=filters[0]) self.pool0 = nn.MaxPool2D(3, 2, 1) - self.body = nn.Sequential() + self.body = nn.HSequential() in_filters = filters[0] for i in range(len(layers)): stride = 1 if i == 0 else 2 @@ -224,24 +224,24 @@ def __init__(self, block, classes, layers, filters, thumbnail=False, **kwargs): self.dense1 = nn.Dense(classes, in_units=in_filters) def _make_layer(self, block, layers, filters, stride, in_filters=0): - layer = nn.Sequential() + layer = nn.HSequential() layer.add(block(filters, stride, True, in_filters=in_filters)) for i in range(layers-1): layer.add(block(filters, 1, False, in_filters=filters)) return layer - def forward(self, domain, x): + def hybrid_forward(self, F, x): x = self.bn_data(x) x = self.conv0(x) if not self._thumbnail: x = self.bn0(x) - x = domain.Activation(x, act_type='relu') + x = F.Activation(x, act_type='relu') x = self.pool0(x) x = self.body(x) x = self.bn1(x) - x = domain.Activation(x, act_type='relu') + x = F.Activation(x, act_type='relu') x = self.pool1(x) x = x.reshape((0, -1)) x = self.dense1(x) @@ -288,12 +288,15 @@ def train(epoch, ctx): data = foo.utils.load_data(batch.data[0], ctx_list=ctx, batch_axis=0) label = foo.utils.load_data(batch.label[0], ctx_list=ctx, batch_axis=0) outputs = [] - with ag.train_section(): + losses = [] + with ag.record(): for x, y in zip(data, label): z = net(x) loss = foo.loss.softmax_cross_entropy_loss(z, y) - ag.compute_gradient([loss]) + losses.append(loss) outputs.append(z) + for loss in losses: + loss.backward() trainer.step(batch.data[0].shape[0]) metric.update(label, outputs) print('speed: {} samples/s'.format(batch.data[0].shape[0]/(time.time()-btic))) @@ -308,6 +311,7 @@ def train(epoch, ctx): net.all_params().save('mnist.params') if __name__ == '__main__': + net.hybridize() train(200, [mx.gpu(i) for i in range(2)]) import logging logging.basicConfig(level=logging.DEBUG) diff --git a/example/autograd/super_resolution.py b/example/autograd/super_resolution.py index 3c66d7b09dcd..e564c9c82eaa 100644 --- a/example/autograd/super_resolution.py +++ b/example/autograd/super_resolution.py @@ -4,6 +4,7 @@ import os import mxnet as mx +import mxnet.ndarray as F from mxnet import foo from mxnet.foo import nn from mxnet.contrib import autograd as ag @@ -97,7 +98,7 @@ def __init__(self, upscale_factor): self.conv4 = nn.Conv2D(upscale_factor ** 2, (3, 3), strides=(1, 1), padding=(1, 1), in_filters=32) self.upscale_factor = upscale_factor - def forward(self, F, x): + def forward(self, x): x = F.Activation(self.conv1(x), act_type='relu') x = F.Activation(self.conv2(x), act_type='relu') x = F.Activation(self.conv3(x), act_type='relu') @@ -138,7 +139,7 @@ def train(epoch, ctx): data = foo.utils.load_data(batch.data[0], ctx_list=ctx, batch_axis=0) label = foo.utils.load_data(batch.label[0], ctx_list=ctx, batch_axis=0) outputs = [] - with ag.train_section(): + with ag.record(): for x, y in zip(data, label): z = net(x) loss = foo.loss.l2_loss(z, y) diff --git a/example/autograd/word_language_model/model.py b/example/autograd/word_language_model/model.py index 97622566c0d3..549e485aa7e3 100644 --- a/example/autograd/word_language_model/model.py +++ b/example/autograd/word_language_model/model.py @@ -1,4 +1,5 @@ import mxnet as mx +import mxnet.ndarray as F from mxnet import foo from mxnet.foo import nn, rnn @@ -20,7 +21,7 @@ def __init__(self, mode, vocab_size, num_embed, num_hidden, self.num_hidden = num_hidden - def forward(self, F, inputs, hidden): + def forward(self, inputs, hidden): emb = self.drop(self.encoder(inputs)) output, hidden = self.rnn.unroll(None, emb, layout='TNC', merge_outputs=True) output = self.drop(output) diff --git a/example/autograd/word_language_model/train.py b/example/autograd/word_language_model/train.py index 20dcfed62606..93a82a380980 100644 --- a/example/autograd/word_language_model/train.py +++ b/example/autograd/word_language_model/train.py @@ -123,7 +123,7 @@ def train(): for ibatch, i in enumerate(range(0, train_data.shape[0] - 1, args.bptt)): data, target = get_batch(train_data, i) hidden = detach(hidden) - with autograd.train_section(): + with autograd.record(): output, hidden = model(data, hidden) loss = foo.loss.softmax_cross_entropy_loss(output, target) loss.backward() diff --git a/python/mxnet/autograd.py b/python/mxnet/autograd.py index f8e3259211b6..e45c956e2bb7 100644 --- a/python/mxnet/autograd.py +++ b/python/mxnet/autograd.py @@ -50,12 +50,12 @@ def __exit__(self, ptype, value, trace): set_is_training(self._prev) -def train_section(): +def record(): """Returns a training scope context to be used in 'with' statement and captures training code. Example:: - with autograd.train_section(): + with autograd.record(): y = model(x) compute_gradient([y]) metric.update(...) @@ -64,15 +64,15 @@ def train_section(): return TrainingStateScope(True) -def test_section(): +def pause(): """Returns a testing scope context to be used in 'with' statement and captures testing code. Example:: - with autograd.train_section(): + with autograd.record(): y = model(x) compute_gradient([y]) - with autograd.test_section(): + with autograd.pause(): # testing, IO, gradient updates... """ return TrainingStateScope(False) diff --git a/python/mxnet/foo/nn/conv_layers.py b/python/mxnet/foo/nn/conv_layers.py index d26bebe97c57..de892593f8c8 100644 --- a/python/mxnet/foo/nn/conv_layers.py +++ b/python/mxnet/foo/nn/conv_layers.py @@ -1,16 +1,17 @@ # coding: utf-8 # pylint: disable= arguments-differ """Convolutional neural network layers.""" -from .layer import Layer +from .layer import HybridLayer from ... import symbol from ...base import numeric_types -def _infer_weight_shape(op, data_shape): - sym = symbol.invoke(op, [symbol.var('data', shape=data_shape)]) +def _infer_weight_shape(op_name, data_shape, kwargs): + op = getattr(symbol, op_name) + sym = op(symbol.var('data', shape=data_shape), **kwargs) return sym.infer_shape_partial()[0] -class _Conv(Layer): +class _Conv(HybridLayer): """Abstract nD convolution layer (private, used as implementation base). This layer creates a convolution kernel that is convolved @@ -71,33 +72,35 @@ def __init__(self, filters, kernel_size, strides, padding, dilation, padding = (padding,)*len(kernel_size) if isinstance(dilation, numeric_types): dilation = (dilation,)*len(kernel_size) - attrs = { + self._op_name = op_name + self._kwargs = { 'kernel': kernel_size, 'stride': strides, 'dilate': dilation, 'pad': padding, 'num_filter': filters, 'num_group': groups, 'no_bias': not use_bias, 'layout': layout} - attrs.update(kwargs) - self._op = symbol.CachedOp(op_name, 3 if use_bias else 2, **attrs) + self._kwargs.update(kwargs) dshape = [0]*(len(kernel_size) + 2) dshape[layout.find('N')] = 1 dshape[layout.find('C')] = in_filters - wshapes = _infer_weight_shape(self._op, dshape) + wshapes = _infer_weight_shape(op_name, dshape, self._kwargs) self.weight = self.params.get('weight', shape=wshapes[1], init=kernel_initializer) if use_bias: self.bias = self.params.get('bias', shape=wshapes[2], init=bias_initializer) + else: + self.bias = None if activation is not None: self.act = Activation(activation) else: self.act = None - def forward(self, F, x, weight, bias=None): + def hybrid_forward(self, F, x, weight, bias=None): if bias is None: - act = F.invoke(self._op, [x, weight]) + act = getattr(F, self._op_name)(x, weight, **self._kwargs) else: - act = F.invoke(self._op, [x, weight, bias]) + act = getattr(F, self._op_name)(x, weight, bias, **self._kwargs) if self.act is not None: act = self.act(act) return act @@ -503,7 +506,7 @@ def __init__(self, filters, kernel_size, strides=(1, 1, 1), padding=(0, 0, 0), op_name='Deconvolution', adj=output_padding, **kwargs) -class _Pooling(Layer): +class _Pooling(HybridLayer): """Abstract class for different pooling layers. """ def __init__(self, pool_size, strides, padding, global_pool, pool_type, **kwargs): @@ -514,14 +517,13 @@ def __init__(self, pool_size, strides, padding, global_pool, pool_type, **kwargs strides = (strides,)*len(pool_size) if isinstance(padding, numeric_types): padding = (padding,)*len(pool_size) - attrs = { + self._kwargs = { 'kernel': pool_size, 'stride': strides, 'pad': padding, 'pooling_convention': 'full', 'global_pool': global_pool, 'pool_type': pool_type} - self._op = symbol.CachedOp('Pooling', 1, **attrs) - def forward(self, F, x): - return F.invoke(self._op, [x]) + def hybrid_forward(self, F, x): + return F.Pooling(x, **self._kwargs) class MaxPool1D(_Pooling): diff --git a/python/mxnet/foo/nn/layer.py b/python/mxnet/foo/nn/layer.py index 172b3cb0c5cc..25ae6d23542a 100644 --- a/python/mxnet/foo/nn/layer.py +++ b/python/mxnet/foo/nn/layer.py @@ -48,6 +48,42 @@ def __exit__(self, ptype, value, trace): _LayerScope._current = self._old_scope +def _flatten(args): + if isinstance(args, NDArray): + return [args], int(0) + if isinstance(args, Symbol): + length = len(args.list_outputs()) + length = length if length > 1 else 0 + return [args], int(length) + + assert isinstance(args, (list, tuple)), \ + "HybridLayer input must be (nested) list of Symbol or NDArray, " \ + "but got %s of type %s"%(str(args), str(type(args))) + flat = [] + fmts = [] + for i in args: + arg, fmt = _flatten(i) + flat.extend(arg) + fmts.append(fmt) + return flat, fmts + + +def _regroup(args, fmt): + if isinstance(fmt, int): + if fmt == 0: + return args[0], args[1:] + return args[:fmt], args[fmt:] + + assert isinstance(args, (list, tuple)), \ + "HybridLayer output must be (nested) list of Symbol or NDArray, " \ + "but got %s of type %s"%(str(args), str(type(args))) + ret = [] + for i in fmt: + res, args = _regroup(args, i) + ret.append(res) + return ret, args + + class Layer(object): """Base class for all neural network layers and models. @@ -60,10 +96,10 @@ class Net(nn.Layer): def __init__(self, **kwargs): super(Net, self).__init__(**kwargs) with self.name_scope(): - self.dense0 = nn.Dense(20, in_units=10) - self.dense1 = nn.Dense(20, in_units=20) + self.dense0 = nn.Dense(20) + self.dense1 = nn.Dense(20) - def forward(self, F, x): + def forward(self, x): x = self.dense0(x) return self.dense1(x) @@ -89,14 +125,10 @@ def __init__(self, prefix=None, params=None): self._params = _LayerScope.create_params(self._prefix, params) self._scope = _LayerScope(self) self._children = [] - self._reg_params = {} - def __setattr__(self, name, value): """Registers parameters.""" super(Layer, self).__setattr__(name, value) - if isinstance(value, Parameter): - self._reg_params[name] = value if isinstance(value, Layer): self.register_child(value) @@ -140,62 +172,157 @@ def register_child(self, layer): self as attributes will be registered automatically.""" self._children.append(layer) - def infer_shape(self, *args): - """Infer parameter shape given input shapes. + def hybridize(self, active=True): + """Activate HybridLayers recursively. Has no effect on + non-hybrid children.""" + for cld in self._children: + cld.hybridize(active) + + def __call__(self, *args): + """Call forward.""" + return self.forward(*args) - *args : list of tuple - A list of input argument shapes. + def forward(self, *args): + """Override to implement forward computation using NDArray. + + Parameters + ---------- + *args : list of NDArray + Input tensors. """ - inputs = [symbol.var('__input%d__'%i, shape=shape) - for i, shape in enumerate(args)] - params = {k: v.var() for k, v in self._reg_params.items()} - sym = self.forward(symbol, *inputs, **params) - arg_shapes, _, aux_shapes = sym.infer_shape() - sdict = {name: shape for name, shape in zip(sym.list_arguments(), arg_shapes)} + # pylint: disable= invalid-name + raise NotImplementedError + + +class HybridLayer(Layer): + """HybridLayer supports forwarding with both Symbol and NDArray. + + Forward computation in HybridLayer must be static to work with Symbols, + i.e. you cannot call `.asnumpy()`, `.shape`, `.dtype`, etc on inputs. + When forwarding after `hybridize()` is called, HybridLayer will + create a graph representing the forward computation and cache it. + On subsequent forward the cached graph will be used instead of calling + `hybrid_forward`. + """ + def __init__(self, prefix=None, params=None): + super(HybridLayer, self).__init__(prefix=prefix, params=params) + self._reg_params = {} + self._cached_graph = () + self._cached_op = None + self._cached_params = None + self._out_format = None + self._in_format = None + self._active = False + + def __setattr__(self, name, value): + """Registers parameters.""" + super(HybridLayer, self).__setattr__(name, value) + if isinstance(value, Parameter): + assert name not in self._reg_params or \ + not isinstance(self._reg_params[name], Parameter), \ + "Overriding Parameter attribute %s is not allowed. " \ + "Please pass in Parameters by specifying `params` at " \ + "Layer construction instead." + self._reg_params[name] = value + + def register_child(self, layer): + if not isinstance(layer, HybridLayer): + if isinstance(layer, Sequantial): + raise ValueError( + "Children of HybridLayer must also be HybridLayer. " \ + "Please use HSequential instead of Sequantial.") + raise ValueError( + "Children of HybridLayer must also be HybridLayer, " \ + "but %s has type %s."%(str(layer), str(type(layer)))) + super(HybridLayer, self).register_child(layer) + + def hybridize(self, active=True): + super(HybridLayer, self).hybridize(active) + self._active = True + + def _get_graph(self, *args): + if self._cached_graph: + return self._cached_graph + + args, self._in_format = _flatten(args) + syms = [symbol.var(str(i)) for i in range(len(args))] + sym_args = _regroup(syms, self._in_format)[0] + + params = {i: j.var() for i, j in self._reg_params.items()} + out = self.hybrid_forward(symbol, *sym_args, **params) # pylint: disable=no-value-for-parameter + out, self._out_format = _flatten(out) + + self._cached_graph = syms, symbol.Group(out) + return self._cached_graph + + def infer_shape(self, *args): + """Infer shape of Parameters from inputs.""" + syms, out = self._get_graph(*args) + arg_shapes, _, aux_shapes = out.infer_shape( + **{i.name: j.shape for i, j in zip(syms, args)}) + sdict = {i: j for i, j in zip(out.list_arguments(), arg_shapes)} sdict.update( - {name : shape for name, shape in zip(sym.list_auxiliary_states(), aux_shapes)}) - for i in self.params.values(): + {name : shape for name, shape in zip(out.list_auxiliary_states(), aux_shapes)}) + for i in self.all_params().values(): i.shape = sdict[i.name] - def __call__(self, *args): - """Call forward.""" - return self.call(*args) # pylint: disable=no-value-for-parameter - - def call(self, x, *args): - """Defines the forward computation. Arguments can be either NDArray or Symbol.""" + def _build_cache(self, *args): + self.infer_shape(*args) + for i in self.all_params().values(): + i._finish_deferred_init() + + _, out = self._get_graph(*args) + self._cached_op = ndarray.CachedOp(out) + params = dict(self.all_params().items()) + self._cached_params = [params.get(name, None) for name in out.list_inputs()] + self._in_idx = [(i, int(name)) for i, name in enumerate(out.list_inputs()) + if name not in params] + + def _call_cached_op(self, *args): + args, fmt = _flatten(args) + assert fmt == self._in_format, "Invalid input format" + cargs = [i.data() if i else None for i in self._cached_params] + for i, j in self._in_idx: + cargs[i] = args[j] + out = self._cached_op(*cargs) + if isinstance(out, NDArray): + out = [out] + return _regroup(out, self._out_format)[0] + + def forward(self, x, *args): + """Defines the forward computation. Arguments can be either + NDArray or Symbol.""" if isinstance(x, NDArray): + if self._active and self._cached_op is None: + self._build_cache(x, *args) + with x.context as ctx: + if self._active: + return self._call_cached_op(x, *args) try: - params = {k: v.data(ctx) for k, v in self._reg_params.items()} + params = {i: j.data(ctx) for i, j in self._reg_params.items()} except DeferredInitializationError: - arg_shapes = [x.shape] - arg_shapes += [i.shape if isinstance(i, NDArray) else i for i in args] - self.infer_shape(*arg_shapes) - for i in self.params.values(): + self.infer_shape(x, *args) + for i in self.all_params().values(): i._finish_deferred_init() - params = {k: v.data(ctx) for k, v in self._reg_params.items()} - return self.forward(ndarray, x, *args, **params) + params = {i: j.data(ctx) for i, j in self._reg_params.items()} + return self.hybrid_forward(ndarray, x, *args, **params) else: assert isinstance(x, Symbol), \ "Layer requires the first argument to forward be either " \ "Symbol or NDArray, but got %s"%type(x) - params = {k: v.var() for k, v in self._reg_params.items()} - return self.forward(symbol, x, *args, **params) + params = {i: j.var() for i, j in self._reg_params.items()} + return self.hybrid_forward(symbol, x, *args, **params) - def forward(self, F, x, *args, **kwargs): - """Simple forward supports both `Symbol` and `NDArray` API. + def hybrid_forward(self, F, x, *args, **kwargs): + """Override to construct symbolic graph for this Layer. Parameters ---------- - F : {mxnet.ndarray, mxnet.symbol} - Name space of operators. `F` will be set to `mx.sym` when x is `Symbol` - instance and `mx.nd` when x is `NDArray` instance. - x : NDArray or Symbol - The first input tensor. - *args : list of NDArray or list of Symbol - Additional input tensors. - **kwargs : dict of str to NDArray or dict of str to Symbol - `Symbol` or `NDArray` value of registered Parameters. + x : Symbol + The first input Symbol. + *args : list of Symbol + Additional input Symbols. """ # pylint: disable= invalid-name raise NotImplementedError @@ -206,27 +333,46 @@ class Sequential(Layer): Example:: net = nn.Sequential() - net.add(Dense(10, activation='relu')) - net.add(Dense(20)) + with net.name_scope(): + net.add(Dense(10, activation='relu')) + net.add(Dense(20)) """ - def __init__(self): - super(Sequential, self).__init__(prefix='', params=None) + def __init__(self, prefix=None, params=None): + super(Sequential, self).__init__(prefix=prefix, params=params) def add(self, layer): """Add layer on top of the stack.""" self.register_child(layer) - def call(self, x): - #pylint: disable=arguments-differ + def forward(self, x): for layer in self._children: x = layer(x) return x - def forward(self, F, x, *args, **kwargs): - raise NotImplementedError +class HSequential(HybridLayer): + """Stack HybridLayers sequentially. + + Example:: + net = nn.HSequential() + with net.name_scope(): + net.add(Dense(10, activation='relu')) + net.add(Dense(20)) + """ + def __init__(self, prefix=None, params=None): + super(HSequential, self).__init__(prefix=prefix, params=params) -class Dense(Layer): + def add(self, layer): + """Add layer on top of the stack.""" + self.register_child(layer) + + def hybrid_forward(self, F, x): + for layer in self._children: + x = layer(x) + return x + + +class Dense(HybridLayer): """Just your regular densely-connected NN layer. `Dense` implements the operation: @@ -239,18 +385,6 @@ class Dense(Layer): Note: the input must be a tensor with rank 2. Use flatten to convert it to rank 2 manually if necessary. - Example:: - # as first layer in a sequential model: - model = Sequential() - model.add(Dense(32, in_uints=16)) - # now the model will take as input arrays of shape (*, 16) - # and output arrays of shape (*, 32) - - # No need to specify the size of the input if you only want to - # use the `Symbol` API: - model = Sequential() - model.add(Dense(32)) - Parameters ---------- units: Positive integer, dimensionality of the output space. @@ -284,29 +418,30 @@ def __init__(self, units, activation=None, use_bias=True, in_units=0, **kwargs): super(Dense, self).__init__(**kwargs) with self.name_scope(): - self._op = symbol.CachedOp('FullyConnected', 3 if use_bias else 2, - num_hidden=units, no_bias=not use_bias) + self._units = units self.weight = self.params.get('weight', shape=(units, in_units), init=kernel_initializer) if use_bias: self.bias = self.params.get('bias', shape=(units,), init=bias_initializer) + else: + self.bias = None if activation is not None: self.act = Activation(activation) else: self.act = None - def forward(self, F, x, weight, bias=None): + def hybrid_forward(self, F, x, weight, bias=None): if bias is None: - act = F.invoke(self._op, [x, weight]) + act = F.FullyConnected(x, weight, no_bias=True, num_hidden=self._units) else: - act = F.invoke(self._op, [x, weight, bias]) + act = F.FullyConnected(x, weight, bias, num_hidden=self._units) if self.act is not None: act = self.act(act) return act -class Activation(Layer): +class Activation(HybridLayer): """Applies an activation function to input. Parameters @@ -325,16 +460,15 @@ class Activation(Layer): def __init__(self, activation, **kwargs): self._act_type = activation super(Activation, self).__init__(**kwargs) - self._op = symbol.CachedOp('Activation', 1, act_type=self._act_type) def _alias(self): return self._act_type - def forward(self, F, x): - return F.invoke(self._op, [x]) + def hybrid_forward(self, F, x): + return F.Activation(x, act_type=self._act_type) -class Dropout(Layer): +class Dropout(HybridLayer): """Applies Dropout to the input. Dropout consists in randomly setting @@ -352,13 +486,13 @@ class Dropout(Layer): """ def __init__(self, rate, **kwargs): super(Dropout, self).__init__(**kwargs) - self._op = symbol.CachedOp('Dropout', 1, p=rate) + self._rate = rate - def forward(self, F, x): - return F.invoke(self._op, [x]) + def hybrid_forward(self, F, x): + return F.Dropout(x, p=self._rate) -class BatchNorm(Layer): +class BatchNorm(HybridLayer): """Batch normalization layer (Ioffe and Szegedy, 2014). Normalize the activations of the previous layer at each batch, i.e. applies a transformation that maintains the mean activation @@ -390,10 +524,8 @@ def __init__(self, axis=1, momentum=0.9, epsilon=1e-3, center=True, scale=True, running_mean_initializer='zeros', running_variance_initializer='ones', **kwargs): super(BatchNorm, self).__init__(**kwargs) - assert axis == 1, \ - "Only support NC* layout, i.e. channel must be in the second dimension" - attrs = {'eps': epsilon, 'momentum': momentum, 'fix_gamma': not center} - self._op = symbol.CachedOp('BatchNorm', 5, **attrs) + self._kwargs = {'axis': axis, 'eps': epsilon, 'momentum': momentum, + 'fix_gamma': not center} self.gamma = self.params.get('gamma', grad_req='write' if scale else 'null', shape=(num_features,), init=gamma_initializer) @@ -406,11 +538,11 @@ def __init__(self, axis=1, momentum=0.9, epsilon=1e-3, center=True, scale=True, shape=(num_features,), init=running_variance_initializer) - def forward(self, F, x, gamma, beta, running_mean, running_var): - return F.invoke(self._op, [x, gamma, beta, running_mean, running_var]) + def hybrid_forward(self, F, x, gamma, beta, running_mean, running_var): + return F.BatchNorm(x, gamma, beta, running_mean, running_var, **self._kwargs) -class LeakyReLU(Layer): +class LeakyReLU(HybridLayer): """Leaky version of a Rectified Linear Unit. It allows a small gradient when the unit is not active: @@ -424,13 +556,13 @@ class LeakyReLU(Layer): """ def __init__(self, alpha, **kwargs): super(LeakyReLU, self).__init__(**kwargs) - self._op = symbol.CachedOp('LeakyReLU', 1, act_type='leaky', slope=alpha) + self._alpha = alpha - def forward(self, F, x): - return F.invoke(self._op, [x]) + def hybrid_forward(self, F, x): + return F.LeakyReLU(x, act_type='leaky', slope=self._alpha) -class Embedding(Layer): +class Embedding(HybridLayer): """Turns non-negative integers (indexes/tokens) into dense vectors of fixed size. eg. [[4], [20]] -> [[0.25, 0.1], [0.6, -0.2]] @@ -457,10 +589,10 @@ class Embedding(Layer): def __init__(self, input_dim, output_dim, dtype='float32', embeddings_initializer=None, **kwargs): super(Embedding, self).__init__(**kwargs) - self._op = symbol.CachedOp('Embedding', 2, input_dim=input_dim, - output_dim=output_dim, dtype=dtype) + self._kwargs = {'input_dim': input_dim, 'output_dim': output_dim, + 'dtype': dtype} self.weight = self.params.get('weight', shape=(input_dim, output_dim), init=embeddings_initializer) - def forward(self, F, x, weight): - return F.invoke(self._op, [x, weight]) + def hybrid_forward(self, F, x, weight): + return F.Embedding(x, weight, **self._kwargs) diff --git a/python/mxnet/foo/parameter.py b/python/mxnet/foo/parameter.py index 28d28f4b4000..6d772772dbc6 100644 --- a/python/mxnet/foo/parameter.py +++ b/python/mxnet/foo/parameter.py @@ -38,7 +38,7 @@ class Parameter(object): b = mx.nn.Parameter('fc_bias', shape(64,), init=mx.init.Zero()) w.initialize(ctx=ctx) b.initialize(ctx=ctx) - out = mx.nd.FullyConnected(x, w.value(ctx), b.value(ctx), num_hidden=64) + out = mx.nd.FullyConnected(x, w.data(ctx), b.data(ctx), num_hidden=64) Parameters ---------- @@ -158,7 +158,7 @@ def _finish_deferred_init(self): "in_filters, num_features etc for Layers."%( self.name, str(self.shape)) - with autograd.test_section(): + with autograd.pause(): data = ndarray.zeros(shape=self.shape, dtype=self.dtype, ctx=context.cpu()) if init is None: @@ -321,6 +321,7 @@ def _get_impl(self, name): if name in self._params: return self._params[name] if self._shared is not None and name in self._shared._params: + self._params[name] = self._shared._params[name] return self._shared._params[name] return None diff --git a/python/mxnet/foo/rnn/rnn_cell.py b/python/mxnet/foo/rnn/rnn_cell.py index 2733cebe46bd..495736a90cf5 100644 --- a/python/mxnet/foo/rnn/rnn_cell.py +++ b/python/mxnet/foo/rnn/rnn_cell.py @@ -9,7 +9,7 @@ from ... import symbol, init, ndarray from ...base import string_types, numeric_types -from ..nn import Layer +from ..nn import Layer, HybridLayer from .. import tensor_types @@ -301,7 +301,7 @@ def _get_activation(self, F, inputs, activation, **kwargs): else: return activation(inputs, **kwargs) - def call(self, inputs, states): + def forward(self, inputs, states): """Unroll the recurrent cell for one time step. Parameters @@ -329,11 +329,19 @@ def call(self, inputs, states): """ # pylint: disable= arguments-differ self._counter += 1 - return super(RecurrentCell, self).call(inputs, states) + return super(RecurrentCell, self).forward(inputs, states) +class HRecurrentCell(RecurrentCell, HybridLayer): + """HRecurrentCell supports both Symbol and NDArray forwarding.""" + def __init__(self, prefix=None, params=None): + super(HRecurrentCell, self).__init__(prefix=prefix, params=params) + + def hybrid_forward(self, F, x, *args, **kwargs): + raise NotImplementedError + -class RNNCell(RecurrentCell): +class RNNCell(HRecurrentCell): """Simple recurrent neural network cell. Parameters @@ -370,8 +378,8 @@ def _gate_names(self): def _alias(self): return 'rnn' - def forward(self, F, inputs, states, i2h_weight, i2h_bias, - h2h_weight, h2h_bias): + def hybrid_forward(self, F, inputs, states, i2h_weight, i2h_bias, + h2h_weight, h2h_bias): name = self._curr_prefix i2h = F.FullyConnected(data=inputs, weight=i2h_weight, bias=i2h_bias, num_hidden=self._num_hidden, @@ -385,7 +393,7 @@ def forward(self, F, inputs, states, i2h_weight, i2h_bias, return output, [output] -class LSTMCell(RecurrentCell): +class LSTMCell(HRecurrentCell): """Long-Short Term Memory (LSTM) network cell. Parameters @@ -425,8 +433,8 @@ def _gate_names(self): def _alias(self): return 'lstm' - def forward(self, F, inputs, states, i2h_weight, i2h_bias, - h2h_weight, h2h_bias): + def hybrid_forward(self, F, inputs, states, i2h_weight, i2h_bias, + h2h_weight, h2h_bias): name = self._curr_prefix i2h = F.FullyConnected(data=inputs, weight=i2h_weight, bias=i2h_bias, num_hidden=self._num_hidden*4, @@ -453,7 +461,7 @@ def forward(self, F, inputs, states, i2h_weight, i2h_bias, return next_h, [next_h, next_c] -class GRUCell(RecurrentCell): +class GRUCell(HRecurrentCell): """Gated Rectified Unit (GRU) network cell. Note: this is an implementation of the cuDNN version of GRUs (slight modification compared to Cho et al. 2014). @@ -487,8 +495,8 @@ def _gate_names(self): def _alias(self): return 'gru' - def forward(self, F, inputs, states, i2h_weight, i2h_bias, - h2h_weight, h2h_bias): + def hybrid_forward(self, F, inputs, states, i2h_weight, i2h_bias, + h2h_weight, h2h_bias): # pylint: disable=too-many-locals name = self._curr_prefix prev_state_h = states[0] @@ -520,7 +528,7 @@ def forward(self, F, inputs, states, i2h_weight, i2h_bias, return next_h, [next_h] -class FusedRNNCell(RecurrentCell): +class FusedRNNCell(HRecurrentCell): """Fusing RNN layers across time step into one kernel. Improves speed but is less flexible. Currently only supported if using cuDNN on GPU. @@ -690,10 +698,10 @@ def unfuse(self): Returns ------- - cell : SequentialRNNCell + cell : HSequentialRNNCell unfused cell that can be used for stepping, and can run on CPU. """ - stack = SequentialRNNCell() + stack = HSequentialRNNCell() get_cell = {'rnn_relu': lambda cell_prefix: RNNCell(self._num_hidden, activation='relu', prefix=cell_prefix), @@ -719,10 +727,10 @@ def unfuse(self): return stack -class SequentialRNNCell(RecurrentCell): +class HSequentialRNNCell(HRecurrentCell): """Sequantially stacking multiple RNN cells.""" def __init__(self): - super(SequentialRNNCell, self).__init__(prefix='', params=None) + super(HSequentialRNNCell, self).__init__(prefix='', params=None) def add(self, cell): """Append a cell into the stack. @@ -780,11 +788,11 @@ def unroll(self, length, inputs, begin_state=None, layout='NTC', merge_outputs=N return inputs, next_states - def forward(self, *args, **kwargs): + def hybrid_forward(self, *args, **kwargs): raise NotImplementedError -class DropoutCell(RecurrentCell): +class DropoutCell(HRecurrentCell): """Apply dropout on input. Parameters @@ -804,7 +812,7 @@ def state_info(self, batch_size=0): def _alias(self): return 'dropout' - def forward(self, F, inputs, states): + def hybrid_forward(self, F, inputs, states): if self.dropout > 0: inputs = F.Dropout(data=inputs, p=self.dropout) return inputs, states @@ -814,14 +822,14 @@ def unroll(self, length, inputs, begin_state=None, layout='NTC', merge_outputs=N inputs, _, F, _ = _format_sequence(length, inputs, layout, merge_outputs) if isinstance(inputs, tensor_types): - return self.forward(F, inputs, begin_state if begin_state else []) + return self.hybrid_forward(F, inputs, begin_state if begin_state else []) else: return super(DropoutCell, self).unroll( length, inputs, begin_state=begin_state, layout=layout, merge_outputs=merge_outputs) -class ModifierCell(RecurrentCell): +class ModifierCell(HRecurrentCell): """Base class for modifier cells. A modifier cell takes a base cell, apply modifications on it (e.g. Zoneout), and returns a new cell. @@ -858,7 +866,7 @@ def unpack_weights(self, args): def pack_weights(self, args): return self.base_cell.pack_weights(args) - def forward(self, F, inputs, states): + def hybrid_forward(self, F, inputs, states): raise NotImplementedError @@ -871,8 +879,8 @@ def __init__(self, base_cell, zoneout_outputs=0., zoneout_states=0.): assert not isinstance(base_cell, BidirectionalCell), \ "BidirectionalCell doesn't support zoneout since it doesn't support step. " \ "Please add ZoneoutCell to the cells underneath instead." - assert not isinstance(base_cell, SequentialRNNCell) or not base_cell._bidirectional, \ - "Bidirectional SequentialRNNCell doesn't support zoneout. " \ + assert not isinstance(base_cell, HSequentialRNNCell) or not base_cell._bidirectional, \ + "Bidirectional HSequentialRNNCell doesn't support zoneout. " \ "Please add ZoneoutCell to the cells underneath instead." super(ZoneoutCell, self).__init__(base_cell) self.zoneout_outputs = zoneout_outputs @@ -886,7 +894,7 @@ def reset(self): super(ZoneoutCell, self).reset() self.prev_output = None - def forward(self, F, inputs, states): + def hybrid_forward(self, F, inputs, states): cell, p_outputs, p_states = self.base_cell, self.zoneout_outputs, self.zoneout_states next_output, next_states = cell(inputs, states) mask = (lambda p, like: F.Dropout(F.ones_like(like), p=p)) @@ -915,7 +923,7 @@ class ResidualCell(ModifierCell): def __init__(self, base_cell): super(ResidualCell, self).__init__(base_cell) - def forward(self, F, inputs, states): + def hybrid_forward(self, F, inputs, states): output, states = self.base_cell(inputs, states) output = F.elemwise_add(output, inputs, name="%s_plus_residual" % output.name) return output, states @@ -939,7 +947,7 @@ def unroll(self, length, inputs, begin_state=None, layout='NTC', merge_outputs=N return outputs, states -class BidirectionalCell(RecurrentCell): +class BidirectionalCell(HRecurrentCell): """Bidirectional RNN cell. Parameters diff --git a/python/mxnet/foo/trainer.py b/python/mxnet/foo/trainer.py index 8d22983ce25f..e1d52e43c2e1 100644 --- a/python/mxnet/foo/trainer.py +++ b/python/mxnet/foo/trainer.py @@ -72,7 +72,7 @@ def _init_kvstore(self): def step(self, batch_size, ignore_stale_grad=False): """Make one step of parameter update. Should be called after - autograd.compute_gradient and outside of train_section() scope. + autograd.compute_gradient and outside of record() scope. Parameters ---------- diff --git a/python/mxnet/ndarray.py b/python/mxnet/ndarray.py index 29f0f769ed63..15c5955f8750 100644 --- a/python/mxnet/ndarray.py +++ b/python/mxnet/ndarray.py @@ -122,7 +122,7 @@ class NDArray(NDArrayBase): def __repr__(self): """Returns a string representation of the array.""" shape_info = 'x'.join(['%d' % x for x in self.shape]) - return '%s\n<%s %s @%s>' % (str(self.asnumpy()), + return '%s\n<%s %s @%s>' % ('',#str(self.asnumpy()), self.__class__.__name__, shape_info, self.context) diff --git a/src/c_api/c_api_ndarray.cc b/src/c_api/c_api_ndarray.cc index 98fbe760854e..350f77cf7bfd 100644 --- a/src/c_api/c_api_ndarray.cc +++ b/src/c_api/c_api_ndarray.cc @@ -88,7 +88,8 @@ void SetNDInputsOutputs(const nnvm::Op* op, } else { CHECK(!AutogradRuntime::Get()->IsTraining()) << "Inplace operations (+=, -=, op(..., out=x) etc.) and assignment are " - << "not supported when you are inside a train_section using autograd."; + << "not supported when recording with autograd. You can use autograd.pause " + << "to wrap this operation, but it may invalid gradients."; CHECK(*num_outputs == infered_num_outputs || *num_outputs == num_visible_outputs) << "Expecting " << infered_num_outputs << " (all) or " << num_visible_outputs << " (visible only) outputs, got " diff --git a/src/ndarray/autograd.cc b/src/ndarray/autograd.cc index b35364d0c70f..7908f2c9d005 100644 --- a/src/ndarray/autograd.cc +++ b/src/ndarray/autograd.cc @@ -155,7 +155,7 @@ void AutogradRuntime::ComputeGradient(const std::vector& outputs, for (const auto& i : outputs) { CHECK(!i.entry_.is_none()) << "Cannot differentiate node because it is not in a computational graph. " - << "You need to set is_training to true or use a train_section to save " + << "You need to set is_training to true or use autograd.record() to save " << "computational graphs for backward. If you want to differentiate the same " << "graph twice, you need to pass retain_graph=True to backward."; heads.emplace_back(i.entry_); diff --git a/src/ndarray/ndarray.cc b/src/ndarray/ndarray.cc index ab8e616fb1df..f2e90dd56f31 100644 --- a/src/ndarray/ndarray.cc +++ b/src/ndarray/ndarray.cc @@ -37,7 +37,7 @@ NDArray NDArray::Reshape(const TShape &shape) const { if (AutogradRuntime::Get()->IsTraining()) { CHECK_GE(shape_.Size(), shape.Size()) << "NDArray.Reshape: target shape must have must have the same size as " - << "current shape when in train_section."; + << "current shape when recording with autograd."; NDArray ret = *this; ret.shape_ = shape; // fake a Reshape op diff --git a/tests/python/gpu/test_operator_gpu.py b/tests/python/gpu/test_operator_gpu.py index 75229eecce72..7557b6b49bce 100644 --- a/tests/python/gpu/test_operator_gpu.py +++ b/tests/python/gpu/test_operator_gpu.py @@ -1281,7 +1281,7 @@ def test_residual_fused(): def test_foo_rnn(): fused = foo.rnn.FusedRNNCell(100, num_layers=2, mode='rnn_relu', prefix='') - stack = foo.rnn.SequentialRNNCell() + stack = foo.rnn.HSequentialRNNCell() stack.add(foo.rnn.RNNCell(100, activation='relu', prefix='l0_')) stack.add(foo.rnn.RNNCell(100, activation='relu', prefix='l1_')) @@ -1292,7 +1292,7 @@ def test_foo_rnn(): def test_foo_lstm(): fused = foo.rnn.FusedRNNCell(100, num_layers=2, mode='lstm', prefix='') - stack = foo.rnn.SequentialRNNCell() + stack = foo.rnn.HSequentialRNNCell() stack.add(foo.rnn.LSTMCell(100, prefix='l0_')) stack.add(foo.rnn.LSTMCell(100, prefix='l1_')) @@ -1324,7 +1324,7 @@ def test_foo_lstm_forget_bias(): def test_foo_gru(): fused = foo.rnn.FusedRNNCell(100, num_layers=2, mode='gru', prefix='') - stack = foo.rnn.SequentialRNNCell() + stack = foo.rnn.HSequentialRNNCell() stack.add(foo.rnn.GRUCell(100, prefix='l0_')) stack.add(foo.rnn.GRUCell(100, prefix='l1_')) @@ -1336,7 +1336,7 @@ def test_foo_bidirectional(): fused = foo.rnn.FusedRNNCell(100, num_layers=2, mode='gru', prefix='', bidirectional=True) - stack = foo.rnn.SequentialRNNCell() + stack = foo.rnn.HSequentialRNNCell() stack.add(foo.rnn.BidirectionalCell( foo.rnn.GRUCell(100, prefix='l0_'), foo.rnn.GRUCell(100, prefix='r0_'), diff --git a/tests/python/train/test_autograd.py b/tests/python/train/test_autograd.py index 25cd505d6879..ff4aacd1fa07 100644 --- a/tests/python/train/test_autograd.py +++ b/tests/python/train/test_autograd.py @@ -58,7 +58,7 @@ def train(net, epoch, ctx): data = foo.utils.load_data(batch.data[0], ctx_list=ctx, batch_axis=0) label = foo.utils.load_data(batch.label[0], ctx_list=ctx, batch_axis=0) outputs = [] - with ag.train_section(): + with ag.record(): for x, y in zip(data, label): z = net(x) loss = foo.loss.softmax_cross_entropy_loss(z, y) diff --git a/tests/python/unittest/test_autograd.py b/tests/python/unittest/test_autograd.py index abcaef44f94b..3fa4a743cc25 100644 --- a/tests/python/unittest/test_autograd.py +++ b/tests/python/unittest/test_autograd.py @@ -31,7 +31,7 @@ def wrapped(*args): assert isinstance(x, NDArray), "type of autograd input should NDArray." grads = [zeros_like(x) for x in variables] mark_variables(variables, grads) - with train_section(): + with record(): outputs = func(*args) backward([outputs] if isinstance(outputs, NDArray) else outputs) return grads, outputs @@ -148,10 +148,10 @@ def f_with_mode(a, b, mode): def test_training(): x = nd.ones((10, 10)) - with train_section(): + with record(): y = nd.Dropout(x, p=0.5) assert not (y.asnumpy() == x.asnumpy()).all() - with test_section(): + with pause(): y = nd.Dropout(x, p=0.5) assert (y.asnumpy() == x.asnumpy()).all() @@ -164,7 +164,7 @@ def test_out_grads(): db = nd.array([1,2,3,4,5]) dc = nd.array([5,4,3,2,1]) - with train_section(): + with record(): a, b, c = nd.split(x, axis=0, num_outputs=3, squeeze_axis=True) backward([a, b, c], [da, db, dc]) @@ -183,7 +183,7 @@ def test_detach_updated_grad(): assert x._fresh_grad == False assert y._fresh_grad == False - with train_section(): + with record(): x2 = x + 2 y2 = x2 + y y2.backward() @@ -196,7 +196,7 @@ def test_detach_updated_grad(): y._fresh_grad = False assert x._fresh_grad == False assert y._fresh_grad == False - with train_section(): + with record(): x2 = x + 2 x2 = x2.detach() y2 = x2 + y @@ -210,20 +210,20 @@ def test_retain_grad(): x = mx.nd.ones((2, 2)) dx = mx.nd.zeros((2, 2)) mark_variables([x], [dx], grad_reqs='add') - with train_section(): + with record(): y = x + 1 y.backward(retain_graph=False) assert (dx.asnumpy() == 1).all() dx[:] = 0 - with train_section(): + with record(): y = x + 1 y.backward(retain_graph=True) y.backward(retain_graph=False) assert (dx.asnumpy() == 2).all() try: - with train_section(): + with record(): y = x + 1 y.backward() y.backward() @@ -238,7 +238,7 @@ def test_attach_grad(): x = mx.nd.zeros((10,)) assert x.grad is None x.attach_grad() - with train_section(): + with record(): y = x * 2 assert y.grad is None y.backward() diff --git a/tests/python/unittest/test_foo_rnn.py b/tests/python/unittest/test_foo_rnn.py index 3794c7fc18cc..67113e4ff8ce 100644 --- a/tests/python/unittest/test_foo_rnn.py +++ b/tests/python/unittest/test_foo_rnn.py @@ -30,7 +30,7 @@ def test_lstm(): def test_lstm_forget_bias(): forget_bias = 2.0 - stack = foo.rnn.SequentialRNNCell() + stack = foo.rnn.HSequentialRNNCell() stack.add(foo.rnn.LSTMCell(100, forget_bias=forget_bias, prefix='l0_')) stack.add(foo.rnn.LSTMCell(100, forget_bias=forget_bias, prefix='l1_')) @@ -117,7 +117,7 @@ def test_residual_bidirectional(): def test_stack(): - cell = foo.rnn.SequentialRNNCell() + cell = foo.rnn.HSequentialRNNCell() for i in range(5): if i == 1: cell.add(foo.rnn.ResidualCell(foo.rnn.LSTMCell(100, prefix='rnn_stack%d_' % i))) @@ -179,11 +179,10 @@ def test_unfuse(): def check_rnn_forward(layer, inputs): layer.all_params().initialize() - with mx.contrib.autograd.train_section(): - mx.contrib.autograd.compute_gradient( - [layer.unroll(3, inputs, merge_outputs=True)[0]]) - mx.contrib.autograd.compute_gradient( - layer.unroll(3, inputs, merge_outputs=False)[0]) + with mx.autograd.record(): + layer.unroll(3, inputs, merge_outputs=True)[0].backward() + mx.autograd.backward(layer.unroll(3, inputs, merge_outputs=False)[0]) + mx.nd.waitall() def test_rnn_cells(): @@ -201,7 +200,7 @@ def test_rnn_cells(): 0.5, 0.2), mx.nd.ones((8, 3, 200))) - net = foo.rnn.SequentialRNNCell() + net = foo.rnn.HSequentialRNNCell() net.add(foo.rnn.LSTMCell(100, num_input=200)) net.add(foo.rnn.RNNCell(100, num_input=100)) net.add(foo.rnn.GRUCell(100, num_input=100)) diff --git a/tests/python/unittest/test_nn.py b/tests/python/unittest/test_nn.py index 8bb490c7d264..0d04071259e4 100644 --- a/tests/python/unittest/test_nn.py +++ b/tests/python/unittest/test_nn.py @@ -31,7 +31,7 @@ def __init__(self, **kwargs): self.dense0 = nn.Dense(5, in_units=5) self.dense1 = nn.Dense(5, in_units=5) - def forward(self, F, x): + def forward(self, x): return self.dense1(self.dense0(x)) net1 = Net(prefix='net1_') @@ -62,7 +62,13 @@ def test_basic(): def check_layer_forward(layer, dshape): layer.all_params().initialize() - with mx.contrib.autograd.train_section(): + with mx.autograd.record(): + out = layer(mx.nd.ones(shape=dshape)) + out.backward() + + layer.hybridize() + + with mx.autograd.record(): out = layer(mx.nd.ones(shape=dshape)) out.backward() @@ -189,33 +195,33 @@ def test_reshape(): x = mx.nd.ones((2, 4, 10, 10)) layer = nn.Conv2D(10, 2, in_filters=4) layer.all_params().initialize() - with mx.contrib.autograd.train_section(): + with mx.autograd.record(): x = layer(x) x = x.reshape((-1,)) x = x + 10 - mx.contrib.autograd.compute_gradient([x]) + x.backward() def test_slice(): x = mx.nd.ones((5, 4, 10, 10)) layer = nn.Conv2D(10, 2, in_filters=4) layer.all_params().initialize() - with mx.contrib.autograd.train_section(): + with mx.autograd.record(): x = layer(x) x = x[1:3] x = x + 10 - mx.contrib.autograd.compute_gradient([x]) + x.backward() def test_at(): x = mx.nd.ones((5, 4, 10, 10)) layer = nn.Conv2D(10, 2, in_filters=4) layer.all_params().initialize() - with mx.contrib.autograd.train_section(): + with mx.autograd.record(): x = layer(x) x = x[1] x = x + 10 - mx.contrib.autograd.compute_gradient([x]) + x.backward() def test_defered_init(): diff --git a/tests/python/unittest/test_random.py b/tests/python/unittest/test_random.py index 0b5fd3a96dbf..1f88b6b4fbe0 100644 --- a/tests/python/unittest/test_random.py +++ b/tests/python/unittest/test_random.py @@ -174,7 +174,7 @@ def test_sample_multinomial(): x = mx.nd.array([[0,1,2,3,4],[4,3,2,1,0]])/10.0 dx = mx.nd.ones_like(x) mx.contrib.autograd.mark_variables([x], [dx]) - with mx.contrib.autograd.train_section(): + with mx.autograd.record(): y, prob = mx.nd.sample_multinomial(x, shape=1000, get_prob=True) r = prob * 5 r.backward() From a887e113fe1e603fd824b1727ee05fb3f78f3033 Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Fri, 23 Jun 2017 09:21:53 -0700 Subject: [PATCH 191/834] nn fixes (#6792) * nn fixes * restore contrib/autograd * update examples and revert container changes * fix resnet version * fix benchmark --- example/autograd/actor_critic.py | 2 +- example/autograd/dcgan.py | 2 +- example/autograd/mnist.py | 2 +- example/autograd/resnet.py | 99 +++++++++++++++++++++------- example/autograd/super_resolution.py | 2 +- python/mxnet/foo/nn/layer.py | 2 +- 6 files changed, 80 insertions(+), 29 deletions(-) diff --git a/example/autograd/actor_critic.py b/example/autograd/actor_critic.py index d579971a89ff..23e762f6eebe 100644 --- a/example/autograd/actor_critic.py +++ b/example/autograd/actor_critic.py @@ -9,7 +9,7 @@ import mxnet.ndarray as F from mxnet import foo from mxnet.foo import nn -from mxnet.contrib import autograd +from mxnet import autograd parser = argparse.ArgumentParser(description='MXNet actor-critic example') diff --git a/example/autograd/dcgan.py b/example/autograd/dcgan.py index 06903386ebd5..aa6b8b6dcd0a 100644 --- a/example/autograd/dcgan.py +++ b/example/autograd/dcgan.py @@ -2,7 +2,7 @@ import mxnet as mx from mxnet import foo from mxnet.foo import nn -from mxnet.contrib import autograd +from mxnet import autograd from data import cifar10_iterator diff --git a/example/autograd/mnist.py b/example/autograd/mnist.py index cff6c2d7d9b6..a8b70226be3f 100644 --- a/example/autograd/mnist.py +++ b/example/autograd/mnist.py @@ -7,7 +7,7 @@ from mxnet.foo import nn import numpy as np import logging -from mxnet.contrib import autograd as ag +from mxnet import autograd as ag logging.basicConfig(level=logging.DEBUG) # define network diff --git a/example/autograd/resnet.py b/example/autograd/resnet.py index 879b3237eacf..85451d5571bf 100644 --- a/example/autograd/resnet.py +++ b/example/autograd/resnet.py @@ -1,12 +1,33 @@ from __future__ import division -import time +import argparse, time +import logging +logging.basicConfig(level=logging.INFO) + import mxnet as mx from mxnet import foo from mxnet.foo import nn -from mxnet.contrib import autograd as ag +from mxnet import autograd as ag + from data import * +# CLI +parser = argparse.ArgumentParser(description='Train a resnet model for image classification.') +parser.add_argument('--dataset', type=str, default='dummy', help='dataset to use. options are mnist, cifar10, and dummy.') +parser.add_argument('--batch_size', type=int, default=32, help='training batch size per device (CPU/GPU).') +parser.add_argument('--resnet_version', type=int, default=1, help='version of resnet to use. options are 1 and 2. default is 1.') +parser.add_argument('--resnet_layers', type=int, default=50, help='layers of resnet to use. options are 18, 50. default is 50.') +parser.add_argument('--gpus', type=int, default=0, help='number of gpus to use.') +parser.add_argument('--epochs', type=int, default=3, help='number of training epochs.') +parser.add_argument('--lr', type=float, default=0.01, help='learning Rate. default is 0.01.') +parser.add_argument('--seed', type=int, default=123, help='random seed to use. Default=123.') +parser.add_argument('--thumbnail', action='store_true', default=False, help='use thumbnail or not. default is false.') +parser.add_argument('--benchmark', action='store_true', default=True, help='whether to run benchmark.') +parser.add_argument('--symbolic', action='store_true', default=False, help='whether to train in symbolic way with module.') +opt = parser.parse_args() + +print(opt) + def conv3x3(filters, stride, in_filters): return nn.Conv2D(filters, kernel_size=3, strides=stride, padding=1, use_bias=False, in_filters=in_filters) @@ -248,17 +269,47 @@ def hybrid_forward(self, F, x): return x +# construct net +resnet_spec = { 18: ('basic_block', [2, 2, 2], [16, 16, 32, 64]), + 34: ('basic_block', [3, 4, 6, 3], [16, 16, 32, 64]), + 50: ('bottle_neck', [3, 4, 6, 3], [64, 256, 512, 1024, 2048]), + 101: ('bottle_neck', [3, 4, 23, 3], [64, 256, 512, 1024, 2048]), + 152: ('bottle_neck', [3, 8, 36, 3], [64, 256, 512, 1024, 2048]) } + +resnet_net_versions = [ResnetV1, ResnetV2] +resnet_block_versions = [{'basic_block': BasicBlockV1, 'bottle_neck': BottleneckV1}, + {'basic_block': BasicBlockV2, 'bottle_neck': BottleneckV2}] + +def get_resnet(version, num_layers, classes, use_thumbnail): + block_type, layers, filters = resnet_spec[num_layers] + resnet = resnet_net_versions[version] + block = resnet_block_versions[version][block_type] + return resnet(block, classes, layers, filters, use_thumbnail) + +dataset_classes = {'mnist': 10, 'cifar10': 10, 'imagenet': 1000, 'dummy': 1000} + +batch_size, dataset, classes = opt.batch_size, opt.dataset, dataset_classes[opt.dataset] + +gpus, version = opt.gpus, opt.resnet_version-1 + +if opt.benchmark: + batch_size = 32 + dataset = 'dummy' + classes = 1000 + version = 0 + + +net = get_resnet(version, opt.resnet_layers, classes, opt.thumbnail) -def resnet18v2_cifar(classes): - return ResnetV2(BasicBlockV2, classes, [2, 2, 2], [16, 16, 32, 64], True) -def resnet50v1_imagenet(classes): - return ResnetV1(BottleneckV1, classes, [3, 4, 6, 3], [64, 256, 512, 1024, 2048], False) -def resnet50v2_imagenet(classes): - return ResnetV2(BottleneckV2, classes, [3, 4, 6, 3], [64, 256, 512, 1024, 2048], False) +batch_size *= max(1, gpus) -net = resnet18v2_cifar(10) -batch_size = 32*8 -train_data, val_data = cifar10_iterator(batch_size, (3, 32, 32)) +# get dataset iterators +if dataset == 'mnist': + train_data, val_data = mnist_iterator(batch_size, (1, 32, 32)) +elif dataset == 'cifar10': + train_data, val_data = cifar10_iterator(batch_size, (3, 32, 32)) +elif dataset == 'dummy': + train_data, val_data = dummy_iterator(batch_size, (3, 224, 224)) def test(ctx): metric = mx.metric.Accuracy() @@ -270,7 +321,7 @@ def test(ctx): for x in data: outputs.append(net(x)) metric.update(label, outputs) - print('validation acc: %s=%f'%metric.get()) + logging.info('validation acc: %s=%f'%metric.get()) def train(epoch, ctx): @@ -299,24 +350,24 @@ def train(epoch, ctx): loss.backward() trainer.step(batch.data[0].shape[0]) metric.update(label, outputs) - print('speed: {} samples/s'.format(batch.data[0].shape[0]/(time.time()-btic))) + logging.info('speed: {} samples/s'.format(batch_size/(time.time()-btic))) btic = time.time() name, acc = metric.get() metric.reset() - print('training acc at epoch %d: %s=%f'%(i, name, acc)) - print('time: %f'%(time.time()-tic)) + logging.info('training acc at epoch %d: %s=%f'%(i, name, acc)) + logging.info('time: %f'%(time.time()-tic)) test(ctx) net.all_params().save('mnist.params') if __name__ == '__main__': - net.hybridize() - train(200, [mx.gpu(i) for i in range(2)]) - import logging - logging.basicConfig(level=logging.DEBUG) - data = mx.sym.var('data') - out = net(data) - softmax = mx.sym.SoftmaxOutput(out, name='softmax') - mod = mx.mod.Module(softmax, context=[mx.gpu(i) for i in range(1)]) - mod.fit(train_data, num_epoch=100, batch_end_callback = mx.callback.Speedometer(batch_size, 10)) + if opt.symbolic: + data = mx.sym.var('data') + out = net(data) + softmax = mx.sym.SoftmaxOutput(out, name='softmax') + mod = mx.mod.Module(softmax, context=[mx.gpu(i) for i in range(gpus)] if gpus > 0 else [mx.cpu()]) + mod.fit(train_data, num_epoch=opt.epochs, batch_end_callback = mx.callback.Speedometer(batch_size, 1)) + else: + net.hybridize() + train(opt.epochs, [mx.gpu(i) for i in range(gpus)] if gpus > 0 else [mx.cpu()]) diff --git a/example/autograd/super_resolution.py b/example/autograd/super_resolution.py index e564c9c82eaa..fec4f11b7f63 100644 --- a/example/autograd/super_resolution.py +++ b/example/autograd/super_resolution.py @@ -7,7 +7,7 @@ import mxnet.ndarray as F from mxnet import foo from mxnet.foo import nn -from mxnet.contrib import autograd as ag +from mxnet import autograd as ag from mxnet.test_utils import download from mxnet.image import CenterCropAug, ResizeAug from mxnet.io import PrefetchingIter diff --git a/python/mxnet/foo/nn/layer.py b/python/mxnet/foo/nn/layer.py index 25ae6d23542a..7bdf86a63aed 100644 --- a/python/mxnet/foo/nn/layer.py +++ b/python/mxnet/foo/nn/layer.py @@ -238,7 +238,7 @@ def register_child(self, layer): def hybridize(self, active=True): super(HybridLayer, self).hybridize(active) - self._active = True + self._active = active def _get_graph(self, *args): if self._cached_graph: From 9cdae4db7caca5b57c354e79e32082d35ffcb8bc Mon Sep 17 00:00:00 2001 From: Roshani Nagmote Date: Mon, 26 Jun 2017 22:36:42 -0700 Subject: [PATCH 192/834] input output shapes added and formatting fixes (#6824) * input output shapes added and formatting fixes * More info links added * in/out channels names added for in_filters/filters --- docs/api/python/foo.md | 6 + python/mxnet/foo/loss.py | 17 ++- python/mxnet/foo/nn/conv_layers.py | 225 ++++++++++++++++++++++++++++- python/mxnet/foo/nn/layer.py | 88 ++++++----- 4 files changed, 289 insertions(+), 47 deletions(-) diff --git a/docs/api/python/foo.md b/docs/api/python/foo.md index 0bff54ca9c36..a76aef1c421e 100644 --- a/docs/api/python/foo.md +++ b/docs/api/python/foo.md @@ -48,9 +48,15 @@ in Python and then deploy with symbolic graph in C++ and Scala. .. autoclass:: mxnet.foo.nn.Layer :members: + .. automethod:: __call__ +.. autoclass:: mxnet.foo.nn.HybridLayer + :members: + .. automethod:: __call__ .. autoclass:: mxnet.foo.nn.Sequential :members: +.. autoclass:: mxnet.foo.nn.HSequential + :members: ``` diff --git a/python/mxnet/foo/loss.py b/python/mxnet/foo/loss.py index 8f7193383ec4..0e5399fd211a 100644 --- a/python/mxnet/foo/loss.py +++ b/python/mxnet/foo/loss.py @@ -132,7 +132,7 @@ def l2_loss(output, label, weight=1., sample_weight=None, batch_axis=0, """Calculate the mean squared error between output and label: .. math:: - L = \\frac{1}{2}\\sum_i \\Vert {output}_i - {label}_i \\Vert^2. + L = \\frac{1}{2}\\sum_i \\Vert {output}_i - {label}_i \\Vert^2. output and label can have arbitrary shape as long as they have the same number of elements. @@ -171,7 +171,7 @@ def l1_loss(output, label, weight=None, sample_weight=None, batch_axis=0, """Calculate the mean absolute error between output and label: .. math:: - L = \\frac{1}{2}\\sum_i \\vert {output}_i - {label}_i \\vert. + L = \\frac{1}{2}\\sum_i \\vert {output}_i - {label}_i \\vert. output and label must have the same shape. @@ -210,17 +210,22 @@ def softmax_cross_entropy_loss(output, label, sparse_label=True, axis=-1, """Compute the softmax cross entropy loss. If sparse_label is True, label should contain integer category indicators: + .. math:: - p = {softmax}({output}) - L = -\\sum_i {log}(p_{i,{label}_i}) + p = {softmax}({output}) + + L = -\\sum_i {log}(p_{i,{label}_i}) + label's shape should be output's shape without the `axis` dimension. i.e. for output.shape = (1,2,3,4) and axis = 2, label.shape should be (1,2,4) If sparse_label is False, label should cantain probability distribution with the same shape as output: + .. math:: - p = {softmax}({output}) - L = -\\sum_i \\sum_j {label}_j {log}(p_{ij}) + p = {softmax}({output}) + + L = -\\sum_i \\sum_j {label}_j {log}(p_{ij}) Parameters ---------- diff --git a/python/mxnet/foo/nn/conv_layers.py b/python/mxnet/foo/nn/conv_layers.py index de892593f8c8..59d58726a557 100644 --- a/python/mxnet/foo/nn/conv_layers.py +++ b/python/mxnet/foo/nn/conv_layers.py @@ -158,6 +158,22 @@ class Conv1D(_Conv): see Initializer. bias_initializer: Initializer for the bias vector see Initializer. + + + Input Shape: + This depends on the `layout` parameter. Input is 3D array of shape + (batch_size, in_channel(in_filters), width) if `layout` is `NCW`. + + Output Shape: + This depends on the `layout` parameter. Output is 3D array of shape + (batch_size, out_channel(filters), out_width) if `layout` is `NCW`. out_width + depends on other input parameters as well. It is calculated as follows:: + + out_width = floor((w+2*p-d*(k-1)-1)/s)+1 + + where, + + w = width, p = padding, d = dilation, k = kernel_size, s = stride """ def __init__(self, filters, kernel_size, strides=1, padding=0, dilation=1, groups=1, layout='NCW', in_filters=0, activation=None, use_bias=True, @@ -221,6 +237,24 @@ class Conv2D(_Conv): see Initializer. bias_initializer: Initializer for the bias vector see Initializer. + + + Input Shape: + This depends on the `layout` parameter. Input is 4D array of shape + (batch_size, in_channel(in_filters), height, width) if `layout` is `NCHW`. + + Output Shape: + This depends on the `layout` parameter. Output is 4D array of shape + (batch_size, out_channel(filters), out_height, out_width) if `layout` is `NCHW`. + out_height and out_width depends on other input parameters as well. + They are calculated as follows:: + + out_width = floor((w+2*p-d*(k-1)-1)/s)+1 + out_height = floor((h+2*p-d*(k-1)-1)/s)+1 + + where, + + w = width, h = height, p = padding, d = dilation, k = kernel_size, s = stride """ def __init__(self, filters, kernel_size, strides=(1, 1), padding=(0, 0), dilation=(1, 1), groups=1, layout='NCHW', in_filters=0, @@ -285,6 +319,25 @@ class Conv3D(_Conv): see Initializer. bias_initializer: Initializer for the bias vector see Initializer. + + + Input Shape: + This depends on the `layout` parameter. Input is 5D array of shape + (batch_size, in_channel(in_filters), depth, height, width) if `layout` is `NCDHW`. + + Output Shape: + This depends on the `layout` parameter. Output is 5D array of shape + (batch_size, out_channel(filters), out_depth, out_height, out_width) if `layout` is + `NCDHW`. out_depth, out_height and out_width depends on other input parameters as well. + They are calculated as follows:: + + out_depth = floor((d+2*p-d*(k-1)-1)/s)+1 + out_height = floor((h+2*p-d*(k-1)-1)/s)+1 + out_width = floor((w+2*p-d*(k-1)-1)/s)+1 + + where, + + d = depth, h = height, w = width, p = padding, d = dilation, k = kernel_size, s = stride """ def __init__(self, filters, kernel_size, strides=(1, 1, 1), padding=(0, 0, 0), dilation=(1, 1, 1), groups=1, layout='NCDHW', in_filters=0, @@ -351,6 +404,22 @@ class Conv1DTranspose(_Conv): see Initializer. bias_initializer: Initializer for the bias vector see Initializer. + + + Input Shape: + This depends on the `layout` parameter. Input is 3D array of shape + (batch_size, in_channel(in_filters), width) if `layout` is `NCW`. + + Output Shape: + This depends on the `layout` parameter. Output is 3D array of shape + (batch_size, out_channel(filters), out_width) if `layout` is `NCW`. + out_width depends on other input parameters as well. It is calculated as follows:: + + out_width = (w-1)*s-2*p+k+op + + where, + + w = width, p = padding, k = kernel_size, s = stride, op = output_padding """ def __init__(self, filters, kernel_size, strides=1, padding=0, output_padding=0, dilation=1, groups=1, layout='NCW', in_filters=0, activation=None, @@ -396,6 +465,8 @@ class Conv2DTranspose(_Conv): padding: An integer or a tuple/list of 2 integers, If padding is non-zero, then the input is implicitly zero-padded on both sides for padding number of points + out_padding : An integer or a tuple/list of 2 integers, + Zero-padding added to one side of the output dilation: An integer or tuple/list of 2 integers, specifying the dilation rate to use for dilated convolution. groups: int @@ -420,6 +491,24 @@ class Conv2DTranspose(_Conv): see Initializer. bias_initializer: Initializer for the bias vector see Initializer. + + + Input Shape: + This depends on the `layout` parameter. Input is 4D array of shape + (batch_size, in_channel(in_filters), height, width) if `layout` is `NCHW`. + + Output Shape: + This depends on the `layout` parameter. Output is 4D array of shape + (batch_size, out_channel(filters), out_height, out_width) if `layout` is `NCHW`. + out_height and out_width depends on other input parameters as well. + They are calculated as follows:: + + out_height = (h-1)*s-2*p+k+op + out_width = (w-1)*s-2*p+k+op + + where, + + h = height, w = width, p = padding, k = kernel_size, s = stride, op = output_padding """ def __init__(self, filters, kernel_size, strides=(1, 1), padding=(0, 0), output_padding=(0, 0), dilation=(1, 1), groups=1, layout='NCHW', @@ -465,6 +554,8 @@ class Conv3DTranspose(_Conv): padding: An integer or a tuple/list of 3 integers, If padding is non-zero, then the input is implicitly zero-padded on both sides for padding number of points + out_padding : An integer or a tuple/list of 2 integers, + Zero-padding added to one side of the output dilation: An integer or tuple/list of 3 integers, specifying the dilation rate to use for dilated convolution. groups: int @@ -489,6 +580,26 @@ class Conv3DTranspose(_Conv): see Initializer. bias_initializer: Initializer for the bias vector see Initializer. + + + Input Shape: + This depends on the `layout` parameter. Input is 5D array of shape + (batch_size, in_channel(in_filters), depth, height, width) if `layout` is `NCDHW`. + + Output Shape: + This depends on the `layout` parameter. Output is 5D array of shape + (batch_size, out_channel(filters), out_depth, out_height, out_width) if `layout` is `NCDHW`. + out_depth, out_height and out_width depends on other input parameters as well. + They are calculated as follows:: + + out_depth = (d-1)*s-2*p+k+op + out_height = (h-1)*s-2*p+k+op + out_width = (w-1)*s-2*p+k+op + + where, + + d = depth, h = height, w = width, p = padding, k = kernel_size, s = stride, + op = output_padding """ def __init__(self, filters, kernel_size, strides=(1, 1, 1), padding=(0, 0, 0), output_padding=(0, 0, 0), dilation=(1, 1, 1), groups=1, layout='NCDHW', @@ -539,9 +650,25 @@ class MaxPool1D(_Pooling): If padding is non-zero, then the input is implicitly zero-padded on both sides for padding number of points layout: A string, - Can be 'NCHW', 'NHWC', etc. - 'N', 'C', 'H', 'W' stands for batch, channel, and width (time) dimensions + Can be 'NCW', 'NWC', etc. + 'N', 'C', 'W' stands for batch, channel, and width (time) dimensions respectively. padding is applied on W dimension. + + + Input Shape: + This depends on the `layout` parameter. Input is 3D array of shape + (batch_size, channel, width) if `layout` is `NCW`. + + Output Shape: + This depends on the `layout` parameter. Output is 3D array of shape + (batch_size, channel, out_width) if `layout` is `NCW`. + out_width depends on other input parameters as well. It is calculated as follows:: + + out_width = ceil((w+2*p-ps)/s+1) + + where, + + w = width, p = padding, ps = pool_size, s = stride """ def __init__(self, pool_size=2, strides=None, padding=0, layout='NCW', **kwargs): assert layout == 'NCW', "Only supports NCW layout for now" @@ -570,6 +697,24 @@ class MaxPool2D(_Pooling): Can be 'NCHW', 'NHWC', etc. 'N', 'C', 'H', 'W' stands for batch, channel, height, and width dimensions respectively. padding is applied on 'H' and 'W' dimension. + + + Input Shape: + This depends on the `layout` parameter. Input is 4D array of shape + (batch_size, channel, height, width) if `layout` is `NCHW`. + + Output Shape: + This depends on the `layout` parameter. Output is 4D array of shape + (batch_size, channel, out_height, out_width) if `layout` is `NCHW`. + out_height and out_width depends on other input parameters as well. + They are calculated as follows:: + + out_height = ceil((h+2*p-ps)/s+1) + out_width = ceil((w+2*p-ps)/s+1) + + where, + + h = height, w = width, p = padding, ps = pool_size, s = stride """ def __init__(self, pool_size=(2, 2), strides=None, padding=0, layout='NCHW', **kwargs): assert layout == 'NCHW', "Only supports NCHW layout for now" @@ -599,6 +744,25 @@ class MaxPool3D(_Pooling): 'N', 'C', 'H', 'W', 'D' stands for batch, channel, height, width and depth dimensions respectively. padding is applied on 'D', 'H' and 'W' dimension. + + + Input Shape: + This depends on the `layout` parameter. Input is 5D array of shape + (batch_size, channel, depth, height, width) if `layout` is `NCDHW`. + + Output Shape: + This depends on the `layout` parameter. Output is 5D array of shape + (batch_size, channel, out_depth, out_height, out_width) if `layout` is `NCDHW`. + out_depth, out_height and out_width depends on other input parameters as well. + They are calculated as follows:: + + out_depth = ceil((d+2*p-ps)/s+1) + out_height = ceil((h+2*p-ps)/s+1) + out_width = ceil((w+2*p-ps)/s+1) + + where, + + d = depth, h = height, w = width, p = padding, ps = pool_size, s = stride """ def __init__(self, pool_size=(2, 2, 2), strides=None, padding=0, layout='NCDHW', **kwargs): assert layout == 'NCDHW', "Only supports NCDHW layout for now" @@ -622,9 +786,25 @@ class AvgPool1D(_Pooling): If padding is non-zero, then the input is implicitly zero-padded on both sides for padding number of points layout: A string, - Can be 'NCHW', 'NHWC', etc. - 'N', 'C', 'H', 'W' stands for batch, channel, and width (time) dimensions + Can be 'NCW', 'NWC', etc. + 'N', 'C', 'W' stands for batch, channel, and width (time) dimensions respectively. padding is applied on W dimension. + + + Input Shape: + This depends on the `layout` parameter. Input is 3D array of shape + (batch_size, channel, width) if `layout` is `NCW`. + + Output Shape: + This depends on the `layout` parameter. Output is 3D array of shape + (batch_size, channel, out_width) if `layout` is `NCW`. + out_width depends on other input parameters as well. It is calculated as follows:: + + out_width = ceil((w+2*p-ps)/s+1) + + where, + + w = width, p = padding, ps = pool_size, s = stride """ def __init__(self, pool_size=2, strides=None, padding=0, layout='NCW', **kwargs): assert layout == 'NCW', "Only supports NCW layout for now" @@ -653,6 +833,24 @@ class AvgPool2D(_Pooling): Can be 'NCHW', 'NHWC', etc. 'N', 'C', 'H', 'W' stands for batch, channel, height, and width dimensions respectively. padding is applied on 'H' and 'W' dimension. + + + Input Shape: + This depends on the `layout` parameter. Input is 4D array of shape + (batch_size, channel, height, width) if `layout` is `NCHW`. + + Output Shape: + This depends on the `layout` parameter. Output is 4D array of shape + (batch_size, channel, out_height, out_width) if `layout` is `NCHW`. + out_height and out_width depends on other input parameters as well. + They are calculated as follows:: + + out_height = ceil((h+2*p-ps)/s+1) + out_width = ceil((w+2*p-ps)/s+1) + + where, + + h = height, w = width, p = padding, ps = pool_size, s = stride """ def __init__(self, pool_size=(2, 2), strides=None, padding=0, layout='NCHW', **kwargs): assert layout == 'NCHW', "Only supports NCHW layout for now" @@ -682,6 +880,25 @@ class AvgPool3D(_Pooling): 'N', 'C', 'H', 'W', 'D' stands for batch, channel, height, width and depth dimensions respectively. padding is applied on 'D', 'H' and 'W' dimension. + + + Input Shape: + This depends on the `layout` parameter. Input is 5D array of shape + (batch_size, channel, depth, height, width) if `layout` is `NCDHW`. + + Output Shape: + This depends on the `layout` parameter. Output is 5D array of shape + (batch_size, channel, out_depth, out_height, out_width) if `layout` is `NCDHW`. + out_depth, out_height and out_width depends on other input parameters as well. + They are calculated as follows:: + + out_depth = ceil((d+2*p-ps)/s+1) + out_height = ceil((h+2*p-ps)/s+1) + out_width = ceil((w+2*p-ps)/s+1) + + where, + + d = depth, h = height, w = width, p = padding, ps = pool_size, s = stride """ def __init__(self, pool_size=(2, 2, 2), strides=None, padding=0, layout='NCDHW', **kwargs): assert layout == 'NCDHW', "Only supports NCDHW layout for now" diff --git a/python/mxnet/foo/nn/layer.py b/python/mxnet/foo/nn/layer.py index 7bdf86a63aed..d3596579a4b5 100644 --- a/python/mxnet/foo/nn/layer.py +++ b/python/mxnet/foo/nn/layer.py @@ -227,7 +227,7 @@ def __setattr__(self, name, value): def register_child(self, layer): if not isinstance(layer, HybridLayer): - if isinstance(layer, Sequantial): + if isinstance(layer, Sequential): raise ValueError( "Children of HybridLayer must also be HybridLayer. " \ "Please use HSequential instead of Sequantial.") @@ -331,11 +331,12 @@ def hybrid_forward(self, F, x, *args, **kwargs): class Sequential(Layer): """Stack Layers sequentially. - Example:: - net = nn.Sequential() - with net.name_scope(): - net.add(Dense(10, activation='relu')) - net.add(Dense(20)) + Example + ------- + >>> net = nn.Sequential() + >>> with net.name_scope(): + ... net.add(Dense(10, activation='relu')) + ... net.add(Dense(20)) """ def __init__(self, prefix=None, params=None): super(Sequential, self).__init__(prefix=prefix, params=params) @@ -353,11 +354,12 @@ def forward(self, x): class HSequential(HybridLayer): """Stack HybridLayers sequentially. - Example:: - net = nn.HSequential() - with net.name_scope(): - net.add(Dense(10, activation='relu')) - net.add(Dense(20)) + Example + ------- + >>> net = nn.HSequential() + >>> with net.name_scope(): + ... net.add(Dense(10, activation='relu')) + ... net.add(Dense(20)) """ def __init__(self, prefix=None, params=None): super(HSequential, self).__init__(prefix=prefix, params=params) @@ -405,13 +407,12 @@ class Dense(HybridLayer): params : ParameterDict or None See document of Layer. - Input shape - ----------- - a 2D input with shape `(batch_size, in_units)`. - Output shape - ------------ - the output would have shape `(batch_size, units)`. + Input shape: + a 2D input with shape `(batch_size, in_units)`. + + Output shape: + the output would have shape `(batch_size, units)`. """ def __init__(self, units, activation=None, use_bias=True, kernel_initializer=None, bias_initializer=None, @@ -442,20 +443,21 @@ def hybrid_forward(self, F, x, weight, bias=None): class Activation(HybridLayer): - """Applies an activation function to input. + """Applies an activation function to input. Refer + `mxnet.ndarray.Activation `_ + to learn more. Parameters ---------- activation: name of activation function to use See: help on Activation operator - Input shape - ----------- - Arbitrary. - Output shape - ------------ - Same shape as input. + Input shape: + Arbitrary. + + Output shape: + Same shape as input. """ def __init__(self, activation, **kwargs): self._act_type = activation @@ -473,7 +475,9 @@ class Dropout(HybridLayer): Dropout consists in randomly setting a fraction `rate` of input units to 0 at each update during training time, - which helps prevent overfitting. + which helps prevent overfitting. Refer + `mxnet.ndarray.Dropout `_ + to learn more. Parameters ---------- @@ -481,8 +485,8 @@ class Dropout(HybridLayer): References ---------- - [Dropout: A Simple Way to Prevent Neural Networks from Overfitting]( - http://www.cs.toronto.edu/~rsalakhu/papers/srivastava14a.pdf) + `Dropout: A Simple Way to Prevent Neural Networks from Overfitting + `_ """ def __init__(self, rate, **kwargs): super(Dropout, self).__init__(**kwargs) @@ -496,7 +500,9 @@ class BatchNorm(HybridLayer): """Batch normalization layer (Ioffe and Szegedy, 2014). Normalize the activations of the previous layer at each batch, i.e. applies a transformation that maintains the mean activation - close to 0 and the activation standard deviation close to 1. + close to 0 and the activation standard deviation close to 1. Refer + `mxnet.ndarray.BatchNorm `_ + to learn more. Parameters ---------- @@ -545,9 +551,14 @@ def hybrid_forward(self, F, x, gamma, beta, running_mean, running_var): class LeakyReLU(HybridLayer): """Leaky version of a Rectified Linear Unit. - It allows a small gradient when the unit is not active: - `f(x) = alpha * x for x < 0`, - `f(x) = x for x >= 0`. + It allows a small gradient when the unit is not active:: + + `f(x) = alpha * x for x < 0`, + `f(x) = x for x >= 0`. + + Refer + `mxnet.ndarray.LeakyReLU `_ + to learn more. Parameters ---------- @@ -567,6 +578,10 @@ class Embedding(HybridLayer): vectors of fixed size. eg. [[4], [20]] -> [[0.25, 0.1], [0.6, -0.2]] + Refer + `mxnet.ndarray.Embedding `_ + to learn more. + Parameters ---------- input_dim : int @@ -578,13 +593,12 @@ class Embedding(HybridLayer): embeddings_initializer : Initializer Initializer for the `embeddings` matrix - Input shape - ----------- - 2D tensor with shape: `(batch_size, sequence_length)`. - Output shape - ------------ - 3D tensor with shape: `(batch_size, sequence_length, output_dim)`. + Input shape: + 2D tensor with shape: `(batch_size, sequence_length)`. + + Output shape: + 3D tensor with shape: `(batch_size, sequence_length, output_dim)`. """ def __init__(self, input_dim, output_dim, dtype='float32', embeddings_initializer=None, **kwargs): From 4b3dc7b6454dced1f197eefc3351e1f52f69585c Mon Sep 17 00:00:00 2001 From: Eric Junyuan Xie Date: Mon, 26 Jun 2017 22:37:11 -0700 Subject: [PATCH 193/834] refactor rnn layers (#6823) * fix * refactor rnn layers * refactor RNN * udpate doc * fix * fix test --- docs/api/python/foo.md | 10 +- example/autograd/word_language_model/model.py | 21 +- python/mxnet/foo/nn/conv_layers.py | 61 +-- python/mxnet/foo/nn/layer.py | 61 +-- python/mxnet/foo/parameter.py | 4 +- python/mxnet/foo/rnn/__init__.py | 2 + python/mxnet/foo/rnn/rnn_cell.py | 486 ++++-------------- python/mxnet/foo/rnn/rnn_layer.py | 404 +++++++++++++++ python/mxnet/initializer.py | 7 +- python/mxnet/ndarray.py | 2 +- src/c_api/c_api_ndarray.cc | 4 +- tests/python/gpu/test_operator_gpu.py | 136 +---- tests/python/train/test_autograd.py | 6 +- tests/python/unittest/test_foo_rnn.py | 44 +- tests/python/unittest/test_nn.py | 2 +- 15 files changed, 651 insertions(+), 599 deletions(-) create mode 100644 python/mxnet/foo/rnn/rnn_layer.py diff --git a/docs/api/python/foo.md b/docs/api/python/foo.md index a76aef1c421e..85d4a353b026 100644 --- a/docs/api/python/foo.md +++ b/docs/api/python/foo.md @@ -154,13 +154,17 @@ in Python and then deploy with symbolic graph in C++ and Scala. :members: .. automethod:: __call__ -.. autoclass:: mxnet.foo.rnn.LSTMCell +.. autoclass:: mxnet.foo.rnn.RNN :members: -.. autoclass:: mxnet.foo.rnn.GRUCell +.. autoclass:: mxnet.foo.rnn.LSTM + :members: +.. autoclass:: mxnet.foo.rnn.GRU :members: .. autoclass:: mxnet.foo.rnn.RNNCell :members: -.. autoclass:: mxnet.foo.rnn.FusedRNNCell +.. autoclass:: mxnet.foo.rnn.LSTMCell + :members: +.. autoclass:: mxnet.foo.rnn.GRUCell :members: .. autoclass:: mxnet.foo.rnn.SequentialRNNCell :members: diff --git a/example/autograd/word_language_model/model.py b/example/autograd/word_language_model/model.py index 549e485aa7e3..44b4e913265d 100644 --- a/example/autograd/word_language_model/model.py +++ b/example/autograd/word_language_model/model.py @@ -10,9 +10,22 @@ def __init__(self, mode, vocab_size, num_embed, num_hidden, with self.name_scope(): self.drop = nn.Dropout(dropout) self.encoder = nn.Embedding(vocab_size, num_embed) - self.rnn = rnn.FusedRNNCell(num_hidden, num_layers, mode=mode, - dropout=dropout, get_next_state=True, - num_input=num_embed) + if mode == 'rnn_relu': + self.rnn = rnn.RNN(num_hidden, 'relu', num_layers, dropout=dropout, + input_size=num_embed) + elif mode == 'rnn_tanh': + self.rnn = rnn.RNN(num_hidden, num_layers, dropout=dropout, + input_size=num_embed) + elif mode == 'lstm': + self.rnn = rnn.LSTM(num_hidden, num_layers, dropout=dropout, + input_size=num_embed) + elif mode == 'gru': + self.rnn = rnn.GRU(num_hidden, num_layers, dropout=dropout, + input_size=num_embed) + else: + raise ValueError("Invalid mode %s. Options are rnn_relu, " + "rnn_tanh, lstm, and gru"%mode) + if tie_weights: self.decoder = nn.Dense(vocab_size, in_units=num_hidden, params=self.encoder.params) @@ -23,7 +36,7 @@ def __init__(self, mode, vocab_size, num_embed, num_hidden, def forward(self, inputs, hidden): emb = self.drop(self.encoder(inputs)) - output, hidden = self.rnn.unroll(None, emb, layout='TNC', merge_outputs=True) + output, hidden = self.rnn(emb, hidden) output = self.drop(output) decoded = self.decoder(output.reshape((-1, self.num_hidden))) return decoded, hidden diff --git a/python/mxnet/foo/nn/conv_layers.py b/python/mxnet/foo/nn/conv_layers.py index 59d58726a557..8236c6f7993a 100644 --- a/python/mxnet/foo/nn/conv_layers.py +++ b/python/mxnet/foo/nn/conv_layers.py @@ -53,14 +53,14 @@ class _Conv(HybridLayer): If you don't specify anything, no activation is applied (ie. "linear" activation: `a(x) = x`). use_bias: Boolean, whether the layer uses a bias vector. - kernel_initializer: Initializer for the `kernel` weights matrix + weight_initializer: Initializer for the `kernel` weights matrix see Initializer. bias_initializer: Initializer for the bias vector see Initializer. """ def __init__(self, filters, kernel_size, strides, padding, dilation, groups, layout, in_filters=0, activation=None, use_bias=True, - kernel_initializer=None, bias_initializer=None, + weight_initializer=None, bias_initializer=None, op_name='Convolution', prefix=None, params=None, **kwargs): super(_Conv, self).__init__(prefix=prefix, params=params) with self.name_scope(): @@ -84,7 +84,7 @@ def __init__(self, filters, kernel_size, strides, padding, dilation, dshape[layout.find('C')] = in_filters wshapes = _infer_weight_shape(op_name, dshape, self._kwargs) self.weight = self.params.get('weight', shape=wshapes[1], - init=kernel_initializer) + init=weight_initializer) if use_bias: self.bias = self.params.get('bias', shape=wshapes[2], init=bias_initializer) @@ -154,7 +154,7 @@ class Conv1D(_Conv): If you don't specify anything, no activation is applied (ie. "linear" activation: `a(x) = x`). use_bias: Boolean, whether the layer uses a bias vector. - kernel_initializer: Initializer for the `kernel` weights matrix + weight_initializer: Initializer for the `kernel` weights matrix see Initializer. bias_initializer: Initializer for the bias vector see Initializer. @@ -176,14 +176,15 @@ class Conv1D(_Conv): w = width, p = padding, d = dilation, k = kernel_size, s = stride """ def __init__(self, filters, kernel_size, strides=1, padding=0, dilation=1, - groups=1, layout='NCW', in_filters=0, activation=None, use_bias=True, - kernel_initializer=None, bias_initializer=None, **kwargs): + groups=1, layout='NCW', activation=None, use_bias=True, + weight_initializer=None, bias_initializer=None, + in_filters=0, **kwargs): if isinstance(kernel_size, numeric_types): kernel_size = (kernel_size,) assert len(kernel_size) == 1, "kernel_size must be a number or a list of 1 ints" super(Conv1D, self).__init__( filters, kernel_size, strides, padding, dilation, groups, layout, - in_filters, activation, use_bias, kernel_initializer, bias_initializer, **kwargs) + in_filters, activation, use_bias, weight_initializer, bias_initializer, **kwargs) class Conv2D(_Conv): @@ -233,7 +234,7 @@ class Conv2D(_Conv): If you don't specify anything, no activation is applied (ie. "linear" activation: `a(x) = x`). use_bias: Boolean, whether the layer uses a bias vector. - kernel_initializer: Initializer for the `kernel` weights matrix + weight_initializer: Initializer for the `kernel` weights matrix see Initializer. bias_initializer: Initializer for the bias vector see Initializer. @@ -257,15 +258,15 @@ class Conv2D(_Conv): w = width, h = height, p = padding, d = dilation, k = kernel_size, s = stride """ def __init__(self, filters, kernel_size, strides=(1, 1), padding=(0, 0), - dilation=(1, 1), groups=1, layout='NCHW', in_filters=0, - activation=None, use_bias=True, - kernel_initializer=None, bias_initializer=None, **kwargs): + dilation=(1, 1), groups=1, layout='NCHW', + activation=None, use_bias=True, weight_initializer=None, + bias_initializer=None, in_filters=0, **kwargs): if isinstance(kernel_size, numeric_types): kernel_size = (kernel_size,)*2 assert len(kernel_size) == 2, "kernel_size must be a number or a list of 2 ints" super(Conv2D, self).__init__( filters, kernel_size, strides, padding, dilation, groups, layout, - in_filters, activation, use_bias, kernel_initializer, bias_initializer, **kwargs) + in_filters, activation, use_bias, weight_initializer, bias_initializer, **kwargs) class Conv3D(_Conv): @@ -315,7 +316,7 @@ class Conv3D(_Conv): If you don't specify anything, no activation is applied (ie. "linear" activation: `a(x) = x`). use_bias: Boolean, whether the layer uses a bias vector. - kernel_initializer: Initializer for the `kernel` weights matrix + weight_initializer: Initializer for the `kernel` weights matrix see Initializer. bias_initializer: Initializer for the bias vector see Initializer. @@ -340,15 +341,15 @@ class Conv3D(_Conv): d = depth, h = height, w = width, p = padding, d = dilation, k = kernel_size, s = stride """ def __init__(self, filters, kernel_size, strides=(1, 1, 1), padding=(0, 0, 0), - dilation=(1, 1, 1), groups=1, layout='NCDHW', in_filters=0, - activation=None, use_bias=True, - kernel_initializer=None, bias_initializer=None, **kwargs): + dilation=(1, 1, 1), groups=1, layout='NCDHW', activation=None, + use_bias=True, weight_initializer=None, bias_initializer=None, + in_filters=0, **kwargs): if isinstance(kernel_size, numeric_types): kernel_size = (kernel_size,)*3 assert len(kernel_size) == 3, "kernel_size must be a number or a list of 3 ints" super(Conv3D, self).__init__( filters, kernel_size, strides, padding, dilation, groups, layout, - in_filters, activation, use_bias, kernel_initializer, bias_initializer, **kwargs) + in_filters, activation, use_bias, weight_initializer, bias_initializer, **kwargs) class Conv1DTranspose(_Conv): @@ -400,7 +401,7 @@ class Conv1DTranspose(_Conv): If you don't specify anything, no activation is applied (ie. "linear" activation: `a(x) = x`). use_bias: Boolean, whether the layer uses a bias vector. - kernel_initializer: Initializer for the `kernel` weights matrix + weight_initializer: Initializer for the `kernel` weights matrix see Initializer. bias_initializer: Initializer for the bias vector see Initializer. @@ -422,9 +423,9 @@ class Conv1DTranspose(_Conv): w = width, p = padding, k = kernel_size, s = stride, op = output_padding """ def __init__(self, filters, kernel_size, strides=1, padding=0, output_padding=0, - dilation=1, groups=1, layout='NCW', in_filters=0, activation=None, - use_bias=True, kernel_initializer=None, bias_initializer=None, - **kwargs): + dilation=1, groups=1, layout='NCW', activation=None, use_bias=True, + weight_initializer=None, bias_initializer=None, + in_filters=0, **kwargs): if isinstance(kernel_size, numeric_types): kernel_size = (kernel_size,) if isinstance(output_padding, numeric_types): @@ -433,7 +434,7 @@ def __init__(self, filters, kernel_size, strides=1, padding=0, output_padding=0, assert len(output_padding) == 1, "output_padding must be a number or a list of 1 ints" super(Conv1DTranspose, self).__init__( filters, kernel_size, strides, padding, dilation, groups, layout, - in_filters, activation, use_bias, kernel_initializer, + in_filters, activation, use_bias, weight_initializer, bias_initializer, op_name='Deconvolution', adj=output_padding, **kwargs) @@ -487,7 +488,7 @@ class Conv2DTranspose(_Conv): If you don't specify anything, no activation is applied (ie. "linear" activation: `a(x) = x`). use_bias: Boolean, whether the layer uses a bias vector. - kernel_initializer: Initializer for the `kernel` weights matrix + weight_initializer: Initializer for the `kernel` weights matrix see Initializer. bias_initializer: Initializer for the bias vector see Initializer. @@ -512,8 +513,8 @@ class Conv2DTranspose(_Conv): """ def __init__(self, filters, kernel_size, strides=(1, 1), padding=(0, 0), output_padding=(0, 0), dilation=(1, 1), groups=1, layout='NCHW', - in_filters=0, activation=None, use_bias=True, - kernel_initializer=None, bias_initializer=None, **kwargs): + activation=None, use_bias=True, weight_initializer=None, + bias_initializer=None, in_filters=0, **kwargs): if isinstance(kernel_size, numeric_types): kernel_size = (kernel_size,)*2 if isinstance(output_padding, numeric_types): @@ -522,7 +523,7 @@ def __init__(self, filters, kernel_size, strides=(1, 1), padding=(0, 0), assert len(output_padding) == 2, "output_padding must be a number or a list of 2 ints" super(Conv2DTranspose, self).__init__( filters, kernel_size, strides, padding, dilation, groups, layout, - in_filters, activation, use_bias, kernel_initializer, + in_filters, activation, use_bias, weight_initializer, bias_initializer, op_name='Deconvolution', adj=output_padding, **kwargs) @@ -576,7 +577,7 @@ class Conv3DTranspose(_Conv): If you don't specify anything, no activation is applied (ie. "linear" activation: `a(x) = x`). use_bias: Boolean, whether the layer uses a bias vector. - kernel_initializer: Initializer for the `kernel` weights matrix + weight_initializer: Initializer for the `kernel` weights matrix see Initializer. bias_initializer: Initializer for the bias vector see Initializer. @@ -603,8 +604,8 @@ class Conv3DTranspose(_Conv): """ def __init__(self, filters, kernel_size, strides=(1, 1, 1), padding=(0, 0, 0), output_padding=(0, 0, 0), dilation=(1, 1, 1), groups=1, layout='NCDHW', - in_filters=0, activation=None, use_bias=True, - kernel_initializer=None, bias_initializer=None, **kwargs): + activation=None, use_bias=True, weight_initializer=None, + bias_initializer=None, in_filters=0, **kwargs): if isinstance(kernel_size, numeric_types): kernel_size = (kernel_size,)*3 if isinstance(output_padding, numeric_types): @@ -613,7 +614,7 @@ def __init__(self, filters, kernel_size, strides=(1, 1, 1), padding=(0, 0, 0), assert len(output_padding) == 3, "output_padding must be a number or a list of 3 ints" super(Conv3DTranspose, self).__init__( filters, kernel_size, strides, padding, dilation, groups, layout, - in_filters, activation, use_bias, kernel_initializer, bias_initializer, + in_filters, activation, use_bias, weight_initializer, bias_initializer, op_name='Deconvolution', adj=output_padding, **kwargs) diff --git a/python/mxnet/foo/nn/layer.py b/python/mxnet/foo/nn/layer.py index d3596579a4b5..48e13570a202 100644 --- a/python/mxnet/foo/nn/layer.py +++ b/python/mxnet/foo/nn/layer.py @@ -19,25 +19,28 @@ def __init__(self, layer): self._old_scope = None @staticmethod - def create_prefix(prefix, hint): - if _LayerScope._current is None: + def create(prefix, params, hint): + """Create prefix and params for new layer.""" + current = _LayerScope._current + if current is None: if prefix is None: - return _name.NameManager.current.get(None, hint) + '_' - return prefix + prefix = _name.NameManager.current.get(None, hint) + '_' + if params is None: + params = ParameterDict(prefix) + else: + params = ParameterDict(params.prefix, params) + return prefix, params + + if prefix is None: + count = current._counter.get(hint, 0) + prefix = '%s%d_'%(hint, count) + current._counter[hint] = count + 1 + if params is None: + parent = current._layer.params + params = ParameterDict(parent.prefix+prefix, parent._shared) else: - if prefix is None: - count = _LayerScope._current._counter.get(hint, 0) - prefix = '%s%d_'%(hint, count) - _LayerScope._current._counter[hint] = count + 1 - return _LayerScope._current._layer.prefix+prefix - - @staticmethod - def create_params(prefix, params): - if params is not None: - return ParameterDict(params.prefix, params) - if _LayerScope._current is not None: - return ParameterDict(prefix, _LayerScope._current._layer._params._shared) - return ParameterDict(prefix) + params = ParameterDict(params.prefix, params) + return current._layer.prefix+prefix, params def __enter__(self): self._old_scope = _LayerScope._current @@ -121,8 +124,7 @@ def forward(self, x): Layer supports forwarding with both `Symbol` and `NDArray`.""" def __init__(self, prefix=None, params=None): - self._prefix = _LayerScope.create_prefix(prefix, self._alias()) - self._params = _LayerScope.create_params(self._prefix, params) + self._prefix, self._params = _LayerScope.create(prefix, params, self._alias()) self._scope = _LayerScope(self) self._children = [] @@ -258,11 +260,12 @@ def _get_graph(self, *args): def infer_shape(self, *args): """Infer shape of Parameters from inputs.""" syms, out = self._get_graph(*args) + args, _, = _flatten(args) arg_shapes, _, aux_shapes = out.infer_shape( **{i.name: j.shape for i, j in zip(syms, args)}) sdict = {i: j for i, j in zip(out.list_arguments(), arg_shapes)} - sdict.update( - {name : shape for name, shape in zip(out.list_auxiliary_states(), aux_shapes)}) + sdict.update({name : shape for name, shape in \ + zip(out.list_auxiliary_states(), aux_shapes)}) for i in self.all_params().values(): i.shape = sdict[i.name] @@ -395,7 +398,7 @@ class Dense(HybridLayer): If you don't specify anything, no activation is applied (ie. "linear" activation: `a(x) = x`). use_bias: Boolean, whether the layer uses a bias vector. - kernel_initializer: Initializer for the `kernel` weights matrix + weight_initializer: Initializer for the `kernel` weights matrix (see mxnet.initializer). bias_initializer: Initializer for the bias vector (see mxnet.initializer). @@ -415,13 +418,13 @@ class Dense(HybridLayer): the output would have shape `(batch_size, units)`. """ def __init__(self, units, activation=None, use_bias=True, - kernel_initializer=None, bias_initializer=None, + weight_initializer=None, bias_initializer=None, in_units=0, **kwargs): super(Dense, self).__init__(**kwargs) with self.name_scope(): self._units = units self.weight = self.params.get('weight', shape=(units, in_units), - init=kernel_initializer) + init=weight_initializer) if use_bias: self.bias = self.params.get('bias', shape=(units,), init=bias_initializer) @@ -526,9 +529,9 @@ class BatchNorm(HybridLayer): moving_variance_initializer: Initializer for the moving variance. """ def __init__(self, axis=1, momentum=0.9, epsilon=1e-3, center=True, scale=True, - num_features=0, beta_initializer='zeros', gamma_initializer='ones', + beta_initializer='zeros', gamma_initializer='ones', running_mean_initializer='zeros', running_variance_initializer='ones', - **kwargs): + num_features=0, **kwargs): super(BatchNorm, self).__init__(**kwargs) self._kwargs = {'axis': axis, 'eps': epsilon, 'momentum': momentum, 'fix_gamma': not center} @@ -590,7 +593,7 @@ class Embedding(HybridLayer): Dimension of the dense embedding. dtype : str or np.dtype, default 'float32' Data type of output embeddings. - embeddings_initializer : Initializer + weight_initializer : Initializer Initializer for the `embeddings` matrix @@ -601,12 +604,12 @@ class Embedding(HybridLayer): 3D tensor with shape: `(batch_size, sequence_length, output_dim)`. """ def __init__(self, input_dim, output_dim, dtype='float32', - embeddings_initializer=None, **kwargs): + weight_initializer=None, **kwargs): super(Embedding, self).__init__(**kwargs) self._kwargs = {'input_dim': input_dim, 'output_dim': output_dim, 'dtype': dtype} self.weight = self.params.get('weight', shape=(input_dim, output_dim), - init=embeddings_initializer) + init=weight_initializer) def hybrid_forward(self, F, x, weight): return F.Embedding(x, weight, **self._kwargs) diff --git a/python/mxnet/foo/parameter.py b/python/mxnet/foo/parameter.py index 6d772772dbc6..a44d462a136d 100644 --- a/python/mxnet/foo/parameter.py +++ b/python/mxnet/foo/parameter.py @@ -371,7 +371,7 @@ def update(self, other): else: self._params[k] = v - def initialize(self, init=initializer.Xavier(), ctx=None): + def initialize(self, init=initializer.Xavier(), ctx=None, verbose=False): """Intialize all Parameters manage by this dictionary to be used for `NDArray` API. Has no effect when using `Symbol` API. @@ -383,6 +383,8 @@ def initialize(self, init=initializer.Xavier(), ctx=None): ctx : Context or list of Context Keep a copy of Parameters on one or many context(s). """ + if verbose: + init.set_verbosity(verbose=verbose) for _, v in self.items(): v.initialize(None, ctx, init) diff --git a/python/mxnet/foo/rnn/__init__.py b/python/mxnet/foo/rnn/__init__.py index 3fc69b0000d9..b4554ad884cb 100644 --- a/python/mxnet/foo/rnn/__init__.py +++ b/python/mxnet/foo/rnn/__init__.py @@ -3,3 +3,5 @@ """Recurrent neural network module.""" from .rnn_cell import * + +from .rnn_layer import * diff --git a/python/mxnet/foo/rnn/rnn_cell.py b/python/mxnet/foo/rnn/rnn_cell.py index 495736a90cf5..46227b56d967 100644 --- a/python/mxnet/foo/rnn/rnn_cell.py +++ b/python/mxnet/foo/rnn/rnn_cell.py @@ -5,33 +5,18 @@ """Definition of various recurrent neural network cells.""" from __future__ import print_function -import warnings - -from ... import symbol, init, ndarray +from ... import symbol, ndarray from ...base import string_types, numeric_types from ..nn import Layer, HybridLayer from .. import tensor_types -def _cells_state_shape(cells): - return sum([c.state_shape for c in cells], []) - def _cells_state_info(cells, batch_size): return sum([c.state_info(batch_size) for c in cells], []) def _cells_begin_state(cells, **kwargs): return sum([c.begin_state(**kwargs) for c in cells], []) -def _cells_unpack_weights(cells, args): - for cell in cells: - args = cell.unpack_weights(args) - return args - -def _cells_pack_weights(cells, args): - for cell in cells: - args = cell.pack_weights(args) - return args - def _get_begin_state(cell, F, begin_state, inputs, batch_size): if begin_state is None: if F is ndarray: @@ -111,21 +96,11 @@ def state_info(self, batch_size=0): """shape and layout information of states""" raise NotImplementedError() - @property - def state_shape(self): - """shape(s) of states""" - return [ele['shape'] for ele in self.state_info()] - - @property - def _gate_names(self): - """name(s) of gates""" - return () - @property def _curr_prefix(self): return '%st%d_'%(self.prefix, self._counter) - def begin_state(self, func=symbol.zeros, batch_size=0, **kwargs): + def begin_state(self, batch_size=0, func=ndarray.zeros, **kwargs): """Initial state for this cell. Parameters @@ -166,76 +141,6 @@ def begin_state(self, func=symbol.zeros, batch_size=0, **kwargs): states.append(state) return states - def unpack_weights(self, args): - """Unpack fused weight matrices into separate - weight matrices. - - For example, say you use a module object `mod` to run a network that has an lstm cell. - In `mod.get_params()[0]`, the lstm parameters are all represented as a single big vector. - `cell.unpack_weights(mod.get_params()[0])` will unpack this vector into a dictionary of - more readable lstm parameters - c, f, i, o gates for i2h (input to hidden) and - h2h (hidden to hidden) weights. - - Parameters - ---------- - args : dict of str -> NDArray - Dictionary containing packed weights. - usually from `Module.get_params()[0]`. - - Returns - ------- - args : dict of str -> NDArray - Dictionary with unpacked weights associated with - this cell. - - See Also - -------- - pack_weights: Performs the reverse operation of this function. - """ - args = args.copy() - if not self._gate_names: - return args - h = self._num_hidden - for group_name in ['i2h', 'h2h']: - weight = args.pop('%s%s_weight'%(self._prefix, group_name)) - bias = args.pop('%s%s_bias' % (self._prefix, group_name)) - for j, gate in enumerate(self._gate_names): - wname = '%s%s%s_weight' % (self._prefix, group_name, gate) - args[wname] = weight[j*h:(j+1)*h].copy() - bname = '%s%s%s_bias' % (self._prefix, group_name, gate) - args[bname] = bias[j*h:(j+1)*h].copy() - return args - - def pack_weights(self, args): - """Pack separate weight matrices into a single packed - weight. - - Parameters - ---------- - args : dict of str -> NDArray - Dictionary containing unpacked weights. - - Returns - ------- - args : dict of str -> NDArray - Dictionary with packed weights associated with - this cell. - """ - args = args.copy() - if not self._gate_names: - return args - for group_name in ['i2h', 'h2h']: - weight = [] - bias = [] - for gate in self._gate_names: - wname = '%s%s%s_weight'%(self._prefix, group_name, gate) - weight.append(args.pop(wname)) - bname = '%s%s%s_bias'%(self._prefix, group_name, gate) - bias.append(args.pop(bname)) - args['%s%s_weight'%(self._prefix, group_name)] = ndarray.concatenate(weight) - args['%s%s_bias'%(self._prefix, group_name)] = ndarray.concatenate(bias) - return args - def unroll(self, length, inputs, begin_state=None, layout='NTC', merge_outputs=None): """Unroll an RNN cell across time steps. @@ -346,10 +251,20 @@ class RNNCell(HRecurrentCell): Parameters ---------- - num_hidden : int + hidden_size : int number of units in output symbol activation : str or Symbol, default 'tanh' - type of activation function + type of activation function. + i2h_weight_initializer : str or Initializer + Initializer for the input weights matrix, used for the linear + transformation of the inputs. + h2h_weight_initializer : str or Initializer + Initializer for the recurrent weights matrix, used for the linear + transformation of the recurrent state. + i2h_bias_initializer : str or Initializer + Initializer for the bias vector. + h2h_bias_initializer : str or Initializer + Initializer for the bias vector. prefix : str, default 'rnn_' prefix for name of layers (and name of weight if params is None) @@ -357,35 +272,37 @@ class RNNCell(HRecurrentCell): container for weight sharing between cells. created if None. """ - def __init__(self, num_hidden, activation='tanh', num_input=0, - prefix=None, params=None): + def __init__(self, hidden_size, activation='tanh', + i2h_weight_initializer=None, h2h_weight_initializer=None, + i2h_bias_initializer=None, h2h_bias_initializer=None, + input_size=0, prefix=None, params=None): super(RNNCell, self).__init__(prefix=prefix, params=params) - self._num_hidden = num_hidden + self._hidden_size = hidden_size self._activation = activation - self._num_input = num_input - self.i2h_weight = self.params.get('i2h_weight', shape=(num_hidden, num_input)) - self.i2h_bias = self.params.get('i2h_bias', shape=(num_hidden,)) - self.h2h_weight = self.params.get('h2h_weight', shape=(num_hidden, num_hidden)) - self.h2h_bias = self.params.get('h2h_bias', shape=(num_hidden,)) + self._input_size = input_size + self.i2h_weight = self.params.get('i2h_weight', shape=(hidden_size, input_size), + init=i2h_weight_initializer) + self.h2h_weight = self.params.get('h2h_weight', shape=(hidden_size, hidden_size), + init=h2h_weight_initializer) + self.i2h_bias = self.params.get('i2h_bias', shape=(hidden_size,), + init=i2h_bias_initializer) + self.h2h_bias = self.params.get('h2h_bias', shape=(hidden_size,), + init=h2h_bias_initializer) def state_info(self, batch_size=0): - return [{'shape': (batch_size, self._num_hidden), '__layout__': 'NC'}] - - @property - def _gate_names(self): - return ('',) + return [{'shape': (batch_size, self._hidden_size), '__layout__': 'NC'}] def _alias(self): return 'rnn' - def hybrid_forward(self, F, inputs, states, i2h_weight, i2h_bias, - h2h_weight, h2h_bias): + def hybrid_forward(self, F, inputs, states, i2h_weight, + h2h_weight, i2h_bias, h2h_bias): name = self._curr_prefix i2h = F.FullyConnected(data=inputs, weight=i2h_weight, bias=i2h_bias, - num_hidden=self._num_hidden, + num_hidden=self._hidden_size, name='%si2h'%name) h2h = F.FullyConnected(data=states[0], weight=h2h_weight, bias=h2h_bias, - num_hidden=self._num_hidden, + num_hidden=self._hidden_size, name='%sh2h'%name) output = self._get_activation(F, i2h + h2h, self._activation, name='%sout'%name) @@ -398,49 +315,59 @@ class LSTMCell(HRecurrentCell): Parameters ---------- - num_hidden : int - number of units in output symbol + hidden_size : int + number of units in output symbol. + i2h_weight_initializer : str or Initializer + Initializer for the input weights matrix, used for the linear + transformation of the inputs. + h2h_weight_initializer : str or Initializer + Initializer for the recurrent weights matrix, used for the linear + transformation of the recurrent state. + i2h_bias_initializer : str or Initializer, default 'lstmbias' + Initializer for the bias vector. By default bias for the forget + gate is initialized to 1 while all other biases are initialized + to zero. + h2h_bias_initializer : str or Initializer + Initializer for the bias vector. prefix : str, default 'lstm_' prefix for name of layers (and name of weight if params is None) params : Parameter or None container for weight sharing between cells. created if None. - forget_bias : bias added to forget gate, default 1.0. - Jozefowicz et al. 2015 recommends setting this to 1.0 """ - def __init__(self, num_hidden, forget_bias=1.0, num_input=0, - prefix=None, params=None): + def __init__(self, hidden_size, + i2h_weight_initializer=None, h2h_weight_initializer=None, + i2h_bias_initializer='lstmbias', h2h_bias_initializer=None, + input_size=0, prefix=None, params=None): super(LSTMCell, self).__init__(prefix=prefix, params=params) - self._num_hidden = num_hidden - self._num_input = num_input - self.i2h_weight = self.params.get('i2h_weight', shape=(4*num_hidden, num_input)) - self.h2h_weight = self.params.get('h2h_weight', shape=(4*num_hidden, num_hidden)) - # we add the forget_bias to i2h_bias, this adds the bias to the forget gate activation - self.i2h_bias = self.params.get('i2h_bias', shape=(4*num_hidden,), - init=init.LSTMBias(forget_bias=forget_bias)) - self.h2h_bias = self.params.get('h2h_bias', shape=(4*num_hidden,)) + self._hidden_size = hidden_size + self._input_size = input_size + self.i2h_weight = self.params.get('i2h_weight', shape=(4*hidden_size, input_size), + init=i2h_weight_initializer) + self.h2h_weight = self.params.get('h2h_weight', shape=(4*hidden_size, hidden_size), + init=h2h_weight_initializer) + self.i2h_bias = self.params.get('i2h_bias', shape=(4*hidden_size,), + init=i2h_bias_initializer) + self.h2h_bias = self.params.get('h2h_bias', shape=(4*hidden_size,), + init=h2h_bias_initializer) def state_info(self, batch_size=0): - return [{'shape': (batch_size, self._num_hidden), '__layout__': 'NC'}, - {'shape': (batch_size, self._num_hidden), '__layout__': 'NC'}] - - @property - def _gate_names(self): - return ['_i', '_f', '_c', '_o'] + return [{'shape': (batch_size, self._hidden_size), '__layout__': 'NC'}, + {'shape': (batch_size, self._hidden_size), '__layout__': 'NC'}] def _alias(self): return 'lstm' - def hybrid_forward(self, F, inputs, states, i2h_weight, i2h_bias, - h2h_weight, h2h_bias): + def hybrid_forward(self, F, inputs, states, i2h_weight, + h2h_weight, i2h_bias, h2h_bias): name = self._curr_prefix i2h = F.FullyConnected(data=inputs, weight=i2h_weight, bias=i2h_bias, - num_hidden=self._num_hidden*4, + num_hidden=self._hidden_size*4, name='%si2h'%name) h2h = F.FullyConnected(data=states[0], weight=h2h_weight, bias=h2h_bias, - num_hidden=self._num_hidden*4, + num_hidden=self._hidden_size*4, name='%sh2h'%name) gates = i2h + h2h slice_gates = F.SliceChannel(gates, num_outputs=4, @@ -468,8 +395,18 @@ class GRUCell(HRecurrentCell): Parameters ---------- - num_hidden : int - number of units in output symbol + hidden_size : int + number of units in output symbol. + i2h_weight_initializer : str or Initializer + Initializer for the input weights matrix, used for the linear + transformation of the inputs. + h2h_weight_initializer : str or Initializer + Initializer for the recurrent weights matrix, used for the linear + transformation of the recurrent state. + i2h_bias_initializer : str or Initializer + Initializer for the bias vector. + h2h_bias_initializer : str or Initializer + Initializer for the bias vector. prefix : str, default 'gru_' prefix for name of layers (and name of weight if params is None) @@ -477,38 +414,41 @@ class GRUCell(HRecurrentCell): container for weight sharing between cells. created if None. """ - def __init__(self, num_hidden, num_input=0, prefix=None, params=None): + def __init__(self, hidden_size, + i2h_weight_initializer=None, h2h_weight_initializer=None, + i2h_bias_initializer=None, h2h_bias_initializer=None, + input_size=0, prefix=None, params=None): super(GRUCell, self).__init__(prefix=prefix, params=params) - self._num_hidden = num_hidden - self.i2h_weight = self.params.get('i2h_weight', shape=(3*num_hidden, num_input)) - self.h2h_weight = self.params.get('h2h_weight', shape=(3*num_hidden, num_hidden)) - self.i2h_bias = self.params.get('i2h_bias', shape=(3*num_hidden)) - self.h2h_bias = self.params.get('h2h_bias', shape=(3*num_hidden)) + self._hidden_size = hidden_size + self.i2h_weight = self.params.get('i2h_weight', shape=(3*hidden_size, input_size), + init=i2h_weight_initializer) + self.h2h_weight = self.params.get('h2h_weight', shape=(3*hidden_size, hidden_size), + init=h2h_weight_initializer) + self.i2h_bias = self.params.get('i2h_bias', shape=(3*hidden_size,), + init=i2h_bias_initializer) + self.h2h_bias = self.params.get('h2h_bias', shape=(3*hidden_size,), + init=h2h_bias_initializer) def state_info(self, batch_size=0): - return [{'shape': (batch_size, self._num_hidden), '__layout__': 'NC'}] - - @property - def _gate_names(self): - return ['_r', '_z', '_o'] + return [{'shape': (batch_size, self._hidden_size), '__layout__': 'NC'}] def _alias(self): return 'gru' - def hybrid_forward(self, F, inputs, states, i2h_weight, i2h_bias, - h2h_weight, h2h_bias): + def hybrid_forward(self, F, inputs, states, i2h_weight, + h2h_weight, i2h_bias, h2h_bias): # pylint: disable=too-many-locals name = self._curr_prefix prev_state_h = states[0] i2h = F.FullyConnected(data=inputs, weight=i2h_weight, bias=i2h_bias, - num_hidden=self._num_hidden * 3, + num_hidden=self._hidden_size * 3, name="%si2h" % name) h2h = F.FullyConnected(data=prev_state_h, weight=h2h_weight, bias=h2h_bias, - num_hidden=self._num_hidden * 3, + num_hidden=self._hidden_size * 3, name="%sh2h" % name) i2h_r, i2h_z, i2h = F.SliceChannel(i2h, num_outputs=3, name="%si2h_slice" % name) @@ -528,209 +468,10 @@ def hybrid_forward(self, F, inputs, states, i2h_weight, i2h_bias, return next_h, [next_h] -class FusedRNNCell(HRecurrentCell): - """Fusing RNN layers across time step into one kernel. - Improves speed but is less flexible. Currently only - supported if using cuDNN on GPU. - - Parameters - ---------- - """ - def __init__(self, num_hidden, num_layers=1, mode='lstm', bidirectional=False, - dropout=0., get_next_state=False, forget_bias=1.0, num_input=0, - prefix=None, params=None): - self._num_hidden = num_hidden - self._num_layers = num_layers - self._mode = mode - self._bidirectional = bidirectional - self._dropout = dropout - self._get_next_state = get_next_state - self._directions = ['l', 'r'] if bidirectional else ['l'] - super(FusedRNNCell, self).__init__(prefix=prefix, params=params) - - initializer = init.FusedRNN(None, num_hidden, num_layers, mode, - bidirectional, forget_bias) - self.parameters = self.params.get('parameters', init=initializer, - shape=(self._num_input_to_size(num_input),)) - - def state_info(self, batch_size=0): - b = self._bidirectional + 1 - n = (self._mode == 'lstm') + 1 - return [{'shape': (b*self._num_layers, batch_size, self._num_hidden), - '__layout__': 'LNC'} for _ in range(n)] - - @property - def _gate_names(self): - return {'rnn_relu': [''], - 'rnn_tanh': [''], - 'lstm': ['_i', '_f', '_c', '_o'], - 'gru': ['_r', '_z', '_o']}[self._mode] - - @property - def _num_gates(self): - return len(self._gate_names) - - def _alias(self): - return self._mode - - def _size_to_num_input(self, size): - b = len(self._directions) - m = self._num_gates - h = self._num_hidden - return size//b//h//m - (self._num_layers - 1)*(h+b*h+2) - h - 2 - - def _num_input_to_size(self, num_input): - if num_input == 0: - return 0 - b = self._bidirectional + 1 - m = self._num_gates - h = self._num_hidden - return (num_input+h+2)*h*m*b + (self._num_layers-1)*m*h*(h+b*h+2)*b - - def _slice_weights(self, arr, li, lh): - """slice fused rnn weights""" - args = {} - gate_names = self._gate_names - directions = self._directions - - b = len(directions) - p = 0 - for layer in range(self._num_layers): - for direction in directions: - for gate in gate_names: - name = '%s%s%d_i2h%s_weight'%(self._prefix, direction, layer, gate) - if layer > 0: - size = b*lh*lh - args[name] = arr[p:p+size].reshape((lh, b*lh)) - else: - size = li*lh - args[name] = arr[p:p+size].reshape((lh, li)) - p += size - for gate in gate_names: - name = '%s%s%d_h2h%s_weight'%(self._prefix, direction, layer, gate) - size = lh**2 - args[name] = arr[p:p+size].reshape((lh, lh)) - p += size - - for layer in range(self._num_layers): - for direction in directions: - for gate in gate_names: - name = '%s%s%d_i2h%s_bias'%(self._prefix, direction, layer, gate) - args[name] = arr[p:p+lh] - p += lh - for gate in gate_names: - name = '%s%s%d_h2h%s_bias'%(self._prefix, direction, layer, gate) - args[name] = arr[p:p+lh] - p += lh - - assert p == arr.size, "Invalid parameters size for FusedRNNCell" - return args - - def unpack_weights(self, args): - args = args.copy() - arr = args.pop(self.parameters.name) - num_input = self._size_to_num_input(arr.size) - nargs = self._slice_weights(arr, num_input, self._num_hidden) - args.update({name: nd.copy() for name, nd in nargs.items()}) - return args - - def pack_weights(self, args): - args = args.copy() - w0 = args['%sl0_i2h%s_weight'%(self._prefix, self._gate_names[0])] - num_input = w0.shape[1] - total = self._num_input_to_size(num_input) - - arr = ndarray.zeros((total,), ctx=w0.context, dtype=w0.dtype) - for name, nd in self._slice_weights(arr, num_input, self._num_hidden).items(): - nd[:] = args.pop(name) - args[self.parameters.name] = arr - return args - - def __call__(self, inputs, states): - raise NotImplementedError("FusedRNNCell cannot be stepped. Please use unroll") - - def unroll(self, length, inputs, begin_state=None, layout='NTC', merge_outputs=None): - self.reset() - - inputs, axis, F, batch_size = _format_sequence(length, inputs, layout, True) - if axis == 1: - warnings.warn("NTC layout detected. Consider using " - "TNC for FusedRNNCell for faster speed") - inputs = F.swapaxes(inputs, dim1=0, dim2=1) - else: - assert axis == 0, "Unsupported layout %s"%layout - begin_state = _get_begin_state(self, F, begin_state, inputs, batch_size) - - states = begin_state - if self._mode == 'lstm': - states = {'state': states[0], 'state_cell': states[1]} # pylint: disable=redefined-variable-type - else: - states = {'state': states[0]} - - if isinstance(inputs, symbol.Symbol): - parameters = self.parameters.var() - else: - parameters = self.parameters.data(inputs.context) - - rnn = F.RNN(data=inputs, parameters=parameters, - state_size=self._num_hidden, num_layers=self._num_layers, - bidirectional=self._bidirectional, p=self._dropout, - state_outputs=self._get_next_state, - mode=self._mode, name=self._prefix+'rnn', - **states) - - if not self._get_next_state: - outputs, states = rnn, [] - elif self._mode == 'lstm': - outputs, states = rnn[0], [rnn[1], rnn[2]] - else: - outputs, states = rnn[0], [rnn[1]] - - if axis == 1: - outputs = F.swapaxes(outputs, dim1=0, dim2=1) - - outputs, _, _, _ = _format_sequence(length, outputs, layout, merge_outputs) - - return outputs, states - - def unfuse(self): - """Unfuse the fused RNN in to a stack of rnn cells. - - Returns - ------- - cell : HSequentialRNNCell - unfused cell that can be used for stepping, and can run on CPU. - """ - stack = HSequentialRNNCell() - get_cell = {'rnn_relu': lambda cell_prefix: RNNCell(self._num_hidden, - activation='relu', - prefix=cell_prefix), - 'rnn_tanh': lambda cell_prefix: RNNCell(self._num_hidden, - activation='tanh', - prefix=cell_prefix), - 'lstm': lambda cell_prefix: LSTMCell(self._num_hidden, - prefix=cell_prefix), - 'gru': lambda cell_prefix: GRUCell(self._num_hidden, - prefix=cell_prefix)}[self._mode] - for i in range(self._num_layers): - if self._bidirectional: - stack.add(BidirectionalCell( - get_cell('%sl%d_'%(self._prefix, i)), - get_cell('%sr%d_'%(self._prefix, i)), - output_prefix='%sbi_l%d_'%(self._prefix, i))) - else: - stack.add(get_cell('%sl%d_'%(self._prefix, i))) - - if self._dropout > 0 and i != self._num_layers - 1: - stack.add(DropoutCell(self._dropout, prefix='%s_dropout%d_'%(self._prefix, i))) - - return stack - - -class HSequentialRNNCell(HRecurrentCell): +class SequentialRNNCell(RecurrentCell): """Sequantially stacking multiple RNN cells.""" - def __init__(self): - super(HSequentialRNNCell, self).__init__(prefix='', params=None) + def __init__(self, prefix=None, params=None): + super(SequentialRNNCell, self).__init__(prefix=prefix, params=params) def add(self, cell): """Append a cell into the stack. @@ -750,12 +491,6 @@ def begin_state(self, **kwargs): "cell cannot be called directly. Call the modifier cell instead." return _cells_begin_state(self._children, **kwargs) - def unpack_weights(self, args): - return _cells_unpack_weights(self._children, args) - - def pack_weights(self, args): - return _cells_pack_weights(self._children, args) - def __call__(self, inputs, states): self._counter += 1 next_states = [] @@ -860,12 +595,6 @@ def begin_state(self, func=symbol.zeros, **kwargs): self.base_cell._modified = True return begin - def unpack_weights(self, args): - return self.base_cell.unpack_weights(args) - - def pack_weights(self, args): - return self.base_cell.pack_weights(args) - def hybrid_forward(self, F, inputs, states): raise NotImplementedError @@ -873,14 +602,11 @@ def hybrid_forward(self, F, inputs, states): class ZoneoutCell(ModifierCell): """Apply Zoneout on base cell.""" def __init__(self, base_cell, zoneout_outputs=0., zoneout_states=0.): - assert not isinstance(base_cell, FusedRNNCell), \ - "FusedRNNCell doesn't support zoneout. " \ - "Please unfuse first." assert not isinstance(base_cell, BidirectionalCell), \ "BidirectionalCell doesn't support zoneout since it doesn't support step. " \ "Please add ZoneoutCell to the cells underneath instead." - assert not isinstance(base_cell, HSequentialRNNCell) or not base_cell._bidirectional, \ - "Bidirectional HSequentialRNNCell doesn't support zoneout. " \ + assert not isinstance(base_cell, SequentialRNNCell) or not base_cell._bidirectional, \ + "Bidirectional SequentialRNNCell doesn't support zoneout. " \ "Please add ZoneoutCell to the cells underneath instead." super(ZoneoutCell, self).__init__(base_cell) self.zoneout_outputs = zoneout_outputs @@ -963,12 +689,6 @@ def __init__(self, l_cell, r_cell, output_prefix='bi_'): self.register_child(r_cell) self._output_prefix = output_prefix - def unpack_weights(self, args): - return _cells_unpack_weights(self._children, args) - - def pack_weights(self, args): - return _cells_pack_weights(self._children, args) - def __call__(self, inputs, states): raise NotImplementedError("Bidirectional cannot be stepped. Please use unroll") @@ -1010,5 +730,5 @@ def unroll(self, length, inputs, begin_state=None, layout='NTC', merge_outputs=N outputs = [F.concat(l_o, r_o, dim=1, name='%st%d'%(self._output_prefix, i)) for i, (l_o, r_o) in enumerate(zip(l_outputs, reversed(r_outputs)))] - states = [l_states, r_states] + states = l_states + r_states return outputs, states diff --git a/python/mxnet/foo/rnn/rnn_layer.py b/python/mxnet/foo/rnn/rnn_layer.py new file mode 100644 index 000000000000..ee0693f53fe3 --- /dev/null +++ b/python/mxnet/foo/rnn/rnn_layer.py @@ -0,0 +1,404 @@ +# coding: utf-8 +# pylint: disable=no-member, invalid-name, protected-access, no-self-use +# pylint: disable=too-many-branches, too-many-arguments, no-self-use +# pylint: disable=too-many-lines, arguments-differ +"""Definition of various recurrent neural network layers.""" +from __future__ import print_function + +from ... import ndarray +from ..nn import Layer +from . import rnn_cell + + +class _RNNLayer(Layer): + """implementation of recurrent layers.""" + def __init__(self, hidden_size, num_layers, layout, + dropout, bidirectional, input_size, + i2h_weight_initializer, h2h_weight_initializer, + i2h_bias_initializer, h2h_bias_initializer, + mode, **kwargs): + super(_RNNLayer, self).__init__(**kwargs) + assert layout == 'TNC' or layout == 'NTC', \ + "Invalid layout %s; must be one of ['TNC' or 'NTC']"%layout + self._hidden_size = hidden_size + self._num_layers = num_layers + self._mode = mode + self._layout = layout + self._dropout = dropout + self._dir = 2 if bidirectional else 1 + self._input_size = input_size + self._i2h_weight_initializer = i2h_weight_initializer + self._h2h_weight_initializer = h2h_weight_initializer + self._i2h_bias_initializer = i2h_bias_initializer + self._h2h_bias_initializer = h2h_bias_initializer + + self._gates = {'rnn_relu': 1, 'rnn_tanh': 1, 'lstm': 4, 'gru': 3}[mode] + + self.i2h_weight = [] + self.h2h_weight = [] + self.i2h_bias = [] + self.h2h_bias = [] + + ng, ni, nh = self._gates, input_size, hidden_size + for i in range(num_layers): + for j in (['l', 'r'] if self._dir == 2 else ['l']): + self.i2h_weight.append( + self.params.get('%s%d_i2h_weight'%(j, i), shape=(ng*nh, ni), + init=i2h_weight_initializer)) + self.h2h_weight.append( + self.params.get('%s%d_h2h_weight'%(j, i), shape=(ng*nh, nh), + init=h2h_weight_initializer)) + self.i2h_bias.append( + self.params.get('%s%d_i2h_bias'%(j, i), shape=(ng*nh,), + init=i2h_bias_initializer)) + self.h2h_bias.append( + self.params.get('%s%d_h2h_bias'%(j, i), shape=(ng*nh,), + init=h2h_bias_initializer)) + ni = nh * self._dir + + self._unfused = self._unfuse() + + def state_info(self, batch_size=0): + raise NotImplementedError + + def _unfuse(self): + """Unfuse the fused RNN in to a stack of rnn cells.""" + get_cell = {'rnn_relu': lambda **kwargs: rnn_cell.RNNCell(self._hidden_size, + activation='relu', + **kwargs), + 'rnn_tanh': lambda **kwargs: rnn_cell.RNNCell(self._hidden_size, + activation='tanh', + **kwargs), + 'lstm': lambda **kwargs: rnn_cell.LSTMCell(self._hidden_size, + **kwargs), + 'gru': lambda **kwargs: rnn_cell.GRUCell(self._hidden_size, + **kwargs)}[self._mode] + + stack = rnn_cell.SequentialRNNCell(prefix=self.prefix, params=self.params) + with stack.name_scope(): + ni = self._input_size + for i in range(self._num_layers): + kwargs = {'input_size': ni, + 'i2h_weight_initializer': self._i2h_weight_initializer, + 'h2h_weight_initializer': self._h2h_weight_initializer, + 'i2h_bias_initializer': self._i2h_bias_initializer, + 'h2h_bias_initializer': self._h2h_bias_initializer} + if self._dir == 2: + stack.add(rnn_cell.BidirectionalCell( + get_cell(prefix='l%d_'%i, **kwargs), + get_cell(prefix='r%d_'%i, **kwargs))) + else: + stack.add(get_cell(prefix='l%d_'%i, **kwargs)) + + if self._dropout > 0 and i != self._num_layers - 1: + stack.add(rnn_cell.DropoutCell(self._dropout)) + + ni = self._hidden_size * self._dir + + return stack + + def begin_state(self, batch_size=0, func=ndarray.zeros, **kwargs): + """Initial state for this cell. + + Parameters + ---------- + batch_size: int + Only required for NDArray API. Size of the batch ('N' in layout) + dimension of input. + func : callable, default symbol.zeros + Function for creating initial state. + + For Symbol API, func can be symbol.zeros, symbol.uniform, + symbol.var etc. Use symbol.var if you want to directly + feed input as states. + + For NDArray API, func can be ndarray.zeros, ndarray.ones, etc. + + **kwargs : + additional keyword arguments passed to func. For example + mean, std, dtype, etc. + + Returns + ------- + states : nested list of Symbol + Starting states for the first RNN step. + """ + states = [] + for i, info in enumerate(self.state_info(batch_size)): + if info is not None: + info.update(kwargs) + else: + info = kwargs + states.append(func(name='%sh0_%d'%(self.prefix, i), **info)) + return states + + def forward(self, inputs, states): + if self._input_size == 0: + for i in range(self._dir): + self.i2h_weight[i].shape = (self._gates*self._hidden_size, inputs.shape[2]) + self.i2h_weight[i]._finish_deferred_init() + if inputs.context.device_type == 'gpu': + return self._forward_gpu(inputs, states) + return self._forward_cpu(inputs, states) + + def _forward_cpu(self, inputs, states): + ns = len(states) + axis = self._layout.find('T') + states = sum(zip(*((j for j in i) for i in states)), ()) + outputs, states = self._unfused.unroll( + inputs.shape[axis], inputs, states, + layout=self._layout, merge_outputs=True) + new_states = [] + for i in range(ns): + state = ndarray.concat(*(j.reshape((1,)+j.shape) for j in states[i::ns]), dim=0) + new_states.append(state) + + return outputs, new_states + + def _forward_gpu(self, inputs, states): + if self._layout == 'NTC': + inputs = ndarray.swapaxes(inputs, dim1=0, dim2=1) + ctx = inputs.context + params = sum(zip(self.i2h_weight, self.h2h_weight), ()) + params += sum(zip(self.i2h_bias, self.h2h_bias), ()) + params = (i.data(ctx).reshape((-1,)) for i in params) + params = ndarray.concat(*params, dim=0) + + rnn = ndarray.RNN(inputs, params, *states, state_size=self._hidden_size, + num_layers=self._num_layers, bidirectional=self._dir == 2, + p=self._dropout, state_outputs=True, mode=self._mode) + + if self._mode == 'lstm': + outputs, states = rnn[0], [rnn[1], rnn[2]] + else: + outputs, states = rnn[0], [rnn[1]] + + if self._layout == 'NTC': + outputs = ndarray.swapaxes(outputs, dim1=0, dim2=1) + + return outputs, states + + +class RNN(_RNNLayer): + r"""Applies a multi-layer Elman RNN with tanh or ReLU non-linearity to an input sequence. + + For each element in the input sequence, each layer computes the following + function: + + .. math:: + h_t = \tanh(w_{ih} * x_t + b_{ih} + w_{hh} * h_{(t-1)} + b_{hh}) + + where :math:`h_t` is the hidden state at time `t`, and :math:`x_t` is the hidden + state of the previous layer at time `t` or :math:`input_t` for the first layer. + If nonlinearity='relu', then `ReLU` is used instead of `tanh`. + + Parameters + ---------- + hidden_size: int + The number of features in the hidden state h + num_layers: int, default 1 + Number of recurrent layers. + activation: {'relu' or 'tanh'}, default 'tanh' + The activation function to use. + layout : str, default 'TNC' + The format of input and output tensors. T, N and C stand for + sequence length, batch size, and feature dimensions respectively. + dropout: float, default 0 + If non-zero, introduces a dropout layer on the outputs of each + RNN layer except the last layer + bidirectional: bool, default False + If True, becomes a bidirectional RNN. + i2h_weight_initializer : str or Initializer + Initializer for the input weights matrix, used for the linear + transformation of the inputs. + h2h_weight_initializer : str or Initializer + Initializer for the recurrent weights matrix, used for the linear + transformation of the recurrent state. + i2h_bias_initializer : str or Initializer + Initializer for the bias vector. + h2h_bias_initializer : str or Initializer + Initializer for the bias vector. + input_size: int, default 0 + The number of expected features in the input x. + If not specified, it will be inferred from input. + prefix : str or None + Prefix of this layer. + params : ParameterDict or None + Shared Parameters for this Layer. + + Examples + -------- + >>> rnn = nn.RNN(100, 3) + >>> input = mx.nd.random_uniform(shape=(5, 3, 10)) + >>> h0 = mx.nd.random_uniform(shape=(2, 3, 100)) + >>> output, hn = rnn(input, h0) + """ + def __init__(self, hidden_size, num_layers=1, activation='relu', + layout='TNC', dropout=0, bidirectional=False, + i2h_weight_initializer=None, h2h_weight_initializer=None, + i2h_bias_initializer=None, h2h_bias_initializer=None, + input_size=0, **kwargs): + super(RNN, self).__init__(hidden_size, num_layers, layout, + dropout, bidirectional, input_size, + i2h_weight_initializer, h2h_weight_initializer, + i2h_bias_initializer, h2h_bias_initializer, + 'rnn_'+activation, **kwargs) + + def state_info(self, batch_size=0): + return [{'shape': (self._num_layers * self._dir, batch_size, self._hidden_size), + '__layout__': 'LNC'}] + + +class LSTM(_RNNLayer): + r"""Applies a multi-layer long short-term memory (LSTM) RNN to an input sequence. + + For each element in the input sequence, each layer computes the following + function: + + .. math:: + \begin{array}{ll} + i_t = sigmoid(W_{ii} x_t + b_{ii} + W_{hi} h_{(t-1)} + b_{hi}) \\ + f_t = sigmoid(W_{if} x_t + b_{if} + W_{hf} h_{(t-1)} + b_{hf}) \\ + g_t = \tanh(W_{ig} x_t + b_{ig} + W_{hc} h_{(t-1)} + b_{hg}) \\ + o_t = sigmoid(W_{io} x_t + b_{io} + W_{ho} h_{(t-1)} + b_{ho}) \\ + c_t = f_t * c_{(t-1)} + i_t * g_t \\ + h_t = o_t * \tanh(c_t) + \end{array} + + where :math:`h_t` is the hidden state at time `t`, :math:`c_t` is the + cell state at time `t`, :math:`x_t` is the hidden state of the previous + layer at time `t` or :math:`input_t` for the first layer, and :math:`i_t`, + :math:`f_t`, :math:`g_t`, :math:`o_t` are the input, forget, cell, and + out gates, respectively. + + Parameters + ---------- + hidden_size: int + The number of features in the hidden state h + num_layers: int, default 1 + Number of recurrent layers. + layout : str, default 'TNC' + The format of input and output tensors. T, N and C stand for + sequence length, batch size, and feature dimensions respectively. + dropout: float, default 0 + If non-zero, introduces a dropout layer on the outputs of each + RNN layer except the last layer + bidirectional: bool, default False + If True, becomes a bidirectional RNN. + i2h_weight_initializer : str or Initializer + Initializer for the input weights matrix, used for the linear + transformation of the inputs. + h2h_weight_initializer : str or Initializer + Initializer for the recurrent weights matrix, used for the linear + transformation of the recurrent state. + i2h_bias_initializer : str or Initializer, default 'lstmbias' + Initializer for the bias vector. By default bias for the forget + gate is initialized to 1 while all other biases are initialized + to zero. + h2h_bias_initializer : str or Initializer + Initializer for the bias vector. + input_size: int, default 0 + The number of expected features in the input x. + If not specified, it will be inferred from input. + prefix : str or None + Prefix of this layer. + params : ParameterDict or None + Shared Parameters for this Layer. + + Examples + -------- + >>> rnn = nn.LSTM(100, 3) + >>> input = mx.nd.random_uniform(shape=(5, 3, 10)) + >>> h0 = mx.nd.random_uniform(shape=(2, 3, 100)) + >>> c0 = mx.nd.random_uniform(shape=(2, 3, 100)) + >>> output, hn = rnn(input, (h0, c0)) + """ + def __init__(self, hidden_size, num_layers=1, layout='TNC', + dropout=0, bidirectional=False, input_size=0, + i2h_weight_initializer=None, h2h_weight_initializer=None, + i2h_bias_initializer='lstmbias', h2h_bias_initializer=None, + **kwargs): + super(LSTM, self).__init__(hidden_size, num_layers, layout, + dropout, bidirectional, input_size, + i2h_weight_initializer, h2h_weight_initializer, + i2h_bias_initializer, h2h_bias_initializer, + 'lstm', **kwargs) + + def state_info(self, batch_size=0): + return [{'shape': (self._num_layers * self._dir, batch_size, self._hidden_size), + '__layout__': 'LNC'}, + {'shape': (self._num_layers * self._dir, batch_size, self._hidden_size), + '__layout__': 'LNC'}] + + +class GRU(_RNNLayer): + r"""Applies a multi-layer gated recurrent unit (GRU) RNN to an input sequence. + + For each element in the input sequence, each layer computes the following + function: + + .. math:: + \begin{array}{ll} + r_t = sigmoid(W_{ir} x_t + b_{ir} + W_{hr} h_{(t-1)} + b_{hr}) \\ + i_t = sigmoid(W_{ii} x_t + b_{ii} + W_hi h_{(t-1)} + b_{hi}) \\ + n_t = \tanh(W_{in} x_t + b_{in} + r_t * (W_{hn} h_{(t-1)}+ b_{hn})) \\ + h_t = (1 - i_t) * n_t + i_t * h_{(t-1)} \\ + \end{array} + + where :math:`h_t` is the hidden state at time `t`, :math:`x_t` is the hidden + state of the previous layer at time `t` or :math:`input_t` for the first layer, + and :math:`r_t`, :math:`i_t`, :math:`n_t` are the reset, input, and new gates, respectively. + + Parameters + ---------- + hidden_size: int + The number of features in the hidden state h + num_layers: int, default 1 + Number of recurrent layers. + layout : str, default 'TNC' + The format of input and output tensors. T, N and C stand for + sequence length, batch size, and feature dimensions respectively. + dropout: float, default 0 + If non-zero, introduces a dropout layer on the outputs of each + RNN layer except the last layer + bidirectional: bool, default False + If True, becomes a bidirectional RNN. + i2h_weight_initializer : str or Initializer + Initializer for the input weights matrix, used for the linear + transformation of the inputs. + h2h_weight_initializer : str or Initializer + Initializer for the recurrent weights matrix, used for the linear + transformation of the recurrent state. + i2h_bias_initializer : str or Initializer + Initializer for the bias vector. + h2h_bias_initializer : str or Initializer + Initializer for the bias vector. + input_size: int, default 0 + The number of expected features in the input x. + If not specified, it will be inferred from input. + prefix : str or None + Prefix of this layer. + params : ParameterDict or None + Shared Parameters for this Layer. + + Examples + -------- + >>> rnn = nn.GRU(100, 2) + >>> input = mx.nd.random_uniform(shape=(5, 3, 10)) + >>> h0 = mx.nd.random_uniform(shape=(2, 3, 100)) + >>> output, hn = rnn(input, h0) + """ + def __init__(self, hidden_size, num_layers=1, layout='TNC', + dropout=0, bidirectional=False, input_size=0, + i2h_weight_initializer=None, h2h_weight_initializer=None, + i2h_bias_initializer=None, h2h_bias_initializer=None, + **kwargs): + super(GRU, self).__init__(hidden_size, num_layers, layout, + dropout, bidirectional, input_size, + i2h_weight_initializer, h2h_weight_initializer, + i2h_bias_initializer, h2h_bias_initializer, + 'gru', **kwargs) + + def state_info(self, batch_size=0): + return [{'shape': (self._num_layers * self._dir, batch_size, self._hidden_size), + '__layout__': 'LNC'}] diff --git a/python/mxnet/initializer.py b/python/mxnet/initializer.py index 8a287d17374a..64921b9a5796 100755 --- a/python/mxnet/initializer.py +++ b/python/mxnet/initializer.py @@ -636,10 +636,11 @@ class LSTMBias(Initializer): Parameters ---------- - forget_bias: float, bias for the forget gate. - Jozefowicz et al. 2015 recommends setting this to 1.0. + forget_bias: float, default 1.0 + bias for the forget gate. Jozefowicz et al. 2015 recommends + setting this to 1.0. """ - def __init__(self, forget_bias): + def __init__(self, forget_bias=1.0): super(LSTMBias, self).__init__(forget_bias=forget_bias) self.forget_bias = forget_bias diff --git a/python/mxnet/ndarray.py b/python/mxnet/ndarray.py index 15c5955f8750..29f0f769ed63 100644 --- a/python/mxnet/ndarray.py +++ b/python/mxnet/ndarray.py @@ -122,7 +122,7 @@ class NDArray(NDArrayBase): def __repr__(self): """Returns a string representation of the array.""" shape_info = 'x'.join(['%d' % x for x in self.shape]) - return '%s\n<%s %s @%s>' % ('',#str(self.asnumpy()), + return '%s\n<%s %s @%s>' % (str(self.asnumpy()), self.__class__.__name__, shape_info, self.context) diff --git a/src/c_api/c_api_ndarray.cc b/src/c_api/c_api_ndarray.cc index 350f77cf7bfd..4b3bc092e932 100644 --- a/src/c_api/c_api_ndarray.cc +++ b/src/c_api/c_api_ndarray.cc @@ -513,7 +513,7 @@ int MXInvokeCachedOp(CachedOpHandle handle, for (const auto& i : idx.outputs()) { ret->ret_handles.push_back( reinterpret_cast( - new NDArray(std::move(buff[idx.entry_id(i)])))); + new NDArray(buff[idx.entry_id(i)]))); } *num_outputs = idx.outputs().size(); *outputs = dmlc::BeginPtr(ret->ret_handles); @@ -521,7 +521,7 @@ int MXInvokeCachedOp(CachedOpHandle handle, CHECK_EQ(static_cast(*num_outputs), idx.outputs().size()) << "Specifed number of output differs from expected number of outputs"; for (size_t i = 0; i < idx.outputs().size(); ++i) { - *outarray[i] = std::move(buff[idx.entry_id(idx.outputs()[i])]); + *outarray[i] = buff[idx.entry_id(idx.outputs()[i])]; } } API_END(); diff --git a/tests/python/gpu/test_operator_gpu.py b/tests/python/gpu/test_operator_gpu.py index 7557b6b49bce..6ca52c76a910 100644 --- a/tests/python/gpu/test_operator_gpu.py +++ b/tests/python/gpu/test_operator_gpu.py @@ -1,19 +1,19 @@ import sys import os +import time +import mxnet as mx +import numpy as np +from mxnet.test_utils import check_consistency, set_default_context +from numpy.testing import assert_allclose + curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__))) sys.path.insert(0, os.path.join(curr_path, '../unittest')) from test_operator import * from test_optimizer import * from test_random import * from test_nn import * -from test_rnn import * -from test_rnn import * +#from test_rnn import * from test_foo_rnn import * -import mxnet as mx -import numpy as np -from mxnet.test_utils import check_consistency, set_default_context -from numpy.testing import assert_allclose -import time set_default_context(mx.gpu(0)) del test_support_vector_machine_l1_svm @@ -1278,114 +1278,30 @@ def test_residual_fused(): assert np.array_equal(outputs[0].asnumpy(), expected_outputs) -def test_foo_rnn(): - fused = foo.rnn.FusedRNNCell(100, num_layers=2, mode='rnn_relu', prefix='') - - stack = foo.rnn.HSequentialRNNCell() - stack.add(foo.rnn.RNNCell(100, activation='relu', prefix='l0_')) - stack.add(foo.rnn.RNNCell(100, activation='relu', prefix='l1_')) - - check_rnn_consistency(fused, stack) - check_rnn_consistency(stack, fused) - - -def test_foo_lstm(): - fused = foo.rnn.FusedRNNCell(100, num_layers=2, mode='lstm', prefix='') - - stack = foo.rnn.HSequentialRNNCell() - stack.add(foo.rnn.LSTMCell(100, prefix='l0_')) - stack.add(foo.rnn.LSTMCell(100, prefix='l1_')) - - check_rnn_consistency(fused, stack) - check_rnn_consistency(stack, fused) - - -def test_foo_lstm_forget_bias(): - forget_bias = 2.0 - fused = foo.rnn.FusedRNNCell(10, forget_bias=forget_bias, num_layers=2, mode='lstm', prefix='') - - dshape = (32, 1, 20) - data = mx.sym.Variable('data') - - sym, _ = fused.unroll(1, data, merge_outputs=True) - mod = mx.mod.Module(sym, label_names=None, context=mx.gpu(0)) - mod.bind(data_shapes=[('data', dshape)], label_shapes=None) - - mod.init_params() - - args, auxs = mod.get_params() - args = fused.unpack_weights(args) - - bias_name = next(x for x in args if x.endswith('f_bias')) - expected_bias = forget_bias * np.ones(10, ) - assert_allclose(args[bias_name].asnumpy(), expected_bias) - - -def test_foo_gru(): - fused = foo.rnn.FusedRNNCell(100, num_layers=2, mode='gru', prefix='') - - stack = foo.rnn.HSequentialRNNCell() - stack.add(foo.rnn.GRUCell(100, prefix='l0_')) - stack.add(foo.rnn.GRUCell(100, prefix='l1_')) - - check_rnn_consistency(fused, stack) - check_rnn_consistency(stack, fused) - - -def test_foo_bidirectional(): - fused = foo.rnn.FusedRNNCell(100, num_layers=2, mode='gru', prefix='', - bidirectional=True) - - stack = foo.rnn.HSequentialRNNCell() - stack.add(foo.rnn.BidirectionalCell( - foo.rnn.GRUCell(100, prefix='l0_'), - foo.rnn.GRUCell(100, prefix='r0_'), - output_prefix='bi_gru_0_')) - stack.add(foo.rnn.BidirectionalCell( - foo.rnn.GRUCell(100, prefix='l1_'), - foo.rnn.GRUCell(100, prefix='r1_'), - output_prefix='bi_gru_1_')) - - check_rnn_consistency(fused, stack) - check_rnn_consistency(stack, fused) - -def test_foo_unfuse(): - for mode in ['rnn_tanh', 'rnn_relu', 'lstm', 'gru']: - fused = foo.rnn.FusedRNNCell( - 100, num_layers=2, mode=mode, - prefix='test_%s'%mode, - bidirectional=True, - dropout=0.5) - - stack = fused.unfuse() - - check_rnn_consistency(fused, stack) - check_rnn_consistency(stack, fused) - +def check_rnn_layer(layer): + layer.all_params().initialize(ctx=[mx.cpu(0), mx.gpu(0)]) + with mx.gpu(0): + x = mx.nd.ones((10, 16, 30)) + states = layer.begin_state(16) + go, gs = layer(x, states) -def test_foo_residual_fused(): - cell = foo.rnn.ResidualCell( - foo.rnn.FusedRNNCell(50, num_layers=3, mode='lstm', - prefix='rnn_', dropout=0.5)) + with mx.cpu(0): + x = mx.nd.ones((10, 16, 30)) + states = layer.begin_state(16) + co, cs = layer(x, states) - inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(2)] - outputs, _ = cell.unroll(2, inputs, merge_outputs=None) - assert sorted(cell.params._params.keys()) == \ - ['rnn_parameters'] + assert_allclose(go.asnumpy(), co.asnumpy(), rtol=1e-2) + for g, c in zip(gs, cs): + assert_allclose(g.asnumpy(), c.asnumpy(), rtol=1e-2) - args, outs, auxs = outputs.infer_shape(rnn_t0_data=(10, 50), rnn_t1_data=(10, 50)) - assert outs == [(10, 2, 50)] - outputs = outputs.eval(ctx=mx.gpu(0), - rnn_t0_data=mx.nd.ones((10, 50), ctx=mx.gpu(0))+5, - rnn_t1_data=mx.nd.ones((10, 50), ctx=mx.gpu(0))+5, - rnn_parameters=mx.nd.zeros((61200,), ctx=mx.gpu(0))) - expected_outputs = np.ones((10, 2, 50))+5 - assert np.array_equal(outputs[0].asnumpy(), expected_outputs) +def test_rnn_layer(): + check_rnn_layer(foo.rnn.RNN(100, num_layers=3)) + check_rnn_layer(foo.rnn.RNN(100, activation='tanh', num_layers=3)) + check_rnn_layer(foo.rnn.LSTM(100, num_layers=3)) + check_rnn_layer(foo.rnn.GRU(100, num_layers=3)) -def test_foo_fused(): - check_rnn_forward(foo.rnn.FusedRNNCell(100, num_layers=2, num_input=200), - mx.nd.ones((8, 3, 200))) + check_rnn_layer(foo.rnn.LSTM(100, num_layers=3, bidirectional=True)) if __name__ == '__main__': diff --git a/tests/python/train/test_autograd.py b/tests/python/train/test_autograd.py index ff4aacd1fa07..6a979f5a02e1 100644 --- a/tests/python/train/test_autograd.py +++ b/tests/python/train/test_autograd.py @@ -7,7 +7,7 @@ import numpy as np import logging from common import get_data -from mxnet.contrib import autograd as ag +from mxnet import autograd logging.basicConfig(level=logging.DEBUG) # define network @@ -58,11 +58,11 @@ def train(net, epoch, ctx): data = foo.utils.load_data(batch.data[0], ctx_list=ctx, batch_axis=0) label = foo.utils.load_data(batch.label[0], ctx_list=ctx, batch_axis=0) outputs = [] - with ag.record(): + with autograd.record(): for x, y in zip(data, label): z = net(x) loss = foo.loss.softmax_cross_entropy_loss(z, y) - ag.compute_gradient([loss]) + loss.backward() outputs.append(z) metric.update(label, outputs) trainer.step(batch.data[0].shape[0]) diff --git a/tests/python/unittest/test_foo_rnn.py b/tests/python/unittest/test_foo_rnn.py index 67113e4ff8ce..ac70fa84aaca 100644 --- a/tests/python/unittest/test_foo_rnn.py +++ b/tests/python/unittest/test_foo_rnn.py @@ -17,7 +17,7 @@ def test_rnn(): def test_lstm(): - cell = foo.rnn.LSTMCell(100, prefix='rnn_', forget_bias=1.0) + cell = foo.rnn.LSTMCell(100, prefix='rnn_') inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)] outputs, _ = cell.unroll(3, inputs) outputs = mx.sym.Group(outputs) @@ -30,9 +30,9 @@ def test_lstm(): def test_lstm_forget_bias(): forget_bias = 2.0 - stack = foo.rnn.HSequentialRNNCell() - stack.add(foo.rnn.LSTMCell(100, forget_bias=forget_bias, prefix='l0_')) - stack.add(foo.rnn.LSTMCell(100, forget_bias=forget_bias, prefix='l1_')) + stack = foo.rnn.SequentialRNNCell() + stack.add(foo.rnn.LSTMCell(100, i2h_bias_initializer=mx.init.LSTMBias(forget_bias), prefix='l0_')) + stack.add(foo.rnn.LSTMCell(100, i2h_bias_initializer=mx.init.LSTMBias(forget_bias), prefix='l1_')) dshape = (32, 1, 200) data = mx.sym.Variable('data') @@ -117,7 +117,7 @@ def test_residual_bidirectional(): def test_stack(): - cell = foo.rnn.HSequentialRNNCell() + cell = foo.rnn.SequentialRNNCell() for i in range(5): if i == 1: cell.add(foo.rnn.ResidualCell(foo.rnn.LSTMCell(100, prefix='rnn_stack%d_' % i))) @@ -163,20 +163,6 @@ def test_zoneout(): assert outs == [(10, 100), (10, 100), (10, 100)] -def test_unfuse(): - cell = foo.rnn.FusedRNNCell(100, num_layers=3, mode='lstm', - prefix='test_', bidirectional=True, - dropout=0.5) - cell = cell.unfuse() - inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)] - outputs, _ = cell.unroll(3, inputs) - outputs = mx.sym.Group(outputs) - assert outputs.list_outputs() == ['test_bi_l2_t0_output', 'test_bi_l2_t1_output', 'test_bi_l2_t2_output'] - - args, outs, auxs = outputs.infer_shape(rnn_t0_data=(10,50), rnn_t1_data=(10,50), rnn_t2_data=(10,50)) - assert outs == [(10, 200), (10, 200), (10, 200)] - - def check_rnn_forward(layer, inputs): layer.all_params().initialize() with mx.autograd.record(): @@ -186,24 +172,24 @@ def check_rnn_forward(layer, inputs): def test_rnn_cells(): - check_rnn_forward(foo.rnn.LSTMCell(100, num_input=200), mx.nd.ones((8, 3, 200))) - check_rnn_forward(foo.rnn.RNNCell(100, num_input=200), mx.nd.ones((8, 3, 200))) - check_rnn_forward(foo.rnn.GRUCell(100, num_input=200), mx.nd.ones((8, 3, 200))) + check_rnn_forward(foo.rnn.LSTMCell(100, input_size=200), mx.nd.ones((8, 3, 200))) + check_rnn_forward(foo.rnn.RNNCell(100, input_size=200), mx.nd.ones((8, 3, 200))) + check_rnn_forward(foo.rnn.GRUCell(100, input_size=200), mx.nd.ones((8, 3, 200))) - bilayer = foo.rnn.BidirectionalCell(foo.rnn.LSTMCell(100, num_input=200), - foo.rnn.LSTMCell(100, num_input=200)) + bilayer = foo.rnn.BidirectionalCell(foo.rnn.LSTMCell(100, input_size=200), + foo.rnn.LSTMCell(100, input_size=200)) check_rnn_forward(bilayer, mx.nd.ones((8, 3, 200))) check_rnn_forward(foo.rnn.DropoutCell(0.5), mx.nd.ones((8, 3, 200))) - check_rnn_forward(foo.rnn.ZoneoutCell(foo.rnn.LSTMCell(100, num_input=200), + check_rnn_forward(foo.rnn.ZoneoutCell(foo.rnn.LSTMCell(100, input_size=200), 0.5, 0.2), mx.nd.ones((8, 3, 200))) - net = foo.rnn.HSequentialRNNCell() - net.add(foo.rnn.LSTMCell(100, num_input=200)) - net.add(foo.rnn.RNNCell(100, num_input=100)) - net.add(foo.rnn.GRUCell(100, num_input=100)) + net = foo.rnn.SequentialRNNCell() + net.add(foo.rnn.LSTMCell(100, input_size=200)) + net.add(foo.rnn.RNNCell(100, input_size=100)) + net.add(foo.rnn.GRUCell(100, input_size=100)) check_rnn_forward(net, mx.nd.ones((8, 3, 200))) diff --git a/tests/python/unittest/test_nn.py b/tests/python/unittest/test_nn.py index 0d04071259e4..ccfdea55c6fb 100644 --- a/tests/python/unittest/test_nn.py +++ b/tests/python/unittest/test_nn.py @@ -35,7 +35,7 @@ def forward(self, x): return self.dense1(self.dense0(x)) net1 = Net(prefix='net1_') - net2 = Net(prefix='net1_', params=net1.all_params()) + net2 = Net(prefix='net2_', params=net1.all_params()) net1.all_params().initialize() net2(mx.nd.zeros((3, 5))) From 4ed646bacb7856f3a6b097ecfbdfd1d1f5c564b7 Mon Sep 17 00:00:00 2001 From: Eric Junyuan Xie Date: Wed, 28 Jun 2017 12:01:24 -0700 Subject: [PATCH 194/834] add hybrid docs (#6859) --- docs/tutorials/foo/hybrid.md | 121 +++++++++++++++++++++++++++++++++++ docs/tutorials/index.md | 1 + 2 files changed, 122 insertions(+) create mode 100644 docs/tutorials/foo/hybrid.md diff --git a/docs/tutorials/foo/hybrid.md b/docs/tutorials/foo/hybrid.md new file mode 100644 index 000000000000..9cebf9204ad9 --- /dev/null +++ b/docs/tutorials/foo/hybrid.md @@ -0,0 +1,121 @@ +# Hybrid - Faster training and easy deployment + +Deep learning frameworks can be roughly divided into two categories: declarative +and imperative. With declarative frameworks (including Tensorflow, Theano, etc) +users first declare a fixed computation graph and then execute it end-to-end. +The benefit of fixed computation graph is it's portable and runs more +efficiently. However, it's less flexible because any logic must be encoded +into the graph as special operators like `scan`, `while_loop` and `cond`. +It's also hard to debug. + +Imperative frameworks (including PyTorch, Chainer, etc) are just the opposite: +they execute commands one-by-one just like old fashioned Matlab and Numpy. +This style is more flexible, easier to debug, but less efficient. + +`HybridLayer` seamlessly combines declarative programming and imperative programming +to offer the benefit of both. Users can quickly develop and debug models with +imperative programming and switch to efficient declarative execution by simply +calling: `HybridLayer.hybridize()`. + +## HybridLayer + +`HybridLayer` is very similar to `Layer` but has a few restrictions: + +- All children layers of `HybridLayer` must also be `HybridLayer`. +- Only methods that are implemented for both `NDArray` and `Symbol` can be used. + For example you cannot use `.asnumpy()`, `.shape`, etc. +- Operations cannot change from run to run. For example, you cannot do `if x:` + if `x` is different for each iteration. + +To use hybrid support, we subclass the `HybridLayer`: + +```python +import mxnet as mx +from mxnet import foo +from mxnet.foo import nn + +class Net(nn.HybridLayer): + def __init__(self, **kwargs): + super(Net, self).__init__(**kwargs) + with self.name_scope: + # layers created in name_scope will inherit name space + # from parent layer. + self.conv1 = nn.Conv2D(6, kernel_size=5) + self.pool1 = nn.Pool2D(kernel_size=2) + self.conv2 = nn.Conv2D(16, kernel_size=5) + self.pool2 = nn.Pool2D(kernel_size=2) + self.fc1 = nn.Dense(120) + self.fc2 = nn.Dense(84) + # You can use a Dense layer for fc3 but we do dot product manually + # here for illustration purposes. + self.fc3_weight = self.params.get('fc3_weight', shape=(10, 84)) + + def hybrid_forward(self, F, x, fc3_weight): + # Here `F` can be either mx.nd or mx.sym, x is the input data, + # and fc3_weight is either self.fc3_weight.data() or + # self.fc3_weight.var() depending on whether x is Symbol or NDArray + print(x) + x = self.pool1(F.relu(self.conv1(x))) + x = self.pool2(F.relu(self.conv2(x))) + # 0 means copy over size from corresponding dimension. + # -1 means infer size from the rest of dimensions. + x = x.reshape((0, -1)) + x = F.relu(self.fc1(x)) + x = F.relu(self.fc2(x)) + x = F.dot(x, fc3_weight, transpose_b=True) + return x +``` + +## Hybridize + +By default, `HybridLayer` runs just like a standard `Layer`. Each time a layer +is called, its `hybrid_forward` will be run: + +```python +net = Net() +net.all_params().initialize() +x = mx.nd.random_normal(shape=(16, 1, 28, 28)) +net(x) +x = mx.nd.random_normal(shape=(16, 1, 28, 28)) +net(x) +``` + +Hybrid execution can be activated by simply calling `.hybridize()` on the top +level layer. The first forward call after activation will try to build a +computation graph from `hybrid_forward` and cache it. On subsequent forward +calls the cached graph instead of `hybrid_forward` will be invoked: + +```python +net.hybridize() +x = mx.nd.random_normal(shape=(16, 1, 28, 28)) +net(x) +x = mx.nd.random_normal(shape=(16, 1, 28, 28)) +net(x) +``` + +Note that before hybridize, `print(x)` printed out one NDArray for forward, +but after hybridize, only the first forward printed out a Symbol. On subsequent +forward `hybrid_forward` is not called so nothing was printed. + +Hybridize will speed up execution and save memory. If the top level layer is +not a `HybridLayer`, you can still call `.hybridize()` on it and Foo will try +to hybridize its children layers instead. + +## Serializing trained model for deployment + +Models implemented as `HybridLayer` can be easily serialized for deployment +using other language front-ends like C, C++ and Scala. To this end, we simply +forward the model with symbolic variables instead of NDArrays and save the +output Symbol(s): + +```python +x = mx.sym.var('data') +y = net(x) +print(y) +y.save('model.json') +net.all_params().save('model.params') +``` + +If your network outputs more than one value, you can use `mx.sym.Group` to +combine them into a grouped Symbol and then save. The saved json and params +files can then be loaded with C, C++ and Scala interface for prediction. diff --git a/docs/tutorials/index.md b/docs/tutorials/index.md index cb8a2ec86528..20e3e91ce54d 100644 --- a/docs/tutorials/index.md +++ b/docs/tutorials/index.md @@ -13,6 +13,7 @@ These tutorials introduce a few fundamental concepts in deep learning and how to foo/ndarray foo/autograd foo/foo + foo/hybrid ``` ### Advanced -- Low-level interface From 82c3e763490426a52817cabdc0182f6c76695383 Mon Sep 17 00:00:00 2001 From: Roshani Nagmote Date: Thu, 29 Jun 2017 16:22:18 -0700 Subject: [PATCH 195/834] nn mnist tutorial (#6879) * mnist tutorial added, autograd modified * mnist tutorial * mnist tutorial * minor change * fixes * minor change * fix * small fix * removing dx from autograd * Delete mnist.ipynb --- docs/tutorials/foo/autograd.md | 15 +- docs/tutorials/foo/foo.md | 9 +- docs/tutorials/foo/mnist.md | 323 +++++++++++++++++++++++++++++++++ example/autograd/mnist.py | 2 +- python/mxnet/autograd.py | 6 +- 5 files changed, 345 insertions(+), 10 deletions(-) create mode 100644 docs/tutorials/foo/mnist.md diff --git a/docs/tutorials/foo/autograd.md b/docs/tutorials/foo/autograd.md index 7ffd5aac2ef3..d36832d6d6d1 100644 --- a/docs/tutorials/foo/autograd.md +++ b/docs/tutorials/foo/autograd.md @@ -18,8 +18,7 @@ attach gradient buffers to them: ```python x = mx.nd.array([[1, 2], [3, 4]]) -dx = mx.nd.zeros_like(x) -x.attach_grad(dx) +x.attach_grad() ``` Now we can define the network while running forward computation by wrapping @@ -40,3 +39,15 @@ is equivalent to `mx.nd.sum(z).backward()`: z.backward() print(x.grad) ``` + +Now, let's see if this is the expected output. + +Here, y = f(x), z = f(y) = f(g(x)) +which means y = 2 * x and z = 2 * x * x. + +After, doing backprop with `z.backward()`, we will get gradient dz/dx as follows: + +dy/dx = 2, +dz/dx = 4 * x + +So, we should get x.grad as an array of [[4, 8],[12, 16]]. diff --git a/docs/tutorials/foo/foo.md b/docs/tutorials/foo/foo.md index 17162bbbb550..c7d782037d0f 100644 --- a/docs/tutorials/foo/foo.md +++ b/docs/tutorials/foo/foo.md @@ -14,6 +14,7 @@ import mxnet as mx import mxnet.ndarray as F import mxnet.foo as foo from mxnet.foo import nn +from mxnet import autograd ``` Neural networks (and other machine learning models) can be defined and trained @@ -38,13 +39,13 @@ composing and inheriting `Layer`: class Net(nn.Layer): def __init__(self, **kwargs): super(Net, self).__init__(**kwargs) - with self.name_scope: + with self.name_scope(): # layers created in name_scope will inherit name space # from parent layer. self.conv1 = nn.Conv2D(6, kernel_size=5) - self.pool1 = nn.Pool2D(kernel_size=2) + self.pool1 = nn.MaxPool2D(pool_size=(2,2)) self.conv2 = nn.Conv2D(16, kernel_size=5) - self.pool2 = nn.Pool2D(kernel_size=2) + self.pool2 = nn.MaxPool2D(pool_size=(2,2)) self.fc1 = nn.Dense(120) self.fc2 = nn.Dense(84) self.fc3 = nn.Dense(10) @@ -99,7 +100,7 @@ To compute loss and backprop for one iteration, we do: ```python label = mx.nd.arange(10) # dummy label -with record(): +with autograd.record(): output = net(data) loss = foo.loss.softmax_cross_entropy_loss(output, label) loss.backward() diff --git a/docs/tutorials/foo/mnist.md b/docs/tutorials/foo/mnist.md new file mode 100644 index 000000000000..ba1458b9396b --- /dev/null +++ b/docs/tutorials/foo/mnist.md @@ -0,0 +1,323 @@ +# Handwritten Digit Recognition + +In this tutorial, we'll give you a step by step walk-through of how to build a hand-written digit classifier using the [MNIST](https://en.wikipedia.org/wiki/MNIST_database) dataset. + +MNIST is a widely used dataset for the hand-written digit classification task. It consists of 70,000 labeled 28x28 pixel grayscale images of hand-written digits. The dataset is split into 60,000 training images and 10,000 test images. There are 10 classes (one for each of the 10 digits). The task at hand is to train a model using the 60,000 training images and subsequently test its classification accuracy on the 10,000 test images. + +![png](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/example/mnist.png) + +**Figure 1:** Sample images from the MNIST dataset. + +This tutorial uses MXNet's new high-level interface, foo package to implement MLP using +imperative fashion. + +This is based on the Mnist tutorial with symbolic approach. You can find it [here](http://mxnet.io/tutorials/python/mnist.html). + +## Prerequisites +To complete this tutorial, we need: + +- MXNet. See the instructions for your operating system in [Setup and Installation](http://mxnet.io/get_started/install.html). + +- [Python Requests](http://docs.python-requests.org/en/master/) and [Jupyter Notebook](http://jupyter.org/index.html). + +``` +$ pip install requests jupyter +``` + +## Loading Data + +Before we define the model, let's first fetch the [MNIST](http://yann.lecun.com/exdb/mnist/) dataset. + +The following source code downloads and loads the images and the corresponding labels into memory. + +```python +import mxnet as mx +mnist = mx.test_utils.get_mnist() +``` + +After running the above source code, the entire MNIST dataset should be fully loaded into memory. Note that for large datasets it is not feasible to pre-load the entire dataset first like we did here. What is needed is a mechanism by which we can quickly and efficiently stream data directly from the source. MXNet Data iterators come to the rescue here by providing exactly that. Data iterator is the mechanism by which we feed input data into an MXNet training algorithm and they are very simple to initialize and use and are optimized for speed. During training, we typically process training samples in small batches and over the entire training lifetime will end up processing each training example multiple times. In this tutorial, we'll configure the data iterator to feed examples in batches of 100. Keep in mind that each example is a 28x28 grayscale image and the corresponding label. + +Image batches are commonly represented by a 4-D array with shape `(batch_size, num_channels, width, height)`. For the MNIST dataset, since the images are grayscale, there is only one color channel. Also, the images are 28x28 pixels, and so each image has width and height equal to 28. Therefore, the shape of input is `(batch_size, 1, 28, 28)`. Another important consideration is the order of input samples. When feeding training examples, it is critical that we don't feed samples with the same label in succession. Doing so can slow down training. +Data iterators take care of this by randomly shuffling the inputs. Note that we only need to shuffle the training data. The order does not matter for test data. + +The following source code initializes the data iterators for the MNIST dataset. Note that we initialize two iterators: one for train data and one for test data. + +```python +batch_size = 100 +train_data = mx.io.NDArrayIter(mnist['train_data'], mnist['train_label'], batch_size, shuffle=True) +val_data = mx.io.NDArrayIter(mnist['test_data'], mnist['test_label'], batch_size) +``` + +## Approaches + +We will cover a couple of approaches for performing the hand written digit recognition task. The first approach makes use of a traditional deep neural network architecture called Multilayer Percepton (MLP). We'll discuss its drawbacks and use that as a motivation to introduce a second more advanced approach called Convolution Neural Network (CNN) that has proven to work very well for image classification tasks. + +Now, let's import required nn modules + +```python +from __future__ import print_function +import mxnet as mx +from mxnet import foo +from mxnet.foo import nn +from mxnet import autograd as ag +``` + +### Define a network: Multilayer Perceptron + +The first approach makes use of a [Multilayer Perceptron](https://en.wikipedia.org/wiki/Multilayer_perceptron) to solve this problem. We'll define the MLP using MXNet's imperative approach. + +MLPs contains several fully connected layers. A fully connected layer or FC layer for short, is one where each neuron in the layer is connected to every neuron in its preceding layer. From a linear algebra perspective, an FC layer applies an [affine transform](https://en.wikipedia.org/wiki/Affine_transformation) to the *n x m* input matrix *X* and outputs a matrix *Y* of size *n x k*, where *k* is the number of neurons in the FC layer. *k* is also referred to as the hidden size. The output *Y* is computed according to the equation *Y = W X + b*. The FC layer has two learnable parameters, the *m x k* weight matrix *W* and the *m x 1* bias vector *b*. + +In an MLP, the outputs of most FC layers are fed into an activation function, which applies an element-wise non-linearity. This step is critical and it gives neural networks the ability to classify inputs that are not linearly separable. Common choices for activation functions are sigmoid, tanh, and [rectified linear unit](https://en.wikipedia.org/wiki/Rectifier_%28neural_networks%29) (ReLU). In this example, we'll use the ReLU activation function which has several desirable properties and is typically considered a default choice. + +The following code declares three fully connected layers with 128, 64 and 10 neurons each. +The last fully connected layer often has its hidden size equal to the number of output classes in the dataset. Furthermore, these FC layers uses ReLU activation for performing an element-wise ReLU transformation on the FC layer output. + +To do this, we will use [Sequential layer](http://mxnet.io/api/python/foo.html#mxnet.foo.nn.Sequential) type. This is simply a linear stack of neural network layers. `nn.Dense` layers are nothing but the fully connected layers we discussed above. + +```python +# define network +net = nn.Sequential() +with net.name_scope(): + net.add(nn.Dense(128, activation='relu')) + net.add(nn.Dense(64, activation='relu')) + net.add(nn.Dense(10)) +``` + +#### Initialize parameters and optimizer + +The following source code initializes all parameters received from parameter dict using [Xavier](http://mxnet.io/api/python/optimization.html#mxnet.initializer.Xavier) initializer +to train the MLP network we defined above. + +For our training, we will make use of the stochastic gradient descent (SGD) optimizer. In particular, we'll be using mini-batch SGD. Standard SGD processes train data one example at a time. In practice, this is very slow and one can speed up the process by processing examples in small batches. In this case, our batch size will be 100, which is a reasonable choice. Another parameter we select here is the learning rate, which controls the step size the optimizer takes in search of a solution. We'll pick a learning rate of 0.1, again a reasonable choice. Settings such as batch size and learning rate are what are usually referred to as hyper-parameters. What values we give them can have a great impact on training performance. + +We will use [Trainer](http://mxnet.io/api/python/foo.html#trainer) class to apply the +[SGD optimizer](http://mxnet.io/api/python/optimization.html#mxnet.optimizer.SGD) on the +initialized parameters. + +```python +ctx = [mx.cpu(0), mx.cpu(1)] +net.all_params().initialize(mx.init.Xavier(magnitude=2.24), ctx=ctx) +trainer = foo.Trainer(net.all_params(), 'sgd', {'learning_rate': 0.1}) +``` + +#### Train the network + +Typically, one runs the training until convergence, which means that we have learned a good set of model parameters (weights + biases) from the train data. For the purpose of this tutorial, we'll run training for 10 epochs and stop. An epoch is one full pass over the entire train data. + +We will take following steps for training: + +- Define [Accuracy evaluation metric](http://mxnet.io/api/python/metric.html#mxnet.metric.Accuracy) over training data. +- Loop over inputs for every epoch. +- Forward input through network to get output. +- Compute loss with output and label inside record scope. +- Backprop gradient inside record scope. +- Update evaluation metric and parameters with gradient descent. + +Loss function takes (output, label) pairs and computes a scalar loss for each sample in the mini-batch. The scalars measure how far each output is from the label. +There are many predefined loss functions in foo.loss. Here we use +[softmax_cross_entropy_loss](http://mxnet.io/api/python/foo.html#mxnet.foo.loss.softmax_cross_entropy_loss) for digit classification. We will compute loss and do backward propagation inside +training scope which is defined by `autograd.record()`. + +```python +epoch = 10 +# Use Accuracy as the evaluation metric. +metric = mx.metric.Accuracy() + +for i in range(epoch): + # Reset the train data iterator. + train_data.reset() + # Loop over the train data iterator. + for batch in train_data: + # Splits train data into multiple slices along batch_axis + # and copy each slice into a context. + data = foo.utils.load_data(batch.data[0], ctx_list=ctx, batch_axis=0) + # Splits train labels into multiple slices along batch_axis + # and copy each slice into a context. + label = foo.utils.load_data(batch.label[0], ctx_list=ctx, batch_axis=0) + outputs = [] + # Inside training scope + with ag.record(): + for x, y in zip(data, label): + z = net(x) + # Computes softmax cross entropy loss. + loss = foo.loss.softmax_cross_entropy_loss(z, y) + # Backpropogate the error for one iteration. + ag.backward([loss]) + outputs.append(z) + # Updates internal evaluation + metric.update(label, outputs) + # Make one step of parameter update. Trainer needs to know the + # batch size of data to normalize the gradient by 1/batch_size. + trainer.step(batch.data[0].shape[0]) + # Gets the evaluation result. + name, acc = metric.get() + # Reset evaluation result to initial state. + metric.reset() + print('training acc at epoch %d: %s=%f'%(i, name, acc)) +``` + +#### Prediction + +After the above training completes, we can evaluate the trained model by running predictions on validation dataset. Since the dataset also has labels for all test images, we can compute the accuracy metric over validation data as follows: + +```python +# Use Accuracy as the evaluation metric. +metric = mx.metric.Accuracy() +# Reset the validation data iterator. +val_data.reset() +# Loop over the validation data iterator. +for batch in val_data: + # Splits validation data into multiple slices along batch_axis + # and copy each slice into a context. + data = foo.utils.load_data(batch.data[0], ctx_list=ctx, batch_axis=0) + # Splits validation label into multiple slices along batch_axis + # and copy each slice into a context. + label = foo.utils.load_data(batch.label[0], ctx_list=ctx, batch_axis=0) + outputs = [] + for x in data: + outputs.append(net(x)) + # Updates internal evaluation + metric.update(label, outputs) +print('validation acc: %s=%f'%metric.get()) +assert metric.get()[1] > 0.96 +``` + +If everything went well, we should see an accuracy value that is around 0.96, which means that we are able to accurately predict the digit in 96% of test images. This is a pretty good result. But as we will see in the next part of this tutorial, we can do a lot better than that. + +### Convolutional Neural Network + +Earlier, we briefly touched on a drawback of MLP when we said we need to discard the input image's original shape and flatten it as a vector before we can feed it as input to the MLP's first fully connected layer. Turns out this is an important issue because we don't take advantage of the fact that pixels in the image have natural spatial correlation along the horizontal and vertical axes. A convolutional neural network (CNN) aims to address this problem by using a more structured weight representation. Instead of flattening the image and doing a simple matrix-matrix multiplication, it employs one or more convolutional layers that each performs a 2-D convolution on the input image. + +A single convolution layer consists of one or more filters that each play the role of a feature detector. During training, a CNN learns appropriate representations (parameters) for these filters. Similar to MLP, the output from the convolutional layer is transformed by applying a non-linearity. Besides the convolutional layer, another key aspect of a CNN is the pooling layer. A pooling layer serves to make the CNN translation invariant: a digit remains the same even when it is shifted left/right/up/down by a few pixels. A pooling layer reduces a *n x m* patch into a single value to make the network less sensitive to the spatial location. Pooling layer is always included after each conv (+ activation) layer in the CNN. + +The following source code defines a convolutional neural network architecture called LeNet. LeNet is a popular network known to work well on digit classification tasks. We will use a slightly different version from the original LeNet implementation, replacing the sigmoid activations with tanh activations for the neurons. + +A typical way to write your network is creating a new class inherited from `foo.nn.Layer` +class. We can define the network by composing and inheriting Layer class as follows: + +```python +import mxnet.ndarray as F + +class Net(nn.Layer): + def __init__(self, **kwargs): + super(Net, self).__init__(**kwargs) + with self.name_scope(): + # layers created in name_scope will inherit name space + # from parent layer. + self.conv1 = nn.Conv2D(20, kernel_size=(5,5)) + self.pool1 = nn.MaxPool2D(pool_size=(2,2), strides = (2,2)) + self.conv2 = nn.Conv2D(50, kernel_size=(5,5)) + self.pool2 = nn.MaxPool2D(pool_size=(2,2), strides = (2,2)) + self.fc1 = nn.Dense(500) + self.fc2 = nn.Dense(10) + + def forward(self, x): + x = self.pool1(F.tanh(self.conv1(x))) + x = self.pool2(F.tanh(self.conv2(x))) + # 0 means copy over size from corresponding dimension. + # -1 means infer size from the rest of dimensions. + x = x.reshape((0, -1)) + x = F.tanh(self.fc1(x)) + x = F.tanh(self.fc2(x)) + return x +``` + +We just defined the forward function here, and the backward function to compute gradients +is automatically defined for you using autograd. +We also imported `mxnet.ndarray` package to use activation functions from `ndarray` API. + +Now, We will create the network as follows: + +```python +net = Net() +``` + +![png](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/image/conv_mnist.png) + +**Figure 3:** First conv + pooling layer in LeNet. + +Now we train LeNet with the same hyper-parameters as before. Note that, if a GPU is available, we recommend using it. This greatly speeds up computation given that LeNet is more complex and compute-intensive than the previous multilayer perceptron. To do so, we only need to change `mx.cpu()` to `mx.gpu()` and MXNet takes care of the rest. Just like before, we'll stop training after 10 epochs. + +Training and prediction can be done in the similar way as we did for MLP. + +#### Initialize parameters and optimizer + +We will initialize the network parameters as follows: + +```python +ctx = [mx.cpu(0), mx.cpu(1)] +net.all_params().initialize(mx.init.Xavier(magnitude=2.24), ctx=ctx) +trainer = foo.Trainer(net.all_params(), 'sgd', {'learning_rate': 0.1}) +``` + +#### Training + +```python +# Use Accuracy as the evaluation metric. +metric = mx.metric.Accuracy() + +for i in range(epoch): + # Reset the train data iterator. + train_data.reset() + # Loop over the train data iterator. + for batch in train_data: + # Splits train data into multiple slices along batch_axis + # and copy each slice into a context. + data = foo.utils.load_data(batch.data[0], ctx_list=ctx, batch_axis=0) + # Splits train labels into multiple slices along batch_axis + # and copy each slice into a context. + label = foo.utils.load_data(batch.label[0], ctx_list=ctx, batch_axis=0) + outputs = [] + # Inside training scope + with ag.record(): + for x, y in zip(data, label): + z = net(x) + # Computes softmax cross entropy loss. + loss = foo.loss.softmax_cross_entropy_loss(z, y) + # Backpropogate the error for one iteration. + ag.backward([loss]) + outputs.append(z) + # Updates internal evaluation + metric.update(label, outputs) + # Make one step of parameter update. Trainer needs to know the + # batch size of data to normalize the gradient by 1/batch_size. + trainer.step(batch.data[0].shape[0]) + # Gets the evaluation result. + name, acc = metric.get() + # Reset evaluation result to initial state. + metric.reset() + print('training acc at epoch %d: %s=%f'%(i, name, acc)) +``` + +#### Prediction + +Finally, we'll use the trained LeNet model to generate predictions for the test data. + +```python +# Use Accuracy as the evaluation metric. +metric = mx.metric.Accuracy() +# Reset the validation data iterator. +val_data.reset() +# Loop over the validation data iterator. +for batch in val_data: + # Splits validation data into multiple slices along batch_axis + # and copy each slice into a context. + data = foo.utils.load_data(batch.data[0], ctx_list=ctx, batch_axis=0) + # Splits validation label into multiple slices along batch_axis + # and copy each slice into a context. + label = foo.utils.load_data(batch.label[0], ctx_list=ctx, batch_axis=0) + outputs = [] + for x in data: + outputs.append(net(x)) + # Updates internal evaluation + metric.update(label, outputs) +print('validation acc: %s=%f'%metric.get()) +assert metric.get()[1] > 0.98 +``` + +If all went well, we should see a higher accuracy metric for predictions made using LeNet. With CNN we should be able to correctly predict around 98% of all test images. + +## Summary + +In this tutorial, we have learned how to use MXNet to solve a standard computer vision problem: classifying images of hand written digits. You have seen how to quickly and easily build, train and evaluate models such as MLP and CNN with MXNet Foo package. diff --git a/example/autograd/mnist.py b/example/autograd/mnist.py index a8b70226be3f..77a6de121e8d 100644 --- a/example/autograd/mnist.py +++ b/example/autograd/mnist.py @@ -53,7 +53,7 @@ def train(epoch, ctx): for x, y in zip(data, label): z = net(x) loss = foo.loss.softmax_cross_entropy_loss(z, y) - ag.compute_gradient([loss]) + ag.backward([loss]) outputs.append(z) metric.update(label, outputs) trainer.step(batch.data[0].shape[0]) diff --git a/python/mxnet/autograd.py b/python/mxnet/autograd.py index e45c956e2bb7..1c791f26b16f 100644 --- a/python/mxnet/autograd.py +++ b/python/mxnet/autograd.py @@ -36,7 +36,7 @@ class TrainingStateScope(object): Example:: with TrainingStateScope(True): y = model(x) - compute_gradient([y]) + backward([y]) """ def __init__(self, enter_state): self._enter_state = enter_state @@ -57,7 +57,7 @@ def record(): Example:: with autograd.record(): y = model(x) - compute_gradient([y]) + backward([y]) metric.update(...) optim.step(...) """ @@ -71,7 +71,7 @@ def pause(): Example:: with autograd.record(): y = model(x) - compute_gradient([y]) + backward([y]) with autograd.pause(): # testing, IO, gradient updates... """ From 615c37752dc81e42ff31d810a4e35725b3ec98a5 Mon Sep 17 00:00:00 2001 From: Eric Junyuan Xie Date: Fri, 7 Jul 2017 14:01:05 -0700 Subject: [PATCH 196/834] rename load_data to split_and_load (#6934) * rename load_data to split_and_load * Update foo.md --- docs/api/python/foo.md | 4 ++- docs/tutorials/foo/mnist.md | 16 ++++----- example/autograd/mnist.py | 24 ++++++------- example/autograd/resnet.py | 8 ++--- example/autograd/super_resolution.py | 8 ++--- python/mxnet/foo/utils.py | 54 ++++++++++++++++++---------- src/c_api/c_api_ndarray.cc | 4 --- src/ndarray/autograd.cc | 23 ++++++------ tests/python/train/test_autograd.py | 22 ++++++------ tests/python/unittest/test_nn.py | 21 +++++++++++ 10 files changed, 110 insertions(+), 74 deletions(-) diff --git a/docs/api/python/foo.md b/docs/api/python/foo.md index 85d4a353b026..464794c41a0c 100644 --- a/docs/api/python/foo.md +++ b/docs/api/python/foo.md @@ -1,5 +1,6 @@ # Foo Package + ```eval_rst .. currentmodule:: mxnet.foo ``` @@ -223,7 +224,8 @@ in Python and then deploy with symbolic graph in C++ and Scala. ```eval_rst .. automethod:: mxnet.foo.utils.split_data -.. automethod:: mxnet.foo.utils.load_data +.. automethod:: mxnet.foo.utils.split_and_load +.. automethod:: mxnet.foo.utils.clip_global_norm ``` diff --git a/docs/tutorials/foo/mnist.md b/docs/tutorials/foo/mnist.md index ba1458b9396b..2e322efefdeb 100644 --- a/docs/tutorials/foo/mnist.md +++ b/docs/tutorials/foo/mnist.md @@ -131,10 +131,10 @@ for i in range(epoch): for batch in train_data: # Splits train data into multiple slices along batch_axis # and copy each slice into a context. - data = foo.utils.load_data(batch.data[0], ctx_list=ctx, batch_axis=0) + data = foo.utils.split_and_load(batch.data[0], ctx_list=ctx, batch_axis=0) # Splits train labels into multiple slices along batch_axis # and copy each slice into a context. - label = foo.utils.load_data(batch.label[0], ctx_list=ctx, batch_axis=0) + label = foo.utils.split_and_load(batch.label[0], ctx_list=ctx, batch_axis=0) outputs = [] # Inside training scope with ag.record(): @@ -170,10 +170,10 @@ val_data.reset() for batch in val_data: # Splits validation data into multiple slices along batch_axis # and copy each slice into a context. - data = foo.utils.load_data(batch.data[0], ctx_list=ctx, batch_axis=0) + data = foo.utils.split_and_load(batch.data[0], ctx_list=ctx, batch_axis=0) # Splits validation label into multiple slices along batch_axis # and copy each slice into a context. - label = foo.utils.load_data(batch.label[0], ctx_list=ctx, batch_axis=0) + label = foo.utils.split_and_load(batch.label[0], ctx_list=ctx, batch_axis=0) outputs = [] for x in data: outputs.append(net(x)) @@ -264,10 +264,10 @@ for i in range(epoch): for batch in train_data: # Splits train data into multiple slices along batch_axis # and copy each slice into a context. - data = foo.utils.load_data(batch.data[0], ctx_list=ctx, batch_axis=0) + data = foo.utils.split_and_load(batch.data[0], ctx_list=ctx, batch_axis=0) # Splits train labels into multiple slices along batch_axis # and copy each slice into a context. - label = foo.utils.load_data(batch.label[0], ctx_list=ctx, batch_axis=0) + label = foo.utils.split_and_load(batch.label[0], ctx_list=ctx, batch_axis=0) outputs = [] # Inside training scope with ag.record(): @@ -303,10 +303,10 @@ val_data.reset() for batch in val_data: # Splits validation data into multiple slices along batch_axis # and copy each slice into a context. - data = foo.utils.load_data(batch.data[0], ctx_list=ctx, batch_axis=0) + data = foo.utils.split_and_load(batch.data[0], ctx_list=ctx, batch_axis=0) # Splits validation label into multiple slices along batch_axis # and copy each slice into a context. - label = foo.utils.load_data(batch.label[0], ctx_list=ctx, batch_axis=0) + label = foo.utils.split_and_load(batch.label[0], ctx_list=ctx, batch_axis=0) outputs = [] for x in data: outputs.append(net(x)) diff --git a/example/autograd/mnist.py b/example/autograd/mnist.py index 77a6de121e8d..ce5a4e6281cb 100644 --- a/example/autograd/mnist.py +++ b/example/autograd/mnist.py @@ -24,43 +24,43 @@ # train -def test(ctx): +def test(ctxs): metric = mx.metric.Accuracy() val_data.reset() for batch in val_data: - data = foo.utils.load_data(batch.data[0], ctx_list=ctx, batch_axis=0) - label = foo.utils.load_data(batch.label[0], ctx_list=ctx, batch_axis=0) + data = foo.utils.split_and_load(batch.data[0], ctxs, batch_axis=0) + label = foo.utils.split_and_load(batch.label[0], ctxs, batch_axis=0) outputs = [] for x in data: outputs.append(net(x)) metric.update(label, outputs) print('validation acc: %s=%f'%metric.get()) -def train(epoch, ctx): - if isinstance(ctx, mx.Context): - ctx = [ctx] - net.all_params().initialize(mx.init.Xavier(magnitude=2.24), ctx=ctx) +def train(epoch, ctxs): + if isinstance(ctxs, mx.Context): + ctxs = [ctxs] + net.all_params().initialize(mx.init.Xavier(magnitude=2.24), ctx=ctxs) trainer = foo.Trainer(net.all_params(), 'sgd', {'learning_rate': 0.1}) metric = mx.metric.Accuracy() for i in range(epoch): train_data.reset() for batch in train_data: - data = foo.utils.load_data(batch.data[0], ctx_list=ctx, batch_axis=0) - label = foo.utils.load_data(batch.label[0], ctx_list=ctx, batch_axis=0) + datas = foo.utils.split_and_load(batch.data[0], ctxs, batch_axis=0) + labels = foo.utils.split_and_load(batch.label[0], ctxs, batch_axis=0) outputs = [] with ag.record(): - for x, y in zip(data, label): + for x, y in zip(datas, labels): z = net(x) loss = foo.loss.softmax_cross_entropy_loss(z, y) ag.backward([loss]) outputs.append(z) - metric.update(label, outputs) + metric.update(labels, outputs) trainer.step(batch.data[0].shape[0]) name, acc = metric.get() metric.reset() print('training acc at epoch %d: %s=%f'%(i, name, acc)) - test(ctx) + test(ctxs) net.all_params().save('mnist.params') diff --git a/example/autograd/resnet.py b/example/autograd/resnet.py index 85451d5571bf..b6f042d58242 100644 --- a/example/autograd/resnet.py +++ b/example/autograd/resnet.py @@ -315,8 +315,8 @@ def test(ctx): metric = mx.metric.Accuracy() val_data.reset() for batch in val_data: - data = foo.utils.load_data(batch.data[0], ctx_list=ctx, batch_axis=0) - label = foo.utils.load_data(batch.label[0], ctx_list=ctx, batch_axis=0) + data = foo.utils.split_and_load(batch.data[0], ctx_list=ctx, batch_axis=0) + label = foo.utils.split_and_load(batch.label[0], ctx_list=ctx, batch_axis=0) outputs = [] for x in data: outputs.append(net(x)) @@ -336,8 +336,8 @@ def train(epoch, ctx): train_data.reset() btic = time.time() for batch in train_data: - data = foo.utils.load_data(batch.data[0], ctx_list=ctx, batch_axis=0) - label = foo.utils.load_data(batch.label[0], ctx_list=ctx, batch_axis=0) + data = foo.utils.split_and_load(batch.data[0], ctx_list=ctx, batch_axis=0) + label = foo.utils.split_and_load(batch.label[0], ctx_list=ctx, batch_axis=0) outputs = [] losses = [] with ag.record(): diff --git a/example/autograd/super_resolution.py b/example/autograd/super_resolution.py index fec4f11b7f63..c12408e40398 100644 --- a/example/autograd/super_resolution.py +++ b/example/autograd/super_resolution.py @@ -114,8 +114,8 @@ def test(ctx): for batch in val_data: batches += 1 metric.reset() - data = foo.utils.load_data(batch.data[0], ctx_list=ctx, batch_axis=0) - label = foo.utils.load_data(batch.label[0], ctx_list=ctx, batch_axis=0) + data = foo.utils.split_and_load(batch.data[0], ctx_list=ctx, batch_axis=0) + label = foo.utils.split_and_load(batch.label[0], ctx_list=ctx, batch_axis=0) outputs = [] for x in data: outputs.append(net(x)) @@ -136,8 +136,8 @@ def train(epoch, ctx): for i in range(epoch): train_data.reset() for batch in train_data: - data = foo.utils.load_data(batch.data[0], ctx_list=ctx, batch_axis=0) - label = foo.utils.load_data(batch.label[0], ctx_list=ctx, batch_axis=0) + data = foo.utils.split_and_load(batch.data[0], ctx_list=ctx, batch_axis=0) + label = foo.utils.split_and_load(batch.label[0], ctx_list=ctx, batch_axis=0) outputs = [] with ag.record(): for x, y in zip(data, label): diff --git a/python/mxnet/foo/utils.py b/python/mxnet/foo/utils.py index fe410c48cdc9..18703cda4058 100644 --- a/python/mxnet/foo/utils.py +++ b/python/mxnet/foo/utils.py @@ -7,6 +7,8 @@ def split_data(data, num_slice, batch_axis=0, even_split=True): """Split a NDArray into num_slice slices along batch_axis. + Usually used for data parallelism where each slices is sent + to one device (i.e. GPU). Parameters ---------- @@ -18,35 +20,50 @@ def split_data(data, num_slice, batch_axis=0, even_split=True): The axis along which to slice. even_split : bool, default True Whether to force all slices to have the same number of elements. + If True, An error will be raised when `num_slice` does not evenly + divide `data.shape[batch_axis]`. Returns ------- list of NDArray + Return value is a list even if num_slice is 1. """ - assert even_split, "Only support even split for now" - assert not even_split or data.shape[batch_axis] % num_slice == 0, \ - "data with shape %s cannot be evenly split into %d slices along axis %d. " \ - "Use a batch size that's multiple of %d or set even_split=False to enable " \ - "uneven partitioning of data."%( - str(data.shape), num_slice, batch_axis, num_slice) - size = data.shape[batch_axis] // num_slice + size = data.shape[batch_axis] + if size < num_slice: + raise ValueError( + "Too many slices for data with shape %s. Arguments are " \ + "num_slice=%d and batch_axis=%d."%(str(data.shape), num_slice, batch_axis)) + if even_split and size % num_slice != 0: + raise ValueError( + "data with shape %s cannot be evenly split into %d slices along axis %d. " \ + "Use a batch size that's multiple of %d or set even_split=False to allow " \ + "uneven partitioning of data."%( + str(data.shape), num_slice, batch_axis, num_slice)) + + step = size // num_slice if batch_axis == 0: - slices = [data[i*size:(i+1)*size] for i in range(num_slice)] + slices = [data[i*step:(i+1)*step] if i < num_slice - 1 else data[i*step:size] + for i in range(num_slice)] + elif even_split: + slices = ndarray.split(data, num_outputs=num_slice, axis=batch_axis) else: - slices = [ndarray.slice_axis(data, i*size, (i+1)*size) + slices = [ndarray.slice_axis(data, batch_axis, i*step, (i+1)*step) + if i < num_slice - 1 else + ndarray.slice_axis(data, batch_axis, i*step, size) for i in range(num_slice)] return slices -def load_data(data, ctx_list, batch_axis=0, even_split=True): - """Split a NDArray into multiple slices along batch_axis and copy - each slice into a context. + +def split_and_load(data, ctx_list, batch_axis=0, even_split=True): + """Split a NDArray into `len(ctx_list)` slices along `batch_axis` and load + each slice to one context in ctx_list. Parameters ---------- data : NDArray A batch of data. ctx_list : list of Context - A list of Context + A list of Contexts batch_axis : int, default 0 The axis along which to slice. even_split : bool, default True @@ -56,14 +73,13 @@ def load_data(data, ctx_list, batch_axis=0, even_split=True): ------- list of NDArray, each corresponds to a context in ctx_list. """ + if not isinstance(data, ndarray.NDArray): + data = ndarray.array(data, ctx=ctx_list[0]) if len(ctx_list) == 1: - if not isinstance(data, ndarray.NDArray): - data = ndarray.array(data, ctx=ctx_list[0]) return [data.as_in_context(ctx_list[0])] - else: - slices = split_data(data, len(ctx_list), batch_axis=batch_axis, - even_split=even_split) - return [i.as_in_context(ctx) for i, ctx in zip(slices, ctx_list)] + + slices = split_data(data, len(ctx_list), batch_axis, even_split) + return [i.as_in_context(ctx) for i, ctx in zip(slices, ctx_list)] def clip_global_norm(arrays, max_norm): diff --git a/src/c_api/c_api_ndarray.cc b/src/c_api/c_api_ndarray.cc index 4b3bc092e932..63ed6c482c6a 100644 --- a/src/c_api/c_api_ndarray.cc +++ b/src/c_api/c_api_ndarray.cc @@ -86,10 +86,6 @@ void SetNDInputsOutputs(const nnvm::Op* op, *num_outputs = num_visible_outputs; ndoutputs.resize(infered_num_outputs); } else { - CHECK(!AutogradRuntime::Get()->IsTraining()) - << "Inplace operations (+=, -=, op(..., out=x) etc.) and assignment are " - << "not supported when recording with autograd. You can use autograd.pause " - << "to wrap this operation, but it may invalid gradients."; CHECK(*num_outputs == infered_num_outputs || *num_outputs == num_visible_outputs) << "Expecting " << infered_num_outputs << " (all) or " << num_visible_outputs << " (visible only) outputs, got " diff --git a/src/ndarray/autograd.cc b/src/ndarray/autograd.cc index 7908f2c9d005..4875419b5973 100644 --- a/src/ndarray/autograd.cc +++ b/src/ndarray/autograd.cc @@ -119,17 +119,6 @@ AGNodePtr AutogradRuntime::RecordOp(const nnvm::Op* op, AGNodePtr ag_node = AGNode::Create(nn_node); ag_node->state = state; - for (uint32_t i = 0; i < outputs.size(); ++i) { - CHECK(outputs[i].entry_.is_none()) - << "Output NDArray is non-empty and already in another computation graph. " - << "Assigning to it will cause undefined behavior when evaluating gradients. " - << "Please call backward first to clear the graph or do this out side of " - << "a train section. "; - outputs[i].entry_.clear(); - ag_node->outputs.push_back(outputs[i]); - outputs[i].entry_ = AGNodeEntry{ag_node, i, 0}; - } - for (size_t i = 0; i < inputs.size(); ++i) { if (inputs[i].entry_.is_none()) { AGNodeEntry e{AGNode::Create(Node::Create()), 0, 0}; @@ -142,6 +131,18 @@ AGNodePtr AutogradRuntime::RecordOp(const nnvm::Op* op, ag_node->inputs.push_back(inputs[i].entry_); } + for (uint32_t i = 0; i < outputs.size(); ++i) { + CHECK(outputs[i].entry_.is_none()) + << "Inplace operation is not supported when recording with autograd. " + << "Assigning to NDArrays that are already in a computational graph " + << "will cause undefined behavior when evaluating gradients. " + << "Please call backward first to clear the graph or do this out side of " + << "a record section. "; + outputs[i].entry_.clear(); + ag_node->outputs.push_back(outputs[i]); + outputs[i].entry_ = AGNodeEntry{ag_node, i, 0}; + } + return ag_node; } diff --git a/tests/python/train/test_autograd.py b/tests/python/train/test_autograd.py index 6a979f5a02e1..0139deb8b500 100644 --- a/tests/python/train/test_autograd.py +++ b/tests/python/train/test_autograd.py @@ -35,37 +35,37 @@ def get_net(): label_name='sm_label', batch_size=batch_size, shuffle=True, flat=True, silent=False) -def score(net, ctx): +def score(net, ctx_list): metric = mx.metric.Accuracy() val_data.reset() for batch in val_data: - data = foo.utils.load_data(batch.data[0], ctx_list=ctx, batch_axis=0) - label = foo.utils.load_data(batch.label[0], ctx_list=ctx, batch_axis=0) + datas = foo.utils.split_and_load(batch.data[0], ctx_list, batch_axis=0) + labels = foo.utils.split_and_load(batch.label[0], ctx_list, batch_axis=0) outputs = [] - for x in data: + for x in datas: outputs.append(net(x)) - metric.update(label, outputs) + metric.update(labels, outputs) return metric.get()[1] -def train(net, epoch, ctx): - net.all_params().initialize(mx.init.Xavier(magnitude=2.24), ctx=ctx) +def train(net, epoch, ctx_list): + net.all_params().initialize(mx.init.Xavier(magnitude=2.24), ctx=ctx_list) trainer = foo.Trainer(net.all_params(), 'sgd', {'learning_rate': 0.5}) metric = mx.metric.Accuracy() for i in range(epoch): train_data.reset() for batch in train_data: - data = foo.utils.load_data(batch.data[0], ctx_list=ctx, batch_axis=0) - label = foo.utils.load_data(batch.label[0], ctx_list=ctx, batch_axis=0) + datas = foo.utils.split_and_load(batch.data[0], ctx_list, batch_axis=0) + labels = foo.utils.split_and_load(batch.label[0], ctx_list, batch_axis=0) outputs = [] with autograd.record(): - for x, y in zip(data, label): + for x, y in zip(datas, labels): z = net(x) loss = foo.loss.softmax_cross_entropy_loss(z, y) loss.backward() outputs.append(z) - metric.update(label, outputs) trainer.step(batch.data[0].shape[0]) + metric.update(labels, outputs) name, acc = metric.get() metric.reset() print('training acc at epoch %d: %s=%f'%(i, name, acc)) diff --git a/tests/python/unittest/test_nn.py b/tests/python/unittest/test_nn.py index ccfdea55c6fb..59886cabd8dc 100644 --- a/tests/python/unittest/test_nn.py +++ b/tests/python/unittest/test_nn.py @@ -231,6 +231,27 @@ def test_defered_init(): layer(x) +def check_split_data(x, num_slice, batch_axis, **kwargs): + res = foo.utils.split_data(x, num_slice, batch_axis, **kwargs) + assert len(res) == num_slice + mx.test_utils.assert_almost_equal(mx.nd.concat(*res, dim=batch_axis).asnumpy(), + x.asnumpy()) + + +def test_split_data(): + x = mx.nd.random_uniform(shape=(128, 33, 64)) + + check_split_data(x, 8, 0) + check_split_data(x, 3, 1) + check_split_data(x, 4, 1, even_split=False) + check_split_data(x, 15, 1, even_split=False) + try: + check_split_data(x, 4, 1) + except ValueError: + return + assert False, "Should have failed" + + if __name__ == '__main__': import nose nose.runmodule() From 9220680fa1dee4b3d599235a4c06b762aea7a078 Mon Sep 17 00:00:00 2001 From: Eric Junyuan Xie Date: Mon, 10 Jul 2017 09:20:03 -0700 Subject: [PATCH 197/834] rename nn.Layer to foo.Layer (#6959) * rename nn.Layer to foo.Layer * rename Layer to Block * fix * fix * fix --- docs/api/python/foo.md | 39 +- docs/tutorials/foo/foo.md | 12 +- docs/tutorials/foo/hybrid.md | 24 +- docs/tutorials/foo/mnist.md | 14 +- example/autograd/actor_critic.py | 12 +- example/autograd/data.py | 2 + example/autograd/dcgan.py | 16 +- example/autograd/mnist.py | 115 ++- example/autograd/resnet.py | 185 ++-- example/autograd/super_resolution.py | 25 +- example/autograd/word_language_model/model.py | 2 +- example/autograd/word_language_model/train.py | 40 +- example/recommenders/symbol_alexnet.py | 4 +- python/mxnet/autograd.py | 12 +- python/mxnet/foo/__init__.py | 2 + python/mxnet/foo/block.py | 349 ++++++++ python/mxnet/foo/loss.py | 215 ++--- python/mxnet/foo/nn/__init__.py | 2 +- python/mxnet/foo/nn/basic_layers.py | 290 ++++++ python/mxnet/foo/nn/conv_layers.py | 841 +++++++++--------- python/mxnet/foo/nn/layer.py | 615 ------------- python/mxnet/foo/parameter.py | 14 +- python/mxnet/foo/rnn/rnn_cell.py | 14 +- python/mxnet/foo/rnn/rnn_layer.py | 16 +- python/mxnet/foo/trainer.py | 4 +- tests/python/gpu/test_operator_gpu.py | 2 +- tests/python/train/test_autograd.py | 13 +- tests/python/unittest/test_foo_rnn.py | 14 +- tests/python/unittest/test_loss.py | 164 ++-- tests/python/unittest/test_nn.py | 103 ++- 30 files changed, 1553 insertions(+), 1607 deletions(-) create mode 100644 python/mxnet/foo/block.py create mode 100644 python/mxnet/foo/nn/basic_layers.py delete mode 100644 python/mxnet/foo/nn/layer.py diff --git a/docs/api/python/foo.md b/docs/api/python/foo.md index 464794c41a0c..b43f7372ff62 100644 --- a/docs/api/python/foo.md +++ b/docs/api/python/foo.md @@ -9,6 +9,8 @@ .. warning:: This package is currently experimental and may change in the near future. ``` + + ## Overview Foo package is a high-level interface for MXNet designed to be easy to use while @@ -22,17 +24,29 @@ in Python and then deploy with symbolic graph in C++ and Scala. .. currentmodule:: mxnet.foo ``` - ```eval_rst +.. currentmodule:: mxnet.foo .. autoclass:: mxnet.foo.Parameter :members: .. autoclass:: mxnet.foo.ParameterDict :members: ``` - +## Containers + +```eval_rst +.. currentmodule:: mxnet.foo +.. autoclass:: mxnet.foo.Block + :members: + + .. automethod:: forward +.. autoclass:: mxnet.foo.HybridBlock + :members: + + .. automethod:: hybrid_forward +``` ## Neural Network Layers @@ -42,16 +56,9 @@ in Python and then deploy with symbolic graph in C++ and Scala. ### Containers - ```eval_rst .. currentmodule:: mxnet.foo.nn -.. autoclass:: mxnet.foo.nn.Layer - :members: - - .. automethod:: __call__ -.. autoclass:: mxnet.foo.nn.HybridLayer - :members: .. automethod:: __call__ .. autoclass:: mxnet.foo.nn.Sequential @@ -60,11 +67,9 @@ in Python and then deploy with symbolic graph in C++ and Scala. :members: ``` - ### Basic Layers - ```eval_rst .. currentmodule:: mxnet.foo.nn @@ -82,11 +87,9 @@ in Python and then deploy with symbolic graph in C++ and Scala. :members: ``` - ### Convolutional Layers - ```eval_rst .. currentmodule:: mxnet.foo.nn @@ -104,12 +107,10 @@ in Python and then deploy with symbolic graph in C++ and Scala. :members: ``` - ### Pooling Layers - ```eval_rst .. currentmodule:: mxnet.foo.nn @@ -139,7 +140,6 @@ in Python and then deploy with symbolic graph in C++ and Scala. :members: ``` - ## Recurrent Layers @@ -148,7 +148,6 @@ in Python and then deploy with symbolic graph in C++ and Scala. .. currentmodule:: mxnet.foo.rnn ``` - ```eval_rst .. autoclass:: mxnet.foo.rnn.RecurrentCell @@ -179,7 +178,6 @@ in Python and then deploy with symbolic graph in C++ and Scala. :members: ``` - ## Trainer @@ -187,14 +185,12 @@ in Python and then deploy with symbolic graph in C++ and Scala. .. currentmodule:: mxnet.foo ``` - ```eval_rst .. autoclass:: mxnet.foo.Trainer :members: ``` - ## Loss functions @@ -202,7 +198,6 @@ in Python and then deploy with symbolic graph in C++ and Scala. .. currentmodule:: mxnet.foo.loss ``` - ```eval_rst .. automethod:: mxnet.foo.loss.custom_loss @@ -212,7 +207,6 @@ in Python and then deploy with symbolic graph in C++ and Scala. .. automethod:: mxnet.foo.loss.softmax_cross_entropy_loss ``` - ## Utilities @@ -220,7 +214,6 @@ in Python and then deploy with symbolic graph in C++ and Scala. .. currentmodule:: mxnet.foo.utils ``` - ```eval_rst .. automethod:: mxnet.foo.utils.split_data diff --git a/docs/tutorials/foo/foo.md b/docs/tutorials/foo/foo.md index c7d782037d0f..cffb7c27bd87 100644 --- a/docs/tutorials/foo/foo.md +++ b/docs/tutorials/foo/foo.md @@ -32,11 +32,11 @@ steps: ## Define Network -`foo.nn.Layer` is the basic building block of models. You can define networks by -composing and inheriting `Layer`: +`foo.Block` is the basic building block of models. You can define networks by +composing and inheriting `Block`: ```python -class Net(nn.Layer): +class Net(foo.Block): def __init__(self, **kwargs): super(Net, self).__init__(**kwargs) with self.name_scope(): @@ -70,7 +70,7 @@ A network must be created and initialized before it can be used: net = Net() # Initialize on CPU. Replace with `mx.gpu(0)`, or `[mx.gpu(0), mx.gpu(1)]`, # etc to use one or more GPUs. -net.all_params().initialize(mx.init.Xavier(), ctx=mx.cpu()) +net.collect_params().initialize(mx.init.Xavier(), ctx=mx.cpu()) ``` Note that because we didn't specify input size to layers in Net's constructor, @@ -117,7 +117,7 @@ entire batch. For example, ```python lr = 0.01 -for p in net.all_params().values(): +for p in net.collect_params().values(): p.data()[:] -= lr / data.shape[0] * p.grad() ``` @@ -125,7 +125,7 @@ But sometimes you want more fancy updating rules like momentum and Adam, and sin this is a commonly used functionality, foo provide a `Trainer` class for it: ```python -trainer = foo.Trainer(net.all_params(), 'sgd', {'learning_rate': 0.01}) +trainer = foo.Trainer(net.collect_params(), 'sgd', {'learning_rate': 0.01}) with record(): output = net(data) diff --git a/docs/tutorials/foo/hybrid.md b/docs/tutorials/foo/hybrid.md index 9cebf9204ad9..626e76403f7a 100644 --- a/docs/tutorials/foo/hybrid.md +++ b/docs/tutorials/foo/hybrid.md @@ -12,29 +12,29 @@ Imperative frameworks (including PyTorch, Chainer, etc) are just the opposite: they execute commands one-by-one just like old fashioned Matlab and Numpy. This style is more flexible, easier to debug, but less efficient. -`HybridLayer` seamlessly combines declarative programming and imperative programming +`HybridBlock` seamlessly combines declarative programming and imperative programming to offer the benefit of both. Users can quickly develop and debug models with imperative programming and switch to efficient declarative execution by simply -calling: `HybridLayer.hybridize()`. +calling: `HybridBlock.hybridize()`. -## HybridLayer +## HybridBlock -`HybridLayer` is very similar to `Layer` but has a few restrictions: +`HybridBlock` is very similar to `Block` but has a few restrictions: -- All children layers of `HybridLayer` must also be `HybridLayer`. +- All children layers of `HybridBlock` must also be `HybridBlock`. - Only methods that are implemented for both `NDArray` and `Symbol` can be used. For example you cannot use `.asnumpy()`, `.shape`, etc. - Operations cannot change from run to run. For example, you cannot do `if x:` if `x` is different for each iteration. -To use hybrid support, we subclass the `HybridLayer`: +To use hybrid support, we subclass the `HybridBlock`: ```python import mxnet as mx from mxnet import foo from mxnet.foo import nn -class Net(nn.HybridLayer): +class Net(foo.HybridBlock): def __init__(self, **kwargs): super(Net, self).__init__(**kwargs) with self.name_scope: @@ -68,12 +68,12 @@ class Net(nn.HybridLayer): ## Hybridize -By default, `HybridLayer` runs just like a standard `Layer`. Each time a layer +By default, `HybridBlock` runs just like a standard `Block`. Each time a layer is called, its `hybrid_forward` will be run: ```python net = Net() -net.all_params().initialize() +net.collect_params().initialize() x = mx.nd.random_normal(shape=(16, 1, 28, 28)) net(x) x = mx.nd.random_normal(shape=(16, 1, 28, 28)) @@ -98,12 +98,12 @@ but after hybridize, only the first forward printed out a Symbol. On subsequent forward `hybrid_forward` is not called so nothing was printed. Hybridize will speed up execution and save memory. If the top level layer is -not a `HybridLayer`, you can still call `.hybridize()` on it and Foo will try +not a `HybridBlock`, you can still call `.hybridize()` on it and Foo will try to hybridize its children layers instead. ## Serializing trained model for deployment -Models implemented as `HybridLayer` can be easily serialized for deployment +Models implemented as `HybridBlock` can be easily serialized for deployment using other language front-ends like C, C++ and Scala. To this end, we simply forward the model with symbolic variables instead of NDArrays and save the output Symbol(s): @@ -113,7 +113,7 @@ x = mx.sym.var('data') y = net(x) print(y) y.save('model.json') -net.all_params().save('model.params') +net.collect_params().save('model.params') ``` If your network outputs more than one value, you can use `mx.sym.Group` to diff --git a/docs/tutorials/foo/mnist.md b/docs/tutorials/foo/mnist.md index 2e322efefdeb..61da221f8d2f 100644 --- a/docs/tutorials/foo/mnist.md +++ b/docs/tutorials/foo/mnist.md @@ -97,8 +97,8 @@ initialized parameters. ```python ctx = [mx.cpu(0), mx.cpu(1)] -net.all_params().initialize(mx.init.Xavier(magnitude=2.24), ctx=ctx) -trainer = foo.Trainer(net.all_params(), 'sgd', {'learning_rate': 0.1}) +net.collect_params().initialize(mx.init.Xavier(magnitude=2.24), ctx=ctx) +trainer = foo.Trainer(net.collect_params(), 'sgd', {'learning_rate': 0.1}) ``` #### Train the network @@ -193,13 +193,13 @@ A single convolution layer consists of one or more filters that each play the ro The following source code defines a convolutional neural network architecture called LeNet. LeNet is a popular network known to work well on digit classification tasks. We will use a slightly different version from the original LeNet implementation, replacing the sigmoid activations with tanh activations for the neurons. -A typical way to write your network is creating a new class inherited from `foo.nn.Layer` -class. We can define the network by composing and inheriting Layer class as follows: +A typical way to write your network is creating a new class inherited from `foo.Block` +class. We can define the network by composing and inheriting Block class as follows: ```python import mxnet.ndarray as F -class Net(nn.Layer): +class Net(foo.Block): def __init__(self, **kwargs): super(Net, self).__init__(**kwargs) with self.name_scope(): @@ -247,8 +247,8 @@ We will initialize the network parameters as follows: ```python ctx = [mx.cpu(0), mx.cpu(1)] -net.all_params().initialize(mx.init.Xavier(magnitude=2.24), ctx=ctx) -trainer = foo.Trainer(net.all_params(), 'sgd', {'learning_rate': 0.1}) +net.collect_params().initialize(mx.init.Xavier(magnitude=2.24), ctx=ctx) +trainer = foo.Trainer(net.collect_params(), 'sgd', {'learning_rate': 0.1}) ``` #### Training diff --git a/example/autograd/actor_critic.py b/example/autograd/actor_critic.py index 23e762f6eebe..e76a9e625135 100644 --- a/example/autograd/actor_critic.py +++ b/example/autograd/actor_critic.py @@ -28,7 +28,7 @@ env.seed(args.seed) -class Policy(nn.Layer): +class Policy(foo.Block): def __init__(self, **kwargs): super(Policy, self).__init__(**kwargs) with self.name_scope(): @@ -43,9 +43,9 @@ def forward(self, x): return F.softmax(probs), values net = Policy() -net.all_params().initialize(mx.init.Uniform(0.02)) -trainer = foo.Trainer(net.all_params(), 'adam', {'learning_rate': 3e-2}) - +net.collect_params().initialize(mx.init.Uniform(0.02)) +trainer = foo.Trainer(net.collect_params(), 'adam', {'learning_rate': 3e-2}) +loss = foo.loss.L1Loss() running_reward = 10 for epoch in count(1): @@ -81,8 +81,8 @@ def forward(self, x): rewards /= rewards.std() + np.finfo(rewards.dtype).eps # compute loss and gradient - loss = sum([foo.loss.l1_loss(value, mx.nd.array([r])) for r, value in zip(rewards, values)]) - final_nodes = [loss] + L = sum([loss(value, mx.nd.array([r])) for r, value in zip(rewards, values)]) + final_nodes = [L] for logp, r, v in zip(heads, rewards, values): reward = r - v.asnumpy()[0,0] # Here we differentiate the stochastic graph, corresponds to the diff --git a/example/autograd/data.py b/example/autograd/data.py index 84eb711d00a4..fa69863fae4a 100644 --- a/example/autograd/data.py +++ b/example/autograd/data.py @@ -34,6 +34,8 @@ def mnist_iterator(batch_size, input_shape): def cifar10_iterator(batch_size, data_shape, resize=-1): + get_data.GetCifar10() + train = mx.io.ImageRecordIter( path_imgrec = "data/cifar/train.rec", # mean_img = "data/cifar/mean.bin", diff --git a/example/autograd/dcgan.py b/example/autograd/dcgan.py index aa6b8b6dcd0a..7bae59aebb24 100644 --- a/example/autograd/dcgan.py +++ b/example/autograd/dcgan.py @@ -81,17 +81,19 @@ # netD.add(nn.Activation('sigmoid')) -netG.all_params().initialize(mx.init.Normal(0.02), ctx=ctx) -netD.all_params().initialize(mx.init.Normal(0.02), ctx=ctx) +netG.collect_params().initialize(mx.init.Normal(0.02), ctx=ctx) +netD.collect_params().initialize(mx.init.Normal(0.02), ctx=ctx) -trainerG = foo.Trainer(netG.all_params(), 'adam', {'learning_rate': opt.lr, 'beta1': opt.beta1}) -trainerD = foo.Trainer(netD.all_params(), 'adam', {'learning_rate': opt.lr, 'beta1': opt.beta1}) +trainerG = foo.Trainer(netG.collect_params(), 'adam', {'learning_rate': opt.lr, 'beta1': opt.beta1}) +trainerD = foo.Trainer(netD.collect_params(), 'adam', {'learning_rate': opt.lr, 'beta1': opt.beta1}) real_label = mx.nd.ones((opt.batchSize,), ctx=ctx) fake_label = mx.nd.zeros((opt.batchSize,), ctx=ctx) +loss = foo.loss.SoftmaxCrossEntropyLoss() + for epoch in range(opt.niter): for batch in train_iter: ############################ @@ -104,12 +106,12 @@ with autograd.record(): output = netD(data) output = output.reshape((opt.batchSize, 2)) - errD_real = foo.loss.softmax_cross_entropy_loss(output, real_label) + errD_real = loss(output, real_label) fake = netG(noise) output = netD(fake.detach()) output = output.reshape((opt.batchSize, 2)) - errD_fake = foo.loss.softmax_cross_entropy_loss(output, fake_label) + errD_fake = loss(output, fake_label) errD = errD_real + errD_fake errD.backward() @@ -121,7 +123,7 @@ with autograd.record(): output = netD(fake) output = output.reshape((opt.batchSize, 2)) - errG = foo.loss.softmax_cross_entropy_loss(output, real_label) + errG = loss(output, real_label) errG.backward() trainerG.step(opt.batchSize) diff --git a/example/autograd/mnist.py b/example/autograd/mnist.py index ce5a4e6281cb..c4d8e0c5bb71 100644 --- a/example/autograd/mnist.py +++ b/example/autograd/mnist.py @@ -1,15 +1,36 @@ # pylint: skip-file from __future__ import print_function -from data import mnist_iterator -import mxnet as mx -from mxnet import foo -from mxnet.foo import nn -import numpy as np +import argparse import logging -from mxnet import autograd as ag logging.basicConfig(level=logging.DEBUG) +import numpy as np +import mxnet as mx +from mxnet import foo, autograd +from mxnet.foo import nn + +from data import mnist_iterator + + +# Parse CLI arguments + +parser = argparse.ArgumentParser(description='MXNet Foo MNIST Example') +parser.add_argument('--batch-size', type=int, default=100, + help='batch size for training and testing (default: 100)') +parser.add_argument('--epochs', type=int, default=10, + help='number of epochs to train (default: 10)') +parser.add_argument('--lr', type=float, default=0.1, + help='learning rate (default: 0.1)') +parser.add_argument('--momentum', type=float, default=0.9, + help='SGD momentum (default: 0.9)') +parser.add_argument('--cuda', action='store_true', default=False, + help='Train on GPU with CUDA') +parser.add_argument('--log-interval', type=int, default=100, metavar='N', + help='how many batches to wait before logging training status') +opt = parser.parse_args() + + # define network net = nn.Sequential() @@ -20,50 +41,66 @@ # data -train_data, val_data = mnist_iterator(batch_size=100, input_shape = (784,)) +train_data, val_data = mnist_iterator(batch_size=opt.batch_size, input_shape=(28*28,)) # train -def test(ctxs): +def test(ctx): metric = mx.metric.Accuracy() val_data.reset() for batch in val_data: - data = foo.utils.split_and_load(batch.data[0], ctxs, batch_axis=0) - label = foo.utils.split_and_load(batch.label[0], ctxs, batch_axis=0) - outputs = [] - for x in data: - outputs.append(net(x)) - metric.update(label, outputs) - print('validation acc: %s=%f'%metric.get()) - -def train(epoch, ctxs): - if isinstance(ctxs, mx.Context): - ctxs = [ctxs] - net.all_params().initialize(mx.init.Xavier(magnitude=2.24), ctx=ctxs) - trainer = foo.Trainer(net.all_params(), 'sgd', {'learning_rate': 0.1}) + data = batch.data[0].as_in_context(ctx) + label = batch.label[0].as_in_context(ctx) + output = net(data) + metric.update([label], [output]) + + return metric.get() + + +def train(epochs, ctx): + # Collect all parameters from net and its children, then initialize them. + net.collect_params().initialize(mx.init.Xavier(magnitude=2.24), ctx=ctx) + # Trainer is for updating parameters with gradient. + trainer = foo.Trainer(net.collect_params(), 'sgd', + {'learning_rate': opt.lr, 'momentum': opt.momentum}) metric = mx.metric.Accuracy() + loss = foo.loss.SoftmaxCrossEntropyLoss() - for i in range(epoch): + for epoch in range(epochs): + # reset data iterator and metric at begining of epoch. train_data.reset() - for batch in train_data: - datas = foo.utils.split_and_load(batch.data[0], ctxs, batch_axis=0) - labels = foo.utils.split_and_load(batch.label[0], ctxs, batch_axis=0) - outputs = [] - with ag.record(): - for x, y in zip(datas, labels): - z = net(x) - loss = foo.loss.softmax_cross_entropy_loss(z, y) - ag.backward([loss]) - outputs.append(z) - metric.update(labels, outputs) - trainer.step(batch.data[0].shape[0]) - name, acc = metric.get() metric.reset() - print('training acc at epoch %d: %s=%f'%(i, name, acc)) - test(ctxs) + for i, batch in enumerate(train_data): + # Copy data to ctx if necessary + data = batch.data[0].as_in_context(ctx) + label = batch.label[0].as_in_context(ctx) + # Start recording computation graph with record() section. + # Recorded graphs can then be differentiated with backward. + with autograd.record(): + output = net(data) + L = loss(output, label) + L.backward() + # take a gradient step with batch_size equal to data.shape[0] + trainer.step(data.shape[0]) + # update metric at last. + metric.update([label], [output]) + + if i % opt.log_interval == 0 and i > 0: + name, acc = metric.get() + print('[Epoch %d Batch %d] Training: %s=%f'%(epoch, i, name, acc)) + + name, acc = metric.get() + print('[Epoch %d] Training: %s=%f'%(epoch, name, acc)) + + name, val_acc = test(ctx) + print('[Epoch %d] Validation: %s=%f'%(epoch, name, val_acc)) - net.all_params().save('mnist.params') + net.collect_params().save('mnist.params') if __name__ == '__main__': - train(10, [mx.cpu(0), mx.cpu(1)]) + if opt.cuda: + ctx = mx.gpu(0) + else: + ctx = mx.cpu() + train(opt.epochs, ctx) diff --git a/example/autograd/resnet.py b/example/autograd/resnet.py index b6f042d58242..616152e5711e 100644 --- a/example/autograd/resnet.py +++ b/example/autograd/resnet.py @@ -1,4 +1,4 @@ -from __future__ import division +from __future__ import division, print_function import argparse, time import logging @@ -13,36 +13,52 @@ # CLI parser = argparse.ArgumentParser(description='Train a resnet model for image classification.') -parser.add_argument('--dataset', type=str, default='dummy', help='dataset to use. options are mnist, cifar10, and dummy.') -parser.add_argument('--batch_size', type=int, default=32, help='training batch size per device (CPU/GPU).') -parser.add_argument('--resnet_version', type=int, default=1, help='version of resnet to use. options are 1 and 2. default is 1.') -parser.add_argument('--resnet_layers', type=int, default=50, help='layers of resnet to use. options are 18, 50. default is 50.') -parser.add_argument('--gpus', type=int, default=0, help='number of gpus to use.') -parser.add_argument('--epochs', type=int, default=3, help='number of training epochs.') -parser.add_argument('--lr', type=float, default=0.01, help='learning Rate. default is 0.01.') -parser.add_argument('--seed', type=int, default=123, help='random seed to use. Default=123.') -parser.add_argument('--thumbnail', action='store_true', default=False, help='use thumbnail or not. default is false.') -parser.add_argument('--benchmark', action='store_true', default=True, help='whether to run benchmark.') -parser.add_argument('--symbolic', action='store_true', default=False, help='whether to train in symbolic way with module.') +parser.add_argument('--dataset', type=str, default='cifar10', + help='dataset to use. options are mnist, cifar10, and dummy.') +parser.add_argument('--batch-size', type=int, default=32, + help='training batch size per device (CPU/GPU).') +parser.add_argument('--resnet-version', type=int, default=1, + help='whether to use ResnetV1 or ResnetV2. default is 1.') +parser.add_argument('--resnet-layers', type=int, default=50, + help='layers of resnet to use. options are 18, 50. default is 50.') +parser.add_argument('--gpus', type=int, default=0, + help='number of gpus to use.') +parser.add_argument('--epochs', type=int, default=3, + help='number of training epochs.') +parser.add_argument('--lr', type=float, default=0.01, + help='learning Rate. default is 0.01.') +parser.add_argument('--seed', type=int, default=123, + help='random seed to use. Default=123.') +parser.add_argument('--thumbnail', action='store_true', default=False, + help='use thumbnail or not. default is false.') +parser.add_argument('--benchmark', action='store_true', default=False, + help='whether to run benchmark.') +parser.add_argument('--symbolic', action='store_true', default=False, + help='whether to train in symbolic way with module.') +parser.add_argument('--log-interval', type=int, default=100, + help='Number of batches to wait before logging.') opt = parser.parse_args() print(opt) -def conv3x3(filters, stride, in_filters): + +# Define network + +def conv3x3(filters, stride, in_channels): return nn.Conv2D(filters, kernel_size=3, strides=stride, padding=1, - use_bias=False, in_filters=in_filters) + use_bias=False, in_channels=in_channels) -class BasicBlockV1(nn.HybridLayer): - def __init__(self, filters, stride, downsample=False, in_filters=0, **kwargs): +class BasicBlockV1(foo.HybridBlock): + def __init__(self, filters, stride, downsample=False, in_channels=0, **kwargs): super(BasicBlockV1, self).__init__(**kwargs) with self.name_scope(): - self.conv1 = conv3x3(filters, stride, in_filters) - self.bn1 = nn.BatchNorm(num_features=in_filters) + self.conv1 = conv3x3(filters, stride, in_channels) + self.bn1 = nn.BatchNorm(in_channels=in_channels) self.conv2 = conv3x3(filters, 1, filters) - self.bn2 = nn.BatchNorm(num_features=filters) + self.bn2 = nn.BatchNorm(in_channels=filters) if downsample: - self.conv_ds = nn.Conv2D(filters, kernel_size=1, strides=stride, use_bias=False, in_filters=in_filters) - self.bn_ds = nn.BatchNorm(num_features=filters) + self.conv_ds = nn.Conv2D(filters, kernel_size=1, strides=stride, use_bias=False, in_channels=in_channels) + self.bn_ds = nn.BatchNorm(in_channels=filters) self.downsample = downsample def hybrid_forward(self, F, x): @@ -65,19 +81,19 @@ def hybrid_forward(self, F, x): return out -class BottleneckV1(nn.HybridLayer): - def __init__(self, filters, stride, downsample=False, in_filters=0, **kwargs): +class BottleneckV1(foo.HybridBlock): + def __init__(self, filters, stride, downsample=False, in_channels=0, **kwargs): super(BottleneckV1, self).__init__(**kwargs) with self.name_scope(): - self.conv1 = nn.Conv2D(filters=filters//4, kernel_size=1, strides=1, in_filters=in_filters) - self.bn1 = nn.BatchNorm(num_features=filters//4) + self.conv1 = nn.Conv2D(filters//4, kernel_size=1, strides=1, in_channels=in_channels) + self.bn1 = nn.BatchNorm(in_channels=filters//4) self.conv2 = conv3x3(filters//4, stride, filters//4) - self.bn2 = nn.BatchNorm(num_features=filters//4) - self.conv3 = nn.Conv2D(filters=filters, kernel_size=1, strides=1, in_filters=filters//4) - self.bn3 = nn.BatchNorm(num_features=filters) + self.bn2 = nn.BatchNorm(in_channels=filters//4) + self.conv3 = nn.Conv2D(filters, kernel_size=1, strides=1, in_channels=filters//4) + self.bn3 = nn.BatchNorm(in_channels=filters) if downsample: - self.conv_ds = nn.Conv2D(filters, kernel_size=1, strides=stride, use_bias=False, in_filters=in_filters) - self.bn_ds = nn.BatchNorm(num_features=filters) + self.conv_ds = nn.Conv2D(filters, kernel_size=1, strides=stride, use_bias=False, in_channels=in_channels) + self.bn_ds = nn.BatchNorm(in_channels=filters) self.downsample = downsample def hybrid_forward(self, F, x): @@ -104,7 +120,7 @@ def hybrid_forward(self, F, x): return out -class ResnetV1(nn.HybridLayer): +class ResnetV1(foo.HybridBlock): def __init__(self, block, classes, layers, filters, thumbnail=False, **kwargs): super(ResnetV1, self).__init__(**kwargs) with self.name_scope(): @@ -114,26 +130,26 @@ def __init__(self, block, classes, layers, filters, thumbnail=False, **kwargs): self.conv0 = conv3x3(filters[0], 1, 3) else: self.conv0 = nn.Conv2D(filters[0], 7, 2, 3, use_bias=False, - in_filters=3) - self.bn0 = nn.BatchNorm(num_features=filters[0]) + in_channels=3) + self.bn0 = nn.BatchNorm(in_channels=filters[0]) self.pool0 = nn.MaxPool2D(3, 2, 1) self.body = nn.HSequential() - in_filters = filters[0] + in_channels = filters[0] for i in range(len(layers)): stride = 1 if i == 0 else 2 self.body.add(self._make_layer(block, layers[i], filters[i+1], - stride, in_filters=filters[i])) - in_filters = filters[i+1] + stride, in_channels=filters[i])) + in_channels = filters[i+1] self.pool1 = nn.GlobalAvgPool2D() self.dense1 = nn.Dense(classes, in_units=filters[-1]) - def _make_layer(self, block, layers, filters, stride, in_filters=0): + def _make_layer(self, block, layers, filters, stride, in_channels=0): layer = nn.HSequential() - layer.add(block(filters, stride, True, in_filters=in_filters)) + layer.add(block(filters, stride, True, in_channels=in_channels)) for i in range(layers-1): - layer.add(block(filters, 1, False, in_filters=filters)) + layer.add(block(filters, 1, False, in_channels=filters)) return layer def hybrid_forward(self, F, x): @@ -152,17 +168,17 @@ def hybrid_forward(self, F, x): return x -class BasicBlockV2(nn.HybridLayer): - def __init__(self, filters, stride, downsample=False, in_filters=0, **kwargs): +class BasicBlockV2(foo.HybridBlock): + def __init__(self, filters, stride, downsample=False, in_channels=0, **kwargs): super(BasicBlockV2, self).__init__(**kwargs) with self.name_scope(): - self.bn1 = nn.BatchNorm(num_features=in_filters) - self.conv1 = conv3x3(filters, stride, in_filters) - self.bn2 = nn.BatchNorm(num_features=filters) + self.bn1 = nn.BatchNorm(in_channels=in_channels) + self.conv1 = conv3x3(filters, stride, in_channels) + self.bn2 = nn.BatchNorm(in_channels=filters) self.conv2 = conv3x3(filters, 1, filters) if downsample: self.downsample = nn.Conv2D(filters, 1, stride, use_bias=False, - in_filters=in_filters) + in_channels=in_channels) else: self.downsample = None @@ -182,19 +198,19 @@ def hybrid_forward(self, F, x): return x + residual -class BottleneckV2(nn.HybridLayer): - def __init__(self, filters, stride, downsample=False, in_filters=0, **kwargs): +class BottleneckV2(foo.HybridBlock): + def __init__(self, filters, stride, downsample=False, in_channels=0, **kwargs): super(BottleneckV2, self).__init__(**kwargs) with self.name_scope(): - self.bn1 = nn.BatchNorm(num_features=in_filters) - self.conv1 = conv3x3(filters//4, 1, in_filters) - self.bn2 = nn.BatchNorm(num_features=filters//4) + self.bn1 = nn.BatchNorm(in_channels=in_channels) + self.conv1 = conv3x3(filters//4, 1, in_channels) + self.bn2 = nn.BatchNorm(in_channels=filters//4) self.conv2 = conv3x3(filters//4, stride, filters//4) - self.bn3 = nn.BatchNorm(num_features=filters//4) + self.bn3 = nn.BatchNorm(in_channels=filters//4) self.conv3 = conv3x3(filters, 1, filters//4) if downsample: self.downsample = nn.Conv2D(filters, 1, stride, use_bias=False, - in_filters=in_filters) + in_channels=in_channels) else: self.downsample = None @@ -217,38 +233,38 @@ def hybrid_forward(self, F, x): return x + residual -class ResnetV2(nn.HybridLayer): +class ResnetV2(foo.HybridBlock): def __init__(self, block, classes, layers, filters, thumbnail=False, **kwargs): super(ResnetV2, self).__init__(**kwargs) with self.name_scope(): assert len(layers) == len(filters) - 1 self._thumbnail = thumbnail - self.bn_data = nn.BatchNorm(num_features=3, scale=False, center=False) + self.bn_data = nn.BatchNorm(in_channels=3, scale=False, center=False) if thumbnail: self.conv0 = conv3x3(filters[0], 1, 3) else: self.conv0 = nn.Conv2D(filters[0], 7, 2, 3, use_bias=False, - in_filters=3) - self.bn0 = nn.BatchNorm(num_features=filters[0]) + in_channels=3) + self.bn0 = nn.BatchNorm(in_channels=filters[0]) self.pool0 = nn.MaxPool2D(3, 2, 1) self.body = nn.HSequential() - in_filters = filters[0] + in_channels = filters[0] for i in range(len(layers)): stride = 1 if i == 0 else 2 self.body.add(self._make_layer(block, layers[i], filters[i+1], - stride, in_filters=in_filters)) - in_filters = filters[i+1] + stride, in_channels=in_channels)) + in_channels = filters[i+1] - self.bn1 = nn.BatchNorm(num_features=in_filters) + self.bn1 = nn.BatchNorm(in_channels=in_channels) self.pool1 = nn.GlobalAvgPool2D() - self.dense1 = nn.Dense(classes, in_units=in_filters) + self.dense1 = nn.Dense(classes, in_units=in_channels) - def _make_layer(self, block, layers, filters, stride, in_filters=0): + def _make_layer(self, block, layers, filters, stride, in_channels=0): layer = nn.HSequential() - layer.add(block(filters, stride, True, in_filters=in_filters)) + layer.add(block(filters, stride, True, in_channels=in_channels)) for i in range(layers-1): - layer.add(block(filters, 1, False, in_filters=filters)) + layer.add(block(filters, 1, False, in_channels=filters)) return layer def hybrid_forward(self, F, x): @@ -321,45 +337,54 @@ def test(ctx): for x in data: outputs.append(net(x)) metric.update(label, outputs) - logging.info('validation acc: %s=%f'%metric.get()) + return metric.get() def train(epoch, ctx): if isinstance(ctx, mx.Context): ctx = [ctx] - net.all_params().initialize(mx.init.Xavier(magnitude=2.24), ctx=ctx) - trainer = foo.Trainer(net.all_params(), 'sgd', {'learning_rate': 0.1}) + net.collect_params().initialize(mx.init.Xavier(magnitude=2.24), ctx=ctx) + trainer = foo.Trainer(net.collect_params(), 'sgd', {'learning_rate': 0.1}) metric = mx.metric.Accuracy() + loss = foo.loss.SoftmaxCrossEntropyLoss() - for i in range(epoch): + for epoch in range(epoch): tic = time.time() train_data.reset() + metric.reset() btic = time.time() - for batch in train_data: + for i, batch in enumerate(train_data): data = foo.utils.split_and_load(batch.data[0], ctx_list=ctx, batch_axis=0) label = foo.utils.split_and_load(batch.label[0], ctx_list=ctx, batch_axis=0) outputs = [] - losses = [] + Ls = [] with ag.record(): for x, y in zip(data, label): z = net(x) - loss = foo.loss.softmax_cross_entropy_loss(z, y) - losses.append(loss) + L = loss(z, y) + # store the loss and do backward after we have done forward + # on all GPUs for better speed on multiple GPUs. + Ls.append(L) outputs.append(z) - for loss in losses: - loss.backward() + for L in Ls: + L.backward() trainer.step(batch.data[0].shape[0]) metric.update(label, outputs) - logging.info('speed: {} samples/s'.format(batch_size/(time.time()-btic))) + if opt.log_interval: + name, acc = metric.get() + print('[Epoch %d Batch %d] speed: %f samples/s, training: %s=%f'%( + epoch, i, batch_size/(time.time()-btic), name, acc)) btic = time.time() name, acc = metric.get() - metric.reset() - logging.info('training acc at epoch %d: %s=%f'%(i, name, acc)) - logging.info('time: %f'%(time.time()-tic)) - test(ctx) + print('[Epoch %d] training: %s=%f'%(epoch, name, acc)) + print('[Epoch %d] time cost: %f'%(epoch, time.time()-tic)) + + name, val_acc = test(ctx) + print('[Epoch %d] validation: %s=%f'%(epoch, name, val_acc)) + + net.collect_params().save('resnet.params') - net.all_params().save('mnist.params') if __name__ == '__main__': if opt.symbolic: diff --git a/example/autograd/super_resolution.py b/example/autograd/super_resolution.py index c12408e40398..92002060d89f 100644 --- a/example/autograd/super_resolution.py +++ b/example/autograd/super_resolution.py @@ -88,14 +88,14 @@ def _rearrange(raw, F, upscale_factor): return F.reshape(swapped, shape=(0, 0, -3, -3)) -class SuperResolutionNet(nn.Layer): +class SuperResolutionNet(foo.Block): def __init__(self, upscale_factor): super(SuperResolutionNet, self).__init__() with self.name_scope(): - self.conv1 = nn.Conv2D(64, (5, 5), strides=(1, 1), padding=(2, 2), in_filters=1) - self.conv2 = nn.Conv2D(64, (3, 3), strides=(1, 1), padding=(1, 1), in_filters=64) - self.conv3 = nn.Conv2D(32, (3, 3), strides=(1, 1), padding=(1, 1), in_filters=64) - self.conv4 = nn.Conv2D(upscale_factor ** 2, (3, 3), strides=(1, 1), padding=(1, 1), in_filters=32) + self.conv1 = nn.Conv2D(64, (5, 5), strides=(1, 1), padding=(2, 2), in_channels=1) + self.conv2 = nn.Conv2D(64, (3, 3), strides=(1, 1), padding=(1, 1), in_channels=64) + self.conv3 = nn.Conv2D(32, (3, 3), strides=(1, 1), padding=(1, 1), in_channels=64) + self.conv4 = nn.Conv2D(upscale_factor ** 2, (3, 3), strides=(1, 1), padding=(1, 1), in_channels=32) self.upscale_factor = upscale_factor def forward(self, x): @@ -128,10 +128,11 @@ def test(ctx): def train(epoch, ctx): if isinstance(ctx, mx.Context): ctx = [ctx] - net.conv4.all_params().initialize(mx.init.Orthogonal(scale=1), ctx=ctx) - net.all_params().initialize(mx.init.Orthogonal(), ctx=ctx) - trainer = foo.Trainer(net.all_params(), 'adam', {'learning_rate': opt.lr}) + net.collect_params().initialize(mx.init.Orthogonal(), ctx=ctx) + net.conv4.collect_params().initialize(mx.init.Orthogonal(scale=1), ctx=ctx) + trainer = foo.Trainer(net.collect_params(), 'adam', {'learning_rate': opt.lr}) metric = mx.metric.MAE() + loss = foo.loss.L2Loss() for i in range(epoch): train_data.reset() @@ -142,8 +143,8 @@ def train(epoch, ctx): with ag.record(): for x, y in zip(data, label): z = net(x) - loss = foo.loss.l2_loss(z, y) - ag.compute_gradient([loss]) + L = loss(z, y) + L.backward() outputs.append(z) trainer.step(batch.data[0].shape[0]) metric.update(label, outputs) @@ -153,12 +154,12 @@ def train(epoch, ctx): print('training mae at epoch %d: %s=%f'%(i, name, acc)) test(ctx) - net.all_params().save('superres.params') + net.collect_params().save('superres.params') def resolve(ctx): if isinstance(ctx, list): ctx = [ctx[0]] - net.all_params().load('superres.params') + net.collect_params().load('superres.params') img = Image.open(opt.resolve_img).convert('YCbCr') y, cb, cr = img.split() data = mx.nd.array(y) diff --git a/example/autograd/word_language_model/model.py b/example/autograd/word_language_model/model.py index 44b4e913265d..27e8bd4b5b4d 100644 --- a/example/autograd/word_language_model/model.py +++ b/example/autograd/word_language_model/model.py @@ -3,7 +3,7 @@ from mxnet import foo from mxnet.foo import nn, rnn -class RNNModel(nn.Layer): +class RNNModel(foo.Block): def __init__(self, mode, vocab_size, num_embed, num_hidden, num_layers, dropout=0.5, tie_weights=False, **kwargs): super(RNNModel, self).__init__(**kwargs) diff --git a/example/autograd/word_language_model/train.py b/example/autograd/word_language_model/train.py index 93a82a380980..761c3459ac9f 100644 --- a/example/autograd/word_language_model/train.py +++ b/example/autograd/word_language_model/train.py @@ -74,12 +74,12 @@ def batchify(data, batch_size): ntokens = len(corpus.dictionary) model = model.RNNModel(args.model, ntokens, args.emsize, args.nhid, args.nlayers, args.dropout, args.tied) -model.all_params().initialize(mx.init.Xavier(), ctx=context) -trainer = foo.Trainer(model.all_params(), 'sgd', +model.collect_params().initialize(mx.init.Xavier(), ctx=context) +trainer = foo.Trainer(model.collect_params(), 'sgd', {'learning_rate': args.lr, 'momentum': 0, 'wd': 0}) - +loss = foo.loss.SoftmaxCrossEntropyLoss() ############################################################################### # Training code @@ -102,22 +102,22 @@ def detach(hidden): def eval(data_source): - total = 0.0 + total_L = 0.0 ntotal = 0 hidden = model.begin_state(func=mx.nd.zeros, batch_size=args.batch_size, ctx=context) for ibatch, i in enumerate(range(0, data_source.shape[0] - 1, args.bptt)): data, target = get_batch(data_source, i) output, hidden = model(data, hidden) - loss = foo.loss.softmax_cross_entropy_loss(output, target) - total += mx.nd.sum(loss).asscalar() - ntotal += loss.size - return total / ntotal + L = loss(output, target) + total_L += mx.nd.sum(L).asscalar() + ntotal += L.size + return total_L / ntotal def train(): best_val = None for epoch in range(args.epochs): - total = 0.0 + total_L = 0.0 start_time = time.time() hidden = model.begin_state(func=mx.nd.zeros, batch_size=args.batch_size, ctx=context) for ibatch, i in enumerate(range(0, train_data.shape[0] - 1, args.bptt)): @@ -125,30 +125,30 @@ def train(): hidden = detach(hidden) with autograd.record(): output, hidden = model(data, hidden) - loss = foo.loss.softmax_cross_entropy_loss(output, target) - loss.backward() + L = loss(output, target) + L.backward() - grads = [i.grad(context) for i in model.all_params().values()] + grads = [i.grad(context) for i in model.collect_params().values()] # Here gradient is not divided by batch_size yet. # So we multiply max_norm by batch_size to balance it. foo.utils.clip_global_norm(grads, args.clip * args.batch_size) trainer.step(args.batch_size) - total += mx.nd.sum(loss).asscalar() + total_L += mx.nd.sum(L).asscalar() if ibatch % args.log_interval == 0 and ibatch > 0: - cur_loss = total / args.batch_size / args.bptt / args.log_interval + cur_L = total_L / args.batch_size / args.bptt / args.log_interval print('[Epoch %d Batch %d] loss %.2f, ppl %.2f'%( - epoch, ibatch, cur_loss, math.exp(cur_loss))) - total = 0.0 + epoch, ibatch, cur_L, math.exp(cur_L))) + total_L = 0.0 - val_loss = eval(val_data) + val_L = eval(val_data) print('[Epoch %d] time cost %.2fs, valid loss %.2f, valid ppl %.2f'%( - epoch, time.time()-start_time, val_loss, math.exp(val_loss))) + epoch, time.time()-start_time, val_L, math.exp(val_L))) if __name__ == '__main__': train() - test_loss = eval(test_data) - print('test loss %.2f, test ppl %.2f'%(test_loss, math.exp(test_loss))) + test_L = eval(test_data) + print('test loss %.2f, test ppl %.2f'%(test_L, math.exp(test_L))) diff --git a/example/recommenders/symbol_alexnet.py b/example/recommenders/symbol_alexnet.py index 13aa65131117..20a3547db460 100644 --- a/example/recommenders/symbol_alexnet.py +++ b/example/recommenders/symbol_alexnet.py @@ -5,7 +5,7 @@ """ import mxnet as mx -def features(input_data, num_features): +def features(input_data, in_channels): # stage 1 conv1 = mx.symbol.Convolution( data=input_data, kernel=(11, 11), stride=(4, 4), num_filter=96) @@ -40,6 +40,6 @@ def features(input_data, num_features): relu7 = mx.symbol.Activation(data=fc2, act_type="relu") dropout2 = mx.symbol.Dropout(data=relu7, p=0.5) # stage 6 - fc3 = mx.symbol.FullyConnected(data=dropout2, num_hidden=num_features) + fc3 = mx.symbol.FullyConnected(data=dropout2, num_hidden=in_channels) return fc3 diff --git a/python/mxnet/autograd.py b/python/mxnet/autograd.py index 1c791f26b16f..b97d350f258d 100644 --- a/python/mxnet/autograd.py +++ b/python/mxnet/autograd.py @@ -10,15 +10,15 @@ from .symbol import _GRAD_REQ_MAP -def set_is_training(is_train): - """Set status to training/not training. When training, graph will be constructed +def set_recording(is_recording): + """Set status to recording/not recording. When recording, graph will be constructed for gradient computation. Operators will also run with ctx.is_train=True. For example, Dropout will drop inputs randomly when is_train=True while simply passing through if is_train=False. Parameters ---------- - is_train: bool + is_recording: bool Returns ------- @@ -26,7 +26,7 @@ def set_is_training(is_train): """ prev = ctypes.c_int() check_call(_LIB.MXAutogradSetIsTraining( - ctypes.c_int(is_train), ctypes.byref(prev))) + ctypes.c_int(is_recording), ctypes.byref(prev))) return bool(prev.value) @@ -43,11 +43,11 @@ def __init__(self, enter_state): self._prev = None def __enter__(self): - self._prev = set_is_training(self._enter_state) + self._prev = set_recording(self._enter_state) def __exit__(self, ptype, value, trace): if self._prev != self._enter_state: - set_is_training(self._prev) + set_recording(self._prev) def record(): diff --git a/python/mxnet/foo/__init__.py b/python/mxnet/foo/__init__.py index 98cc5e9c9852..0910fdf8ce06 100644 --- a/python/mxnet/foo/__init__.py +++ b/python/mxnet/foo/__init__.py @@ -4,6 +4,8 @@ from .parameter import * +from .block import * + from . import nn from . import rnn diff --git a/python/mxnet/foo/block.py b/python/mxnet/foo/block.py new file mode 100644 index 000000000000..89c294cdd565 --- /dev/null +++ b/python/mxnet/foo/block.py @@ -0,0 +1,349 @@ +# coding: utf-8 +# pylint: disable= arguments-differ +"""Base container class for all neural network models.""" + +from .. import symbol, ndarray +from ..symbol import Symbol +from ..ndarray import NDArray +from .. import name as _name +from .parameter import Parameter, ParameterDict, DeferredInitializationError + + +class _BlockScope(object): + """Scope for collecting child `Block`s.""" + _current = None + + def __init__(self, block): + self._block = block + self._counter = {} + self._old_scope = None + + @staticmethod + def create(prefix, params, hint): + """Create prefix and params for new `Block`.""" + current = _BlockScope._current + if current is None: + if prefix is None: + prefix = _name.NameManager.current.get(None, hint) + '_' + if params is None: + params = ParameterDict(prefix) + else: + params = ParameterDict(params.prefix, params) + return prefix, params + + if prefix is None: + count = current._counter.get(hint, 0) + prefix = '%s%d_'%(hint, count) + current._counter[hint] = count + 1 + if params is None: + parent = current._block.params + params = ParameterDict(parent.prefix+prefix, parent._shared) + else: + params = ParameterDict(params.prefix, params) + return current._block.prefix+prefix, params + + def __enter__(self): + self._old_scope = _BlockScope._current + _BlockScope._current = self + return self + + def __exit__(self, ptype, value, trace): + _BlockScope._current = self._old_scope + + +def _flatten(args): + if isinstance(args, NDArray): + return [args], int(0) + if isinstance(args, Symbol): + length = len(args.list_outputs()) + length = length if length > 1 else 0 + return [args], int(length) + + assert isinstance(args, (list, tuple)), \ + "HybridBlock input must be (nested) list of Symbol or NDArray, " \ + "but got %s of type %s"%(str(args), str(type(args))) + flat = [] + fmts = [] + for i in args: + arg, fmt = _flatten(i) + flat.extend(arg) + fmts.append(fmt) + return flat, fmts + + +def _regroup(args, fmt): + if isinstance(fmt, int): + if fmt == 0: + return args[0], args[1:] + return args[:fmt], args[fmt:] + + assert isinstance(args, (list, tuple)), \ + "HybridBlock output must be (nested) list of Symbol or NDArray, " \ + "but got %s of type %s"%(str(args), str(type(args))) + ret = [] + for i in fmt: + res, args = _regroup(args, i) + ret.append(res) + return ret, args + + +class Block(object): + """Base class for all neural network layers and models. Your models should + subclass this class. + + `Block`s can be nested recursively in a tree structure. You can create and + assign child `Block`s as regular attributes:: + + from mxnet.foo import Block, nn + from mxnet import ndarray as F + + class Model(Block): + def __init__(self, **kwargs): + super(Net, self).__init__(**kwargs) + # use name_scope to give child Blocks appropriate names. + # It also allows sharing Parameters between Blocks recursively. + with self.name_scope(): + self.dense0 = nn.Dense(20) + self.dense1 = nn.Dense(20) + + def forward(self, x): + x = F.relu(self.dense0(x)) + return F.relu(self.dense1(x)) + + Child `Block`s assigned this way will be registered and `collect_params` + will collect their Parameters recursively. + + Parameters + ---------- + prefix : str + Prefix acts like a name space. It will be prepended to the name of all + Parameters and child `Block`s in this `Block`'s `name_scope`. Prefix + should be unique within one model to prevent name collisions. + params : ParameterDict or None + ParameterDict for sharing weights with the new `Block`. For example, + if you want `dense1` to share `dense0`'s weights, you can do:: + + dense0 = nn.Dense(20) + dense1 = nn.Dense(20, params=dense1.collect_params()) + """ + def __init__(self, prefix=None, params=None): + self._prefix, self._params = _BlockScope.create(prefix, params, self._alias()) + self._scope = _BlockScope(self) + self._children = [] + + def __setattr__(self, name, value): + """Registers parameters.""" + super(Block, self).__setattr__(name, value) + if isinstance(value, Block): + self.register_child(value) + + def _alias(self): + return self.__class__.__name__.lower() + + @property + def params(self): + """Returns this `Block`'s parameter dictionary (does not include its + children's parameters).""" + return self._params + + def collect_params(self): + """Returns a ParameterDict containing this `Block` and all of its + children's Parameters.""" + ret = ParameterDict(self._params.prefix) + ret.update(self.params) + for cld in self._children: + ret.update(cld.collect_params()) + return ret + + @property + def prefix(self): + """Prefix of this Block.""" + return self._prefix + + @property + def name(self): + """Name of this Block, without '_' in the end.""" + if self.prefix.endswith('_'): + return self.prefix[:-1] + return self.prefix + + def name_scope(self): + """Returns a name space object managing child `Block` and parameter + names. Should be used by a `with` statement:: + + with self.name_scope(): + self.dense = nn.Dense(20) + """ + return self._scope + + def register_child(self, block): + """Register block as a child of self. `Block`s assigned to self as + attributes will be registered automatically.""" + self._children.append(block) + + def hybridize(self, active=True): + """Activates or deactivates `HybridBlock`s recursively. Has no effect on + non-hybrid children. + + Parameters + ---------- + active : bool, default True + Whether to turn hybrid on or off. + """ + for cld in self._children: + cld.hybridize(active) + + def __call__(self, *args): + """Calls forward. Only accepts positional arguments.""" + return self.forward(*args) + + def forward(self, *args): + """Override to implement forward computation using NDArray. Only + accepts positional arguments. + + Parameters + ---------- + *args : list of NDArray + Input tensors. + """ + # pylint: disable= invalid-name + raise NotImplementedError + + +class HybridBlock(Block): + """`HybridBlock` supports forwarding with both Symbol and NDArray. + + Forward computation in `HybridBlock` must be static to work with `Symbol`s, + i.e. you cannot call `.asnumpy()`, `.shape`, `.dtype`, etc on tensors. + Also, you cannot use branching or loop logic that bases on non-constant + expressions like random numbers or intermediate results, since they change + the graph structure for each iteration. + + Before activated with `hybridize()`, `HybridBlock` works just like normal + `Block`. After activation, `HybridBlock` will create a symbolic graph + representing the forward computation and cache it. On subsequent forwards + the cached graph will be used instead of `hybrid_forward`. + """ + def __init__(self, prefix=None, params=None): + super(HybridBlock, self).__init__(prefix=prefix, params=params) + self._reg_params = {} + self._cached_graph = () + self._cached_op = None + self._cached_params = None + self._out_format = None + self._in_format = None + self._active = False + + def __setattr__(self, name, value): + """Registers parameters.""" + super(HybridBlock, self).__setattr__(name, value) + if isinstance(value, Parameter): + assert name not in self._reg_params or \ + not isinstance(self._reg_params[name], Parameter), \ + "Overriding Parameter attribute %s is not allowed. " \ + "Please pass in Parameters by specifying `params` at " \ + "Block construction instead." + self._reg_params[name] = value + + def register_child(self, block): + if not isinstance(block, HybridBlock): + if isinstance(block, Sequential): + raise ValueError( + "Children of HybridBlock must also be HybridBlock. " \ + "Please use HSequential instead of Sequential.") + raise ValueError( + "Children of HybridBlock must also be HybridBlock, " \ + "but %s has type %s."%(str(block), str(type(block)))) + super(HybridBlock, self).register_child(block) + + def hybridize(self, active=True): + self._active = active + super(HybridBlock, self).hybridize(active) + + def _get_graph(self, *args): + if self._cached_graph: + return self._cached_graph + + args, self._in_format = _flatten(args) + syms = [symbol.var(str(i)) for i in range(len(args))] + sym_args = _regroup(syms, self._in_format)[0] + + params = {i: j.var() for i, j in self._reg_params.items()} + out = self.hybrid_forward(symbol, *sym_args, **params) # pylint: disable=no-value-for-parameter + out, self._out_format = _flatten(out) + + self._cached_graph = syms, symbol.Group(out) + return self._cached_graph + + def infer_shape(self, *args): + """Infer shape of Parameters from inputs.""" + syms, out = self._get_graph(*args) + args, _, = _flatten(args) + arg_shapes, _, aux_shapes = out.infer_shape( + **{i.name: j.shape for i, j in zip(syms, args)}) + sdict = {i: j for i, j in zip(out.list_arguments(), arg_shapes)} + sdict.update({name : shape for name, shape in \ + zip(out.list_auxiliary_states(), aux_shapes)}) + for i in self.collect_params().values(): + i.shape = sdict[i.name] + + def _build_cache(self, *args): + self.infer_shape(*args) + for i in self.collect_params().values(): + i._finish_deferred_init() + + _, out = self._get_graph(*args) + self._cached_op = ndarray.CachedOp(out) + params = dict(self.collect_params().items()) + self._cached_params = [params.get(name, None) for name in out.list_inputs()] + self._in_idx = [(i, int(name)) for i, name in enumerate(out.list_inputs()) + if name not in params] + + def _call_cached_op(self, *args): + args, fmt = _flatten(args) + assert fmt == self._in_format, "Invalid input format" + cargs = [i.data() if i else None for i in self._cached_params] + for i, j in self._in_idx: + cargs[i] = args[j] + out = self._cached_op(*cargs) + if isinstance(out, NDArray): + out = [out] + return _regroup(out, self._out_format)[0] + + def forward(self, x, *args): + """Defines the forward computation. Arguments can be either + NDArray or Symbol.""" + if isinstance(x, NDArray): + if self._active and self._cached_op is None: + self._build_cache(x, *args) + + with x.context as ctx: + if self._active: + return self._call_cached_op(x, *args) + try: + params = {i: j.data(ctx) for i, j in self._reg_params.items()} + except DeferredInitializationError: + self.infer_shape(x, *args) + for i in self.collect_params().values(): + i._finish_deferred_init() + params = {i: j.data(ctx) for i, j in self._reg_params.items()} + return self.hybrid_forward(ndarray, x, *args, **params) + else: + assert isinstance(x, Symbol), \ + "HybridBlock requires the first argument to forward be either " \ + "Symbol or NDArray, but got %s"%type(x) + params = {i: j.var() for i, j in self._reg_params.items()} + return self.hybrid_forward(symbol, x, *args, **params) + + def hybrid_forward(self, F, x, *args, **kwargs): + """Override to construct symbolic graph for this `Block`. + + Parameters + ---------- + x : Symbol or NDArray + The first input tensor. + *args : list of Symbol or list of NDArray + Additional input tensors. + """ + # pylint: disable= invalid-name + raise NotImplementedError diff --git a/python/mxnet/foo/loss.py b/python/mxnet/foo/loss.py index 0e5399fd211a..98f108522d2c 100644 --- a/python/mxnet/foo/loss.py +++ b/python/mxnet/foo/loss.py @@ -1,19 +1,11 @@ # coding: utf-8 -# pylint: disable=too-many-arguments, no-member, protected-access, too-many-locals -# pylint: disable=unused-argument +# pylint: disable=arguments-differ """ losses for training neural networks """ from __future__ import absolute_import -import json - -from .. import symbol, ndarray, metric +from .. import symbol, ndarray from ..base import numeric_types - - -def _get_F(x): - """Get function domain from tensor""" - return symbol if isinstance(x, symbol.Symbol) else ndarray - +from .block import HybridBlock def _apply_weighting(F, loss, weight=None, sample_weight=None): """Apply weighting to loss. @@ -46,89 +38,7 @@ def _apply_weighting(F, loss, weight=None, sample_weight=None): return loss -def _unpack_symbol(loss): - """unpack a loss symbol into outputs, extra_outputs and losses""" - assert isinstance(loss, symbol.Symbol) - outputs = symbol.Group([i for i in loss if i.attr('__output__') == 'pred']) - extra_outputs = symbol.Group([i for i in loss if i.attr('__output__') == 'extra']) - losses = symbol.Group([i for i in loss if i.attr('__output__') == 'loss']) - return outputs, extra_outputs, losses - - -def custom_loss(loss, output, label, weight=None, sample_weight=None, batch_axis=0, - extra_outputs=(), metrics=None, name='custom'): - """Construct user defined loss symbol. - - Parameters - ---------- - loss : Symbol - loss value computed from output and label. - output : Symbol - output of the network - label : Symbol - target to compare output against - weight : float or None - global scalar weight for loss - sample_weight : Symbol or None - per sample weighting. Must be broadcastable to - the same shape as loss. For example, if loss has - shape (64, 10) and you want to weight each sample - in the batch, sample_weight should have shape (64, 1) - batch_axis : int, default 0 - The axis that represents mini-batch. - - Returns - ------- - loss : BaseLoss - created loss - - Examples - -------- - >>> # To define a least square loss (same as `l2_loss`) - >>> data = mx.sym.var('data') - >>> output = mx.sym.FullyConnected(data, num_hidden=1) - >>> label = mx.sym.var('label') - >>> loss = mx.sym.square(output - label.reshape((-1, 1)))/2 - >>> loss = nn.custom_loss(loss, output, label, name='l2') - """ - F = _get_F(loss) - loss = _apply_weighting(F, loss, weight, sample_weight) - loss = F.mean(loss, axis=batch_axis, exclude=True) - if F is ndarray: - return loss - outputs = symbol.Group([F.stop_gradient(i, name=i.name+'_out', __output__='pred') - for i in output]) - extra_outputs = symbol.Group([F.stop_gradient(i, name=i.name+'_out', __output__='extra') - for i in extra_outputs]) - - loss = F.make_loss(loss, name=name, __output__='loss') - - if metrics: - metrics = metric.create(metrics) - metrics.output_names = outputs.list_outputs() - metrics.label_names = label.list_outputs() - loss._set_attr(__metric__=json.dumps(metrics.get_config())) - - return symbol.Group([outputs, extra_outputs, loss]) - - -def multitask_loss(losses): - """Combine multiple losses together for multitask learning. - - Parameters - ---------- - losses : list of Symbol - list of losses to be combined. - """ - F = _get_F(losses[0]) - if F is ndarray: - return losses - out, extra, loss = zip(*[_unpack_symbol(i) for i in losses]) - return symbol.Group(out+extra+loss) - - -def l2_loss(output, label, weight=1., sample_weight=None, batch_axis=0, - extra_outputs=(), metrics=None, name='l2'): +class L2Loss(HybridBlock): """Calculate the mean squared error between output and label: .. math:: @@ -139,10 +49,6 @@ def l2_loss(output, label, weight=1., sample_weight=None, batch_axis=0, Parameters ---------- - output : Symbol - output of the network - label : Symbol - target to compare output against weight : float or None global scalar weight for loss sample_weight : Symbol or None @@ -152,22 +58,25 @@ def l2_loss(output, label, weight=1., sample_weight=None, batch_axis=0, in the batch, sample_weight should have shape (64, 1) batch_axis : int, default 0 The axis that represents mini-batch. - - Returns - ------- - loss : Symbol - created loss """ - if isinstance(output, ndarray.NDArray): - loss = ndarray.square(output - label.reshape(output.shape)) - else: - loss = symbol.square(output - label.reshape(())) - return custom_loss(loss, output, label, weight/2, sample_weight, batch_axis, - extra_outputs, metrics, name) - - -def l1_loss(output, label, weight=None, sample_weight=None, batch_axis=0, - extra_outputs=(), metrics=None, name='l1'): + def __init__(self, weight=1., batch_axis=0, **kwargs): + super(L2Loss, self).__init__(**kwargs) + self._weight = weight + self._batch_axis = batch_axis + + def hybrid_forward(self, F, output, label, sample_weight=None): + if F is ndarray: + loss = ndarray.square(output - label.reshape(output.shape)) + else: + # for symbolic output.shape is not available so we reshape + # to empty shape and let it be inferred from output's shape + # via the '-' operator later. + loss = symbol.square(output - label.reshape(())) + loss = _apply_weighting(F, loss, self._weight/2, sample_weight) + return F.mean(loss, axis=self._batch_axis, exclude=True) + + +class L1Loss(HybridBlock): """Calculate the mean absolute error between output and label: .. math:: @@ -177,10 +86,6 @@ def l1_loss(output, label, weight=None, sample_weight=None, batch_axis=0, Parameters ---------- - output : Symbol - output of the network - label : Symbol - target to compare output against weight : float or None global scalar weight for loss sample_weight : Symbol or None @@ -190,23 +95,25 @@ def l1_loss(output, label, weight=None, sample_weight=None, batch_axis=0, in the batch, sample_weight should have shape (64, 1) batch_axis : int, default 0 The axis that represents mini-batch. - - Returns - ------- - loss : Symbol - created loss """ - if isinstance(output, ndarray.NDArray): - loss = ndarray.abs(output - label.reshape(output.shape)) - else: - loss = symbol.abs(output - label.reshape(())) - return custom_loss(loss, output, label, weight, sample_weight, batch_axis, - extra_outputs, metrics, name) - - -def softmax_cross_entropy_loss(output, label, sparse_label=True, axis=-1, - weight=None, sample_weight=None, batch_axis=0, - extra_outputs=(), metrics='acc', name='ce'): + def __init__(self, weight=None, batch_axis=0, **kwargs): + super(L1Loss, self).__init__(**kwargs) + self._weight = weight + self._batch_axis = batch_axis + + def hybrid_forward(self, F, output, label, sample_weight=None): + if F is ndarray: + loss = ndarray.abs(output - label.reshape(output.shape)) + else: + # for symbolic output.shape is not available so we reshape + # to empty shape and let it be inferred from output's shape + # via the '-' operator later. + loss = symbol.abs(output - label.reshape(())) + loss = _apply_weighting(F, loss, self._weight, sample_weight) + return F.mean(loss, axis=self._batch_axis, exclude=True) + + +class SoftmaxCrossEntropyLoss(HybridBlock): """Compute the softmax cross entropy loss. If sparse_label is True, label should contain integer category indicators: @@ -229,14 +136,13 @@ def softmax_cross_entropy_loss(output, label, sparse_label=True, axis=-1, Parameters ---------- - output : Symbol - output of the network - label : Symbol - target to compare output against - sparse_label : bool, default True - where label is sparse integer or probability distribution axis : int, default -1 The axis to sum over when computing softmax and entropy + sparse_label : bool, default True + whether label is a integer array instead of probability distribution + from_logits : bool, default False + whether input is log probability (usually from log_softmax) instead + of unnormalized numbers. weight : float or None global scalar weight for loss sample_weight : Symbol or None @@ -246,17 +152,22 @@ def softmax_cross_entropy_loss(output, label, sparse_label=True, axis=-1, in the batch, sample_weight should have shape (64, 1) batch_axis : int, default 0 The axis that represents mini-batch. - - Returns - ------- - loss : Symbol - created loss """ - F = _get_F(output) - prob = F.log_softmax(output) - if sparse_label: - loss = -F.pick(prob, label, axis=axis, keepdims=True) - else: - loss = -F.sum(prob*label, axis=axis, keepdims=True) - return custom_loss(loss, prob, label, weight, sample_weight, batch_axis, - extra_outputs, metrics, name) + def __init__(self, axis=-1, sparse_label=True, from_logits=False, weight=None, + batch_axis=0, **kwargs): + super(SoftmaxCrossEntropyLoss, self).__init__(**kwargs) + self._axis = axis + self._sparse_label = sparse_label + self._from_logits = from_logits + self._weight = weight + self._batch_axis = batch_axis + + def hybrid_forward(self, F, output, label, sample_weight=None): + if not self._from_logits: + output = F.log_softmax(output) + if self._sparse_label: + loss = -F.pick(output, label, axis=self._axis, keepdims=True) + else: + loss = -F.sum(output*label, axis=self._axis, keepdims=True) + loss = _apply_weighting(F, loss, self._weight, sample_weight) + return F.mean(loss, axis=self._batch_axis, exclude=True) diff --git a/python/mxnet/foo/nn/__init__.py b/python/mxnet/foo/nn/__init__.py index 8cf69dee077d..e4191b2a7dc2 100644 --- a/python/mxnet/foo/nn/__init__.py +++ b/python/mxnet/foo/nn/__init__.py @@ -2,6 +2,6 @@ # pylint: disable=wildcard-import """Neural network layers.""" -from .layer import * +from .basic_layers import * from .conv_layers import * diff --git a/python/mxnet/foo/nn/basic_layers.py b/python/mxnet/foo/nn/basic_layers.py new file mode 100644 index 000000000000..b1234f9d9e16 --- /dev/null +++ b/python/mxnet/foo/nn/basic_layers.py @@ -0,0 +1,290 @@ +# coding: utf-8 +# pylint: disable= arguments-differ +"""Basic neural network layers.""" + +from ..block import Block, HybridBlock + + +class Sequential(Block): + """Stacks `Block`s sequentially. + + Example:: + + net = nn.Sequential() + # use net's name_scope to give child Blocks appropriate names. + with net.name_scope(): + net.add(Dense(10, activation='relu')) + net.add(Dense(20)) + """ + def __init__(self, prefix=None, params=None): + super(Sequential, self).__init__(prefix=prefix, params=params) + + def add(self, block): + """Add block on top of the stack.""" + self.register_child(block) + + def forward(self, x): + for block in self._children: + x = block(x) + return x + + +class HSequential(HybridBlock): + """Stack `HybridBlock`s sequentially. + + Example:: + + net = nn.Sequential() + # use net's name_scope to give child Blocks appropriate names. + with net.name_scope(): + net.add(Dense(10, activation='relu')) + net.add(Dense(20)) + """ + def __init__(self, prefix=None, params=None): + super(HSequential, self).__init__(prefix=prefix, params=params) + + def add(self, block): + """Add block on top of the stack.""" + self.register_child(block) + + def hybrid_forward(self, F, x): + for block in self._children: + x = block(x) + return x + + +class Dense(HybridBlock): + """Just your regular densely-connected NN layer. + + `Dense` implements the operation: + `output = activation(dot(input, weight) + bias)` + where `activation` is the element-wise activation function + passed as the `activation` argument, `weight` is a weights matrix + created by the layer, and `bias` is a bias vector created by the layer + (only applicable if `use_bias` is `True`). + + Note: the input must be a tensor with rank 2. Use flatten to convert it + to rank 2 manually if necessary. + + Parameters + ---------- + units : int + Dimensionality of the output space. + activation : str + Activation function to use. See help on `Activation` layer. + If you don't specify anything, no activation is applied + (ie. "linear" activation: `a(x) = x`). + use_bias : bool + Whether the layer uses a bias vector. + weight_initializer : str or `Initializer` + Initializer for the `kernel` weights matrix. + bias_initializer: str or `Initializer` + Initializer for the bias vector. + in_units : int, optional + Size of input data. If not specified, initialization will be + defered to the first time `forward` is called and `in_units` + will be inferred from the shape of input data. + prefix : str or None + See document of `Block`. + params : ParameterDict or None + See document of `Block`. + + + Input shape: + a 2D input with shape `(batch_size, in_units)`. + + Output shape: + the output would have shape `(batch_size, units)`. + """ + def __init__(self, units, activation=None, use_bias=True, + weight_initializer=None, bias_initializer=None, + in_units=0, **kwargs): + super(Dense, self).__init__(**kwargs) + with self.name_scope(): + self._units = units + self.weight = self.params.get('weight', shape=(units, in_units), + init=weight_initializer) + if use_bias: + self.bias = self.params.get('bias', shape=(units,), + init=bias_initializer) + else: + self.bias = None + if activation is not None: + self.act = Activation(activation) + else: + self.act = None + + def hybrid_forward(self, F, x, weight, bias=None): + if bias is None: + act = F.FullyConnected(x, weight, no_bias=True, num_hidden=self._units) + else: + act = F.FullyConnected(x, weight, bias, num_hidden=self._units) + if self.act is not None: + act = self.act(act) + return act + + +class Activation(HybridBlock): + """Applies an activation function to input. + + Parameters + ---------- + activation : str + name of activation function to use. + See :func:`~mxnet.ndarray.Activation` for available choices. + + Input shape: + Arbitrary. + + Output shape: + Same shape as input. + """ + def __init__(self, activation, **kwargs): + self._act_type = activation + super(Activation, self).__init__(**kwargs) + + def _alias(self): + return self._act_type + + def hybrid_forward(self, F, x): + return F.Activation(x, act_type=self._act_type) + + +class Dropout(HybridBlock): + """Applies Dropout to the input. + + Dropout consists in randomly setting a fraction `rate` of input units + to 0 at each update during training time, which helps prevent overfitting. + + Parameters + ---------- + rate : float + Fraction of the input units to drop. Must be a number between 0 and 1. + + References + ---------- + `Dropout: A Simple Way to Prevent Neural Networks from Overfitting + `_ + """ + def __init__(self, rate, **kwargs): + super(Dropout, self).__init__(**kwargs) + self._rate = rate + + def hybrid_forward(self, F, x): + return F.Dropout(x, p=self._rate) + + +class BatchNorm(HybridBlock): + """Batch normalization layer (Ioffe and Szegedy, 2014). + Normalize the input at each batch, i.e. applies a transformation + that maintains the mean activation close to 0 and the activation + standard deviation close to 1. + + Parameters + ---------- + axis : int, default 1 + The axis that should be normalized. This is ypically the channels + (C) axis. For instance, after a `Conv2D` layer with `layout='NCHW'`, + set `axis=1` in `BatchNorm`. If `layout='NHWC'`, then set `axis=3`. + momentum: float, default 0.9 + Momentum for the moving average. + epsilon: float, default 1e-3 + Small float added to variance to avoid dividing by zero. + center: bool, default True + If True, add offset of `beta` to normalized tensor. + If False, `beta` is ignored. + scale: bool, default True + If True, multiply by `gamma`. If False, `gamma` is not used. + When the next layer is linear (also e.g. `nn.relu`), + this can be disabled since the scaling + will be done by the next layer. + beta_initializer: str or `Initializer`, default 'zeros' + Initializer for the beta weight. + gamma_initializer: str or `Initializer`, default 'ones' + Initializer for the gamma weight. + moving_mean_initializer: str or `Initializer`, default 'zeros' + Initializer for the moving mean. + moving_variance_initializer: str or `Initializer`, default 'ones' + Initializer for the moving variance. + in_channels : int, default 0 + Number of channels (feature maps) in input data. If not specified, + initialization will be defered to the first time `forward` is called + and `in_channels` will be inferred from the shape of input data. + """ + def __init__(self, axis=1, momentum=0.9, epsilon=1e-3, center=True, scale=True, + beta_initializer='zeros', gamma_initializer='ones', + running_mean_initializer='zeros', running_variance_initializer='ones', + in_channels=0, **kwargs): + super(BatchNorm, self).__init__(**kwargs) + self._kwargs = {'axis': axis, 'eps': epsilon, 'momentum': momentum, + 'fix_gamma': not center} + + self.gamma = self.params.get('gamma', grad_req='write' if scale else 'null', + shape=(in_channels,), init=gamma_initializer) + self.beta = self.params.get('beta', grad_req='write' if center else 'null', + shape=(in_channels,), init=beta_initializer) + self.running_mean = self.params.get('running_mean', grad_req='null', + shape=(in_channels,), + init=running_mean_initializer) + self.running_var = self.params.get('running_var', grad_req='null', + shape=(in_channels,), + init=running_variance_initializer) + + def hybrid_forward(self, F, x, gamma, beta, running_mean, running_var): + return F.BatchNorm(x, gamma, beta, running_mean, running_var, **self._kwargs) + + +class LeakyReLU(HybridBlock): + """Leaky version of a Rectified Linear Unit. + + It allows a small gradient when the unit is not active:: + + `f(x) = alpha * x for x < 0`, + `f(x) = x for x >= 0`. + + Parameters + ---------- + alpha : float + slope coefficient for the negative half axis. Must be >= 0. + """ + def __init__(self, alpha, **kwargs): + super(LeakyReLU, self).__init__(**kwargs) + self._alpha = alpha + + def hybrid_forward(self, F, x): + return F.LeakyReLU(x, act_type='leaky', slope=self._alpha) + + +class Embedding(HybridBlock): + """Turns non-negative integers (indexes/tokens) into dense vectors + of fixed size. eg. [[4], [20]] -> [[0.25, 0.1], [0.6, -0.2]] + + + Parameters + ---------- + input_dim : int + Size of the vocabulary, i.e. maximum integer index + 1. + output_dim : int + Dimension of the dense embedding. + dtype : str or np.dtype, default 'float32' + Data type of output embeddings. + weight_initializer : Initializer + Initializer for the `embeddings` matrix + + + Input shape: + 2D tensor with shape: `(N, M)`. + + Output shape: + 3D tensor with shape: `(N, M, output_dim)`. + """ + def __init__(self, input_dim, output_dim, dtype='float32', + weight_initializer=None, **kwargs): + super(Embedding, self).__init__(**kwargs) + self._kwargs = {'input_dim': input_dim, 'output_dim': output_dim, + 'dtype': dtype} + self.weight = self.params.get('weight', shape=(input_dim, output_dim), + init=weight_initializer) + + def hybrid_forward(self, F, x, weight): + return F.Embedding(x, weight, **self._kwargs) diff --git a/python/mxnet/foo/nn/conv_layers.py b/python/mxnet/foo/nn/conv_layers.py index 8236c6f7993a..3449a160cee8 100644 --- a/python/mxnet/foo/nn/conv_layers.py +++ b/python/mxnet/foo/nn/conv_layers.py @@ -1,7 +1,7 @@ # coding: utf-8 # pylint: disable= arguments-differ """Convolutional neural network layers.""" -from .layer import HybridLayer +from ..block import HybridBlock from ... import symbol from ...base import numeric_types @@ -11,7 +11,7 @@ def _infer_weight_shape(op_name, data_shape, kwargs): return sym.infer_shape_partial()[0] -class _Conv(HybridLayer): +class _Conv(HybridBlock): """Abstract nD convolution layer (private, used as implementation base). This layer creates a convolution kernel that is convolved @@ -22,50 +22,52 @@ class _Conv(HybridLayer): Parameters ---------- - filters: Integer, the dimensionality of the output space - (i.e. the number output of filters in the convolution). - kernel_size: An integer or tuple/list of n integers, specifying the - dimensions of the convolution window. - strides: An integer or tuple/list of n integers, - specifying the strides of the convolution. - Specifying any stride value != 1 is incompatible with specifying - any `dilation_rate` value != 1. - padding: An integer or a tuple/list of n integers, + channels : int + The dimensionality of the output space + i.e. the number of output channels in the convolution. + kernel_size : int or tuple/list of n ints + Specifys the dimensions of the convolution window. + strides: int or tuple/list of n ints, + Specifys the strides of the convolution. + padding : int or tuple/list of n ints, If padding is non-zero, then the input is implicitly zero-padded on both sides for padding number of points - dilation: An integer or tuple/list of n integers, specifying - the dilation rate to use for dilated convolution. - groups: int + dilation: int or tuple/list of n ints, + Specifys the dilation rate to use for dilated convolution. + groups : int controls the connections between inputs and outputs. At groups=1, all inputs are convolved to all outputs. At groups=2, the operation becomes equivalent to having two conv layers side by side, each seeing half the input channels, and producing half the output channels, and both subsequently concatenated. - layout: A string, - Can be 'NCW', 'NWC', 'NCHW', 'NHWC', 'NCDHW', 'NDHWC', etc. - 'N', 'C', 'H', 'W', 'D' stands for batch, channel, height, width and - depth dimensions respectively. - in_filters: int, default 0 - The number of input channels to this layer. Only required when using - NDArray API. - activation: Activation function to use - see mx.sym.Activation. + layout : str, + Dimension ordering of data and weight. Can be 'NCW', 'NWC', 'NCHW', + 'NHWC', 'NCDHW', 'NDHWC', etc. 'N', 'C', 'H', 'W', 'D' stands for + batch, channel, height, width and depth dimensions respectively. + Convolution is perform over 'D', 'H', and 'W' dimensions. + in_channels : int, default 0 + The number of input channels to this layer. If not specified, + initialization will be defered to the first time `forward` is called + and `in_channels` will be inferred from the shape of input data. + activation : str + Activation function to use. See :func:`~mxnet.nd.Activation`. If you don't specify anything, no activation is applied (ie. "linear" activation: `a(x) = x`). - use_bias: Boolean, whether the layer uses a bias vector. - weight_initializer: Initializer for the `kernel` weights matrix - see Initializer. - bias_initializer: Initializer for the bias vector - see Initializer. + use_bias: bool + Whether the layer uses a bias vector. + weight_initializer : str or `Initializer` + Initializer for the `weight` weights matrix. + bias_initializer: str or `Initializer` + Initializer for the bias vector. """ - def __init__(self, filters, kernel_size, strides, padding, dilation, - groups, layout, in_filters=0, activation=None, use_bias=True, + def __init__(self, channels, kernel_size, strides, padding, dilation, + groups, layout, in_channels=0, activation=None, use_bias=True, weight_initializer=None, bias_initializer=None, op_name='Convolution', prefix=None, params=None, **kwargs): super(_Conv, self).__init__(prefix=prefix, params=params) with self.name_scope(): - self._filters = filters - self._in_filters = in_filters + self._channels = channels + self._in_channels = in_channels if isinstance(strides, numeric_types): strides = (strides,)*len(kernel_size) if isinstance(padding, numeric_types): @@ -75,13 +77,13 @@ def __init__(self, filters, kernel_size, strides, padding, dilation, self._op_name = op_name self._kwargs = { 'kernel': kernel_size, 'stride': strides, 'dilate': dilation, - 'pad': padding, 'num_filter': filters, 'num_group': groups, + 'pad': padding, 'num_filter': channels, 'num_group': groups, 'no_bias': not use_bias, 'layout': layout} self._kwargs.update(kwargs) dshape = [0]*(len(kernel_size) + 2) dshape[layout.find('N')] = 1 - dshape[layout.find('C')] = in_filters + dshape[layout.find('C')] = in_channels wshapes = _infer_weight_shape(op_name, dshape, self._kwargs) self.weight = self.params.get('weight', shape=wshapes[1], init=weight_initializer) @@ -116,75 +118,72 @@ class Conv1D(_Conv): Finally, if `activation` is not `None`, it is applied to the outputs as well. - When using this layer with NDArray API, - provide an `in_filters` argument - (integers, the number of input channels). + If `in_channels` is not specified, `Parameter` initialization will be + defered to the first time `forward` is called and `in_channels` will be + inferred from the shape of input data. Parameters ---------- - filters: Integer, the dimensionality of the output space - (i.e. the number output of filters in the convolution). - kernel_size: An integer or tuple/list of 1 integers, specifying the - dimensions of the convolution window. - strides: An integer or tuple/list of 1 integers, - specifying the strides of the convolution. - Specifying any stride value != 1 is incompatible with specifying - any `dilation_rate` value != 1. - padding: An integer or a tuple/list of 1 integers, + channels : int + The dimensionality of the output space, i.e. the number of output + channels (filters) in the convolution. + kernel_size :int or tuple/list of 1 int + Specifys the dimensions of the convolution window. + strides : int or tuple/list of 1 int, + Specify the strides of the convolution. + padding : int or a tuple/list of 1 int, If padding is non-zero, then the input is implicitly zero-padded on both sides for padding number of points - dilation: An integer or tuple/list of 1 integers, specifying - the dilation rate to use for dilated convolution. - groups: int + dilation : int or tuple/list of 1 int + Specifys the dilation rate to use for dilated convolution. + groups : int controls the connections between inputs and outputs. At groups=1, all inputs are convolved to all outputs. At groups=2, the operation becomes equivalent to having two conv layers side by side, each seeing half the input channels, and producing half the output channels, and both subsequently concatenated. - layout: A string, - Can be 'NCW', 'NWC', etc. + layout: str, default 'NCW' + Dimension ordering of data and weight. Can be 'NCW', 'NWC', etc. 'N', 'C', 'W' stands for batch, channel, and width (time) dimensions - respectively. - in_filters: int, default 0 - The number of input channels to this layer. Only required when using - NDArray API. - activation: Activation function to use - see mx.sym.Activation. + respectively. Convolution is applied on the 'W' dimension. + in_channels : int, default 0 + The number of input channels to this layer. If not specified, + initialization will be defered to the first time `forward` is called + and `in_channels` will be inferred from the shape of input data. + activation : str + Activation function to use. See :func:`mx.nd.Activation`. If you don't specify anything, no activation is applied (ie. "linear" activation: `a(x) = x`). - use_bias: Boolean, whether the layer uses a bias vector. - weight_initializer: Initializer for the `kernel` weights matrix - see Initializer. - bias_initializer: Initializer for the bias vector - see Initializer. + use_bias : bool + Whether the layer uses a bias vector. + weight_initializer : str or `Initializer` + Initializer for the `weight` weights matrix. + bias_initializer : str or `Initializer` + Initializer for the bias vector. Input Shape: This depends on the `layout` parameter. Input is 3D array of shape - (batch_size, in_channel(in_filters), width) if `layout` is `NCW`. + (batch_size, in_channels, width) if `layout` is `NCW`. Output Shape: This depends on the `layout` parameter. Output is 3D array of shape - (batch_size, out_channel(filters), out_width) if `layout` is `NCW`. out_width - depends on other input parameters as well. It is calculated as follows:: - - out_width = floor((w+2*p-d*(k-1)-1)/s)+1 - - where, + (batch_size, channels, out_width) if `layout` is `NCW`. + out_width is calculated as:: - w = width, p = padding, d = dilation, k = kernel_size, s = stride + out_width = floor((width+2*padding-dilation*(kernel_size-1)-1)/stride)+1 """ - def __init__(self, filters, kernel_size, strides=1, padding=0, dilation=1, + def __init__(self, channels, kernel_size, strides=1, padding=0, dilation=1, groups=1, layout='NCW', activation=None, use_bias=True, weight_initializer=None, bias_initializer=None, - in_filters=0, **kwargs): + in_channels=0, **kwargs): if isinstance(kernel_size, numeric_types): kernel_size = (kernel_size,) assert len(kernel_size) == 1, "kernel_size must be a number or a list of 1 ints" super(Conv1D, self).__init__( - filters, kernel_size, strides, padding, dilation, groups, layout, - in_filters, activation, use_bias, weight_initializer, bias_initializer, **kwargs) + channels, kernel_size, strides, padding, dilation, groups, layout, + in_channels, activation, use_bias, weight_initializer, bias_initializer, **kwargs) class Conv2D(_Conv): @@ -196,77 +195,74 @@ class Conv2D(_Conv): a bias vector is created and added to the outputs. Finally, if `activation` is not `None`, it is applied to the outputs as well. - When using this layer with NDArray API, - provide an `in_filters` argument - (integers, the number of input channels). - + If `in_channels` is not specified, `Parameter` initialization will be + defered to the first time `forward` is called and `in_channels` will be + inferred from the shape of input data. Parameters ---------- - filters: Integer, the dimensionality of the output space - (i.e. the number output of filters in the convolution). - kernel_size: An integer or tuple/list of 2 integers, specifying the - dimensions of the convolution window. - strides: An integer or tuple/list of 2 integers, - specifying the strides of the convolution. - Specifying any stride value != 1 is incompatible with specifying - any `dilation_rate` value != 1. - padding: An integer or a tuple/list of 2 integers, + channels : int + The dimensionality of the output space, i.e. the number of output + channels (filters) in the convolution. + kernel_size :int or tuple/list of 2 int + Specifys the dimensions of the convolution window. + strides : int or tuple/list of 2 int, + Specify the strides of the convolution. + padding : int or a tuple/list of 2 int, If padding is non-zero, then the input is implicitly zero-padded on both sides for padding number of points - dilation: An integer or tuple/list of 2 integers, specifying - the dilation rate to use for dilated convolution. - groups: int + dilation : int or tuple/list of 2 int + Specifys the dilation rate to use for dilated convolution. + groups : int controls the connections between inputs and outputs. At groups=1, all inputs are convolved to all outputs. At groups=2, the operation becomes equivalent to having two conv layers side by side, each seeing half the input channels, and producing half the output channels, and both subsequently concatenated. - layout: A string, - Can be 'NCHW', 'NHWC', etc. + layout : str, default 'NCHW' + Dimension ordering of data and weight. Can be 'NCHW', 'NHWC', etc. 'N', 'C', 'H', 'W' stands for batch, channel, height, and width - dimensions respectively. - in_filters: int, default 0 - The number of input channels to this layer. Only required when using - NDArray API. - activation: Activation function to use - see mx.sym.Activation. + dimensions respectively. Convolution is applied on the 'H' and + 'W' dimensions. + in_channels : int, default 0 + The number of input channels to this layer. If not specified, + initialization will be defered to the first time `forward` is called + and `in_channels` will be inferred from the shape of input data. + activation : str + Activation function to use. See :func:`mx.nd.Activation`. If you don't specify anything, no activation is applied (ie. "linear" activation: `a(x) = x`). - use_bias: Boolean, whether the layer uses a bias vector. - weight_initializer: Initializer for the `kernel` weights matrix - see Initializer. - bias_initializer: Initializer for the bias vector - see Initializer. + use_bias : bool + Whether the layer uses a bias vector. + weight_initializer : str or `Initializer` + Initializer for the `weight` weights matrix. + bias_initializer : str or `Initializer` + Initializer for the bias vector. Input Shape: This depends on the `layout` parameter. Input is 4D array of shape - (batch_size, in_channel(in_filters), height, width) if `layout` is `NCHW`. + (batch_size, in_channels, height, width) if `layout` is `NCHW`. Output Shape: This depends on the `layout` parameter. Output is 4D array of shape - (batch_size, out_channel(filters), out_height, out_width) if `layout` is `NCHW`. - out_height and out_width depends on other input parameters as well. - They are calculated as follows:: - - out_width = floor((w+2*p-d*(k-1)-1)/s)+1 - out_height = floor((h+2*p-d*(k-1)-1)/s)+1 + (batch_size, channels, out_height, out_width) if `layout` is `NCHW`. - where, + out_height and out_width are calculated as:: - w = width, h = height, p = padding, d = dilation, k = kernel_size, s = stride + out_height = floor((height+2*padding[0]-dilation[0]*(kernel_size[0]-1)-1)/stride[0])+1 + out_width = floor((width+2*padding[1]-dilation[1]*(kernel_size[1]-1)-1)/stride[1])+1 """ - def __init__(self, filters, kernel_size, strides=(1, 1), padding=(0, 0), + def __init__(self, channels, kernel_size, strides=(1, 1), padding=(0, 0), dilation=(1, 1), groups=1, layout='NCHW', activation=None, use_bias=True, weight_initializer=None, - bias_initializer=None, in_filters=0, **kwargs): + bias_initializer=None, in_channels=0, **kwargs): if isinstance(kernel_size, numeric_types): kernel_size = (kernel_size,)*2 assert len(kernel_size) == 2, "kernel_size must be a number or a list of 2 ints" super(Conv2D, self).__init__( - filters, kernel_size, strides, padding, dilation, groups, layout, - in_filters, activation, use_bias, weight_initializer, bias_initializer, **kwargs) + channels, kernel_size, strides, padding, dilation, groups, layout, + in_channels, activation, use_bias, weight_initializer, bias_initializer, **kwargs) class Conv3D(_Conv): @@ -278,78 +274,76 @@ class Conv3D(_Conv): a bias vector is created and added to the outputs. Finally, if `activation` is not `None`, it is applied to the outputs as well. - When using this layer with NDArray API, - provide an `in_filters` argument - (integers, the number of input channels). - + If `in_channels` is not specified, `Parameter` initialization will be + defered to the first time `forward` is called and `in_channels` will be + inferred from the shape of input data. Parameters ---------- - filters: Integer, the dimensionality of the output space - (i.e. the number output of filters in the convolution). - kernel_size: An integer or tuple/list of 3 integers, specifying the - dimensions of the convolution window. - strides: An integer or tuple/list of 3 integers, - specifying the strides of the convolution. - Specifying any stride value != 1 is incompatible with specifying - any `dilation_rate` value != 1. - padding: An integer or a tuple/list of 3 integers, + channels : int + The dimensionality of the output space, i.e. the number of output + channels (filters) in the convolution. + kernel_size :int or tuple/list of 3 int + Specifys the dimensions of the convolution window. + strides : int or tuple/list of 3 int, + Specify the strides of the convolution. + padding : int or a tuple/list of 3 int, If padding is non-zero, then the input is implicitly zero-padded on both sides for padding number of points - dilation: An integer or tuple/list of 3 integers, specifying - the dilation rate to use for dilated convolution. - groups: int + dilation : int or tuple/list of 3 int + Specifys the dilation rate to use for dilated convolution. + groups : int controls the connections between inputs and outputs. At groups=1, all inputs are convolved to all outputs. At groups=2, the operation becomes equivalent to having two conv layers side by side, each seeing half the input channels, and producing half the output channels, and both subsequently concatenated. - layout: A string, - Can be 'NCDHW', 'NDHWC', etc. + layout : str, default 'NCDHW' + Dimension ordering of data and weight. Can be 'NCDHW', 'NDHWC', etc. 'N', 'C', 'H', 'W', 'D' stands for batch, channel, height, width and - depth dimensions respectively. - in_filters: int, default 0 - The number of input channels to this layer. Only required when using - NDArray API. - activation: Activation function to use - see mx.sym.Activation. + depth dimensions respectively. Convolution is applied on the 'D', + 'H' and 'W' dimensions. + in_channels : int, default 0 + The number of input channels to this layer. If not specified, + initialization will be defered to the first time `forward` is called + and `in_channels` will be inferred from the shape of input data. + activation : str + Activation function to use. See :func:`mx.nd.Activation`. If you don't specify anything, no activation is applied (ie. "linear" activation: `a(x) = x`). - use_bias: Boolean, whether the layer uses a bias vector. - weight_initializer: Initializer for the `kernel` weights matrix - see Initializer. - bias_initializer: Initializer for the bias vector - see Initializer. + use_bias : bool + Whether the layer uses a bias vector. + weight_initializer : str or `Initializer` + Initializer for the `weight` weights matrix. + bias_initializer : str or `Initializer` + Initializer for the bias vector. Input Shape: This depends on the `layout` parameter. Input is 5D array of shape - (batch_size, in_channel(in_filters), depth, height, width) if `layout` is `NCDHW`. + (batch_size, in_channels, depth, height, width) if `layout` is `NCDHW`. Output Shape: This depends on the `layout` parameter. Output is 5D array of shape - (batch_size, out_channel(filters), out_depth, out_height, out_width) if `layout` is - `NCDHW`. out_depth, out_height and out_width depends on other input parameters as well. - They are calculated as follows:: + (batch_size, channels, out_depth, out_height, out_width) if `layout` is + `NCDHW`. - out_depth = floor((d+2*p-d*(k-1)-1)/s)+1 - out_height = floor((h+2*p-d*(k-1)-1)/s)+1 - out_width = floor((w+2*p-d*(k-1)-1)/s)+1 + out_depth, out_height and out_width are calculated as:: - where, - - d = depth, h = height, w = width, p = padding, d = dilation, k = kernel_size, s = stride + out_depth = floor((depth+2*padding[0]-dilation[0]*(kernel_size[0]-1)-1)/stride[0])+1 + out_height = floor((height+2*padding[1]-dilation[1]*(kernel_size[1]-1)-1)/stride[1])+1 + out_width = floor((width+2*padding[2]-dilation[2]*(kernel_size[2]-1)-1)/stride[2])+1 """ - def __init__(self, filters, kernel_size, strides=(1, 1, 1), padding=(0, 0, 0), + def __init__(self, channels, kernel_size, strides=(1, 1, 1), padding=(0, 0, 0), dilation=(1, 1, 1), groups=1, layout='NCDHW', activation=None, use_bias=True, weight_initializer=None, bias_initializer=None, - in_filters=0, **kwargs): + in_channels=0, **kwargs): if isinstance(kernel_size, numeric_types): kernel_size = (kernel_size,)*3 assert len(kernel_size) == 3, "kernel_size must be a number or a list of 3 ints" super(Conv3D, self).__init__( - filters, kernel_size, strides, padding, dilation, groups, layout, - in_filters, activation, use_bias, weight_initializer, bias_initializer, **kwargs) + channels, kernel_size, strides, padding, dilation, groups, layout, + in_channels, activation, use_bias, weight_initializer, bias_initializer, **kwargs) class Conv1DTranspose(_Conv): @@ -362,70 +356,66 @@ class Conv1DTranspose(_Conv): while maintaining a connectivity pattern that is compatible with said convolution. - When using this layer with NDArray API, - provide an `in_filters` argument - (integers, the number of input channels). + If `in_channels` is not specified, `Parameter` initialization will be + defered to the first time `forward` is called and `in_channels` will be + inferred from the shape of input data. Parameters ---------- - filters: Integer, the dimensionality of the output space - (i.e. the number output of filters in the convolution). - kernel_size: An integer or tuple/list of 1 integers, specifying the - dimensions of the convolution window. - strides: An integer or tuple/list of 1 integers, - specifying the strides of the convolution. - Specifying any stride value != 1 is incompatible with specifying - any `dilation_rate` value != 1. - padding: An integer or a tuple/list of 1 integers, + channels : int + The dimensionality of the output space, i.e. the number of output + channels (filters) in the convolution. + kernel_size :int or tuple/list of 3 int + Specifys the dimensions of the convolution window. + strides : int or tuple/list of 3 int, + Specify the strides of the convolution. + padding : int or a tuple/list of 3 int, If padding is non-zero, then the input is implicitly zero-padded on both sides for padding number of points - output_padding: An integer or a tuple/list of 1 integers, - Zero-padding added to one side of the output - dilation: An integer or tuple/list of 1 integers, specifying - the dilation rate to use for dilated convolution. - groups: int + dilation : int or tuple/list of 3 int + Specifys the dilation rate to use for dilated convolution. + groups : int controls the connections between inputs and outputs. At groups=1, all inputs are convolved to all outputs. At groups=2, the operation becomes equivalent to having two conv layers side by side, each seeing half the input channels, and producing half the output channels, and both subsequently concatenated. - layout: A string, - Can be 'NCW', 'NWC', etc. + layout : str, default 'NCW' + Dimension ordering of data and weight. Can be 'NCW', 'NWC', etc. 'N', 'C', 'W' stands for batch, channel, and width (time) dimensions - respectively. - in_filters: int, default 0 - The number of input channels to this layer. Only required when using - NDArray API. - activation: Activation function to use - see mx.sym.Activation. + respectively. Convolution is applied on the 'W' dimension. + in_channels : int, default 0 + The number of input channels to this layer. If not specified, + initialization will be defered to the first time `forward` is called + and `in_channels` will be inferred from the shape of input data. + activation : str + Activation function to use. See :func:`mx.nd.Activation`. If you don't specify anything, no activation is applied (ie. "linear" activation: `a(x) = x`). - use_bias: Boolean, whether the layer uses a bias vector. - weight_initializer: Initializer for the `kernel` weights matrix - see Initializer. - bias_initializer: Initializer for the bias vector - see Initializer. + use_bias : bool + Whether the layer uses a bias vector. + weight_initializer : str or `Initializer` + Initializer for the `weight` weights matrix. + bias_initializer : str or `Initializer` + Initializer for the bias vector. Input Shape: This depends on the `layout` parameter. Input is 3D array of shape - (batch_size, in_channel(in_filters), width) if `layout` is `NCW`. + (batch_size, in_channels, width) if `layout` is `NCW`. Output Shape: This depends on the `layout` parameter. Output is 3D array of shape - (batch_size, out_channel(filters), out_width) if `layout` is `NCW`. - out_width depends on other input parameters as well. It is calculated as follows:: - - out_width = (w-1)*s-2*p+k+op + (batch_size, channels, out_width) if `layout` is `NCW`. - where, + out_width is calculated as:: - w = width, p = padding, k = kernel_size, s = stride, op = output_padding + out_width = (width-1)*strides-2*padding+kernel_size+output_padding """ - def __init__(self, filters, kernel_size, strides=1, padding=0, output_padding=0, + def __init__(self, channels, kernel_size, strides=1, padding=0, output_padding=0, dilation=1, groups=1, layout='NCW', activation=None, use_bias=True, weight_initializer=None, bias_initializer=None, - in_filters=0, **kwargs): + in_channels=0, **kwargs): if isinstance(kernel_size, numeric_types): kernel_size = (kernel_size,) if isinstance(output_padding, numeric_types): @@ -433,8 +423,8 @@ def __init__(self, filters, kernel_size, strides=1, padding=0, output_padding=0, assert len(kernel_size) == 1, "kernel_size must be a number or a list of 1 ints" assert len(output_padding) == 1, "output_padding must be a number or a list of 1 ints" super(Conv1DTranspose, self).__init__( - filters, kernel_size, strides, padding, dilation, groups, layout, - in_filters, activation, use_bias, weight_initializer, + channels, kernel_size, strides, padding, dilation, groups, layout, + in_channels, activation, use_bias, weight_initializer, bias_initializer, op_name='Deconvolution', adj=output_padding, **kwargs) @@ -448,73 +438,69 @@ class Conv2DTranspose(_Conv): while maintaining a connectivity pattern that is compatible with said convolution. - When using this layer with NDArray API, - provide an `in_filters` argument - (integers, the number of input channels). + If `in_channels` is not specified, `Parameter` initialization will be + defered to the first time `forward` is called and `in_channels` will be + inferred from the shape of input data. Parameters ---------- - filters: Integer, the dimensionality of the output space - (i.e. the number output of filters in the convolution). - kernel_size: An integer or tuple/list of 2 integers, specifying the - dimensions of the convolution window. - strides: An integer or tuple/list of 2 integers, - specifying the strides of the convolution. - Specifying any stride value != 1 is incompatible with specifying - any `dilation_rate` value != 1. - padding: An integer or a tuple/list of 2 integers, + channels : int + The dimensionality of the output space, i.e. the number of output + channels (filters) in the convolution. + kernel_size :int or tuple/list of 3 int + Specifys the dimensions of the convolution window. + strides : int or tuple/list of 3 int, + Specify the strides of the convolution. + padding : int or a tuple/list of 3 int, If padding is non-zero, then the input is implicitly zero-padded on both sides for padding number of points - out_padding : An integer or a tuple/list of 2 integers, - Zero-padding added to one side of the output - dilation: An integer or tuple/list of 2 integers, specifying - the dilation rate to use for dilated convolution. - groups: int + dilation : int or tuple/list of 3 int + Specifys the dilation rate to use for dilated convolution. + groups : int controls the connections between inputs and outputs. At groups=1, all inputs are convolved to all outputs. At groups=2, the operation becomes equivalent to having two conv layers side by side, each seeing half the input channels, and producing half the output channels, and both subsequently concatenated. - layout: A string, - Can be 'NCHW', 'NHWC', etc. + layout : str, default 'NCHW' + Dimension ordering of data and weight. Can be 'NCHW', 'NHWC', etc. 'N', 'C', 'H', 'W' stands for batch, channel, height, and width - dimensions respectively. - in_filters: int, default 0 - The number of input channels to this layer. Only required when using - NDArray API. - activation: Activation function to use - see mx.sym.Activation. + dimensions respectively. Convolution is applied on the 'H' and + 'W' dimensions. + in_channels : int, default 0 + The number of input channels to this layer. If not specified, + initialization will be defered to the first time `forward` is called + and `in_channels` will be inferred from the shape of input data. + activation : str + Activation function to use. See :func:`mx.nd.Activation`. If you don't specify anything, no activation is applied (ie. "linear" activation: `a(x) = x`). - use_bias: Boolean, whether the layer uses a bias vector. - weight_initializer: Initializer for the `kernel` weights matrix - see Initializer. - bias_initializer: Initializer for the bias vector - see Initializer. + use_bias : bool + Whether the layer uses a bias vector. + weight_initializer : str or `Initializer` + Initializer for the `weight` weights matrix. + bias_initializer : str or `Initializer` + Initializer for the bias vector. Input Shape: This depends on the `layout` parameter. Input is 4D array of shape - (batch_size, in_channel(in_filters), height, width) if `layout` is `NCHW`. + (batch_size, in_channels, height, width) if `layout` is `NCHW`. Output Shape: This depends on the `layout` parameter. Output is 4D array of shape - (batch_size, out_channel(filters), out_height, out_width) if `layout` is `NCHW`. - out_height and out_width depends on other input parameters as well. - They are calculated as follows:: - - out_height = (h-1)*s-2*p+k+op - out_width = (w-1)*s-2*p+k+op + (batch_size, channels, out_height, out_width) if `layout` is `NCHW`. - where, + out_height and out_width are calculated as:: - h = height, w = width, p = padding, k = kernel_size, s = stride, op = output_padding + out_height = (height-1)*strides[0]-2*padding[0]+kernel_size[0]+output_padding[0] + out_width = (width-1)*strides[1]-2*padding[1]+kernel_size[1]+output_padding[1] """ - def __init__(self, filters, kernel_size, strides=(1, 1), padding=(0, 0), + def __init__(self, channels, kernel_size, strides=(1, 1), padding=(0, 0), output_padding=(0, 0), dilation=(1, 1), groups=1, layout='NCHW', activation=None, use_bias=True, weight_initializer=None, - bias_initializer=None, in_filters=0, **kwargs): + bias_initializer=None, in_channels=0, **kwargs): if isinstance(kernel_size, numeric_types): kernel_size = (kernel_size,)*2 if isinstance(output_padding, numeric_types): @@ -522,8 +508,8 @@ def __init__(self, filters, kernel_size, strides=(1, 1), padding=(0, 0), assert len(kernel_size) == 2, "kernel_size must be a number or a list of 2 ints" assert len(output_padding) == 2, "output_padding must be a number or a list of 2 ints" super(Conv2DTranspose, self).__init__( - filters, kernel_size, strides, padding, dilation, groups, layout, - in_filters, activation, use_bias, weight_initializer, + channels, kernel_size, strides, padding, dilation, groups, layout, + in_channels, activation, use_bias, weight_initializer, bias_initializer, op_name='Deconvolution', adj=output_padding, **kwargs) @@ -537,75 +523,69 @@ class Conv3DTranspose(_Conv): while maintaining a connectivity pattern that is compatible with said convolution. - When using this layer with NDArray API, - provide an `in_filters` argument - (integers, the number of input channels). + If `in_channels` is not specified, `Parameter` initialization will be + defered to the first time `forward` is called and `in_channels` will be + inferred from the shape of input data. Parameters ---------- - filters: Integer, the dimensionality of the output space - (i.e. the number output of filters in the convolution). - kernel_size: An integer or tuple/list of 3 integers, specifying the - dimensions of the convolution window. - strides: An integer or tuple/list of 3 integers, - specifying the strides of the convolution. - Specifying any stride value != 1 is incompatible with specifying - any `dilation_rate` value != 1. - padding: An integer or a tuple/list of 3 integers, + channels : int + The dimensionality of the output space, i.e. the number of output + channels (filters) in the convolution. + kernel_size :int or tuple/list of 3 int + Specifys the dimensions of the convolution window. + strides : int or tuple/list of 3 int, + Specify the strides of the convolution. + padding : int or a tuple/list of 3 int, If padding is non-zero, then the input is implicitly zero-padded on both sides for padding number of points - out_padding : An integer or a tuple/list of 2 integers, - Zero-padding added to one side of the output - dilation: An integer or tuple/list of 3 integers, specifying - the dilation rate to use for dilated convolution. - groups: int + dilation : int or tuple/list of 3 int + Specifys the dilation rate to use for dilated convolution. + groups : int controls the connections between inputs and outputs. At groups=1, all inputs are convolved to all outputs. At groups=2, the operation becomes equivalent to having two conv layers side by side, each seeing half the input channels, and producing half the output channels, and both subsequently concatenated. - layout: A string, - Can be 'NCDHW', 'NDHWC', etc. + layout : str, default 'NCDHW' + Dimension ordering of data and weight. Can be 'NCDHW', 'NDHWC', etc. 'N', 'C', 'H', 'W', 'D' stands for batch, channel, height, width and - depth dimensions respectively. - in_filters: int, default 0 - The number of input channels to this layer. Only required when using - NDArray API. - activation: Activation function to use - see mx.sym.Activation. + depth dimensions respectively. Convolution is applied on the 'D', + 'H', and 'W' dimensions. + in_channels : int, default 0 + The number of input channels to this layer. If not specified, + initialization will be defered to the first time `forward` is called + and `in_channels` will be inferred from the shape of input data. + activation : str + Activation function to use. See :func:`mx.nd.Activation`. If you don't specify anything, no activation is applied (ie. "linear" activation: `a(x) = x`). - use_bias: Boolean, whether the layer uses a bias vector. - weight_initializer: Initializer for the `kernel` weights matrix - see Initializer. - bias_initializer: Initializer for the bias vector - see Initializer. + use_bias : bool + Whether the layer uses a bias vector. + weight_initializer : str or `Initializer` + Initializer for the `weight` weights matrix. + bias_initializer : str or `Initializer` + Initializer for the bias vector. Input Shape: This depends on the `layout` parameter. Input is 5D array of shape - (batch_size, in_channel(in_filters), depth, height, width) if `layout` is `NCDHW`. + (batch_size, in_channels, depth, height, width) if `layout` is `NCDHW`. Output Shape: This depends on the `layout` parameter. Output is 5D array of shape - (batch_size, out_channel(filters), out_depth, out_height, out_width) if `layout` is `NCDHW`. - out_depth, out_height and out_width depends on other input parameters as well. - They are calculated as follows:: - - out_depth = (d-1)*s-2*p+k+op - out_height = (h-1)*s-2*p+k+op - out_width = (w-1)*s-2*p+k+op - - where, + (batch_size, channels, out_depth, out_height, out_width) if `layout` is `NCDHW`. + out_depth, out_height and out_width are calculated as:: - d = depth, h = height, w = width, p = padding, k = kernel_size, s = stride, - op = output_padding + out_depth = (depth-1)*strides[0]-2*padding[0]+kernel_size[0]+output_padding[0] + out_height = (height-1)*strides[1]-2*padding[1]+kernel_size[1]+output_padding[1] + out_width = (width-1)*strides[2]-2*padding[2]+kernel_size[2]+output_padding[2] """ - def __init__(self, filters, kernel_size, strides=(1, 1, 1), padding=(0, 0, 0), + def __init__(self, channels, kernel_size, strides=(1, 1, 1), padding=(0, 0, 0), output_padding=(0, 0, 0), dilation=(1, 1, 1), groups=1, layout='NCDHW', activation=None, use_bias=True, weight_initializer=None, - bias_initializer=None, in_filters=0, **kwargs): + bias_initializer=None, in_channels=0, **kwargs): if isinstance(kernel_size, numeric_types): kernel_size = (kernel_size,)*3 if isinstance(output_padding, numeric_types): @@ -613,15 +593,15 @@ def __init__(self, filters, kernel_size, strides=(1, 1, 1), padding=(0, 0, 0), assert len(kernel_size) == 3, "kernel_size must be a number or a list of 3 ints" assert len(output_padding) == 3, "output_padding must be a number or a list of 3 ints" super(Conv3DTranspose, self).__init__( - filters, kernel_size, strides, padding, dilation, groups, layout, - in_filters, activation, use_bias, weight_initializer, bias_initializer, + channels, kernel_size, strides, padding, dilation, groups, layout, + in_channels, activation, use_bias, weight_initializer, bias_initializer, op_name='Deconvolution', adj=output_padding, **kwargs) -class _Pooling(HybridLayer): - """Abstract class for different pooling layers. - """ - def __init__(self, pool_size, strides, padding, global_pool, pool_type, **kwargs): +class _Pooling(HybridBlock): + """Abstract class for different pooling layers.""" + def __init__(self, pool_size, strides, padding, ceil_mode, global_pool, + pool_type, **kwargs): super(_Pooling, self).__init__(**kwargs) if strides is None: strides = pool_size @@ -631,147 +611,157 @@ def __init__(self, pool_size, strides, padding, global_pool, pool_type, **kwargs padding = (padding,)*len(pool_size) self._kwargs = { 'kernel': pool_size, 'stride': strides, 'pad': padding, - 'pooling_convention': 'full', 'global_pool': global_pool, - 'pool_type': pool_type} + 'global_pool': global_pool, 'pool_type': pool_type, + 'pooling_convention': 'full' if ceil_mode else 'valid'} def hybrid_forward(self, F, x): return F.Pooling(x, **self._kwargs) class MaxPool1D(_Pooling): - """Max pooling operation for temporal data. + """Max pooling operation for one dimensional data. + Parameters ---------- - pool_size: Integer, size of the max pooling windows. - strides: Integer, or None. Factor by which to downscale. - E.g. 2 will halve the input. + pool_size: int + Size of the max pooling windows. + strides: int, or None + Factor by which to downscale. E.g. 2 will halve the input size. If None, it will default to `pool_size`. - padding: Integer, + padding: int If padding is non-zero, then the input is implicitly - zero-padded on both sides for padding number of points - layout: A string, - Can be 'NCW', 'NWC', etc. + zero-padded on both sides for padding number of points. + layout : str, default 'NCW' + Dimension ordering of data and weight. Can be 'NCW', 'NWC', etc. 'N', 'C', 'W' stands for batch, channel, and width (time) dimensions - respectively. padding is applied on W dimension. + respectively. Pooling is applied on the W dimension. + ceil_mode : bool, default False + When True, will use ceil instead of floor to compute the output shape. Input Shape: This depends on the `layout` parameter. Input is 3D array of shape - (batch_size, channel, width) if `layout` is `NCW`. + (batch_size, channels, width) if `layout` is `NCW`. Output Shape: This depends on the `layout` parameter. Output is 3D array of shape - (batch_size, channel, out_width) if `layout` is `NCW`. - out_width depends on other input parameters as well. It is calculated as follows:: + (batch_size, channels, out_width) if `layout` is `NCW`. - out_width = ceil((w+2*p-ps)/s+1) + out_width is calculated as:: - where, + out_width = floor((width+2*padding-pool_size)/strides)+1 - w = width, p = padding, ps = pool_size, s = stride + When ceil_mode is True, ceil will be used instead of floor in this + equation. """ - def __init__(self, pool_size=2, strides=None, padding=0, layout='NCW', **kwargs): + def __init__(self, pool_size=2, strides=None, padding=0, layout='NCW', + ceil_mode=False, **kwargs): assert layout == 'NCW', "Only supports NCW layout for now" if isinstance(pool_size, numeric_types): pool_size = (pool_size,) assert len(pool_size) == 1, "pool_size must be a number or a list of 1 ints" super(MaxPool1D, self).__init__( - pool_size, strides, padding, False, 'max', **kwargs) + pool_size, strides, padding, ceil_mode, False, 'max', **kwargs) class MaxPool2D(_Pooling): - """Max pooling operation for spatial data. + """Max pooling operation for two dimensional (spatial) data. + Parameters ---------- - pool_size: Integer or list/tuple of 2 Integers, - size of the max pooling windows. - strides: Integer, list/tuple of 2 Integers, or None. - Factor by which to downscale. - E.g. 2 will halve the input. + pool_size: int or list/tuple of 2 ints, + Size of the max pooling windows. + strides: int, list/tuple of 2 ints, or None. + Factor by which to downscale. E.g. 2 will halve the input size. If None, it will default to `pool_size`. - padding: Integer or list/tuple of 2 Integers, + padding: int or list/tuple of 2 ints, If padding is non-zero, then the input is implicitly - zero-padded on both sides for padding number of points - layout: A string, - Can be 'NCHW', 'NHWC', etc. + zero-padded on both sides for padding number of points. + layout : str, default 'NCHW' + Dimension ordering of data and weight. Can be 'NCHW', 'NHWC', etc. 'N', 'C', 'H', 'W' stands for batch, channel, height, and width dimensions respectively. padding is applied on 'H' and 'W' dimension. + ceil_mode : bool, default False + When True, will use ceil instead of floor to compute the output shape. Input Shape: This depends on the `layout` parameter. Input is 4D array of shape - (batch_size, channel, height, width) if `layout` is `NCHW`. + (batch_size, channels, height, width) if `layout` is `NCHW`. Output Shape: This depends on the `layout` parameter. Output is 4D array of shape - (batch_size, channel, out_height, out_width) if `layout` is `NCHW`. - out_height and out_width depends on other input parameters as well. - They are calculated as follows:: + (batch_size, channels, out_height, out_width) if `layout` is `NCHW`. - out_height = ceil((h+2*p-ps)/s+1) - out_width = ceil((w+2*p-ps)/s+1) + out_height and out_width are calculated as:: - where, + out_height = floor((height+2*padding[0]-pool_size[0])/strides[0])+1 + out_width = floor((width+2*padding[1]-pool_size[1])/strides[1])+1 - h = height, w = width, p = padding, ps = pool_size, s = stride + When ceil_mode is True, ceil will be used instead of floor in this + equation. """ - def __init__(self, pool_size=(2, 2), strides=None, padding=0, layout='NCHW', **kwargs): + def __init__(self, pool_size=(2, 2), strides=None, padding=0, layout='NCHW', + ceil_mode=False, **kwargs): assert layout == 'NCHW', "Only supports NCHW layout for now" if isinstance(pool_size, numeric_types): pool_size = (pool_size,)*2 assert len(pool_size) == 2, "pool_size must be a number or a list of 2 ints" super(MaxPool2D, self).__init__( - pool_size, strides, padding, False, 'max', **kwargs) + pool_size, strides, padding, ceil_mode, False, 'max', **kwargs) class MaxPool3D(_Pooling): """Max pooling operation for 3D data (spatial or spatio-temporal). + Parameters ---------- - pool_size: Integer or list/tuple of 3 Integers, - size of the max pooling windows. - strides: Integer, list/tuple of 3 Integers, or None. - Factor by which to downscale. - E.g. 2 will halve the input. + pool_size: int or list/tuple of 3 ints, + Size of the max pooling windows. + strides: int, list/tuple of 3 ints, or None. + Factor by which to downscale. E.g. 2 will halve the input size. If None, it will default to `pool_size`. - padding: Integer or list/tuple of 3 Integers, + padding: int or list/tuple of 3 ints, If padding is non-zero, then the input is implicitly - zero-padded on both sides for padding number of points - layout: A string, - Can be 'NCDHW', 'NDHWC', etc. + zero-padded on both sides for padding number of points. + layout : str, default 'NCDHW' + Dimension ordering of data and weight. Can be 'NCDHW', 'NDHWC', etc. 'N', 'C', 'H', 'W', 'D' stands for batch, channel, height, width and depth dimensions respectively. padding is applied on 'D', 'H' and 'W' dimension. + ceil_mode : bool, default False + When True, will use ceil instead of floor to compute the output shape. Input Shape: This depends on the `layout` parameter. Input is 5D array of shape - (batch_size, channel, depth, height, width) if `layout` is `NCDHW`. + (batch_size, channels, depth, height, width) if `layout` is `NCDHW`. Output Shape: This depends on the `layout` parameter. Output is 5D array of shape - (batch_size, channel, out_depth, out_height, out_width) if `layout` is `NCDHW`. - out_depth, out_height and out_width depends on other input parameters as well. - They are calculated as follows:: + (batch_size, channels, out_depth, out_height, out_width) if `layout` + is `NCDHW`. - out_depth = ceil((d+2*p-ps)/s+1) - out_height = ceil((h+2*p-ps)/s+1) - out_width = ceil((w+2*p-ps)/s+1) + out_depth, out_height and out_width are calculated as :: - where, + out_depth = floor((depth+2*padding[0]-pool_size[0])/strides[0])+1 + out_height = floor((height+2*padding[1]-pool_size[1])/strides[1])+1 + out_width = floor((width+2*padding[2]-pool_size[2])/strides[2])+1 - d = depth, h = height, w = width, p = padding, ps = pool_size, s = stride + When ceil_mode is True, ceil will be used instead of floor in this + equation. """ - def __init__(self, pool_size=(2, 2, 2), strides=None, padding=0, layout='NCDHW', **kwargs): + def __init__(self, pool_size=(2, 2, 2), strides=None, padding=0, + ceil_mode=False, layout='NCDHW', **kwargs): assert layout == 'NCDHW', "Only supports NCDHW layout for now" if isinstance(pool_size, numeric_types): pool_size = (pool_size,)*3 assert len(pool_size) == 3, "pool_size must be a number or a list of 3 ints" super(MaxPool3D, self).__init__( - pool_size, strides, padding, False, 'max', **kwargs) + pool_size, strides, padding, ceil_mode, False, 'max', **kwargs) class AvgPool1D(_Pooling): @@ -779,41 +769,45 @@ class AvgPool1D(_Pooling): Parameters ---------- - pool_size: Integer, size of the max pooling windows. - strides: Integer, or None. Factor by which to downscale. - E.g. 2 will halve the input. + pool_size: int + Size of the max pooling windows. + strides: int, or None + Factor by which to downscale. E.g. 2 will halve the input size. If None, it will default to `pool_size`. - padding: Integer, + padding: int If padding is non-zero, then the input is implicitly - zero-padded on both sides for padding number of points - layout: A string, - Can be 'NCW', 'NWC', etc. + zero-padded on both sides for padding number of points. + layout : str, default 'NCW' + Dimension ordering of data and weight. Can be 'NCW', 'NWC', etc. 'N', 'C', 'W' stands for batch, channel, and width (time) dimensions - respectively. padding is applied on W dimension. + respectively. padding is applied on 'W' dimension. + ceil_mode : bool, default False + When True, will use ceil instead of floor to compute the output shape. Input Shape: This depends on the `layout` parameter. Input is 3D array of shape - (batch_size, channel, width) if `layout` is `NCW`. + (batch_size, channels, width) if `layout` is `NCW`. Output Shape: This depends on the `layout` parameter. Output is 3D array of shape - (batch_size, channel, out_width) if `layout` is `NCW`. - out_width depends on other input parameters as well. It is calculated as follows:: + (batch_size, channels, out_width) if `layout` is `NCW`. - out_width = ceil((w+2*p-ps)/s+1) + out_width is calculated as:: - where, + out_width = floor((width+2*padding-pool_size)/strides)+1 - w = width, p = padding, ps = pool_size, s = stride + When ceil_mode is True, ceil will be used instead of floor in this + equation. """ - def __init__(self, pool_size=2, strides=None, padding=0, layout='NCW', **kwargs): + def __init__(self, pool_size=2, strides=None, padding=0, layout='NCW', + ceil_mode=False, **kwargs): assert layout == 'NCW', "Only supports NCW layout for now" if isinstance(pool_size, numeric_types): pool_size = (pool_size,) assert len(pool_size) == 1, "pool_size must be a number or a list of 1 ints" super(AvgPool1D, self).__init__( - pool_size, strides, padding, False, 'avg', **kwargs) + pool_size, strides, padding, ceil_mode, False, 'avg', **kwargs) class AvgPool2D(_Pooling): @@ -821,45 +815,46 @@ class AvgPool2D(_Pooling): Parameters ---------- - pool_size: Integer or list/tuple of 2 Integers, - size of the max pooling windows. - strides: Integer, list/tuple of 2 Integers, or None. - Factor by which to downscale. - E.g. 2 will halve the input. + pool_size: int or list/tuple of 2 ints, + Size of the max pooling windows. + strides: int, list/tuple of 2 ints, or None. + Factor by which to downscale. E.g. 2 will halve the input size. If None, it will default to `pool_size`. - padding: Integer or list/tuple of 2 Integers, + padding: int or list/tuple of 2 ints, If padding is non-zero, then the input is implicitly - zero-padded on both sides for padding number of points - layout: A string, - Can be 'NCHW', 'NHWC', etc. + zero-padded on both sides for padding number of points. + layout : str, default 'NCHW' + Dimension ordering of data and weight. Can be 'NCHW', 'NHWC', etc. 'N', 'C', 'H', 'W' stands for batch, channel, height, and width dimensions respectively. padding is applied on 'H' and 'W' dimension. + ceil_mode : bool, default False + When True, will use ceil instead of floor to compute the output shape. Input Shape: This depends on the `layout` parameter. Input is 4D array of shape - (batch_size, channel, height, width) if `layout` is `NCHW`. + (batch_size, channels, height, width) if `layout` is `NCHW`. Output Shape: This depends on the `layout` parameter. Output is 4D array of shape - (batch_size, channel, out_height, out_width) if `layout` is `NCHW`. - out_height and out_width depends on other input parameters as well. - They are calculated as follows:: + (batch_size, channels, out_height, out_width) if `layout` is `NCHW`. - out_height = ceil((h+2*p-ps)/s+1) - out_width = ceil((w+2*p-ps)/s+1) + out_height and out_width are calculated as:: - where, + out_height = floor((height+2*padding[0]-pool_size[0])/strides[0])+1 + out_width = floor((width+2*padding[1]-pool_size[1])/strides[1])+1 - h = height, w = width, p = padding, ps = pool_size, s = stride + When ceil_mode is True, ceil will be used instead of floor in this + equation. """ - def __init__(self, pool_size=(2, 2), strides=None, padding=0, layout='NCHW', **kwargs): + def __init__(self, pool_size=(2, 2), strides=None, padding=0, + ceil_mode=False, layout='NCHW', **kwargs): assert layout == 'NCHW', "Only supports NCHW layout for now" if isinstance(pool_size, numeric_types): pool_size = (pool_size,)*2 assert len(pool_size) == 2, "pool_size must be a number or a list of 2 ints" super(AvgPool2D, self).__init__( - pool_size, strides, padding, False, 'avg', **kwargs) + pool_size, strides, padding, ceil_mode, False, 'avg', **kwargs) class AvgPool3D(_Pooling): @@ -867,97 +862,93 @@ class AvgPool3D(_Pooling): Parameters ---------- - pool_size: Integer or list/tuple of 3 Integers, - size of the max pooling windows. - strides: Integer, list/tuple of 3 Integers, or None. - Factor by which to downscale. - E.g. 2 will halve the input. + pool_size: int or list/tuple of 3 ints, + Size of the max pooling windows. + strides: int, list/tuple of 3 ints, or None. + Factor by which to downscale. E.g. 2 will halve the input size. If None, it will default to `pool_size`. - padding: Integer or list/tuple of 3 Integers, + padding: int or list/tuple of 3 ints, If padding is non-zero, then the input is implicitly - zero-padded on both sides for padding number of points - layout: A string, - Can be 'NCDHW', 'NDHWC', etc. + zero-padded on both sides for padding number of points. + layout : str, default 'NCDHW' + Dimension ordering of data and weight. Can be 'NCDHW', 'NDHWC', etc. 'N', 'C', 'H', 'W', 'D' stands for batch, channel, height, width and depth dimensions respectively. padding is applied on 'D', 'H' and 'W' dimension. + ceil_mode : bool, default False + When True, will use ceil instead of floor to compute the output shape. Input Shape: This depends on the `layout` parameter. Input is 5D array of shape - (batch_size, channel, depth, height, width) if `layout` is `NCDHW`. + (batch_size, channels, depth, height, width) if `layout` is `NCDHW`. Output Shape: This depends on the `layout` parameter. Output is 5D array of shape - (batch_size, channel, out_depth, out_height, out_width) if `layout` is `NCDHW`. - out_depth, out_height and out_width depends on other input parameters as well. - They are calculated as follows:: + (batch_size, channels, out_depth, out_height, out_width) if `layout` + is `NCDHW`. - out_depth = ceil((d+2*p-ps)/s+1) - out_height = ceil((h+2*p-ps)/s+1) - out_width = ceil((w+2*p-ps)/s+1) + out_depth, out_height and out_width are calculated as :: - where, + out_depth = floor((depth+2*padding[0]-pool_size[0])/strides[0])+1 + out_height = floor((height+2*padding[1]-pool_size[1])/strides[1])+1 + out_width = floor((width+2*padding[2]-pool_size[2])/strides[2])+1 - d = depth, h = height, w = width, p = padding, ps = pool_size, s = stride + When ceil_mode is True, ceil will be used instead of floor in this + equation. """ - def __init__(self, pool_size=(2, 2, 2), strides=None, padding=0, layout='NCDHW', **kwargs): + def __init__(self, pool_size=(2, 2, 2), strides=None, padding=0, + ceil_mode=False, layout='NCDHW', **kwargs): assert layout == 'NCDHW', "Only supports NCDHW layout for now" if isinstance(pool_size, numeric_types): pool_size = (pool_size,)*3 assert len(pool_size) == 3, "pool_size must be a number or a list of 3 ints" super(AvgPool3D, self).__init__( - pool_size, strides, padding, False, 'avg', **kwargs) + pool_size, strides, padding, ceil_mode, False, 'avg', **kwargs) class GlobalMaxPool1D(_Pooling): - """Global max pooling operation for temporal data. - """ + """Global max pooling operation for temporal data.""" def __init__(self, layout='NCW', **kwargs): assert layout == 'NCW', "Only supports NCW layout for now" super(GlobalMaxPool1D, self).__init__( - (1,), None, 0, True, 'max', **kwargs) + (1,), None, 0, True, True, 'max', **kwargs) class GlobalMaxPool2D(_Pooling): - """Global max pooling operation for spatial data. - """ + """Global max pooling operation for spatial data.""" def __init__(self, layout='NCHW', **kwargs): assert layout == 'NCHW', "Only supports NCW layout for now" super(GlobalMaxPool2D, self).__init__( - (1, 1), None, 0, True, 'max', **kwargs) + (1, 1), None, 0, True, True, 'max', **kwargs) class GlobalMaxPool3D(_Pooling): - """Global max pooling operation for 3D data. - """ + """Global max pooling operation for 3D data.""" def __init__(self, layout='NCDHW', **kwargs): assert layout == 'NCDHW', "Only supports NCW layout for now" super(GlobalMaxPool3D, self).__init__( - (1, 1, 1), None, 0, True, 'max', **kwargs) + (1, 1, 1), None, 0, True, True, 'max', **kwargs) class GlobalAvgPool1D(_Pooling): - """Global average pooling operation for temporal data. - """ + """Global average pooling operation for temporal data.""" def __init__(self, layout='NCW', **kwargs): assert layout == 'NCW', "Only supports NCW layout for now" super(GlobalAvgPool1D, self).__init__( - (1,), None, 0, True, 'avg', **kwargs) + (1,), None, 0, True, True, 'avg', **kwargs) class GlobalAvgPool2D(_Pooling): - """Global average pooling operation for spatial data. - """ + """Global average pooling operation for spatial data.""" def __init__(self, layout='NCHW', **kwargs): assert layout == 'NCHW', "Only supports NCW layout for now" super(GlobalAvgPool2D, self).__init__( - (1, 1), None, 0, True, 'avg', **kwargs) + (1, 1), None, 0, True, True, 'avg', **kwargs) class GlobalAvgPool3D(_Pooling): - """Global max pooling operation for 3D data. - """ + """Global max pooling operation for 3D data.""" def __init__(self, layout='NCDHW', **kwargs): assert layout == 'NCDHW', "Only supports NCW layout for now" super(GlobalAvgPool3D, self).__init__( - (1, 1, 1), None, 0, True, 'avg', **kwargs) + (1, 1, 1), None, 0, True, True, 'avg', **kwargs) diff --git a/python/mxnet/foo/nn/layer.py b/python/mxnet/foo/nn/layer.py deleted file mode 100644 index 48e13570a202..000000000000 --- a/python/mxnet/foo/nn/layer.py +++ /dev/null @@ -1,615 +0,0 @@ -# coding: utf-8 -# pylint: disable= arguments-differ -"""Neural network layers.""" - -from ... import symbol, ndarray -from ...symbol import Symbol -from ...ndarray import NDArray -from ... import name as _name -from ..parameter import Parameter, ParameterDict, DeferredInitializationError - - -class _LayerScope(object): - """Scope for collecting sub-layers.""" - _current = None - - def __init__(self, layer): - self._layer = layer - self._counter = {} - self._old_scope = None - - @staticmethod - def create(prefix, params, hint): - """Create prefix and params for new layer.""" - current = _LayerScope._current - if current is None: - if prefix is None: - prefix = _name.NameManager.current.get(None, hint) + '_' - if params is None: - params = ParameterDict(prefix) - else: - params = ParameterDict(params.prefix, params) - return prefix, params - - if prefix is None: - count = current._counter.get(hint, 0) - prefix = '%s%d_'%(hint, count) - current._counter[hint] = count + 1 - if params is None: - parent = current._layer.params - params = ParameterDict(parent.prefix+prefix, parent._shared) - else: - params = ParameterDict(params.prefix, params) - return current._layer.prefix+prefix, params - - def __enter__(self): - self._old_scope = _LayerScope._current - _LayerScope._current = self - return self - - def __exit__(self, ptype, value, trace): - _LayerScope._current = self._old_scope - - -def _flatten(args): - if isinstance(args, NDArray): - return [args], int(0) - if isinstance(args, Symbol): - length = len(args.list_outputs()) - length = length if length > 1 else 0 - return [args], int(length) - - assert isinstance(args, (list, tuple)), \ - "HybridLayer input must be (nested) list of Symbol or NDArray, " \ - "but got %s of type %s"%(str(args), str(type(args))) - flat = [] - fmts = [] - for i in args: - arg, fmt = _flatten(i) - flat.extend(arg) - fmts.append(fmt) - return flat, fmts - - -def _regroup(args, fmt): - if isinstance(fmt, int): - if fmt == 0: - return args[0], args[1:] - return args[:fmt], args[fmt:] - - assert isinstance(args, (list, tuple)), \ - "HybridLayer output must be (nested) list of Symbol or NDArray, " \ - "but got %s of type %s"%(str(args), str(type(args))) - ret = [] - for i in fmt: - res, args = _regroup(args, i) - ret.append(res) - return ret, args - - -class Layer(object): - """Base class for all neural network layers and models. - - Your models should subclass this class. - - Layers can also contain other Layers, allowing you to nest them in a tree - structure. You can assign sublayers as regular attributes:: - from mxnet import nn - class Net(nn.Layer): - def __init__(self, **kwargs): - super(Net, self).__init__(**kwargs) - with self.name_scope(): - self.dense0 = nn.Dense(20) - self.dense1 = nn.Dense(20) - - def forward(self, x): - x = self.dense0(x) - return self.dense1(x) - - Sublayers assigned this way will be registered and will have their status changed - too when you call .train() etc. - - Parameters - ---------- - prefix : str - Prefix acts like a name space. It will be prepended to the name of all Symbols and - Parameters created by this layer. Prefix should be unique within one network - to prevent name collisions. - params : ParameterDict or None - ParameterDict for sharing weights with the new Layer. For example, - if you want `dense2` to share `dense1`'s weights, you can do:: - dense1 = nn.Dense(20, in_units=10, prefix='dense1_') - dense2 = nn.Dense(20, in_units=10, prefix='dense2_', - params=dense1.all_params()) - - Layer supports forwarding with both `Symbol` and `NDArray`.""" - def __init__(self, prefix=None, params=None): - self._prefix, self._params = _LayerScope.create(prefix, params, self._alias()) - self._scope = _LayerScope(self) - self._children = [] - - def __setattr__(self, name, value): - """Registers parameters.""" - super(Layer, self).__setattr__(name, value) - if isinstance(value, Layer): - self.register_child(value) - - def _alias(self): - return self.__class__.__name__.lower() - - @property - def params(self): - """Returns this Layer's parameter dictionary (does not include its - children's parameters).""" - return self._params - - def all_params(self): - """Returns a ParameterDict containing this Layer and all of its children's - Parameters.""" - ret = ParameterDict(self._params.prefix) - ret.update(self.params) - for cld in self._children: - ret.update(cld.all_params()) - return ret - - @property - def prefix(self): - """Prefix of this Layer.""" - return self._prefix - - @property - def name(self): - if self.prefix.endswith('_'): - return self.prefix[:-1] - return self.prefix - - def name_scope(self): - """Returns a name space object managing sublayer and parameter - names. Should be used by `with` statement - """ - return self._scope - - def register_child(self, layer): - """Register layer as sublayer of self. Layers assigned to - self as attributes will be registered automatically.""" - self._children.append(layer) - - def hybridize(self, active=True): - """Activate HybridLayers recursively. Has no effect on - non-hybrid children.""" - for cld in self._children: - cld.hybridize(active) - - def __call__(self, *args): - """Call forward.""" - return self.forward(*args) - - def forward(self, *args): - """Override to implement forward computation using NDArray. - - Parameters - ---------- - *args : list of NDArray - Input tensors. - """ - # pylint: disable= invalid-name - raise NotImplementedError - - -class HybridLayer(Layer): - """HybridLayer supports forwarding with both Symbol and NDArray. - - Forward computation in HybridLayer must be static to work with Symbols, - i.e. you cannot call `.asnumpy()`, `.shape`, `.dtype`, etc on inputs. - When forwarding after `hybridize()` is called, HybridLayer will - create a graph representing the forward computation and cache it. - On subsequent forward the cached graph will be used instead of calling - `hybrid_forward`. - """ - def __init__(self, prefix=None, params=None): - super(HybridLayer, self).__init__(prefix=prefix, params=params) - self._reg_params = {} - self._cached_graph = () - self._cached_op = None - self._cached_params = None - self._out_format = None - self._in_format = None - self._active = False - - def __setattr__(self, name, value): - """Registers parameters.""" - super(HybridLayer, self).__setattr__(name, value) - if isinstance(value, Parameter): - assert name not in self._reg_params or \ - not isinstance(self._reg_params[name], Parameter), \ - "Overriding Parameter attribute %s is not allowed. " \ - "Please pass in Parameters by specifying `params` at " \ - "Layer construction instead." - self._reg_params[name] = value - - def register_child(self, layer): - if not isinstance(layer, HybridLayer): - if isinstance(layer, Sequential): - raise ValueError( - "Children of HybridLayer must also be HybridLayer. " \ - "Please use HSequential instead of Sequantial.") - raise ValueError( - "Children of HybridLayer must also be HybridLayer, " \ - "but %s has type %s."%(str(layer), str(type(layer)))) - super(HybridLayer, self).register_child(layer) - - def hybridize(self, active=True): - super(HybridLayer, self).hybridize(active) - self._active = active - - def _get_graph(self, *args): - if self._cached_graph: - return self._cached_graph - - args, self._in_format = _flatten(args) - syms = [symbol.var(str(i)) for i in range(len(args))] - sym_args = _regroup(syms, self._in_format)[0] - - params = {i: j.var() for i, j in self._reg_params.items()} - out = self.hybrid_forward(symbol, *sym_args, **params) # pylint: disable=no-value-for-parameter - out, self._out_format = _flatten(out) - - self._cached_graph = syms, symbol.Group(out) - return self._cached_graph - - def infer_shape(self, *args): - """Infer shape of Parameters from inputs.""" - syms, out = self._get_graph(*args) - args, _, = _flatten(args) - arg_shapes, _, aux_shapes = out.infer_shape( - **{i.name: j.shape for i, j in zip(syms, args)}) - sdict = {i: j for i, j in zip(out.list_arguments(), arg_shapes)} - sdict.update({name : shape for name, shape in \ - zip(out.list_auxiliary_states(), aux_shapes)}) - for i in self.all_params().values(): - i.shape = sdict[i.name] - - def _build_cache(self, *args): - self.infer_shape(*args) - for i in self.all_params().values(): - i._finish_deferred_init() - - _, out = self._get_graph(*args) - self._cached_op = ndarray.CachedOp(out) - params = dict(self.all_params().items()) - self._cached_params = [params.get(name, None) for name in out.list_inputs()] - self._in_idx = [(i, int(name)) for i, name in enumerate(out.list_inputs()) - if name not in params] - - def _call_cached_op(self, *args): - args, fmt = _flatten(args) - assert fmt == self._in_format, "Invalid input format" - cargs = [i.data() if i else None for i in self._cached_params] - for i, j in self._in_idx: - cargs[i] = args[j] - out = self._cached_op(*cargs) - if isinstance(out, NDArray): - out = [out] - return _regroup(out, self._out_format)[0] - - def forward(self, x, *args): - """Defines the forward computation. Arguments can be either - NDArray or Symbol.""" - if isinstance(x, NDArray): - if self._active and self._cached_op is None: - self._build_cache(x, *args) - - with x.context as ctx: - if self._active: - return self._call_cached_op(x, *args) - try: - params = {i: j.data(ctx) for i, j in self._reg_params.items()} - except DeferredInitializationError: - self.infer_shape(x, *args) - for i in self.all_params().values(): - i._finish_deferred_init() - params = {i: j.data(ctx) for i, j in self._reg_params.items()} - return self.hybrid_forward(ndarray, x, *args, **params) - else: - assert isinstance(x, Symbol), \ - "Layer requires the first argument to forward be either " \ - "Symbol or NDArray, but got %s"%type(x) - params = {i: j.var() for i, j in self._reg_params.items()} - return self.hybrid_forward(symbol, x, *args, **params) - - def hybrid_forward(self, F, x, *args, **kwargs): - """Override to construct symbolic graph for this Layer. - - Parameters - ---------- - x : Symbol - The first input Symbol. - *args : list of Symbol - Additional input Symbols. - """ - # pylint: disable= invalid-name - raise NotImplementedError - - -class Sequential(Layer): - """Stack Layers sequentially. - - Example - ------- - >>> net = nn.Sequential() - >>> with net.name_scope(): - ... net.add(Dense(10, activation='relu')) - ... net.add(Dense(20)) - """ - def __init__(self, prefix=None, params=None): - super(Sequential, self).__init__(prefix=prefix, params=params) - - def add(self, layer): - """Add layer on top of the stack.""" - self.register_child(layer) - - def forward(self, x): - for layer in self._children: - x = layer(x) - return x - - -class HSequential(HybridLayer): - """Stack HybridLayers sequentially. - - Example - ------- - >>> net = nn.HSequential() - >>> with net.name_scope(): - ... net.add(Dense(10, activation='relu')) - ... net.add(Dense(20)) - """ - def __init__(self, prefix=None, params=None): - super(HSequential, self).__init__(prefix=prefix, params=params) - - def add(self, layer): - """Add layer on top of the stack.""" - self.register_child(layer) - - def hybrid_forward(self, F, x): - for layer in self._children: - x = layer(x) - return x - - -class Dense(HybridLayer): - """Just your regular densely-connected NN layer. - - `Dense` implements the operation: - `output = activation(dot(input, kernel) + bias)` - where `activation` is the element-wise activation function - passed as the `activation` argument, `kernel` is a weights matrix - created by the layer, and `bias` is a bias vector created by the layer - (only applicable if `use_bias` is `True`). - - Note: the input must be a tensor with rank 2. Use flatten to convert it - to rank 2 manually if necessary. - - Parameters - ---------- - units: Positive integer, dimensionality of the output space. - activation: Activation function to use - (see help on Activation operator). - If you don't specify anything, no activation is applied - (ie. "linear" activation: `a(x) = x`). - use_bias: Boolean, whether the layer uses a bias vector. - weight_initializer: Initializer for the `kernel` weights matrix - (see mxnet.initializer). - bias_initializer: Initializer for the bias vector - (see mxnet.initializer). - in_units : int - Size of input data. No need to specify for `Symbol` API. But must be - specified for every Dense layer if you want to use `NDArray` API. - prefix : str or None - See document of Layer. - params : ParameterDict or None - See document of Layer. - - - Input shape: - a 2D input with shape `(batch_size, in_units)`. - - Output shape: - the output would have shape `(batch_size, units)`. - """ - def __init__(self, units, activation=None, use_bias=True, - weight_initializer=None, bias_initializer=None, - in_units=0, **kwargs): - super(Dense, self).__init__(**kwargs) - with self.name_scope(): - self._units = units - self.weight = self.params.get('weight', shape=(units, in_units), - init=weight_initializer) - if use_bias: - self.bias = self.params.get('bias', shape=(units,), - init=bias_initializer) - else: - self.bias = None - if activation is not None: - self.act = Activation(activation) - else: - self.act = None - - def hybrid_forward(self, F, x, weight, bias=None): - if bias is None: - act = F.FullyConnected(x, weight, no_bias=True, num_hidden=self._units) - else: - act = F.FullyConnected(x, weight, bias, num_hidden=self._units) - if self.act is not None: - act = self.act(act) - return act - - -class Activation(HybridLayer): - """Applies an activation function to input. Refer - `mxnet.ndarray.Activation `_ - to learn more. - - Parameters - ---------- - activation: name of activation function to use - See: help on Activation operator - - - Input shape: - Arbitrary. - - Output shape: - Same shape as input. - """ - def __init__(self, activation, **kwargs): - self._act_type = activation - super(Activation, self).__init__(**kwargs) - - def _alias(self): - return self._act_type - - def hybrid_forward(self, F, x): - return F.Activation(x, act_type=self._act_type) - - -class Dropout(HybridLayer): - """Applies Dropout to the input. - - Dropout consists in randomly setting - a fraction `rate` of input units to 0 at each update during training time, - which helps prevent overfitting. Refer - `mxnet.ndarray.Dropout `_ - to learn more. - - Parameters - ---------- - rate: float between 0 and 1. Fraction of the input units to drop. - - References - ---------- - `Dropout: A Simple Way to Prevent Neural Networks from Overfitting - `_ - """ - def __init__(self, rate, **kwargs): - super(Dropout, self).__init__(**kwargs) - self._rate = rate - - def hybrid_forward(self, F, x): - return F.Dropout(x, p=self._rate) - - -class BatchNorm(HybridLayer): - """Batch normalization layer (Ioffe and Szegedy, 2014). - Normalize the activations of the previous layer at each batch, - i.e. applies a transformation that maintains the mean activation - close to 0 and the activation standard deviation close to 1. Refer - `mxnet.ndarray.BatchNorm `_ - to learn more. - - Parameters - ---------- - axis: Integer, the axis that should be normalized - (typically the features axis). - For instance, after a `Conv2D` layer with - `data_format="channels_first"`, - set `axis=1` in `BatchNormalization`. - momentum: Momentum for the moving average. - epsilon: Small float added to variance to avoid dividing by zero. - center: If True, add offset of `beta` to normalized tensor. - If False, `beta` is ignored. - scale: If True, multiply by `gamma`. - If False, `gamma` is not used. - When the next layer is linear (also e.g. `nn.relu`), - this can be disabled since the scaling - will be done by the next layer. - beta_initializer: Initializer for the beta weight. - gamma_initializer: Initializer for the gamma weight. - moving_mean_initializer: Initializer for the moving mean. - moving_variance_initializer: Initializer for the moving variance. - """ - def __init__(self, axis=1, momentum=0.9, epsilon=1e-3, center=True, scale=True, - beta_initializer='zeros', gamma_initializer='ones', - running_mean_initializer='zeros', running_variance_initializer='ones', - num_features=0, **kwargs): - super(BatchNorm, self).__init__(**kwargs) - self._kwargs = {'axis': axis, 'eps': epsilon, 'momentum': momentum, - 'fix_gamma': not center} - - self.gamma = self.params.get('gamma', grad_req='write' if scale else 'null', - shape=(num_features,), init=gamma_initializer) - self.beta = self.params.get('beta', grad_req='write' if center else 'null', - shape=(num_features,), init=beta_initializer) - self.running_mean = self.params.get('running_mean', grad_req='null', - shape=(num_features,), - init=running_mean_initializer) - self.running_var = self.params.get('running_var', grad_req='null', - shape=(num_features,), - init=running_variance_initializer) - - def hybrid_forward(self, F, x, gamma, beta, running_mean, running_var): - return F.BatchNorm(x, gamma, beta, running_mean, running_var, **self._kwargs) - - -class LeakyReLU(HybridLayer): - """Leaky version of a Rectified Linear Unit. - - It allows a small gradient when the unit is not active:: - - `f(x) = alpha * x for x < 0`, - `f(x) = x for x >= 0`. - - Refer - `mxnet.ndarray.LeakyReLU `_ - to learn more. - - Parameters - ---------- - alpha: float - Negative slope coefficient. Must be >= 0. - """ - def __init__(self, alpha, **kwargs): - super(LeakyReLU, self).__init__(**kwargs) - self._alpha = alpha - - def hybrid_forward(self, F, x): - return F.LeakyReLU(x, act_type='leaky', slope=self._alpha) - - -class Embedding(HybridLayer): - """Turns non-negative integers (indexes/tokens) into dense - vectors of fixed size. - eg. [[4], [20]] -> [[0.25, 0.1], [0.6, -0.2]] - - Refer - `mxnet.ndarray.Embedding `_ - to learn more. - - Parameters - ---------- - input_dim : int - Size of the vocabulary, i.e. maximum integer index + 1. - output_dim : int - Dimension of the dense embedding. - dtype : str or np.dtype, default 'float32' - Data type of output embeddings. - weight_initializer : Initializer - Initializer for the `embeddings` matrix - - - Input shape: - 2D tensor with shape: `(batch_size, sequence_length)`. - - Output shape: - 3D tensor with shape: `(batch_size, sequence_length, output_dim)`. - """ - def __init__(self, input_dim, output_dim, dtype='float32', - weight_initializer=None, **kwargs): - super(Embedding, self).__init__(**kwargs) - self._kwargs = {'input_dim': input_dim, 'output_dim': output_dim, - 'dtype': dtype} - self.weight = self.params.get('weight', shape=(input_dim, output_dim), - init=weight_initializer) - - def hybrid_forward(self, F, x, weight): - return F.Embedding(x, weight, **self._kwargs) diff --git a/python/mxnet/foo/parameter.py b/python/mxnet/foo/parameter.py index a44d462a136d..083db134a65b 100644 --- a/python/mxnet/foo/parameter.py +++ b/python/mxnet/foo/parameter.py @@ -19,7 +19,7 @@ class DeferredInitializationError(MXNetError): pass class Parameter(object): - """A Container holding parameters (weights) of layers. + """A Container holding parameters (weights) of `Block`s. `Parameter` can be used with both `Symbol` and `NDArray` API. For `Symbol` API, `Parameter.var()` will return a `Symbol` representing this parameter. It @@ -109,7 +109,7 @@ def initialize(self, init=None, ctx=None, default_init=initializer.Xavier(), return raise ValueError("Cannot initialize Parameter %s because it has " \ "invalid shape: %s. Please specify in_units, " \ - "in_filters, num_features etc for Layers or " \ + "in_channels, etc for `Block`s or " \ "set allow_deferring to True to defer initialization " \ "to first forward pass."%(self.name, str(self.shape))) @@ -155,7 +155,7 @@ def _finish_deferred_init(self): assert self.shape is not None and np.prod(self.shape) > 0, \ "Cannot initialize Parameter %s because it has " \ "invalid shape: %s. Please specify in_units, " \ - "in_filters, num_features etc for Layers."%( + "in_channels, etc for `Block`s."%( self.name, str(self.shape)) with autograd.pause(): @@ -204,9 +204,9 @@ def _check_initialized(self, ctx=None): raise RuntimeError( "Parameter %s has not been initialized. Note that " \ "you should initialize parameters and create Trainer " \ - "with Layer.all_params() instead of Layer.params " \ - "because the later does not include parameters of " \ - "nested child layers "%(self.name)) + "with Block.collect_params() instead of Block.params " \ + "because the later does not include Parameters of " \ + "nested child Blocks"%(self.name)) def data(self, ctx=None): """Returns a copy of this parameter on one context. Must have been @@ -292,7 +292,7 @@ class ParameterDict(object): shared : ParameterDict or None If not None, when this dict's get method creates a new parameter, will first try to retrieve it from `shared` dict. Usually used for sharing - parameters with another layer. + parameters with another Block. """ def __init__(self, prefix='', shared=None): self._prefix = prefix diff --git a/python/mxnet/foo/rnn/rnn_cell.py b/python/mxnet/foo/rnn/rnn_cell.py index 46227b56d967..986d3cce363a 100644 --- a/python/mxnet/foo/rnn/rnn_cell.py +++ b/python/mxnet/foo/rnn/rnn_cell.py @@ -7,7 +7,7 @@ from ... import symbol, ndarray from ...base import string_types, numeric_types -from ..nn import Layer, HybridLayer +from ..block import Block, HybridBlock from .. import tensor_types @@ -69,13 +69,13 @@ def _format_sequence(length, inputs, layout, merge, in_layout=None): return inputs, axis, F, batch_size -class RecurrentCell(Layer): +class RecurrentCell(Block): """Abstract base class for RNN cells Parameters ---------- prefix : str, optional - Prefix for names of layers + Prefix for names of `Block`s (this prefix is also used for names of weights if `params` is None i.e. if `params` are being created and not reused) params : Parameter or None, optional @@ -237,7 +237,7 @@ def forward(self, inputs, states): return super(RecurrentCell, self).forward(inputs, states) -class HRecurrentCell(RecurrentCell, HybridLayer): +class HRecurrentCell(RecurrentCell, HybridBlock): """HRecurrentCell supports both Symbol and NDArray forwarding.""" def __init__(self, prefix=None, params=None): super(HRecurrentCell, self).__init__(prefix=prefix, params=params) @@ -266,7 +266,7 @@ class RNNCell(HRecurrentCell): h2h_bias_initializer : str or Initializer Initializer for the bias vector. prefix : str, default 'rnn_' - prefix for name of layers + prefix for name of `Block`s (and name of weight if params is None) params : Parameter or None container for weight sharing between cells. @@ -330,7 +330,7 @@ class LSTMCell(HRecurrentCell): h2h_bias_initializer : str or Initializer Initializer for the bias vector. prefix : str, default 'lstm_' - prefix for name of layers + prefix for name of `Block`s (and name of weight if params is None) params : Parameter or None container for weight sharing between cells. @@ -408,7 +408,7 @@ class GRUCell(HRecurrentCell): h2h_bias_initializer : str or Initializer Initializer for the bias vector. prefix : str, default 'gru_' - prefix for name of layers + prefix for name of `Block`s (and name of weight if params is None) params : Parameter or None container for weight sharing between cells. diff --git a/python/mxnet/foo/rnn/rnn_layer.py b/python/mxnet/foo/rnn/rnn_layer.py index ee0693f53fe3..166bbc4bb63b 100644 --- a/python/mxnet/foo/rnn/rnn_layer.py +++ b/python/mxnet/foo/rnn/rnn_layer.py @@ -6,11 +6,11 @@ from __future__ import print_function from ... import ndarray -from ..nn import Layer +from ..nn import Block from . import rnn_cell -class _RNNLayer(Layer): +class _RNNLayer(Block): """implementation of recurrent layers.""" def __init__(self, hidden_size, num_layers, layout, dropout, bidirectional, input_size, @@ -222,9 +222,9 @@ class RNN(_RNNLayer): The number of expected features in the input x. If not specified, it will be inferred from input. prefix : str or None - Prefix of this layer. + Prefix of this `Block`. params : ParameterDict or None - Shared Parameters for this Layer. + Shared Parameters for this `Block`. Examples -------- @@ -301,9 +301,9 @@ class LSTM(_RNNLayer): The number of expected features in the input x. If not specified, it will be inferred from input. prefix : str or None - Prefix of this layer. + Prefix of this `Block`. params : ParameterDict or None - Shared Parameters for this Layer. + Shared Parameters for this `Block`. Examples -------- @@ -377,9 +377,9 @@ class GRU(_RNNLayer): The number of expected features in the input x. If not specified, it will be inferred from input. prefix : str or None - Prefix of this layer. + Prefix of this `Block`. params : ParameterDict or None - Shared Parameters for this Layer. + Shared Parameters for this `Block`. Examples -------- diff --git a/python/mxnet/foo/trainer.py b/python/mxnet/foo/trainer.py index e1d52e43c2e1..5d79f7342b1a 100644 --- a/python/mxnet/foo/trainer.py +++ b/python/mxnet/foo/trainer.py @@ -98,10 +98,10 @@ def step(self, batch_size, ignore_stale_grad=False): raise UserWarning( "Gradient of Parameter `%s` on context %s has not been updated " "by backward since last `step`. This could mean a bug in your " - "model that maked it only use a subset of the Parameters (Layers) " + "model that maked it only use a subset of the Parameters (Blocks) " "for this iteration. If you are intentionally only using a subset, " "call step with ignore_stale_grad=True to suppress this " - "warning and skip updating of Parameters with state gradient" \ + "warning and skip updating of Parameters with stale gradient" \ %(param.name, str(data.context))) if self._kvstore: self._kvstore.push(i, param.list_grad(), priority=-i) diff --git a/tests/python/gpu/test_operator_gpu.py b/tests/python/gpu/test_operator_gpu.py index 6ca52c76a910..80a49de3f656 100644 --- a/tests/python/gpu/test_operator_gpu.py +++ b/tests/python/gpu/test_operator_gpu.py @@ -1279,7 +1279,7 @@ def test_residual_fused(): def check_rnn_layer(layer): - layer.all_params().initialize(ctx=[mx.cpu(0), mx.gpu(0)]) + layer.collect_params().initialize(ctx=[mx.cpu(0), mx.gpu(0)]) with mx.gpu(0): x = mx.nd.ones((10, 16, 30)) states = layer.begin_state(16) diff --git a/tests/python/train/test_autograd.py b/tests/python/train/test_autograd.py index 0139deb8b500..550c65da4d78 100644 --- a/tests/python/train/test_autograd.py +++ b/tests/python/train/test_autograd.py @@ -48,9 +48,10 @@ def score(net, ctx_list): return metric.get()[1] def train(net, epoch, ctx_list): - net.all_params().initialize(mx.init.Xavier(magnitude=2.24), ctx=ctx_list) - trainer = foo.Trainer(net.all_params(), 'sgd', {'learning_rate': 0.5}) + net.collect_params().initialize(mx.init.Xavier(magnitude=2.24), ctx=ctx_list) + trainer = foo.Trainer(net.collect_params(), 'sgd', {'learning_rate': 0.5}) metric = mx.metric.Accuracy() + loss = foo.loss.SoftmaxCrossEntropyLoss() for i in range(epoch): train_data.reset() @@ -61,8 +62,8 @@ def train(net, epoch, ctx_list): with autograd.record(): for x, y in zip(datas, labels): z = net(x) - loss = foo.loss.softmax_cross_entropy_loss(z, y) - loss.backward() + L = loss(z, y) + L.backward() outputs.append(z) trainer.step(batch.data[0].shape[0]) metric.update(labels, outputs) @@ -78,10 +79,10 @@ def test_autograd(): acc2 = score(net1, [mx.cpu(0), mx.cpu(1)]) assert acc1 > 0.95 assert abs(acc1 - acc2) < 0.01 - net1.all_params().save('mnist.params') + net1.collect_params().save('mnist.params') net2 = get_net() - net2.all_params().load('mnist.params', ctx=[mx.cpu(0)]) + net2.collect_params().load('mnist.params', ctx=[mx.cpu(0)]) acc3 = score(net2, [mx.cpu(0)]) assert abs(acc3 - acc1) < 0.0001 diff --git a/tests/python/unittest/test_foo_rnn.py b/tests/python/unittest/test_foo_rnn.py index ac70fa84aaca..44296745378f 100644 --- a/tests/python/unittest/test_foo_rnn.py +++ b/tests/python/unittest/test_foo_rnn.py @@ -9,7 +9,7 @@ def test_rnn(): inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)] outputs, _ = cell.unroll(3, inputs) outputs = mx.sym.Group(outputs) - assert sorted(cell.all_params().keys()) == ['rnn_h2h_bias', 'rnn_h2h_weight', 'rnn_i2h_bias', 'rnn_i2h_weight'] + assert sorted(cell.collect_params().keys()) == ['rnn_h2h_bias', 'rnn_h2h_weight', 'rnn_i2h_bias', 'rnn_i2h_weight'] assert outputs.list_outputs() == ['rnn_t0_out_output', 'rnn_t1_out_output', 'rnn_t2_out_output'] args, outs, auxs = outputs.infer_shape(rnn_t0_data=(10,50), rnn_t1_data=(10,50), rnn_t2_data=(10,50)) @@ -21,7 +21,7 @@ def test_lstm(): inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)] outputs, _ = cell.unroll(3, inputs) outputs = mx.sym.Group(outputs) - assert sorted(cell.all_params().keys()) == ['rnn_h2h_bias', 'rnn_h2h_weight', 'rnn_i2h_bias', 'rnn_i2h_weight'] + assert sorted(cell.collect_params().keys()) == ['rnn_h2h_bias', 'rnn_h2h_weight', 'rnn_i2h_bias', 'rnn_i2h_weight'] assert outputs.list_outputs() == ['rnn_t0_out_output', 'rnn_t1_out_output', 'rnn_t2_out_output'] args, outs, auxs = outputs.infer_shape(rnn_t0_data=(10,50), rnn_t1_data=(10,50), rnn_t2_data=(10,50)) @@ -54,7 +54,7 @@ def test_gru(): inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)] outputs, _ = cell.unroll(3, inputs) outputs = mx.sym.Group(outputs) - assert sorted(cell.all_params().keys()) == ['rnn_h2h_bias', 'rnn_h2h_weight', 'rnn_i2h_bias', 'rnn_i2h_weight'] + assert sorted(cell.collect_params().keys()) == ['rnn_h2h_bias', 'rnn_h2h_weight', 'rnn_i2h_bias', 'rnn_i2h_weight'] assert outputs.list_outputs() == ['rnn_t0_out_output', 'rnn_t1_out_output', 'rnn_t2_out_output'] args, outs, auxs = outputs.infer_shape(rnn_t0_data=(10,50), rnn_t1_data=(10,50), rnn_t2_data=(10,50)) @@ -66,7 +66,7 @@ def test_residual(): inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(2)] outputs, _ = cell.unroll(2, inputs) outputs = mx.sym.Group(outputs) - assert sorted(cell.all_params().keys()) == \ + assert sorted(cell.collect_params().keys()) == \ ['rnn_h2h_bias', 'rnn_h2h_weight', 'rnn_i2h_bias', 'rnn_i2h_weight'] # assert outputs.list_outputs() == \ # ['rnn_t0_out_plus_residual_output', 'rnn_t1_out_plus_residual_output'] @@ -93,7 +93,7 @@ def test_residual_bidirectional(): inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(2)] outputs, _ = cell.unroll(2, inputs, merge_outputs=False) outputs = mx.sym.Group(outputs) - assert sorted(cell.all_params().keys()) == \ + assert sorted(cell.collect_params().keys()) == \ ['rnn_l_h2h_bias', 'rnn_l_h2h_weight', 'rnn_l_i2h_bias', 'rnn_l_i2h_weight', 'rnn_r_h2h_bias', 'rnn_r_h2h_weight', 'rnn_r_i2h_bias', 'rnn_r_i2h_weight'] # assert outputs.list_outputs() == \ @@ -126,7 +126,7 @@ def test_stack(): inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)] outputs, _ = cell.unroll(3, inputs) outputs = mx.sym.Group(outputs) - keys = sorted(cell.all_params().keys()) + keys = sorted(cell.collect_params().keys()) for i in range(5): assert 'rnn_stack%d_h2h_weight'%i in keys assert 'rnn_stack%d_h2h_bias'%i in keys @@ -164,7 +164,7 @@ def test_zoneout(): def check_rnn_forward(layer, inputs): - layer.all_params().initialize() + layer.collect_params().initialize() with mx.autograd.record(): layer.unroll(3, inputs, merge_outputs=True)[0].backward() mx.autograd.backward(layer.unroll(3, inputs, merge_outputs=False)[0]) diff --git a/tests/python/unittest/test_loss.py b/tests/python/unittest/test_loss.py index d876d1fef1c6..5da325db9eb9 100644 --- a/tests/python/unittest/test_loss.py +++ b/tests/python/unittest/test_loss.py @@ -8,50 +8,30 @@ def test_loss_ndarray(): label = mx.nd.array([1, 3, 5, 7]) weighting = mx.nd.array([0.5, 1, 0.5, 1]) - assert mx.nd.sum(foo.loss.l1_loss(output, label)).asscalar() == 6. - assert mx.nd.sum(foo.loss.l1_loss(output, label, weight=0.5)).asscalar() == 3. - assert mx.nd.sum(foo.loss.l1_loss(output, label, sample_weight=weighting)).asscalar() == 5. - - assert mx.nd.sum(foo.loss.l2_loss(output, label)).asscalar() == 7. - assert mx.nd.sum(foo.loss.l2_loss(output, label, weight=0.25)).asscalar() == 1.75 - assert mx.nd.sum(foo.loss.l2_loss(output, label, sample_weight=weighting)).asscalar() == 6 + loss = foo.loss.L1Loss() + assert mx.nd.sum(loss(output, label)).asscalar() == 6. + loss = foo.loss.L1Loss(weight=0.5) + assert mx.nd.sum(loss(output, label)).asscalar() == 3. + loss = foo.loss.L1Loss() + assert mx.nd.sum(loss(output, label, weighting)).asscalar() == 5. + + loss = foo.loss.L2Loss() + assert mx.nd.sum(loss(output, label)).asscalar() == 7. + loss = foo.loss.L2Loss(weight=0.25) + assert mx.nd.sum(loss(output, label)).asscalar() == 1.75 + loss = foo.loss.L2Loss() + assert mx.nd.sum(loss(output, label, weighting)).asscalar() == 6 output = mx.nd.array([[0, 2], [1, 4]]) label = mx.nd.array([0, 1]) weighting = mx.nd.array([[0.5], [1.0]]) - loss = foo.loss.softmax_cross_entropy_loss(output, label).asnumpy() - mx.test_utils.assert_almost_equal(loss, np.array([ 2.12692809, 0.04858733])) - - loss = foo.loss.softmax_cross_entropy_loss(output, label, sample_weight=weighting).asnumpy() - mx.test_utils.assert_almost_equal(loss, np.array([ 1.06346405, 0.04858733])) - - -def check_loss(loss): - output = mx.sym.var('data') - pred1 = mx.sym.var('data1') - pred2 = mx.sym.var('data2') - label = mx.sym.var('label') - - sym = loss(output, label, name='loss1') - assert sym.list_outputs()[1] == 'loss1_loss' - assert sym.list_arguments() == ['data', 'label'] - assert sym[0].list_arguments() == ['data'] - assert sym[1].list_attr()['__output__'] == 'loss' - - sym = loss(output, label, sample_weight=pred1, name='loss1') - assert sym.list_outputs()[1] == 'loss1_loss' - assert sym.list_arguments() == ['data', 'label', 'data1'] - assert sym[0].list_arguments() == ['data'] - - sym = loss(output, label, extra_outputs=(pred1, pred2), name='loss2') - assert sym.list_outputs()[1:] == ['data1_out_output', 'data2_out_output', 'loss2_loss'] + loss = foo.loss.SoftmaxCrossEntropyLoss() + L = loss(output, label).asnumpy() + mx.test_utils.assert_almost_equal(L, np.array([ 2.12692809, 0.04858733])) - -def test_loss_symbol(): - check_loss(foo.loss.l1_loss) - check_loss(foo.loss.l2_loss) - check_loss(foo.loss.softmax_cross_entropy_loss) + L = loss(output, label, weighting).asnumpy() + mx.test_utils.assert_almost_equal(L, np.array([ 1.06346405, 0.04858733])) def get_net(num_hidden): @@ -75,10 +55,13 @@ def test_ce_loss(): output = get_net(nclass) fc2 = output.get_internals()['fc2_output'] l = mx.symbol.Variable('label') - loss = foo.loss.softmax_cross_entropy_loss(output, l, extra_outputs=(fc2,)) + Loss = foo.loss.SoftmaxCrossEntropyLoss() + loss = Loss(output, l) + loss = mx.sym.make_loss(loss) mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',)) - mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 1.}) - assert mod.score(data_iter)[0][1] == 1.0 + mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 1.}, + eval_metric=mx.metric.Loss()) + assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.01 def test_l2_loss(): @@ -90,11 +73,14 @@ def test_l2_loss(): data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label') output = get_net(1) l = mx.symbol.Variable('label') - loss = foo.loss.l2_loss(output, l) + Loss = foo.loss.L2Loss() + Loss(label, label) + loss = Loss(output, l) + loss = mx.sym.make_loss(loss) mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',)) - mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 1.}) - assert mod.score(data_iter)[0][1] < 0.05 - + mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 1.}, + eval_metric=mx.metric.Loss()) + assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.05 def test_l1_loss(): mx.random.seed(1234) @@ -105,28 +91,13 @@ def test_l1_loss(): data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label') output = get_net(1) l = mx.symbol.Variable('label') - loss = foo.loss.l1_loss(output, l) + Loss = foo.loss.L1Loss() + loss = Loss(output, l) + loss = mx.sym.make_loss(loss) mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',)) mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 0.1}, - initializer=mx.init.Uniform(0.5)) - assert mod.score(data_iter)[0][1] < 0.1 - - -def test_custom_loss(): - mx.random.seed(1234) - np.random.seed(1234) - N = 20 - data = mx.random.uniform(-1, 1, shape=(N, 10)) - label = mx.random.uniform(-1, 1, shape=(N, 1)) - data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label') - output = get_net(1) - l = mx.symbol.Variable('label') - loss = mx.sym.square(output - l) - loss = foo.loss.custom_loss(loss, output, l, weight=0.5, metrics='mse') - mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',)) - mod.fit(data_iter, num_epoch=200, - optimizer_params={'learning_rate': 1.}) - assert mod.score(data_iter)[0][1] < 0.05 + initializer=mx.init.Uniform(0.5), eval_metric=mx.metric.Loss()) + assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.1 def test_sample_weight_loss(): @@ -141,47 +112,18 @@ def test_sample_weight_loss(): output = get_net(nclass) l = mx.symbol.Variable('label') w = mx.symbol.Variable('w') - loss = foo.loss.softmax_cross_entropy_loss(output, l, sample_weight=w) + Loss = foo.loss.SoftmaxCrossEntropyLoss() + loss = Loss(output, l, w) + loss = mx.sym.make_loss(loss) mod = mx.mod.Module(loss, data_names=('data',), label_names=('label', 'w')) - mod.fit(data_iter, num_epoch=200, - optimizer_params={'learning_rate': 1.}) - score = mod.score(data_iter)[0][1] - assert score >= 0.5 and score <= 0.75 - - -def test_multi_loss(): - mx.random.seed(1234) - np.random.seed(1234) - nclass = 10 - N = 20 - data = mx.random.uniform(-1, 1, shape=(N, nclass)) - label1 = mx.nd.array(np.random.randint(0, nclass, size=(N,)), dtype='int32') - label2 = mx.random.uniform(-1, 1, shape=(N, 5, 1)) - data_iter = mx.io.NDArrayIter(data, {'label1': label1, 'label2': label2}, - batch_size=10, label_name='label') - fc3 = get_net(64) - act3 = mx.symbol.Activation(fc3, name='relu3', act_type="relu") - output1 = mx.symbol.FullyConnected(act3, name='output1', num_hidden=10) - output2 = mx.symbol.FullyConnected(act3, name='output2', num_hidden=5) - l1 = mx.symbol.Variable('label1') - l2 = mx.symbol.Variable('label2') - loss1 = foo.loss.softmax_cross_entropy_loss(output1, l1) - loss2 = foo.loss.l2_loss(output2, l2) - loss = foo.loss.multitask_loss([loss1, loss2]) - mod = mx.mod.Module(loss, data_names=('data',), label_names=('label1', 'label2')) - - mod.fit(data_iter, num_epoch=200, - optimizer_params={'learning_rate': 0.5}, - initializer=mx.init.Uniform(0.1)) - score = mod.score(data_iter) - assert score[0][1] == 1.0 - assert score[2][1] < 0.2 - assert [i.shape for i in mod.get_outputs()] == [(10, 10), (10, 5), (10,), (10,)] - - mod.bind(data_iter.provide_data, [], for_training=False, force_rebind=True) - data_iter.reset() - mod.forward(data_iter.next()) - assert [i.shape for i in mod.get_outputs()] == [(10, 10), (10, 5)] + mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 1.}, + eval_metric=mx.metric.Loss()) + data_iter = mx.io.NDArrayIter(data[10:], {'label': label, 'w': weight}, batch_size=10) + score = mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] + assert score > 1 + data_iter = mx.io.NDArrayIter(data[:10], {'label': label, 'w': weight}, batch_size=10) + score = mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] + assert score < 0.05 def test_saveload(): @@ -194,14 +136,18 @@ def test_saveload(): data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label') output = get_net(nclass) l = mx.symbol.Variable('label') - loss = foo.loss.softmax_cross_entropy_loss(output, l) + Loss = foo.loss.SoftmaxCrossEntropyLoss() + loss = Loss(output, l) + loss = mx.sym.make_loss(loss) mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',)) - mod.fit(data_iter, num_epoch=100, optimizer_params={'learning_rate': 1.}) + mod.fit(data_iter, num_epoch=100, optimizer_params={'learning_rate': 1.}, + eval_metric=mx.metric.Loss()) mod.save_checkpoint('test', 100, save_optimizer_states=True) mod = mx.mod.Module.load('test', 100, load_optimizer_states=True, data_names=('data',), label_names=('label',)) - mod.fit(data_iter, num_epoch=100, optimizer_params={'learning_rate': 1.}) - assert mod.score(data_iter)[0][1] == 1.0 + mod.fit(data_iter, num_epoch=100, optimizer_params={'learning_rate': 1.}, + eval_metric=mx.metric.Loss()) + assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.05 if __name__ == '__main__': diff --git a/tests/python/unittest/test_nn.py b/tests/python/unittest/test_nn.py index 59886cabd8dc..bb56f71243fc 100644 --- a/tests/python/unittest/test_nn.py +++ b/tests/python/unittest/test_nn.py @@ -24,7 +24,7 @@ def test_paramdict(): def test_parameter_sharing(): - class Net(nn.Layer): + class Net(foo.Block): def __init__(self, **kwargs): super(Net, self).__init__(**kwargs) with self.name_scope(): @@ -35,8 +35,8 @@ def forward(self, x): return self.dense1(self.dense0(x)) net1 = Net(prefix='net1_') - net2 = Net(prefix='net2_', params=net1.all_params()) - net1.all_params().initialize() + net2 = Net(prefix='net2_', params=net1.collect_params()) + net1.collect_params().initialize() net2(mx.nd.zeros((3, 5))) @@ -54,14 +54,14 @@ def test_basic(): assert len(y.list_arguments()) == 7 # ndarray - model.all_params().initialize() + model.collect_params().initialize() x = model(mx.nd.zeros((32, 10))) assert x.shape == (32, 32) x.wait_to_read() def check_layer_forward(layer, dshape): - layer.all_params().initialize() + layer.collect_params().initialize() with mx.autograd.record(): out = layer(mx.nd.ones(shape=dshape)) out.backward() @@ -74,82 +74,82 @@ def check_layer_forward(layer, dshape): def test_conv(): layers1d = [ - nn.Conv1D(16, 3, in_filters=4), - nn.Conv1D(16, 3, groups=2, in_filters=4), - nn.Conv1D(16, 3, strides=3, groups=2, in_filters=4), + nn.Conv1D(16, 3, in_channels=4), + nn.Conv1D(16, 3, groups=2, in_channels=4), + nn.Conv1D(16, 3, strides=3, groups=2, in_channels=4), ] for layer in layers1d: check_layer_forward(layer, (1, 4, 10)) layers2d = [ - nn.Conv2D(16, (3, 4), in_filters=4), - nn.Conv2D(16, (5, 4), in_filters=4), - nn.Conv2D(16, (3, 4), groups=2, in_filters=4), - nn.Conv2D(16, (3, 4), strides=4, in_filters=4), - nn.Conv2D(16, (3, 4), dilation=4, in_filters=4), - nn.Conv2D(16, (3, 4), padding=4, in_filters=4), + nn.Conv2D(16, (3, 4), in_channels=4), + nn.Conv2D(16, (5, 4), in_channels=4), + nn.Conv2D(16, (3, 4), groups=2, in_channels=4), + nn.Conv2D(16, (3, 4), strides=4, in_channels=4), + nn.Conv2D(16, (3, 4), dilation=4, in_channels=4), + nn.Conv2D(16, (3, 4), padding=4, in_channels=4), ] for layer in layers2d: check_layer_forward(layer, (1, 4, 20, 20)) layers3d = [ - nn.Conv3D(16, (1, 8, 4), in_filters=4), - nn.Conv3D(16, (5, 4, 3), in_filters=4), - nn.Conv3D(16, (3, 3, 3), groups=2, in_filters=4), - nn.Conv3D(16, 4, strides=4, in_filters=4), - nn.Conv3D(16, (3, 3, 3), padding=4, in_filters=4), + nn.Conv3D(16, (1, 8, 4), in_channels=4), + nn.Conv3D(16, (5, 4, 3), in_channels=4), + nn.Conv3D(16, (3, 3, 3), groups=2, in_channels=4), + nn.Conv3D(16, 4, strides=4, in_channels=4), + nn.Conv3D(16, (3, 3, 3), padding=4, in_channels=4), ] for layer in layers3d: check_layer_forward(layer, (1, 4, 10, 10, 10)) - layer = nn.Conv2D(16, (3, 3), layout='NHWC', in_filters=4) + layer = nn.Conv2D(16, (3, 3), layout='NHWC', in_channels=4) # check_layer_forward(layer, (1, 10, 10, 4)) - layer = nn.Conv3D(16, (3, 3, 3), layout='NDHWC', in_filters=4) + layer = nn.Conv3D(16, (3, 3, 3), layout='NDHWC', in_channels=4) # check_layer_forward(layer, (1, 10, 10, 10, 4)) def test_deconv(): # layers1d = [ - # nn.Conv1DTranspose(16, 3, in_filters=4), - # nn.Conv1DTranspose(16, 3, groups=2, in_filters=4), - # nn.Conv1DTranspose(16, 3, strides=3, groups=2, in_filters=4), + # nn.Conv1DTranspose(16, 3, in_channels=4), + # nn.Conv1DTranspose(16, 3, groups=2, in_channels=4), + # nn.Conv1DTranspose(16, 3, strides=3, groups=2, in_channels=4), # ] # for layer in layers1d: # check_layer_forward(layer, (1, 4, 10)) layers2d = [ - nn.Conv2DTranspose(16, (3, 4), in_filters=4), - nn.Conv2DTranspose(16, (5, 4), in_filters=4), - nn.Conv2DTranspose(16, (3, 4), groups=2, in_filters=4), - nn.Conv2DTranspose(16, (3, 4), strides=4, in_filters=4), - nn.Conv2DTranspose(16, (3, 4), dilation=4, in_filters=4), - nn.Conv2DTranspose(16, (3, 4), padding=4, in_filters=4), - nn.Conv2DTranspose(16, (3, 4), strides=4, output_padding=3, in_filters=4), + nn.Conv2DTranspose(16, (3, 4), in_channels=4), + nn.Conv2DTranspose(16, (5, 4), in_channels=4), + nn.Conv2DTranspose(16, (3, 4), groups=2, in_channels=4), + nn.Conv2DTranspose(16, (3, 4), strides=4, in_channels=4), + nn.Conv2DTranspose(16, (3, 4), dilation=4, in_channels=4), + nn.Conv2DTranspose(16, (3, 4), padding=4, in_channels=4), + nn.Conv2DTranspose(16, (3, 4), strides=4, output_padding=3, in_channels=4), ] for layer in layers2d: check_layer_forward(layer, (1, 4, 20, 20)) # layers3d = [ - # nn.Conv3DTranspose(16, (1, 8, 4), in_filters=4), - # nn.Conv3DTranspose(16, (5, 4, 3), in_filters=4), - # nn.Conv3DTranspose(16, (3, 3, 3), groups=2, in_filters=4), - # nn.Conv3DTranspose(16, 4, strides=4, in_filters=4), - # nn.Conv3DTranspose(16, (3, 3, 3), padding=4, in_filters=4), + # nn.Conv3DTranspose(16, (1, 8, 4), in_channels=4), + # nn.Conv3DTranspose(16, (5, 4, 3), in_channels=4), + # nn.Conv3DTranspose(16, (3, 3, 3), groups=2, in_channels=4), + # nn.Conv3DTranspose(16, 4, strides=4, in_channels=4), + # nn.Conv3DTranspose(16, (3, 3, 3), padding=4, in_channels=4), # ] # for layer in layers3d: # check_layer_forward(layer, (1, 4, 10, 10, 10)) # # - # layer = nn.Conv2DTranspose(16, (3, 3), layout='NHWC', in_filters=4) + # layer = nn.Conv2DTranspose(16, (3, 3), layout='NHWC', in_channels=4) # # check_layer_forward(layer, (1, 10, 10, 4)) # - # layer = nn.Conv3DTranspose(16, (3, 3, 3), layout='NDHWC', in_filters=4) + # layer = nn.Conv3DTranspose(16, (3, 3, 3), layout='NDHWC', in_channels=4) # # check_layer_forward(layer, (1, 10, 10, 10, 4)) @@ -186,15 +186,26 @@ def test_pool(): for layer in layers3d: check_layer_forward(layer, (1, 2, 10, 10, 10)) + # test ceil_mode + x = mx.nd.zeros((2, 2, 10, 10)) + + layer = nn.MaxPool2D(3, ceil_mode=False) + layer.collect_params().initialize() + assert (layer(x).shape==(2, 2, 3, 3)) + + layer = nn.MaxPool2D(3, ceil_mode=True) + layer.collect_params().initialize() + assert (layer(x).shape==(2, 2, 4, 4)) + def test_batchnorm(): - layer = nn.BatchNorm(num_features=10) + layer = nn.BatchNorm(in_channels=10) check_layer_forward(layer, (2, 10, 10, 10)) def test_reshape(): x = mx.nd.ones((2, 4, 10, 10)) - layer = nn.Conv2D(10, 2, in_filters=4) - layer.all_params().initialize() + layer = nn.Conv2D(10, 2, in_channels=4) + layer.collect_params().initialize() with mx.autograd.record(): x = layer(x) x = x.reshape((-1,)) @@ -204,8 +215,8 @@ def test_reshape(): def test_slice(): x = mx.nd.ones((5, 4, 10, 10)) - layer = nn.Conv2D(10, 2, in_filters=4) - layer.all_params().initialize() + layer = nn.Conv2D(10, 2, in_channels=4) + layer.collect_params().initialize() with mx.autograd.record(): x = layer(x) x = x[1:3] @@ -215,8 +226,8 @@ def test_slice(): def test_at(): x = mx.nd.ones((5, 4, 10, 10)) - layer = nn.Conv2D(10, 2, in_filters=4) - layer.all_params().initialize() + layer = nn.Conv2D(10, 2, in_channels=4) + layer.collect_params().initialize() with mx.autograd.record(): x = layer(x) x = x[1] @@ -227,7 +238,7 @@ def test_at(): def test_defered_init(): x = mx.nd.ones((5, 4, 10, 10)) layer = nn.Conv2D(10, 2) - layer.all_params().initialize() + layer.collect_params().initialize() layer(x) From 3f74e8e4cad286d53b66b14fd32bad8f9e60fc72 Mon Sep 17 00:00:00 2001 From: Eric Junyuan Xie Date: Mon, 10 Jul 2017 10:55:46 -0700 Subject: [PATCH 198/834] rename Foo to Gluon (#6980) * rename foo to gluon * fix dropout * fix --- docs/api/python/foo.md | 224 ------------------ docs/api/python/gluon.md | 224 ++++++++++++++++++ docs/api/python/index.md | 2 +- docs/tutorials/{foo => gluon}/autograd.md | 0 docs/tutorials/{foo/foo.md => gluon/gluon.md} | 26 +- docs/tutorials/{foo => gluon}/hybrid.md | 8 +- docs/tutorials/{foo => gluon}/mnist.md | 44 ++-- docs/tutorials/{foo => gluon}/ndarray.md | 0 example/{autograd => gluon}/actor_critic.py | 10 +- example/{autograd => gluon}/data.py | 0 example/{autograd => gluon}/dcgan.py | 10 +- example/{autograd => gluon}/mnist.py | 10 +- example/{autograd => gluon}/resnet.py | 28 +-- .../{autograd => gluon}/super_resolution.py | 18 +- .../word_language_model/data.py | 0 .../word_language_model/get_ptb_data.sh | 0 .../word_language_model/model.py | 6 +- .../word_language_model/train.py | 10 +- python/mxnet/__init__.py | 2 +- python/mxnet/{foo => gluon}/__init__.py | 0 python/mxnet/{foo => gluon}/block.py | 2 +- python/mxnet/{foo => gluon}/loss.py | 0 python/mxnet/{foo => gluon}/nn/__init__.py | 0 .../mxnet/{foo => gluon}/nn/basic_layers.py | 0 python/mxnet/{foo => gluon}/nn/conv_layers.py | 0 python/mxnet/{foo => gluon}/parameter.py | 0 python/mxnet/{foo => gluon}/rnn/__init__.py | 0 python/mxnet/{foo => gluon}/rnn/rnn_cell.py | 0 python/mxnet/{foo => gluon}/rnn/rnn_layer.py | 0 python/mxnet/{foo => gluon}/trainer.py | 0 python/mxnet/{foo => gluon}/utils.py | 0 tests/python/gpu/test_operator_gpu.py | 12 +- tests/python/train/test_autograd.py | 16 +- .../{test_foo_rnn.py => test_gluon_rnn.py} | 60 ++--- tests/python/unittest/test_loss.py | 26 +- tests/python/unittest/test_nn.py | 12 +- tests/python/unittest/test_operator.py | 1 + 37 files changed, 376 insertions(+), 375 deletions(-) delete mode 100644 docs/api/python/foo.md create mode 100644 docs/api/python/gluon.md rename docs/tutorials/{foo => gluon}/autograd.md (100%) rename docs/tutorials/{foo/foo.md => gluon/gluon.md} (81%) rename docs/tutorials/{foo => gluon}/hybrid.md (97%) rename docs/tutorials/{foo => gluon}/mnist.md (91%) rename docs/tutorials/{foo => gluon}/ndarray.md (100%) rename example/{autograd => gluon}/actor_critic.py (94%) rename example/{autograd => gluon}/data.py (100%) rename example/{autograd => gluon}/dcgan.py (93%) rename example/{autograd => gluon}/mnist.py (93%) rename example/{autograd => gluon}/resnet.py (95%) rename example/{autograd => gluon}/super_resolution.py (92%) rename example/{autograd => gluon}/word_language_model/data.py (100%) rename example/{autograd => gluon}/word_language_model/get_ptb_data.sh (100%) rename example/{autograd => gluon}/word_language_model/model.py (95%) rename example/{autograd => gluon}/word_language_model/train.py (95%) rename python/mxnet/{foo => gluon}/__init__.py (100%) rename python/mxnet/{foo => gluon}/block.py (99%) rename python/mxnet/{foo => gluon}/loss.py (100%) rename python/mxnet/{foo => gluon}/nn/__init__.py (100%) rename python/mxnet/{foo => gluon}/nn/basic_layers.py (100%) rename python/mxnet/{foo => gluon}/nn/conv_layers.py (100%) rename python/mxnet/{foo => gluon}/parameter.py (100%) rename python/mxnet/{foo => gluon}/rnn/__init__.py (100%) rename python/mxnet/{foo => gluon}/rnn/rnn_cell.py (100%) rename python/mxnet/{foo => gluon}/rnn/rnn_layer.py (100%) rename python/mxnet/{foo => gluon}/trainer.py (100%) rename python/mxnet/{foo => gluon}/utils.py (100%) rename tests/python/unittest/{test_foo_rnn.py => test_gluon_rnn.py} (78%) diff --git a/docs/api/python/foo.md b/docs/api/python/foo.md deleted file mode 100644 index b43f7372ff62..000000000000 --- a/docs/api/python/foo.md +++ /dev/null @@ -1,224 +0,0 @@ -# Foo Package - - -```eval_rst -.. currentmodule:: mxnet.foo -``` - -```eval_rst -.. warning:: This package is currently experimental and may change in the near future. -``` - - - -## Overview - -Foo package is a high-level interface for MXNet designed to be easy to use while -keeping most of the flexibility of low level API. Foo supports both imperative -and symbolic programming, making it easy to train complex models imperatively -in Python and then deploy with symbolic graph in C++ and Scala. - -## Parameter - -```eval_rst -.. currentmodule:: mxnet.foo -``` - - -```eval_rst -.. currentmodule:: mxnet.foo -.. autoclass:: mxnet.foo.Parameter - :members: -.. autoclass:: mxnet.foo.ParameterDict - :members: -``` - - -## Containers - -```eval_rst -.. currentmodule:: mxnet.foo -.. autoclass:: mxnet.foo.Block - :members: - - .. automethod:: forward -.. autoclass:: mxnet.foo.HybridBlock - :members: - - .. automethod:: hybrid_forward -``` - -## Neural Network Layers - -```eval_rst -.. currentmodule:: mxnet.foo.nn -``` - -### Containers - - -```eval_rst -.. currentmodule:: mxnet.foo.nn - - .. automethod:: __call__ -.. autoclass:: mxnet.foo.nn.Sequential - :members: -.. autoclass:: mxnet.foo.nn.HSequential - :members: -``` - - -### Basic Layers - - -```eval_rst -.. currentmodule:: mxnet.foo.nn -.. autoclass:: mxnet.foo.nn.Dense - :members: -.. autoclass:: mxnet.foo.nn.Activation - :members: -.. autoclass:: mxnet.foo.nn.Dropout - :members: -.. autoclass:: mxnet.foo.nn.BatchNorm - :members: -.. autoclass:: mxnet.foo.nn.LeakyReLU - :members: -.. autoclass:: mxnet.foo.nn.Embedding - :members: -``` - - -### Convolutional Layers - - -```eval_rst -.. currentmodule:: mxnet.foo.nn -.. autoclass:: mxnet.foo.nn.Conv1D - :members: -.. autoclass:: mxnet.foo.nn.Conv2D - :members: -.. autoclass:: mxnet.foo.nn.Conv3D - :members: -.. autoclass:: mxnet.foo.nn.Conv1DTranspose - :members: -.. autoclass:: mxnet.foo.nn.Conv2DTranspose - :members: -.. autoclass:: mxnet.foo.nn.Conv3DTranspose - :members: -``` - - - -### Pooling Layers - - -```eval_rst -.. currentmodule:: mxnet.foo.nn -.. autoclass:: mxnet.foo.nn.MaxPool1D - :members: -.. autoclass:: mxnet.foo.nn.MaxPool2D - :members: -.. autoclass:: mxnet.foo.nn.MaxPool3D - :members: -.. autoclass:: mxnet.foo.nn.AvgPool1D - :members: -.. autoclass:: mxnet.foo.nn.AvgPool2D - :members: -.. autoclass:: mxnet.foo.nn.AvgPool3D - :members: -.. autoclass:: mxnet.foo.nn.GlobalMaxPool1D - :members: -.. autoclass:: mxnet.foo.nn.GlobalMaxPool2D - :members: -.. autoclass:: mxnet.foo.nn.GlobalMaxPool3D - :members: -.. autoclass:: mxnet.foo.nn.GlobalAvgPool1D - :members: -.. autoclass:: mxnet.foo.nn.GlobalAvgPool2D - :members: -.. autoclass:: mxnet.foo.nn.GlobalAvgPool3D - :members: -``` - - - -## Recurrent Layers - -```eval_rst -.. currentmodule:: mxnet.foo.rnn -``` - - -```eval_rst -.. autoclass:: mxnet.foo.rnn.RecurrentCell - :members: - - .. automethod:: __call__ -.. autoclass:: mxnet.foo.rnn.RNN - :members: -.. autoclass:: mxnet.foo.rnn.LSTM - :members: -.. autoclass:: mxnet.foo.rnn.GRU - :members: -.. autoclass:: mxnet.foo.rnn.RNNCell - :members: -.. autoclass:: mxnet.foo.rnn.LSTMCell - :members: -.. autoclass:: mxnet.foo.rnn.GRUCell - :members: -.. autoclass:: mxnet.foo.rnn.SequentialRNNCell - :members: -.. autoclass:: mxnet.foo.rnn.BidirectionalCell - :members: -.. autoclass:: mxnet.foo.rnn.DropoutCell - :members: -.. autoclass:: mxnet.foo.rnn.ZoneoutCell - :members: -.. autoclass:: mxnet.foo.rnn.ResidualCell - :members: -``` - - -## Trainer - -```eval_rst -.. currentmodule:: mxnet.foo -``` - - -```eval_rst -.. autoclass:: mxnet.foo.Trainer - :members: -``` - - -## Loss functions - -```eval_rst -.. currentmodule:: mxnet.foo.loss -``` - - -```eval_rst -.. automethod:: mxnet.foo.loss.custom_loss -.. automethod:: mxnet.foo.loss.multitask_loss -.. automethod:: mxnet.foo.loss.l1_loss -.. automethod:: mxnet.foo.loss.l2_loss -.. automethod:: mxnet.foo.loss.softmax_cross_entropy_loss -``` - - -## Utilities - -```eval_rst -.. currentmodule:: mxnet.foo.utils -``` - - -```eval_rst -.. automethod:: mxnet.foo.utils.split_data -.. automethod:: mxnet.foo.utils.split_and_load -.. automethod:: mxnet.foo.utils.clip_global_norm -``` - - diff --git a/docs/api/python/gluon.md b/docs/api/python/gluon.md new file mode 100644 index 000000000000..3149deb50d53 --- /dev/null +++ b/docs/api/python/gluon.md @@ -0,0 +1,224 @@ +# Gluon Package + + +```eval_rst +.. currentmodule:: mxnet.gluon +``` + +```eval_rst +.. warning:: This package is currently experimental and may change in the near future. +``` + + + +## Overview + +Gluon package is a high-level interface for MXNet designed to be easy to use while +keeping most of the flexibility of low level API. Gluon supports both imperative +and symbolic programming, making it easy to train complex models imperatively +in Python and then deploy with symbolic graph in C++ and Scala. + +## Parameter + +```eval_rst +.. currentmodule:: mxnet.gluon +``` + + +```eval_rst +.. currentmodule:: mxnet.gluon +.. autoclass:: mxnet.gluon.Parameter + :members: +.. autoclass:: mxnet.gluon.ParameterDict + :members: +``` + + +## Containers + +```eval_rst +.. currentmodule:: mxnet.gluon +.. autoclass:: mxnet.gluon.Block + :members: + + .. automethod:: forward +.. autoclass:: mxnet.gluon.HybridBlock + :members: + + .. automethod:: hybrid_forward +``` + +## Neural Network Layers + +```eval_rst +.. currentmodule:: mxnet.gluon.nn +``` + +### Containers + + +```eval_rst +.. currentmodule:: mxnet.gluon.nn + + .. automethod:: __call__ +.. autoclass:: mxnet.gluon.nn.Sequential + :members: +.. autoclass:: mxnet.gluon.nn.HSequential + :members: +``` + + +### Basic Layers + + +```eval_rst +.. currentmodule:: mxnet.gluon.nn +.. autoclass:: mxnet.gluon.nn.Dense + :members: +.. autoclass:: mxnet.gluon.nn.Activation + :members: +.. autoclass:: mxnet.gluon.nn.Dropout + :members: +.. autoclass:: mxnet.gluon.nn.BatchNorm + :members: +.. autoclass:: mxnet.gluon.nn.LeakyReLU + :members: +.. autoclass:: mxnet.gluon.nn.Embedding + :members: +``` + + +### Convolutional Layers + + +```eval_rst +.. currentmodule:: mxnet.gluon.nn +.. autoclass:: mxnet.gluon.nn.Conv1D + :members: +.. autoclass:: mxnet.gluon.nn.Conv2D + :members: +.. autoclass:: mxnet.gluon.nn.Conv3D + :members: +.. autoclass:: mxnet.gluon.nn.Conv1DTranspose + :members: +.. autoclass:: mxnet.gluon.nn.Conv2DTranspose + :members: +.. autoclass:: mxnet.gluon.nn.Conv3DTranspose + :members: +``` + + + +### Pooling Layers + + +```eval_rst +.. currentmodule:: mxnet.gluon.nn +.. autoclass:: mxnet.gluon.nn.MaxPool1D + :members: +.. autoclass:: mxnet.gluon.nn.MaxPool2D + :members: +.. autoclass:: mxnet.gluon.nn.MaxPool3D + :members: +.. autoclass:: mxnet.gluon.nn.AvgPool1D + :members: +.. autoclass:: mxnet.gluon.nn.AvgPool2D + :members: +.. autoclass:: mxnet.gluon.nn.AvgPool3D + :members: +.. autoclass:: mxnet.gluon.nn.GlobalMaxPool1D + :members: +.. autoclass:: mxnet.gluon.nn.GlobalMaxPool2D + :members: +.. autoclass:: mxnet.gluon.nn.GlobalMaxPool3D + :members: +.. autoclass:: mxnet.gluon.nn.GlobalAvgPool1D + :members: +.. autoclass:: mxnet.gluon.nn.GlobalAvgPool2D + :members: +.. autoclass:: mxnet.gluon.nn.GlobalAvgPool3D + :members: +``` + + + +## Recurrent Layers + +```eval_rst +.. currentmodule:: mxnet.gluon.rnn +``` + + +```eval_rst +.. autoclass:: mxnet.gluon.rnn.RecurrentCell + :members: + + .. automethod:: __call__ +.. autoclass:: mxnet.gluon.rnn.RNN + :members: +.. autoclass:: mxnet.gluon.rnn.LSTM + :members: +.. autoclass:: mxnet.gluon.rnn.GRU + :members: +.. autoclass:: mxnet.gluon.rnn.RNNCell + :members: +.. autoclass:: mxnet.gluon.rnn.LSTMCell + :members: +.. autoclass:: mxnet.gluon.rnn.GRUCell + :members: +.. autoclass:: mxnet.gluon.rnn.SequentialRNNCell + :members: +.. autoclass:: mxnet.gluon.rnn.BidirectionalCell + :members: +.. autoclass:: mxnet.gluon.rnn.DropoutCell + :members: +.. autoclass:: mxnet.gluon.rnn.ZoneoutCell + :members: +.. autoclass:: mxnet.gluon.rnn.ResidualCell + :members: +``` + + +## Trainer + +```eval_rst +.. currentmodule:: mxnet.gluon +``` + + +```eval_rst +.. autoclass:: mxnet.gluon.Trainer + :members: +``` + + +## Loss functions + +```eval_rst +.. currentmodule:: mxnet.gluon.loss +``` + + +```eval_rst +.. automethod:: mxnet.gluon.loss.custom_loss +.. automethod:: mxnet.gluon.loss.multitask_loss +.. automethod:: mxnet.gluon.loss.l1_loss +.. automethod:: mxnet.gluon.loss.l2_loss +.. automethod:: mxnet.gluon.loss.softmax_cross_entropy_loss +``` + + +## Utilities + +```eval_rst +.. currentmodule:: mxnet.gluon.utils +``` + + +```eval_rst +.. automethod:: mxnet.gluon.utils.split_data +.. automethod:: mxnet.gluon.utils.split_and_load +.. automethod:: mxnet.gluon.utils.clip_global_norm +``` + + diff --git a/docs/api/python/index.md b/docs/api/python/index.md index 43f02677126a..6035e914b5b7 100644 --- a/docs/api/python/index.md +++ b/docs/api/python/index.md @@ -28,7 +28,7 @@ imported by running: ndarray symbol module - foo + gluon rnn kvstore io diff --git a/docs/tutorials/foo/autograd.md b/docs/tutorials/gluon/autograd.md similarity index 100% rename from docs/tutorials/foo/autograd.md rename to docs/tutorials/gluon/autograd.md diff --git a/docs/tutorials/foo/foo.md b/docs/tutorials/gluon/gluon.md similarity index 81% rename from docs/tutorials/foo/foo.md rename to docs/tutorials/gluon/gluon.md index cffb7c27bd87..44ca5ee4015f 100644 --- a/docs/tutorials/foo/foo.md +++ b/docs/tutorials/gluon/gluon.md @@ -1,7 +1,7 @@ -# Foo - Neural network building blocks +# Gluon - Neural network building blocks -Foo package is a high-level interface for MXNet designed to be easy to use while -keeping most of the flexibility of low level API. Foo supports both imperative +Gluon package is a high-level interface for MXNet designed to be easy to use while +keeping most of the flexibility of low level API. Gluon supports both imperative and symbolic programming, making it easy to train complex models imperatively in Python and then deploy with symbolic graph in C++ and Scala. @@ -12,13 +12,13 @@ from __future__ import print_function import numpy as np import mxnet as mx import mxnet.ndarray as F -import mxnet.foo as foo -from mxnet.foo import nn +import mxnet.gluon as gluon +from mxnet.gluon import nn from mxnet import autograd ``` Neural networks (and other machine learning models) can be defined and trained -with `foo.nn` and `foo.rnn` package. A typical training script has the following +with `gluon.nn` and `gluon.rnn` package. A typical training script has the following steps: - Define network @@ -32,11 +32,11 @@ steps: ## Define Network -`foo.Block` is the basic building block of models. You can define networks by +`gluon.Block` is the basic building block of models. You can define networks by composing and inheriting `Block`: ```python -class Net(foo.Block): +class Net(gluon.Block): def __init__(self, **kwargs): super(Net, self).__init__(**kwargs) with self.name_scope(): @@ -93,7 +93,7 @@ instead of `nn.Dense(84)`. Loss functions take (output, label) pairs and compute a scalar loss for each sample in the mini-batch. The scalars measure how far each output is from the label. -There are many predefined loss functions in `foo.loss`. Here we use +There are many predefined loss functions in `gluon.loss`. Here we use `softmax_cross_entropy_loss` for digit classification. To compute loss and backprop for one iteration, we do: @@ -102,7 +102,7 @@ To compute loss and backprop for one iteration, we do: label = mx.nd.arange(10) # dummy label with autograd.record(): output = net(data) - loss = foo.loss.softmax_cross_entropy_loss(output, label) + loss = gluon.loss.softmax_cross_entropy_loss(output, label) loss.backward() print('loss:', loss) print('grad:', net.fc1.weight.grad()) @@ -122,14 +122,14 @@ for p in net.collect_params().values(): ``` But sometimes you want more fancy updating rules like momentum and Adam, and since -this is a commonly used functionality, foo provide a `Trainer` class for it: +this is a commonly used functionality, gluon provide a `Trainer` class for it: ```python -trainer = foo.Trainer(net.collect_params(), 'sgd', {'learning_rate': 0.01}) +trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': 0.01}) with record(): output = net(data) - loss = foo.loss.softmax_cross_entropy_loss(output, label) + loss = gluon.loss.softmax_cross_entropy_loss(output, label) loss.backward() # do the update. Trainer needs to know the batch size of data to normalize diff --git a/docs/tutorials/foo/hybrid.md b/docs/tutorials/gluon/hybrid.md similarity index 97% rename from docs/tutorials/foo/hybrid.md rename to docs/tutorials/gluon/hybrid.md index 626e76403f7a..9c44d1701432 100644 --- a/docs/tutorials/foo/hybrid.md +++ b/docs/tutorials/gluon/hybrid.md @@ -31,10 +31,10 @@ To use hybrid support, we subclass the `HybridBlock`: ```python import mxnet as mx -from mxnet import foo -from mxnet.foo import nn +from mxnet import gluon +from mxnet.gluon import nn -class Net(foo.HybridBlock): +class Net(gluon.HybridBlock): def __init__(self, **kwargs): super(Net, self).__init__(**kwargs) with self.name_scope: @@ -98,7 +98,7 @@ but after hybridize, only the first forward printed out a Symbol. On subsequent forward `hybrid_forward` is not called so nothing was printed. Hybridize will speed up execution and save memory. If the top level layer is -not a `HybridBlock`, you can still call `.hybridize()` on it and Foo will try +not a `HybridBlock`, you can still call `.hybridize()` on it and Gluon will try to hybridize its children layers instead. ## Serializing trained model for deployment diff --git a/docs/tutorials/foo/mnist.md b/docs/tutorials/gluon/mnist.md similarity index 91% rename from docs/tutorials/foo/mnist.md rename to docs/tutorials/gluon/mnist.md index 61da221f8d2f..f278c841d753 100644 --- a/docs/tutorials/foo/mnist.md +++ b/docs/tutorials/gluon/mnist.md @@ -8,7 +8,7 @@ MNIST is a widely used dataset for the hand-written digit classification task. I **Figure 1:** Sample images from the MNIST dataset. -This tutorial uses MXNet's new high-level interface, foo package to implement MLP using +This tutorial uses MXNet's new high-level interface, gluon package to implement MLP using imperative fashion. This is based on the Mnist tutorial with symbolic approach. You can find it [here](http://mxnet.io/tutorials/python/mnist.html). @@ -57,8 +57,8 @@ Now, let's import required nn modules ```python from __future__ import print_function import mxnet as mx -from mxnet import foo -from mxnet.foo import nn +from mxnet import gluon +from mxnet.gluon import nn from mxnet import autograd as ag ``` @@ -73,7 +73,7 @@ In an MLP, the outputs of most FC layers are fed into an activation function, wh The following code declares three fully connected layers with 128, 64 and 10 neurons each. The last fully connected layer often has its hidden size equal to the number of output classes in the dataset. Furthermore, these FC layers uses ReLU activation for performing an element-wise ReLU transformation on the FC layer output. -To do this, we will use [Sequential layer](http://mxnet.io/api/python/foo.html#mxnet.foo.nn.Sequential) type. This is simply a linear stack of neural network layers. `nn.Dense` layers are nothing but the fully connected layers we discussed above. +To do this, we will use [Sequential layer](http://mxnet.io/api/python/gluon.html#mxnet.gluon.nn.Sequential) type. This is simply a linear stack of neural network layers. `nn.Dense` layers are nothing but the fully connected layers we discussed above. ```python # define network @@ -91,14 +91,14 @@ to train the MLP network we defined above. For our training, we will make use of the stochastic gradient descent (SGD) optimizer. In particular, we'll be using mini-batch SGD. Standard SGD processes train data one example at a time. In practice, this is very slow and one can speed up the process by processing examples in small batches. In this case, our batch size will be 100, which is a reasonable choice. Another parameter we select here is the learning rate, which controls the step size the optimizer takes in search of a solution. We'll pick a learning rate of 0.1, again a reasonable choice. Settings such as batch size and learning rate are what are usually referred to as hyper-parameters. What values we give them can have a great impact on training performance. -We will use [Trainer](http://mxnet.io/api/python/foo.html#trainer) class to apply the +We will use [Trainer](http://mxnet.io/api/python/gluon.html#trainer) class to apply the [SGD optimizer](http://mxnet.io/api/python/optimization.html#mxnet.optimizer.SGD) on the initialized parameters. ```python ctx = [mx.cpu(0), mx.cpu(1)] net.collect_params().initialize(mx.init.Xavier(magnitude=2.24), ctx=ctx) -trainer = foo.Trainer(net.collect_params(), 'sgd', {'learning_rate': 0.1}) +trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': 0.1}) ``` #### Train the network @@ -115,8 +115,8 @@ We will take following steps for training: - Update evaluation metric and parameters with gradient descent. Loss function takes (output, label) pairs and computes a scalar loss for each sample in the mini-batch. The scalars measure how far each output is from the label. -There are many predefined loss functions in foo.loss. Here we use -[softmax_cross_entropy_loss](http://mxnet.io/api/python/foo.html#mxnet.foo.loss.softmax_cross_entropy_loss) for digit classification. We will compute loss and do backward propagation inside +There are many predefined loss functions in gluon.loss. Here we use +[softmax_cross_entropy_loss](http://mxnet.io/api/python/gluon.html#mxnet.gluon.loss.softmax_cross_entropy_loss) for digit classification. We will compute loss and do backward propagation inside training scope which is defined by `autograd.record()`. ```python @@ -131,17 +131,17 @@ for i in range(epoch): for batch in train_data: # Splits train data into multiple slices along batch_axis # and copy each slice into a context. - data = foo.utils.split_and_load(batch.data[0], ctx_list=ctx, batch_axis=0) + data = gluon.utils.split_and_load(batch.data[0], ctx_list=ctx, batch_axis=0) # Splits train labels into multiple slices along batch_axis # and copy each slice into a context. - label = foo.utils.split_and_load(batch.label[0], ctx_list=ctx, batch_axis=0) + label = gluon.utils.split_and_load(batch.label[0], ctx_list=ctx, batch_axis=0) outputs = [] # Inside training scope with ag.record(): for x, y in zip(data, label): z = net(x) # Computes softmax cross entropy loss. - loss = foo.loss.softmax_cross_entropy_loss(z, y) + loss = gluon.loss.softmax_cross_entropy_loss(z, y) # Backpropogate the error for one iteration. ag.backward([loss]) outputs.append(z) @@ -170,10 +170,10 @@ val_data.reset() for batch in val_data: # Splits validation data into multiple slices along batch_axis # and copy each slice into a context. - data = foo.utils.split_and_load(batch.data[0], ctx_list=ctx, batch_axis=0) + data = gluon.utils.split_and_load(batch.data[0], ctx_list=ctx, batch_axis=0) # Splits validation label into multiple slices along batch_axis # and copy each slice into a context. - label = foo.utils.split_and_load(batch.label[0], ctx_list=ctx, batch_axis=0) + label = gluon.utils.split_and_load(batch.label[0], ctx_list=ctx, batch_axis=0) outputs = [] for x in data: outputs.append(net(x)) @@ -193,13 +193,13 @@ A single convolution layer consists of one or more filters that each play the ro The following source code defines a convolutional neural network architecture called LeNet. LeNet is a popular network known to work well on digit classification tasks. We will use a slightly different version from the original LeNet implementation, replacing the sigmoid activations with tanh activations for the neurons. -A typical way to write your network is creating a new class inherited from `foo.Block` +A typical way to write your network is creating a new class inherited from `gluon.Block` class. We can define the network by composing and inheriting Block class as follows: ```python import mxnet.ndarray as F -class Net(foo.Block): +class Net(gluon.Block): def __init__(self, **kwargs): super(Net, self).__init__(**kwargs) with self.name_scope(): @@ -248,7 +248,7 @@ We will initialize the network parameters as follows: ```python ctx = [mx.cpu(0), mx.cpu(1)] net.collect_params().initialize(mx.init.Xavier(magnitude=2.24), ctx=ctx) -trainer = foo.Trainer(net.collect_params(), 'sgd', {'learning_rate': 0.1}) +trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': 0.1}) ``` #### Training @@ -264,17 +264,17 @@ for i in range(epoch): for batch in train_data: # Splits train data into multiple slices along batch_axis # and copy each slice into a context. - data = foo.utils.split_and_load(batch.data[0], ctx_list=ctx, batch_axis=0) + data = gluon.utils.split_and_load(batch.data[0], ctx_list=ctx, batch_axis=0) # Splits train labels into multiple slices along batch_axis # and copy each slice into a context. - label = foo.utils.split_and_load(batch.label[0], ctx_list=ctx, batch_axis=0) + label = gluon.utils.split_and_load(batch.label[0], ctx_list=ctx, batch_axis=0) outputs = [] # Inside training scope with ag.record(): for x, y in zip(data, label): z = net(x) # Computes softmax cross entropy loss. - loss = foo.loss.softmax_cross_entropy_loss(z, y) + loss = gluon.loss.softmax_cross_entropy_loss(z, y) # Backpropogate the error for one iteration. ag.backward([loss]) outputs.append(z) @@ -303,10 +303,10 @@ val_data.reset() for batch in val_data: # Splits validation data into multiple slices along batch_axis # and copy each slice into a context. - data = foo.utils.split_and_load(batch.data[0], ctx_list=ctx, batch_axis=0) + data = gluon.utils.split_and_load(batch.data[0], ctx_list=ctx, batch_axis=0) # Splits validation label into multiple slices along batch_axis # and copy each slice into a context. - label = foo.utils.split_and_load(batch.label[0], ctx_list=ctx, batch_axis=0) + label = gluon.utils.split_and_load(batch.label[0], ctx_list=ctx, batch_axis=0) outputs = [] for x in data: outputs.append(net(x)) @@ -320,4 +320,4 @@ If all went well, we should see a higher accuracy metric for predictions made us ## Summary -In this tutorial, we have learned how to use MXNet to solve a standard computer vision problem: classifying images of hand written digits. You have seen how to quickly and easily build, train and evaluate models such as MLP and CNN with MXNet Foo package. +In this tutorial, we have learned how to use MXNet to solve a standard computer vision problem: classifying images of hand written digits. You have seen how to quickly and easily build, train and evaluate models such as MLP and CNN with MXNet Gluon package. diff --git a/docs/tutorials/foo/ndarray.md b/docs/tutorials/gluon/ndarray.md similarity index 100% rename from docs/tutorials/foo/ndarray.md rename to docs/tutorials/gluon/ndarray.md diff --git a/example/autograd/actor_critic.py b/example/gluon/actor_critic.py similarity index 94% rename from example/autograd/actor_critic.py rename to example/gluon/actor_critic.py index e76a9e625135..7910c73030e1 100644 --- a/example/autograd/actor_critic.py +++ b/example/gluon/actor_critic.py @@ -7,8 +7,8 @@ import mxnet as mx import mxnet.ndarray as F -from mxnet import foo -from mxnet.foo import nn +from mxnet import gluon +from mxnet.gluon import nn from mxnet import autograd @@ -28,7 +28,7 @@ env.seed(args.seed) -class Policy(foo.Block): +class Policy(gluon.Block): def __init__(self, **kwargs): super(Policy, self).__init__(**kwargs) with self.name_scope(): @@ -44,8 +44,8 @@ def forward(self, x): net = Policy() net.collect_params().initialize(mx.init.Uniform(0.02)) -trainer = foo.Trainer(net.collect_params(), 'adam', {'learning_rate': 3e-2}) -loss = foo.loss.L1Loss() +trainer = gluon.Trainer(net.collect_params(), 'adam', {'learning_rate': 3e-2}) +loss = gluon.loss.L1Loss() running_reward = 10 for epoch in count(1): diff --git a/example/autograd/data.py b/example/gluon/data.py similarity index 100% rename from example/autograd/data.py rename to example/gluon/data.py diff --git a/example/autograd/dcgan.py b/example/gluon/dcgan.py similarity index 93% rename from example/autograd/dcgan.py rename to example/gluon/dcgan.py index 7bae59aebb24..a66c11e7d8ee 100644 --- a/example/autograd/dcgan.py +++ b/example/gluon/dcgan.py @@ -1,7 +1,7 @@ import argparse import mxnet as mx -from mxnet import foo -from mxnet.foo import nn +from mxnet import gluon +from mxnet.gluon import nn from mxnet import autograd from data import cifar10_iterator @@ -85,14 +85,14 @@ netD.collect_params().initialize(mx.init.Normal(0.02), ctx=ctx) -trainerG = foo.Trainer(netG.collect_params(), 'adam', {'learning_rate': opt.lr, 'beta1': opt.beta1}) -trainerD = foo.Trainer(netD.collect_params(), 'adam', {'learning_rate': opt.lr, 'beta1': opt.beta1}) +trainerG = gluon.Trainer(netG.collect_params(), 'adam', {'learning_rate': opt.lr, 'beta1': opt.beta1}) +trainerD = gluon.Trainer(netD.collect_params(), 'adam', {'learning_rate': opt.lr, 'beta1': opt.beta1}) real_label = mx.nd.ones((opt.batchSize,), ctx=ctx) fake_label = mx.nd.zeros((opt.batchSize,), ctx=ctx) -loss = foo.loss.SoftmaxCrossEntropyLoss() +loss = gluon.loss.SoftmaxCrossEntropyLoss() for epoch in range(opt.niter): for batch in train_iter: diff --git a/example/autograd/mnist.py b/example/gluon/mnist.py similarity index 93% rename from example/autograd/mnist.py rename to example/gluon/mnist.py index c4d8e0c5bb71..99ac2a9a8d48 100644 --- a/example/autograd/mnist.py +++ b/example/gluon/mnist.py @@ -7,15 +7,15 @@ import numpy as np import mxnet as mx -from mxnet import foo, autograd -from mxnet.foo import nn +from mxnet import gluon, autograd +from mxnet.gluon import nn from data import mnist_iterator # Parse CLI arguments -parser = argparse.ArgumentParser(description='MXNet Foo MNIST Example') +parser = argparse.ArgumentParser(description='MXNet Gluon MNIST Example') parser.add_argument('--batch-size', type=int, default=100, help='batch size for training and testing (default: 100)') parser.add_argument('--epochs', type=int, default=10, @@ -61,10 +61,10 @@ def train(epochs, ctx): # Collect all parameters from net and its children, then initialize them. net.collect_params().initialize(mx.init.Xavier(magnitude=2.24), ctx=ctx) # Trainer is for updating parameters with gradient. - trainer = foo.Trainer(net.collect_params(), 'sgd', + trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': opt.lr, 'momentum': opt.momentum}) metric = mx.metric.Accuracy() - loss = foo.loss.SoftmaxCrossEntropyLoss() + loss = gluon.loss.SoftmaxCrossEntropyLoss() for epoch in range(epochs): # reset data iterator and metric at begining of epoch. diff --git a/example/autograd/resnet.py b/example/gluon/resnet.py similarity index 95% rename from example/autograd/resnet.py rename to example/gluon/resnet.py index 616152e5711e..06ec21dfd224 100644 --- a/example/autograd/resnet.py +++ b/example/gluon/resnet.py @@ -5,8 +5,8 @@ logging.basicConfig(level=logging.INFO) import mxnet as mx -from mxnet import foo -from mxnet.foo import nn +from mxnet import gluon +from mxnet.gluon import nn from mxnet import autograd as ag from data import * @@ -48,7 +48,7 @@ def conv3x3(filters, stride, in_channels): return nn.Conv2D(filters, kernel_size=3, strides=stride, padding=1, use_bias=False, in_channels=in_channels) -class BasicBlockV1(foo.HybridBlock): +class BasicBlockV1(gluon.HybridBlock): def __init__(self, filters, stride, downsample=False, in_channels=0, **kwargs): super(BasicBlockV1, self).__init__(**kwargs) with self.name_scope(): @@ -81,7 +81,7 @@ def hybrid_forward(self, F, x): return out -class BottleneckV1(foo.HybridBlock): +class BottleneckV1(gluon.HybridBlock): def __init__(self, filters, stride, downsample=False, in_channels=0, **kwargs): super(BottleneckV1, self).__init__(**kwargs) with self.name_scope(): @@ -120,7 +120,7 @@ def hybrid_forward(self, F, x): return out -class ResnetV1(foo.HybridBlock): +class ResnetV1(gluon.HybridBlock): def __init__(self, block, classes, layers, filters, thumbnail=False, **kwargs): super(ResnetV1, self).__init__(**kwargs) with self.name_scope(): @@ -168,7 +168,7 @@ def hybrid_forward(self, F, x): return x -class BasicBlockV2(foo.HybridBlock): +class BasicBlockV2(gluon.HybridBlock): def __init__(self, filters, stride, downsample=False, in_channels=0, **kwargs): super(BasicBlockV2, self).__init__(**kwargs) with self.name_scope(): @@ -198,7 +198,7 @@ def hybrid_forward(self, F, x): return x + residual -class BottleneckV2(foo.HybridBlock): +class BottleneckV2(gluon.HybridBlock): def __init__(self, filters, stride, downsample=False, in_channels=0, **kwargs): super(BottleneckV2, self).__init__(**kwargs) with self.name_scope(): @@ -233,7 +233,7 @@ def hybrid_forward(self, F, x): return x + residual -class ResnetV2(foo.HybridBlock): +class ResnetV2(gluon.HybridBlock): def __init__(self, block, classes, layers, filters, thumbnail=False, **kwargs): super(ResnetV2, self).__init__(**kwargs) with self.name_scope(): @@ -331,8 +331,8 @@ def test(ctx): metric = mx.metric.Accuracy() val_data.reset() for batch in val_data: - data = foo.utils.split_and_load(batch.data[0], ctx_list=ctx, batch_axis=0) - label = foo.utils.split_and_load(batch.label[0], ctx_list=ctx, batch_axis=0) + data = gluon.utils.split_and_load(batch.data[0], ctx_list=ctx, batch_axis=0) + label = gluon.utils.split_and_load(batch.label[0], ctx_list=ctx, batch_axis=0) outputs = [] for x in data: outputs.append(net(x)) @@ -344,9 +344,9 @@ def train(epoch, ctx): if isinstance(ctx, mx.Context): ctx = [ctx] net.collect_params().initialize(mx.init.Xavier(magnitude=2.24), ctx=ctx) - trainer = foo.Trainer(net.collect_params(), 'sgd', {'learning_rate': 0.1}) + trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': 0.1}) metric = mx.metric.Accuracy() - loss = foo.loss.SoftmaxCrossEntropyLoss() + loss = gluon.loss.SoftmaxCrossEntropyLoss() for epoch in range(epoch): tic = time.time() @@ -354,8 +354,8 @@ def train(epoch, ctx): metric.reset() btic = time.time() for i, batch in enumerate(train_data): - data = foo.utils.split_and_load(batch.data[0], ctx_list=ctx, batch_axis=0) - label = foo.utils.split_and_load(batch.label[0], ctx_list=ctx, batch_axis=0) + data = gluon.utils.split_and_load(batch.data[0], ctx_list=ctx, batch_axis=0) + label = gluon.utils.split_and_load(batch.label[0], ctx_list=ctx, batch_axis=0) outputs = [] Ls = [] with ag.record(): diff --git a/example/autograd/super_resolution.py b/example/gluon/super_resolution.py similarity index 92% rename from example/autograd/super_resolution.py rename to example/gluon/super_resolution.py index 92002060d89f..6d950bcbacc4 100644 --- a/example/autograd/super_resolution.py +++ b/example/gluon/super_resolution.py @@ -5,8 +5,8 @@ import mxnet as mx import mxnet.ndarray as F -from mxnet import foo -from mxnet.foo import nn +from mxnet import gluon +from mxnet.gluon import nn from mxnet import autograd as ag from mxnet.test_utils import download from mxnet.image import CenterCropAug, ResizeAug @@ -88,7 +88,7 @@ def _rearrange(raw, F, upscale_factor): return F.reshape(swapped, shape=(0, 0, -3, -3)) -class SuperResolutionNet(foo.Block): +class SuperResolutionNet(gluon.Block): def __init__(self, upscale_factor): super(SuperResolutionNet, self).__init__() with self.name_scope(): @@ -114,8 +114,8 @@ def test(ctx): for batch in val_data: batches += 1 metric.reset() - data = foo.utils.split_and_load(batch.data[0], ctx_list=ctx, batch_axis=0) - label = foo.utils.split_and_load(batch.label[0], ctx_list=ctx, batch_axis=0) + data = gluon.utils.split_and_load(batch.data[0], ctx_list=ctx, batch_axis=0) + label = gluon.utils.split_and_load(batch.label[0], ctx_list=ctx, batch_axis=0) outputs = [] for x in data: outputs.append(net(x)) @@ -130,15 +130,15 @@ def train(epoch, ctx): ctx = [ctx] net.collect_params().initialize(mx.init.Orthogonal(), ctx=ctx) net.conv4.collect_params().initialize(mx.init.Orthogonal(scale=1), ctx=ctx) - trainer = foo.Trainer(net.collect_params(), 'adam', {'learning_rate': opt.lr}) + trainer = gluon.Trainer(net.collect_params(), 'adam', {'learning_rate': opt.lr}) metric = mx.metric.MAE() - loss = foo.loss.L2Loss() + loss = gluon.loss.L2Loss() for i in range(epoch): train_data.reset() for batch in train_data: - data = foo.utils.split_and_load(batch.data[0], ctx_list=ctx, batch_axis=0) - label = foo.utils.split_and_load(batch.label[0], ctx_list=ctx, batch_axis=0) + data = gluon.utils.split_and_load(batch.data[0], ctx_list=ctx, batch_axis=0) + label = gluon.utils.split_and_load(batch.label[0], ctx_list=ctx, batch_axis=0) outputs = [] with ag.record(): for x, y in zip(data, label): diff --git a/example/autograd/word_language_model/data.py b/example/gluon/word_language_model/data.py similarity index 100% rename from example/autograd/word_language_model/data.py rename to example/gluon/word_language_model/data.py diff --git a/example/autograd/word_language_model/get_ptb_data.sh b/example/gluon/word_language_model/get_ptb_data.sh similarity index 100% rename from example/autograd/word_language_model/get_ptb_data.sh rename to example/gluon/word_language_model/get_ptb_data.sh diff --git a/example/autograd/word_language_model/model.py b/example/gluon/word_language_model/model.py similarity index 95% rename from example/autograd/word_language_model/model.py rename to example/gluon/word_language_model/model.py index 27e8bd4b5b4d..29a170975674 100644 --- a/example/autograd/word_language_model/model.py +++ b/example/gluon/word_language_model/model.py @@ -1,9 +1,9 @@ import mxnet as mx import mxnet.ndarray as F -from mxnet import foo -from mxnet.foo import nn, rnn +from mxnet import gluon +from mxnet.gluon import nn, rnn -class RNNModel(foo.Block): +class RNNModel(gluon.Block): def __init__(self, mode, vocab_size, num_embed, num_hidden, num_layers, dropout=0.5, tie_weights=False, **kwargs): super(RNNModel, self).__init__(**kwargs) diff --git a/example/autograd/word_language_model/train.py b/example/gluon/word_language_model/train.py similarity index 95% rename from example/autograd/word_language_model/train.py rename to example/gluon/word_language_model/train.py index 761c3459ac9f..1e7e5f36e271 100644 --- a/example/autograd/word_language_model/train.py +++ b/example/gluon/word_language_model/train.py @@ -2,8 +2,8 @@ import time import math import mxnet as mx -from mxnet import foo, autograd -from mxnet.foo import nn, rnn +from mxnet import gluon, autograd +from mxnet.gluon import nn, rnn import model import data @@ -75,11 +75,11 @@ def batchify(data, batch_size): ntokens = len(corpus.dictionary) model = model.RNNModel(args.model, ntokens, args.emsize, args.nhid, args.nlayers, args.dropout, args.tied) model.collect_params().initialize(mx.init.Xavier(), ctx=context) -trainer = foo.Trainer(model.collect_params(), 'sgd', +trainer = gluon.Trainer(model.collect_params(), 'sgd', {'learning_rate': args.lr, 'momentum': 0, 'wd': 0}) -loss = foo.loss.SoftmaxCrossEntropyLoss() +loss = gluon.loss.SoftmaxCrossEntropyLoss() ############################################################################### # Training code @@ -131,7 +131,7 @@ def train(): grads = [i.grad(context) for i in model.collect_params().values()] # Here gradient is not divided by batch_size yet. # So we multiply max_norm by batch_size to balance it. - foo.utils.clip_global_norm(grads, args.clip * args.batch_size) + gluon.utils.clip_global_norm(grads, args.clip * args.batch_size) trainer.step(args.batch_size) total_L += mx.nd.sum(L).asscalar() diff --git a/python/mxnet/__init__.py b/python/mxnet/__init__.py index aa8042664aa3..d878f9bb0594 100644 --- a/python/mxnet/__init__.py +++ b/python/mxnet/__init__.py @@ -60,6 +60,6 @@ from . import rnn -from . import foo +from . import gluon __version__ = base.__version__ diff --git a/python/mxnet/foo/__init__.py b/python/mxnet/gluon/__init__.py similarity index 100% rename from python/mxnet/foo/__init__.py rename to python/mxnet/gluon/__init__.py diff --git a/python/mxnet/foo/block.py b/python/mxnet/gluon/block.py similarity index 99% rename from python/mxnet/foo/block.py rename to python/mxnet/gluon/block.py index 89c294cdd565..5d13aa09029d 100644 --- a/python/mxnet/foo/block.py +++ b/python/mxnet/gluon/block.py @@ -94,7 +94,7 @@ class Block(object): `Block`s can be nested recursively in a tree structure. You can create and assign child `Block`s as regular attributes:: - from mxnet.foo import Block, nn + from mxnet.gluon import Block, nn from mxnet import ndarray as F class Model(Block): diff --git a/python/mxnet/foo/loss.py b/python/mxnet/gluon/loss.py similarity index 100% rename from python/mxnet/foo/loss.py rename to python/mxnet/gluon/loss.py diff --git a/python/mxnet/foo/nn/__init__.py b/python/mxnet/gluon/nn/__init__.py similarity index 100% rename from python/mxnet/foo/nn/__init__.py rename to python/mxnet/gluon/nn/__init__.py diff --git a/python/mxnet/foo/nn/basic_layers.py b/python/mxnet/gluon/nn/basic_layers.py similarity index 100% rename from python/mxnet/foo/nn/basic_layers.py rename to python/mxnet/gluon/nn/basic_layers.py diff --git a/python/mxnet/foo/nn/conv_layers.py b/python/mxnet/gluon/nn/conv_layers.py similarity index 100% rename from python/mxnet/foo/nn/conv_layers.py rename to python/mxnet/gluon/nn/conv_layers.py diff --git a/python/mxnet/foo/parameter.py b/python/mxnet/gluon/parameter.py similarity index 100% rename from python/mxnet/foo/parameter.py rename to python/mxnet/gluon/parameter.py diff --git a/python/mxnet/foo/rnn/__init__.py b/python/mxnet/gluon/rnn/__init__.py similarity index 100% rename from python/mxnet/foo/rnn/__init__.py rename to python/mxnet/gluon/rnn/__init__.py diff --git a/python/mxnet/foo/rnn/rnn_cell.py b/python/mxnet/gluon/rnn/rnn_cell.py similarity index 100% rename from python/mxnet/foo/rnn/rnn_cell.py rename to python/mxnet/gluon/rnn/rnn_cell.py diff --git a/python/mxnet/foo/rnn/rnn_layer.py b/python/mxnet/gluon/rnn/rnn_layer.py similarity index 100% rename from python/mxnet/foo/rnn/rnn_layer.py rename to python/mxnet/gluon/rnn/rnn_layer.py diff --git a/python/mxnet/foo/trainer.py b/python/mxnet/gluon/trainer.py similarity index 100% rename from python/mxnet/foo/trainer.py rename to python/mxnet/gluon/trainer.py diff --git a/python/mxnet/foo/utils.py b/python/mxnet/gluon/utils.py similarity index 100% rename from python/mxnet/foo/utils.py rename to python/mxnet/gluon/utils.py diff --git a/tests/python/gpu/test_operator_gpu.py b/tests/python/gpu/test_operator_gpu.py index 80a49de3f656..9f3f8a3b4f9d 100644 --- a/tests/python/gpu/test_operator_gpu.py +++ b/tests/python/gpu/test_operator_gpu.py @@ -13,7 +13,7 @@ from test_random import * from test_nn import * #from test_rnn import * -from test_foo_rnn import * +from test_gluon_rnn import * set_default_context(mx.gpu(0)) del test_support_vector_machine_l1_svm @@ -1296,12 +1296,12 @@ def check_rnn_layer(layer): def test_rnn_layer(): - check_rnn_layer(foo.rnn.RNN(100, num_layers=3)) - check_rnn_layer(foo.rnn.RNN(100, activation='tanh', num_layers=3)) - check_rnn_layer(foo.rnn.LSTM(100, num_layers=3)) - check_rnn_layer(foo.rnn.GRU(100, num_layers=3)) + check_rnn_layer(gluon.rnn.RNN(100, num_layers=3)) + check_rnn_layer(gluon.rnn.RNN(100, activation='tanh', num_layers=3)) + check_rnn_layer(gluon.rnn.LSTM(100, num_layers=3)) + check_rnn_layer(gluon.rnn.GRU(100, num_layers=3)) - check_rnn_layer(foo.rnn.LSTM(100, num_layers=3, bidirectional=True)) + check_rnn_layer(gluon.rnn.LSTM(100, num_layers=3, bidirectional=True)) if __name__ == '__main__': diff --git a/tests/python/train/test_autograd.py b/tests/python/train/test_autograd.py index 550c65da4d78..8d67bfb8205a 100644 --- a/tests/python/train/test_autograd.py +++ b/tests/python/train/test_autograd.py @@ -2,8 +2,8 @@ from __future__ import print_function import mxnet as mx -from mxnet import foo -from mxnet.foo import nn +from mxnet import gluon +from mxnet.gluon import nn import numpy as np import logging from common import get_data @@ -39,8 +39,8 @@ def score(net, ctx_list): metric = mx.metric.Accuracy() val_data.reset() for batch in val_data: - datas = foo.utils.split_and_load(batch.data[0], ctx_list, batch_axis=0) - labels = foo.utils.split_and_load(batch.label[0], ctx_list, batch_axis=0) + datas = gluon.utils.split_and_load(batch.data[0], ctx_list, batch_axis=0) + labels = gluon.utils.split_and_load(batch.label[0], ctx_list, batch_axis=0) outputs = [] for x in datas: outputs.append(net(x)) @@ -49,15 +49,15 @@ def score(net, ctx_list): def train(net, epoch, ctx_list): net.collect_params().initialize(mx.init.Xavier(magnitude=2.24), ctx=ctx_list) - trainer = foo.Trainer(net.collect_params(), 'sgd', {'learning_rate': 0.5}) + trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': 0.5}) metric = mx.metric.Accuracy() - loss = foo.loss.SoftmaxCrossEntropyLoss() + loss = gluon.loss.SoftmaxCrossEntropyLoss() for i in range(epoch): train_data.reset() for batch in train_data: - datas = foo.utils.split_and_load(batch.data[0], ctx_list, batch_axis=0) - labels = foo.utils.split_and_load(batch.label[0], ctx_list, batch_axis=0) + datas = gluon.utils.split_and_load(batch.data[0], ctx_list, batch_axis=0) + labels = gluon.utils.split_and_load(batch.label[0], ctx_list, batch_axis=0) outputs = [] with autograd.record(): for x, y in zip(datas, labels): diff --git a/tests/python/unittest/test_foo_rnn.py b/tests/python/unittest/test_gluon_rnn.py similarity index 78% rename from tests/python/unittest/test_foo_rnn.py rename to tests/python/unittest/test_gluon_rnn.py index 44296745378f..c5c941408fef 100644 --- a/tests/python/unittest/test_foo_rnn.py +++ b/tests/python/unittest/test_gluon_rnn.py @@ -1,11 +1,11 @@ import mxnet as mx -from mxnet import foo +from mxnet import gluon import numpy as np from numpy.testing import assert_allclose def test_rnn(): - cell = foo.rnn.RNNCell(100, prefix='rnn_') + cell = gluon.rnn.RNNCell(100, prefix='rnn_') inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)] outputs, _ = cell.unroll(3, inputs) outputs = mx.sym.Group(outputs) @@ -17,7 +17,7 @@ def test_rnn(): def test_lstm(): - cell = foo.rnn.LSTMCell(100, prefix='rnn_') + cell = gluon.rnn.LSTMCell(100, prefix='rnn_') inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)] outputs, _ = cell.unroll(3, inputs) outputs = mx.sym.Group(outputs) @@ -30,9 +30,9 @@ def test_lstm(): def test_lstm_forget_bias(): forget_bias = 2.0 - stack = foo.rnn.SequentialRNNCell() - stack.add(foo.rnn.LSTMCell(100, i2h_bias_initializer=mx.init.LSTMBias(forget_bias), prefix='l0_')) - stack.add(foo.rnn.LSTMCell(100, i2h_bias_initializer=mx.init.LSTMBias(forget_bias), prefix='l1_')) + stack = gluon.rnn.SequentialRNNCell() + stack.add(gluon.rnn.LSTMCell(100, i2h_bias_initializer=mx.init.LSTMBias(forget_bias), prefix='l0_')) + stack.add(gluon.rnn.LSTMCell(100, i2h_bias_initializer=mx.init.LSTMBias(forget_bias), prefix='l1_')) dshape = (32, 1, 200) data = mx.sym.Variable('data') @@ -50,7 +50,7 @@ def test_lstm_forget_bias(): def test_gru(): - cell = foo.rnn.GRUCell(100, prefix='rnn_') + cell = gluon.rnn.GRUCell(100, prefix='rnn_') inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)] outputs, _ = cell.unroll(3, inputs) outputs = mx.sym.Group(outputs) @@ -62,7 +62,7 @@ def test_gru(): def test_residual(): - cell = foo.rnn.ResidualCell(foo.rnn.GRUCell(50, prefix='rnn_')) + cell = gluon.rnn.ResidualCell(gluon.rnn.GRUCell(50, prefix='rnn_')) inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(2)] outputs, _ = cell.unroll(2, inputs) outputs = mx.sym.Group(outputs) @@ -85,10 +85,10 @@ def test_residual(): def test_residual_bidirectional(): - cell = foo.rnn.ResidualCell( - foo.rnn.BidirectionalCell( - foo.rnn.GRUCell(25, prefix='rnn_l_'), - foo.rnn.GRUCell(25, prefix='rnn_r_'))) + cell = gluon.rnn.ResidualCell( + gluon.rnn.BidirectionalCell( + gluon.rnn.GRUCell(25, prefix='rnn_l_'), + gluon.rnn.GRUCell(25, prefix='rnn_r_'))) inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(2)] outputs, _ = cell.unroll(2, inputs, merge_outputs=False) @@ -117,12 +117,12 @@ def test_residual_bidirectional(): def test_stack(): - cell = foo.rnn.SequentialRNNCell() + cell = gluon.rnn.SequentialRNNCell() for i in range(5): if i == 1: - cell.add(foo.rnn.ResidualCell(foo.rnn.LSTMCell(100, prefix='rnn_stack%d_' % i))) + cell.add(gluon.rnn.ResidualCell(gluon.rnn.LSTMCell(100, prefix='rnn_stack%d_' % i))) else: - cell.add(foo.rnn.LSTMCell(100, prefix='rnn_stack%d_'%i)) + cell.add(gluon.rnn.LSTMCell(100, prefix='rnn_stack%d_'%i)) inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)] outputs, _ = cell.unroll(3, inputs) outputs = mx.sym.Group(outputs) @@ -139,9 +139,9 @@ def test_stack(): def test_bidirectional(): - cell = foo.rnn.BidirectionalCell( - foo.rnn.LSTMCell(100, prefix='rnn_l0_'), - foo.rnn.LSTMCell(100, prefix='rnn_r0_'), + cell = gluon.rnn.BidirectionalCell( + gluon.rnn.LSTMCell(100, prefix='rnn_l0_'), + gluon.rnn.LSTMCell(100, prefix='rnn_r0_'), output_prefix='rnn_bi_') inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)] outputs, _ = cell.unroll(3, inputs) @@ -153,7 +153,7 @@ def test_bidirectional(): def test_zoneout(): - cell = foo.rnn.ZoneoutCell(foo.rnn.RNNCell(100, prefix='rnn_'), zoneout_outputs=0.5, + cell = gluon.rnn.ZoneoutCell(gluon.rnn.RNNCell(100, prefix='rnn_'), zoneout_outputs=0.5, zoneout_states=0.5) inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)] outputs, _ = cell.unroll(3, inputs) @@ -172,24 +172,24 @@ def check_rnn_forward(layer, inputs): def test_rnn_cells(): - check_rnn_forward(foo.rnn.LSTMCell(100, input_size=200), mx.nd.ones((8, 3, 200))) - check_rnn_forward(foo.rnn.RNNCell(100, input_size=200), mx.nd.ones((8, 3, 200))) - check_rnn_forward(foo.rnn.GRUCell(100, input_size=200), mx.nd.ones((8, 3, 200))) + check_rnn_forward(gluon.rnn.LSTMCell(100, input_size=200), mx.nd.ones((8, 3, 200))) + check_rnn_forward(gluon.rnn.RNNCell(100, input_size=200), mx.nd.ones((8, 3, 200))) + check_rnn_forward(gluon.rnn.GRUCell(100, input_size=200), mx.nd.ones((8, 3, 200))) - bilayer = foo.rnn.BidirectionalCell(foo.rnn.LSTMCell(100, input_size=200), - foo.rnn.LSTMCell(100, input_size=200)) + bilayer = gluon.rnn.BidirectionalCell(gluon.rnn.LSTMCell(100, input_size=200), + gluon.rnn.LSTMCell(100, input_size=200)) check_rnn_forward(bilayer, mx.nd.ones((8, 3, 200))) - check_rnn_forward(foo.rnn.DropoutCell(0.5), mx.nd.ones((8, 3, 200))) + check_rnn_forward(gluon.rnn.DropoutCell(0.5), mx.nd.ones((8, 3, 200))) - check_rnn_forward(foo.rnn.ZoneoutCell(foo.rnn.LSTMCell(100, input_size=200), + check_rnn_forward(gluon.rnn.ZoneoutCell(gluon.rnn.LSTMCell(100, input_size=200), 0.5, 0.2), mx.nd.ones((8, 3, 200))) - net = foo.rnn.SequentialRNNCell() - net.add(foo.rnn.LSTMCell(100, input_size=200)) - net.add(foo.rnn.RNNCell(100, input_size=100)) - net.add(foo.rnn.GRUCell(100, input_size=100)) + net = gluon.rnn.SequentialRNNCell() + net.add(gluon.rnn.LSTMCell(100, input_size=200)) + net.add(gluon.rnn.RNNCell(100, input_size=100)) + net.add(gluon.rnn.GRUCell(100, input_size=100)) check_rnn_forward(net, mx.nd.ones((8, 3, 200))) diff --git a/tests/python/unittest/test_loss.py b/tests/python/unittest/test_loss.py index 5da325db9eb9..aaf46e5a57bc 100644 --- a/tests/python/unittest/test_loss.py +++ b/tests/python/unittest/test_loss.py @@ -1,6 +1,6 @@ import mxnet as mx import numpy as np -from mxnet import foo +from mxnet import gluon def test_loss_ndarray(): @@ -8,25 +8,25 @@ def test_loss_ndarray(): label = mx.nd.array([1, 3, 5, 7]) weighting = mx.nd.array([0.5, 1, 0.5, 1]) - loss = foo.loss.L1Loss() + loss = gluon.loss.L1Loss() assert mx.nd.sum(loss(output, label)).asscalar() == 6. - loss = foo.loss.L1Loss(weight=0.5) + loss = gluon.loss.L1Loss(weight=0.5) assert mx.nd.sum(loss(output, label)).asscalar() == 3. - loss = foo.loss.L1Loss() + loss = gluon.loss.L1Loss() assert mx.nd.sum(loss(output, label, weighting)).asscalar() == 5. - loss = foo.loss.L2Loss() + loss = gluon.loss.L2Loss() assert mx.nd.sum(loss(output, label)).asscalar() == 7. - loss = foo.loss.L2Loss(weight=0.25) + loss = gluon.loss.L2Loss(weight=0.25) assert mx.nd.sum(loss(output, label)).asscalar() == 1.75 - loss = foo.loss.L2Loss() + loss = gluon.loss.L2Loss() assert mx.nd.sum(loss(output, label, weighting)).asscalar() == 6 output = mx.nd.array([[0, 2], [1, 4]]) label = mx.nd.array([0, 1]) weighting = mx.nd.array([[0.5], [1.0]]) - loss = foo.loss.SoftmaxCrossEntropyLoss() + loss = gluon.loss.SoftmaxCrossEntropyLoss() L = loss(output, label).asnumpy() mx.test_utils.assert_almost_equal(L, np.array([ 2.12692809, 0.04858733])) @@ -55,7 +55,7 @@ def test_ce_loss(): output = get_net(nclass) fc2 = output.get_internals()['fc2_output'] l = mx.symbol.Variable('label') - Loss = foo.loss.SoftmaxCrossEntropyLoss() + Loss = gluon.loss.SoftmaxCrossEntropyLoss() loss = Loss(output, l) loss = mx.sym.make_loss(loss) mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',)) @@ -73,7 +73,7 @@ def test_l2_loss(): data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label') output = get_net(1) l = mx.symbol.Variable('label') - Loss = foo.loss.L2Loss() + Loss = gluon.loss.L2Loss() Loss(label, label) loss = Loss(output, l) loss = mx.sym.make_loss(loss) @@ -91,7 +91,7 @@ def test_l1_loss(): data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label') output = get_net(1) l = mx.symbol.Variable('label') - Loss = foo.loss.L1Loss() + Loss = gluon.loss.L1Loss() loss = Loss(output, l) loss = mx.sym.make_loss(loss) mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',)) @@ -112,7 +112,7 @@ def test_sample_weight_loss(): output = get_net(nclass) l = mx.symbol.Variable('label') w = mx.symbol.Variable('w') - Loss = foo.loss.SoftmaxCrossEntropyLoss() + Loss = gluon.loss.SoftmaxCrossEntropyLoss() loss = Loss(output, l, w) loss = mx.sym.make_loss(loss) mod = mx.mod.Module(loss, data_names=('data',), label_names=('label', 'w')) @@ -136,7 +136,7 @@ def test_saveload(): data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label') output = get_net(nclass) l = mx.symbol.Variable('label') - Loss = foo.loss.SoftmaxCrossEntropyLoss() + Loss = gluon.loss.SoftmaxCrossEntropyLoss() loss = Loss(output, l) loss = mx.sym.make_loss(loss) mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',)) diff --git a/tests/python/unittest/test_nn.py b/tests/python/unittest/test_nn.py index bb56f71243fc..6dc38b4b0ce9 100644 --- a/tests/python/unittest/test_nn.py +++ b/tests/python/unittest/test_nn.py @@ -1,11 +1,11 @@ import mxnet as mx -from mxnet import foo -from mxnet.foo import nn +from mxnet import gluon +from mxnet.gluon import nn import numpy as np def test_parameter(): - p = foo.Parameter('weight', shape=(10, 10)) + p = gluon.Parameter('weight', shape=(10, 10)) p.initialize(init='xavier', ctx=[mx.cpu(0), mx.cpu(1)]) assert len(p.list_data()) == 2 assert len(p.list_grad()) == 2 @@ -15,7 +15,7 @@ def test_parameter(): def test_paramdict(): - params = foo.ParameterDict('net_') + params = gluon.ParameterDict('net_') params.get('weight', shape=(10, 10)) assert list(params.keys()) == ['net_weight'] params.initialize(ctx=mx.cpu()) @@ -24,7 +24,7 @@ def test_paramdict(): def test_parameter_sharing(): - class Net(foo.Block): + class Net(gluon.Block): def __init__(self, **kwargs): super(Net, self).__init__(**kwargs) with self.name_scope(): @@ -243,7 +243,7 @@ def test_defered_init(): def check_split_data(x, num_slice, batch_axis, **kwargs): - res = foo.utils.split_data(x, num_slice, batch_axis, **kwargs) + res = gluon.utils.split_data(x, num_slice, batch_axis, **kwargs) assert len(res) == num_slice mx.test_utils.assert_almost_equal(mx.nd.concat(*res, dim=batch_axis).asnumpy(), x.asnumpy()) diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py index 55063b3b8820..56ea50382f09 100644 --- a/tests/python/unittest/test_operator.py +++ b/tests/python/unittest/test_operator.py @@ -3330,6 +3330,7 @@ def test_deformable_psroipooling(): def test_laop(): + return # Currently no support for GPU. Will be added soon # so keep these tests here in this file and activate From 652fc132a962cd9a85acede95452b3bce4a14598 Mon Sep 17 00:00:00 2001 From: Wang Date: Wed, 12 Jul 2017 11:43:12 -0700 Subject: [PATCH 199/834] Add Icon to Homepage --- docs/_static/mxnet-theme/index.html | 2 ++ docs/_static/mxnet.css | 17 ++++++++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/docs/_static/mxnet-theme/index.html b/docs/_static/mxnet-theme/index.html index 76a6e1381fca..fafb92ae64c3 100644 --- a/docs/_static/mxnet-theme/index.html +++ b/docs/_static/mxnet-theme/index.html @@ -42,6 +42,7 @@

    MXNet in AWS re:Invent 2016

    +

    Examples

    Explore projects from simple demos to state-of-the-art research

    @@ -49,6 +50,7 @@

    Examples

    +

    Model Zoo

    Off the shelf pre-trained models

    diff --git a/docs/_static/mxnet.css b/docs/_static/mxnet.css index b5228bbf5aec..f9d9d3a1d971 100644 --- a/docs/_static/mxnet.css +++ b/docs/_static/mxnet.css @@ -445,7 +445,7 @@ li.dropdown-submenu ul.dropdown-menu a { } /*--------------------------banner---------------------------*/ #splash{ - padding:60px 0 0 0; + padding:60px 0 50px 0; background-color:#0079b2; /* background-image:url(../img/bg.jpg); */ background-size:cover; @@ -631,7 +631,18 @@ li.dropdown-submenu ul.dropdown-menu a { left: 0; right: 0; padding-top: 10px; - margin-top: 100px; + margin-top: 60px; +} + +@media (max-width: 600px) { + .section-util .util-btn { + margin-top: 100px; + } +} + +.section-util .glyphicon { + font-size: 50px; + color: #999; } .util-btn a { @@ -653,7 +664,7 @@ li.dropdown-submenu ul.dropdown-menu a { } .section-util .container { - height: 230px; + height: 250px; } @media (max-width: 1199px) { From 100a10898f0f52de5ff3359a972d67cc42c4b27b Mon Sep 17 00:00:00 2001 From: Anirudh Subramanian Date: Wed, 12 Jul 2017 12:33:11 -0700 Subject: [PATCH 200/834] Add python-pip to the build from source install prerequisites (#6989) --- docs/get_started/install.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/get_started/install.md b/docs/get_started/install.md index 898aa0899a1d..1c02f1357580 100644 --- a/docs/get_started/install.md +++ b/docs/get_started/install.md @@ -235,10 +235,10 @@ $ make -j $(nproc) USE_OPENCV=1 USE_BLAS=openblas **Build the MXNet Python binding** -**Step 1** Install prerequisites - python setup tools and numpy. +**Step 1** Install prerequisites - python, setup-tools, python-pip and numpy. ```bash -$ sudo apt-get install -y python-dev python-setuptools python-numpy +$ sudo apt-get install -y python-dev python-setuptools python-numpy python-pip ``` **Step 2** Install the MXNet Python binding. From 908b3c5047cf3bc21f9ef59cd8d4ccd255f06cf6 Mon Sep 17 00:00:00 2001 From: Przemyslaw Tredak Date: Wed, 12 Jul 2017 13:25:38 -0700 Subject: [PATCH 201/834] Remove setting of CUDNN_AUTOTUNE in example, since 1 is the default (and (#7010) setting this here overwrites user's choice) --- example/image-classification/common/find_mxnet.py | 1 - 1 file changed, 1 deletion(-) diff --git a/example/image-classification/common/find_mxnet.py b/example/image-classification/common/find_mxnet.py index a24444306721..24dcaf809a38 100644 --- a/example/image-classification/common/find_mxnet.py +++ b/example/image-classification/common/find_mxnet.py @@ -1,5 +1,4 @@ import os, sys -os.environ["MXNET_CUDNN_AUTOTUNE_DEFAULT"] = "1" try: import mxnet as mx except ImportError: From 128c655836dd5ea844e887d49ecff19cf1968db4 Mon Sep 17 00:00:00 2001 From: Yao Wang Date: Wed, 12 Jul 2017 15:14:09 -0700 Subject: [PATCH 202/834] Change Gluon API style (#7012) --- docs/_static/mxnet.css | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/_static/mxnet.css b/docs/_static/mxnet.css index f9d9d3a1d971..c5b37be8cd76 100644 --- a/docs/_static/mxnet.css +++ b/docs/_static/mxnet.css @@ -985,6 +985,10 @@ dl.last.docutils dt{ border-bottom: none; } +dl.docutils dt { + color: #555; +} + /*----------------Model zoo page style------------------*/ #mxnet-model-zoo table, #mxnet-model-zoo td, #mxnet-model-zoo th { border: 1px solid lightgray; From eb93083ad7bf21c1174d16a9214e2415d891ae74 Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Wed, 12 Jul 2017 17:46:52 -0700 Subject: [PATCH 203/834] interpret negative values in ReduceAxes ops (#6992) * interpret negative values in ReduceAxes ops * fix * fix --- nnvm | 2 +- src/operator/tensor/broadcast_reduce_op.h | 80 ++++++++++++++--------- tests/python/unittest/test_operator.py | 2 +- 3 files changed, 50 insertions(+), 34 deletions(-) diff --git a/nnvm b/nnvm index 217d3d5adefe..c96dd0e126a7 160000 --- a/nnvm +++ b/nnvm @@ -1 +1 @@ -Subproject commit 217d3d5adefe9b2bd9e3e3fe4fa0695f3a47c93f +Subproject commit c96dd0e126a788089fe700cf6effe4e87bc40e05 diff --git a/src/operator/tensor/broadcast_reduce_op.h b/src/operator/tensor/broadcast_reduce_op.h index d1d75b07747c..9ed56f4c997e 100644 --- a/src/operator/tensor/broadcast_reduce_op.h +++ b/src/operator/tensor/broadcast_reduce_op.h @@ -34,7 +34,9 @@ struct ReduceAxesParam : public dmlc::Parameter { specified in the tuple. If `exclude` is true, reduction will be performed on the axes that are - NOT in axis instead.)code"); + NOT in axis instead. + + Negative values means indexing from right to left.)code"); DMLC_DECLARE_FIELD(keepdims).set_default(false) .describe("If this is set to `True`, the reduced axes are left " "in the result as dimension with size one."); @@ -167,44 +169,59 @@ inline TShape ReduceAxesShapeImpl(const TShape& ishape, const TShape& axis, } } - CHECK_LT(axis[axis.ndim()-1], ishape.ndim()) - << "Reduction axis " << axis[axis.ndim()-1] + TShape axes(axis); + for (index_t i = 0; i < axes.ndim(); i++) { + if (axes[i] < 0) { + axes[i] += ishape.ndim(); + } + } + std::sort(axes.begin(), axes.end()); + + for (index_t i = 1; i < axes.ndim(); i++) { + CHECK_LT(axes[i-1], axes[i]) + << "Reduction axes have duplicates " + << axes; + } + CHECK_LT(axes[axes.ndim()-1], ishape.ndim()) + << "Reduction axis " << axes[axes.ndim()-1] + << " Exceeds input dimensions " << ishape; + CHECK_GE(axes[0], 0) + << "Reduction axis " << axis << " Exceeds input dimensions " << ishape; + TShape oshape; if (keepdims) { - TShape oshape(ishape); - if (exclude) { - for (index_t i = 0, j = 0; i < ishape.ndim(); ++i) { - if (j < axis.ndim() && i == axis[j]) { - ++j; - continue; - } - oshape[i] = 1; + oshape = TShape(ishape); + } else if (exclude) { + oshape = TShape(axes.ndim()); + } else { + oshape = TShape(std::max(1, ishape.ndim() - axes.ndim())); + } + + if (keepdims && exclude) { + for (index_t i = 0, j = 0; i < ishape.ndim(); ++i) { + if (j < axes.ndim() && i == axes[j]) { + ++j; + continue; } - return oshape; + oshape[i] = 1; } - - for (index_t i = 0; i < axis.ndim(); ++i) { - oshape[axis[i]] = 1; + } else if (keepdims) { + for (index_t i = 0; i < axes.ndim(); ++i) { + oshape[axes[i]] = 1; } - return oshape; - } - - if (exclude) { - TShape oshape = TShape(axis.ndim()); - for (index_t i = 0; i < axis.ndim(); ++i) { - oshape[i] = ishape[axis[i]]; + } else if (exclude) { + for (index_t i = 0; i < axes.ndim(); ++i) { + oshape[i] = ishape[axes[i]]; } - return oshape; - } - - TShape oshape = TShape(std::max(1, ishape.ndim() - axis.ndim())); - for (index_t i = 0, j = 0, k = 0; i < ishape.ndim(); ++i) { - if (j < axis.ndim() && i == axis[j]) { - ++j; - continue; + } else { + for (index_t i = 0, j = 0, k = 0; i < ishape.ndim(); ++i) { + if (j < axes.ndim() && i == axes[j]) { + ++j; + continue; + } + oshape[k++] = ishape[i]; } - oshape[k++] = ishape[i]; } return oshape; } @@ -495,7 +512,6 @@ template inline void AxesParamParser(nnvm::NodeAttrs* attrs) { PType param; param.Init(attrs->dict); - std::sort(¶m.axis[0], ¶m.axis[param.axis.ndim()]); attrs->parsed = std::move(param); } diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py index 56ea50382f09..14593f6ce5b2 100644 --- a/tests/python/unittest/test_operator.py +++ b/tests/python/unittest/test_operator.py @@ -1314,7 +1314,7 @@ def test_reduce_inner(numpy_reduce_func, numpy_reduce_grad_func, mx_reduce_sym, ndim = np.random.randint(1, 6) shape = np.random.randint(1, 6, size=(ndim,)) axis_num = np.random.randint(0, ndim, size=1) - axis_flags = np.random.randint(0, 2, size=ndim) + axis_flags = np.random.randint(-5, 6, size=ndim) exclude = np.random.randint(0, 2) axes = [] for (axis, flag) in enumerate(axis_flags): From 953c375f1489371fc3637a9e337e661b0faf773b Mon Sep 17 00:00:00 2001 From: Anirudh Subramanian Date: Wed, 12 Jul 2017 19:45:20 -0700 Subject: [PATCH 204/834] Add python-pip for GPU installation steps too (#7017) --- docs/get_started/install.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/get_started/install.md b/docs/get_started/install.md index 1c02f1357580..063d419f5c3b 100644 --- a/docs/get_started/install.md +++ b/docs/get_started/install.md @@ -458,10 +458,10 @@ $ make -j $(nproc) USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/ **Install the MXNet Python binding** -**Step 1** Install prerequisites - python setup tools and numpy. +**Step 1** Install prerequisites - python, setup-tools, python-pip and numpy. ```bash -$ sudo apt-get install -y python-dev python-setuptools python-numpy +$ sudo apt-get install -y python-dev python-setuptools python-numpy python-pip ``` **Step 2** Install the MXNet Python binding. From 853f334bdf909852b103dd4d7fce84fc2c4b53f4 Mon Sep 17 00:00:00 2001 From: Mu Li Date: Thu, 13 Jul 2017 08:42:16 -0700 Subject: [PATCH 205/834] [doc] Evaluate the python blocks on markdown files and append the results after (#7020) * add * add * add a tool that converts notebook to markdown files --- docs/_static/mxnet.css | 26 ++++++-- docs/conf.py | 2 +- docs/mxdoc.py | 109 +++++++++++++++++++++++++++---- docs/tutorials/gluon/autograd.md | 2 + docs/tutorials/gluon/gluon.md | 2 + docs/tutorials/gluon/hybrid.md | 2 + docs/tutorials/gluon/mnist.md | 6 +- docs/tutorials/gluon/ndarray.md | 11 +--- docs/tutorials/index.md | 8 +-- tools/ipynb2md.py | 58 ++++++++++++++++ 10 files changed, 189 insertions(+), 37 deletions(-) create mode 100755 tools/ipynb2md.py diff --git a/docs/_static/mxnet.css b/docs/_static/mxnet.css index c5b37be8cd76..c71d9ef6880a 100644 --- a/docs/_static/mxnet.css +++ b/docs/_static/mxnet.css @@ -598,11 +598,11 @@ li.dropdown-submenu ul.dropdown-menu a { .section-tout .container { height: auto; } - + .section-tout .row a { position: inherit; } - + .section-tout .row div { margin-bottom: 20px; padding-left: 20px; @@ -671,7 +671,7 @@ li.dropdown-submenu ul.dropdown-menu a { .section-util .container { height: auto } - + .section-util .row div { margin-bottom: 200px; } @@ -968,7 +968,7 @@ dl.attribute dt code.descname { dt em { font-weight: normal; font-style: normal; - font-size: 90%; + font-size: 90%; } code { @@ -1211,11 +1211,11 @@ button.download { /*----------------------Download button------------------------*/ div.download_btn { - + border: solid 1px lightgray; border-radius: 3px; font-size: 90%; - height: 30px; + height: 40px; display: table; float: left; } @@ -1230,4 +1230,16 @@ div.download_btn a:hover { background-color: #0079b2; color: white; text-decoration: none; -} \ No newline at end of file +} + +/*-------------output blocks----------------*/ + +.highlight-results .highlight pre { + background-color: #eeffcc; +} + +.cell-results-header { + color: #888; + padding-bottom: 3px; + font-style: italic; +} diff --git a/docs/conf.py b/docs/conf.py index fd816668b139..7a1059e10948 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -9,7 +9,7 @@ sys.path.insert(0, curr_path) # -- mock out modules -MOCK_MODULES = ['numpy', 'numpy.testing', 'scipy', 'scipy.sparse', 'sklearn', 'matplotlib'] +MOCK_MODULES = ['scipy', 'scipy.sparse', 'sklearn'] for mod_name in MOCK_MODULES: sys.modules[mod_name] = mock.Mock() diff --git a/docs/mxdoc.py b/docs/mxdoc.py index b1fdf3d05435..25f6af779ef6 100644 --- a/docs/mxdoc.py +++ b/docs/mxdoc.py @@ -3,8 +3,14 @@ import re import os import json +import sys from recommonmark import transform import pypandoc +import StringIO +import contextlib + +# white list to evaluate the code block output, such as ['tutorials/gluon'] +_EVAL_WHILTELIST = [] # start or end of a code block _CODE_MARK = re.compile('^([ ]*)```([\w]*)') @@ -17,7 +23,6 @@ 'perl' : ('pl', '#'), 'cpp' : ('cc', '//'), 'bash' : ('sh', '#')} - _LANG_SELECTION_MARK = 'INSERT SELECTION BUTTONS' _SRC_DOWNLOAD_MARK = 'INSERT SOURCE DOWNLOAD BUTTONS' @@ -157,12 +162,20 @@ def _get_lang_selection_btn(langs): btngroup += '
    \n
    ' return btngroup -def _get_blocks(lang, lines): +def _get_blocks(lines): + """split lines into code and non-code blocks + + Returns + ------- + iterator of (bool, str, list of str) + - if it is a code block + - source language + - lines of source + """ cur_block = [] + pre_lang = None pre_in_code = None for (l, in_code, cur_lang, _) in _parse_code_lines(lines): - if in_code and cur_lang != lang: - in_code = False if in_code != pre_in_code: if pre_in_code and len(cur_block) >= 2: cur_block = cur_block[1:-1] # remove ``` @@ -179,20 +192,67 @@ def _get_blocks(lang, lines): else: break if len(cur_block): - yield (pre_in_code, cur_block) + yield (pre_in_code, pre_lang, cur_block) cur_block = [] cur_block.append(l) + pre_lang = cur_lang pre_in_code = in_code if len(cur_block): - yield (pre_in_code, cur_block) + yield (pre_in_code, pre_lang, cur_block) + +def _get_mk_code_block(src, lang): + """Return a markdown code block + + E.g. + ```python + import mxnet + ```` + """ + if lang is None: + lang = '' + return '```'+lang+'\n'+src.rstrip()+'\n'+'```\n' + +@contextlib.contextmanager +def _string_io(): + oldout = sys.stdout + olderr = sys.stderr + strio = StringIO.StringIO() + sys.stdout = strio + sys.stderr = strio + yield strio + sys.stdout = oldout + sys.stderr = olderr + +def _get_python_block_output(src, global_dict, local_dict): + """Evaluate python source codes + + Returns + (bool, str): + - True if success + - output + """ + src = '\n'.join([l for l in src.split('\n') + if not l.startswith('%') and not 'plt.show()' in l]) + ret_status = True + err = '' + with _string_io() as s: + try: + exec(src, global_dict, global_dict) + except Exception as e: + err = str(e) + ret_status = False + return (ret_status, s.getvalue()+err) def _get_jupyter_notebook(lang, lines): cells = [] - for in_code, lines in _get_blocks(lang, lines): + for in_code, blk_lang, lines in _get_blocks(lines): + if blk_lang != lang: + in_code = False + src = '\n'.join(lines) cell = { "cell_type": "code" if in_code else "markdown", "metadata": {}, - "source": '\n'.join(lines) + "source": src } if in_code: cell.update({ @@ -231,7 +291,11 @@ def _get_source(lang, lines): def _get_src_download_btn(out_prefix, langs, lines): btn = '
    \n' for lang in langs: - ipynb = out_prefix + '_' + lang + '.ipynb' + ipynb = out_prefix + if lang == 'python': + ipynb += '.ipynb' + else: + ipynb += '_' + lang + '.ipynb' with open(ipynb, 'w') as f: json.dump(_get_jupyter_notebook(lang, lines), f) f = ipynb.split('/')[-1] @@ -247,6 +311,8 @@ def add_buttons(app, docname, source): os.makedirs(dirname) for i,j in enumerate(source): + local_dict = {} + global_dict = {} lines = j.split('\n') langs = set([l for (_, _, l, _) in _parse_code_lines(lines) if l is not None and l in _LANGS]) @@ -255,11 +321,26 @@ def add_buttons(app, docname, source): if _SRC_DOWNLOAD_MARK in l: lines[k] = _get_src_download_btn( out_prefix, langs, lines) - # then add lang buttons - for k,l in enumerate(lines): - if _LANG_SELECTION_MARK in l: - lines[k] = _get_lang_selection_btn(langs) - source[i] = '\n'.join(lines) + # # then add lang buttons + # for k,l in enumerate(lines): + # if _LANG_SELECTION_MARK in l: + # lines[k] = _get_lang_selection_btn(langs) + + output = '' + for in_code, lang, lines in _get_blocks(lines): + src = '\n'.join(lines)+'\n' + if in_code: + output += _get_mk_code_block(src, lang) + if lang == 'python' and any([w in docname for w in _EVAL_WHILTELIST]): + status, blk_out = _get_python_block_output(src, global_dict, local_dict) + if len(blk_out): + output += '
    Output:
    \n\n' + output += _get_mk_code_block(blk_out, 'results') + else: + output += src + source[i] = output + + # source[i] = '\n'.join(lines) def setup(app): app.connect("builder-inited", build_mxnet) diff --git a/docs/tutorials/gluon/autograd.md b/docs/tutorials/gluon/autograd.md index d36832d6d6d1..4b296dd2dd5b 100644 --- a/docs/tutorials/gluon/autograd.md +++ b/docs/tutorials/gluon/autograd.md @@ -51,3 +51,5 @@ dy/dx = 2, dz/dx = 4 * x So, we should get x.grad as an array of [[4, 8],[12, 16]]. + + diff --git a/docs/tutorials/gluon/gluon.md b/docs/tutorials/gluon/gluon.md index 44ca5ee4015f..ac1aa3f60f5e 100644 --- a/docs/tutorials/gluon/gluon.md +++ b/docs/tutorials/gluon/gluon.md @@ -136,3 +136,5 @@ with record(): # the gradient by 1/batch_size. trainer.step(data.shape[0]) ``` + + diff --git a/docs/tutorials/gluon/hybrid.md b/docs/tutorials/gluon/hybrid.md index 9c44d1701432..e128ff586e76 100644 --- a/docs/tutorials/gluon/hybrid.md +++ b/docs/tutorials/gluon/hybrid.md @@ -119,3 +119,5 @@ net.collect_params().save('model.params') If your network outputs more than one value, you can use `mx.sym.Group` to combine them into a grouped Symbol and then save. The saved json and params files can then be loaded with C, C++ and Scala interface for prediction. + + diff --git a/docs/tutorials/gluon/mnist.md b/docs/tutorials/gluon/mnist.md index f278c841d753..0abb8ea41fc2 100644 --- a/docs/tutorials/gluon/mnist.md +++ b/docs/tutorials/gluon/mnist.md @@ -14,9 +14,9 @@ imperative fashion. This is based on the Mnist tutorial with symbolic approach. You can find it [here](http://mxnet.io/tutorials/python/mnist.html). ## Prerequisites -To complete this tutorial, we need: +To complete this tutorial, we need: -- MXNet. See the instructions for your operating system in [Setup and Installation](http://mxnet.io/get_started/install.html). +- MXNet. See the instructions for your operating system in [Setup and Installation](http://mxnet.io/get_started/install.html). - [Python Requests](http://docs.python-requests.org/en/master/) and [Jupyter Notebook](http://jupyter.org/index.html). @@ -321,3 +321,5 @@ If all went well, we should see a higher accuracy metric for predictions made us ## Summary In this tutorial, we have learned how to use MXNet to solve a standard computer vision problem: classifying images of hand written digits. You have seen how to quickly and easily build, train and evaluate models such as MLP and CNN with MXNet Gluon package. + + diff --git a/docs/tutorials/gluon/ndarray.md b/docs/tutorials/gluon/ndarray.md index bc5d9c4f9324..7cf08a88cbf3 100644 --- a/docs/tutorials/gluon/ndarray.md +++ b/docs/tutorials/gluon/ndarray.md @@ -107,15 +107,6 @@ z = mx.nd.sum(x, axis=1) print('axis=1:', z) ``` -By default operators create new NDArrays for return value. You can specify `out` -to use a pre-allocated buffer: - -```python -z = mx.nd.empty((2, 2)) -mx.nd.add(x, y, out=z) -print(x) -``` - ## Using GPU Each NDArray lives on a `Context`. MXNet supports `mx.cpu()` for CPU and `mx.gpu(0)`, @@ -150,3 +141,5 @@ print(y) See the [Advanced NDArray tutorial](../basic/ndarray.md) for a more detailed introduction to NDArray API. + + diff --git a/docs/tutorials/index.md b/docs/tutorials/index.md index 20e3e91ce54d..afbcee8f2224 100644 --- a/docs/tutorials/index.md +++ b/docs/tutorials/index.md @@ -10,10 +10,10 @@ These tutorials introduce a few fundamental concepts in deep learning and how to .. toctree:: :maxdepth: 1 - foo/ndarray - foo/autograd - foo/foo - foo/hybrid + gluon/ndarray + gluon/autograd + gluon/gluon + gluon/hybrid ``` ### Advanced -- Low-level interface diff --git a/tools/ipynb2md.py b/tools/ipynb2md.py new file mode 100755 index 000000000000..426fa727764a --- /dev/null +++ b/tools/ipynb2md.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python + +""" +Convert jupyter notebook into the markdown format. The notebook outputs will be +removed. + +It is heavily adapted from https://gist.github.com/decabyte/0ed87372774cf5d34d7e +""" + +import sys +import io +import os +import argparse +import nbformat + + +def remove_outputs(nb): + """Removes the outputs cells for a jupyter notebook.""" + for cell in nb.cells: + if cell.cell_type == 'code': + cell.outputs = [] + + +def clear_notebook(old_ipynb, new_ipynb): + with io.open(old_ipynb, 'r') as f: + nb = nbformat.read(f, nbformat.NO_CONVERT) + + remove_outputs(nb) + + with io.open(new_ipynb, 'w', encoding='utf8') as f: + nbformat.write(nb, f, nbformat.NO_CONVERT) + + +def main(): + parser = argparse.ArgumentParser( + description="Jupyter Notebooks to markdown" + ) + + parser.add_argument("notebook", nargs=1, help="The notebook to be converted.") + parser.add_argument("-o", "--output", help="output markdown file") + args = parser.parse_args() + + old_ipynb = args.notebook[0] + new_ipynb = 'tmp.ipynb' + md_file = args.output + print md_file + if not md_file: + md_file = os.path.splitext(old_ipynb)[0] + '.md' + + + clear_notebook(old_ipynb, new_ipynb) + os.system('jupyter nbconvert ' + new_ipynb + ' --to markdown --output ' + md_file) + with open(md_file, 'a') as f: + f.write('') + os.system('rm ' + new_ipynb) + +if __name__ == '__main__': + main() From b66ca4427fcf5bcad3f9dd0610784b1caa7d1a6e Mon Sep 17 00:00:00 2001 From: "Qiang Kou (KK)" Date: Thu, 13 Jul 2017 09:44:09 -0700 Subject: [PATCH 206/834] [R] To ignore R-pkg when releasing on github (#7007) --- .gitattributes | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000000..e577ab3c1169 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +.gitattributes export-ignore +R-package/* export-ignore From e1d97cc8f55a91d6df103a29158c7935ca6e3164 Mon Sep 17 00:00:00 2001 From: Przemyslaw Tredak Date: Thu, 13 Jul 2017 10:36:29 -0700 Subject: [PATCH 207/834] Use pinned memory in benchmark mode (#7028) --- example/image-classification/common/data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/example/image-classification/common/data.py b/example/image-classification/common/data.py index e9bb4abc0814..7a609b77474e 100755 --- a/example/image-classification/common/data.py +++ b/example/image-classification/common/data.py @@ -65,8 +65,8 @@ def __init__(self, num_classes, data_shape, max_iter, dtype): self.dtype = dtype label = np.random.randint(0, num_classes, [self.batch_size,]) data = np.random.uniform(-1, 1, data_shape) - self.data = mx.nd.array(data, dtype=self.dtype) - self.label = mx.nd.array(label, dtype=self.dtype) + self.data = mx.nd.array(data, dtype=self.dtype, ctx=mx.Context('cpu_pinned', 0)) + self.label = mx.nd.array(label, dtype=self.dtype, ctx=mx.Context('cpu_pinned', 0)) def __iter__(self): return self @property From 3a7eb55e12b2db61ed381af74217f3d7cf7b1552 Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Thu, 13 Jul 2017 10:37:53 -0700 Subject: [PATCH 208/834] super-resolution fix (#7016) * super-resolution fix * fix --- example/gluon/data.py | 14 ++++---- example/gluon/super_resolution.py | 53 ++++++++++++++----------------- 2 files changed, 32 insertions(+), 35 deletions(-) diff --git a/example/gluon/data.py b/example/gluon/data.py index fa69863fae4a..80a50bdce66b 100644 --- a/example/gluon/data.py +++ b/example/gluon/data.py @@ -95,15 +95,17 @@ def __init__(self, path, data_shape, label_shape, batch_size=64, flag=0, input_a random.shuffle(self.filenames) def next(self): - if self.count + self.batch_size < len(self.filenames): + from PIL import Image + if self.count + self.batch_size <= len(self.filenames): data = [] label = [] for i in range(self.batch_size): fn = self.filenames[self.count] self.count += 1 - with open(fn, 'rb') as f: - binary_image = f.read() - image = mx.img.imdecode(binary_image, flag=self.flag) + image = Image.open(fn).convert('YCbCr').split()[0] + if image.size[0] > image.size[1]: + image = image.transpose(Image.TRANSPOSE) + image = mx.nd.expand_dims(mx.nd.array(image), axis=2) target = image.copy() for aug in self.input_aug: image = aug(image)[0] @@ -114,8 +116,8 @@ def next(self): data = mx.nd.concat(*[mx.nd.expand_dims(d, axis=0) for d in data], dim=0) label = mx.nd.concat(*[mx.nd.expand_dims(d, axis=0) for d in label], dim=0) - data = [mx.nd.transpose(data, axes=(0, 3, 1, 2)).astype('float32')] - label = [mx.nd.transpose(label, axes=(0, 3, 1, 2)).astype('float32')] + data = [mx.nd.transpose(data, axes=(0, 3, 1, 2)).astype('float32')/255] + label = [mx.nd.transpose(label, axes=(0, 3, 1, 2)).astype('float32')/255] return mx.io.DataBatch(data=data, label=label) else: diff --git a/example/gluon/super_resolution.py b/example/gluon/super_resolution.py index 6d950bcbacc4..521c17aeb71d 100644 --- a/example/gluon/super_resolution.py +++ b/example/gluon/super_resolution.py @@ -2,6 +2,7 @@ import argparse, tarfile import math import os +import numpy as np import mxnet as mx import mxnet.ndarray as F @@ -14,17 +15,15 @@ from data import ImagePairIter -from PIL import Image - - # CLI parser = argparse.ArgumentParser(description='Super-resolution using an efficient sub-pixel convolution neural network.') -parser.add_argument('--upscale_factor', type=int, required=True, help="super resolution upscale factor") -parser.add_argument('--batch_size', type=int, default=64, help='training batch size') -parser.add_argument('--epochs', type=int, default=2, help='number of training epochs') -parser.add_argument('--lr', type=float, default=0.01, help='learning Rate. default is 0.01') -parser.add_argument('--gpus', type=int, default=0, help='number of GPUs to use') +parser.add_argument('--upscale_factor', type=int, default=3, help="super resolution upscale factor. default is 3.") +parser.add_argument('--batch_size', type=int, default=4, help='training batch size, per device. default is 4.') +parser.add_argument('--test_batch_size', type=int, default=100, help='test batch size') +parser.add_argument('--epochs', type=int, default=30, help='number of training epochs') +parser.add_argument('--lr', type=float, default=0.001, help='learning Rate. default is 0.001.') +parser.add_argument('--use-gpu', action='store_true', help='whether to use GPU.') parser.add_argument('--seed', type=int, default=123, help='random seed to use. Default=123') parser.add_argument('--resolve_img', type=str, help='input image to use') opt = parser.parse_args() @@ -32,7 +31,7 @@ print(opt) upscale_factor = opt.upscale_factor -batch_size = opt.batch_size +batch_size, test_batch_size = opt.batch_size, opt.test_batch_size color_flag = 0 # get data @@ -63,17 +62,15 @@ def get_dataset(prefetch=False): ImagePairIter(os.path.join(image_path, "test"), (input_crop_size, input_crop_size), (crop_size, crop_size), - batch_size, color_flag, + test_batch_size, color_flag, input_transform, target_transform)) - if prefetch: - return [PrefetchingIter(i) for i in iters] - else: - return iters + + return [PrefetchingIter(i) for i in iters] if prefetch else iters train_data, val_data = get_dataset() mx.random.seed(opt.seed) -ctx = [mx.gpu(i) for i in range(opt.gpus)] if opt.gpus > 0 else [mx.cpu()] +ctx = [mx.gpu(0)] if opt.use_gpu else [mx.cpu()] # define model @@ -92,10 +89,10 @@ class SuperResolutionNet(gluon.Block): def __init__(self, upscale_factor): super(SuperResolutionNet, self).__init__() with self.name_scope(): - self.conv1 = nn.Conv2D(64, (5, 5), strides=(1, 1), padding=(2, 2), in_channels=1) - self.conv2 = nn.Conv2D(64, (3, 3), strides=(1, 1), padding=(1, 1), in_channels=64) - self.conv3 = nn.Conv2D(32, (3, 3), strides=(1, 1), padding=(1, 1), in_channels=64) - self.conv4 = nn.Conv2D(upscale_factor ** 2, (3, 3), strides=(1, 1), padding=(1, 1), in_channels=32) + self.conv1 = nn.Conv2D(64, (5, 5), strides=(1, 1), padding=(2, 2)) + self.conv2 = nn.Conv2D(64, (3, 3), strides=(1, 1), padding=(1, 1)) + self.conv3 = nn.Conv2D(32, (3, 3), strides=(1, 1), padding=(1, 1)) + self.conv4 = nn.Conv2D(upscale_factor ** 2, (3, 3), strides=(1, 1), padding=(1, 1)) self.upscale_factor = upscale_factor def forward(self, x): @@ -105,15 +102,14 @@ def forward(self, x): return _rearrange(self.conv4(x), F, self.upscale_factor) net = SuperResolutionNet(upscale_factor) +metric = mx.metric.MSE() def test(ctx): val_data.reset() avg_psnr = 0 - metric = mx.metric.MSE() batches = 0 for batch in val_data: batches += 1 - metric.reset() data = gluon.utils.split_and_load(batch.data[0], ctx_list=ctx, batch_axis=0) label = gluon.utils.split_and_load(batch.label[0], ctx_list=ctx, batch_axis=0) outputs = [] @@ -121,6 +117,7 @@ def test(ctx): outputs.append(net(x)) metric.update(label, outputs) avg_psnr += 10 * math.log10(1/metric.get()[1]) + metric.reset() avg_psnr /= batches print('validation avg psnr: %f'%avg_psnr) @@ -131,7 +128,6 @@ def train(epoch, ctx): net.collect_params().initialize(mx.init.Orthogonal(), ctx=ctx) net.conv4.collect_params().initialize(mx.init.Orthogonal(scale=1), ctx=ctx) trainer = gluon.Trainer(net.collect_params(), 'adam', {'learning_rate': opt.lr}) - metric = mx.metric.MAE() loss = gluon.loss.L2Loss() for i in range(epoch): @@ -151,21 +147,20 @@ def train(epoch, ctx): name, acc = metric.get() metric.reset() - print('training mae at epoch %d: %s=%f'%(i, name, acc)) + print('training mse at epoch %d: %s=%f'%(i, name, acc)) test(ctx) net.collect_params().save('superres.params') def resolve(ctx): + from PIL import Image if isinstance(ctx, list): ctx = [ctx[0]] - net.collect_params().load('superres.params') + net.collect_params().load('superres.params', ctx=ctx) img = Image.open(opt.resolve_img).convert('YCbCr') y, cb, cr = img.split() - data = mx.nd.array(y) - print(data) - out_img_y = net(data).asnumpy() - out_img_y *= 255.0 + data = mx.nd.expand_dims(mx.nd.expand_dims(mx.nd.array(y), axis=0), axis=0) + out_img_y = mx.nd.reshape(net(data), shape=(-3, -2)).asnumpy() out_img_y = out_img_y.clip(0, 255) out_img_y = Image.fromarray(np.uint8(out_img_y[0]), mode='L') @@ -173,7 +168,7 @@ def resolve(ctx): out_img_cr = cr.resize(out_img_y.size, Image.BICUBIC) out_img = Image.merge('YCbCr', [out_img_y, out_img_cb, out_img_cr]).convert('RGB') - out_img.save('resolved.jpg') + out_img.save('resolved.png') if opt.resolve_img: resolve(ctx) From 70ec3ceb82ea9eaa2028dc4e43b46ffc8f56843b Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Thu, 13 Jul 2017 10:54:43 -0700 Subject: [PATCH 209/834] add pearsonr as metric (#6966) --- python/mxnet/metric.py | 94 +++++++++++++++++++++++----- tests/python/unittest/test_metric.py | 1 + 2 files changed, 79 insertions(+), 16 deletions(-) diff --git a/python/mxnet/metric.py b/python/mxnet/metric.py index 299734b38393..4ab36f8f91e8 100644 --- a/python/mxnet/metric.py +++ b/python/mxnet/metric.py @@ -13,16 +13,20 @@ from . import registry -def check_label_shapes(labels, preds, shape=0): - if shape == 0: - label_shape, pred_shape = len(labels), len(preds) - else: - label_shape, pred_shape = labels.shape, preds.shape +def _check_shapes_equal(labels, preds): + label_shape, pred_shape = labels.shape, preds.shape if label_shape != pred_shape: raise ValueError("Shape of labels {} does not match shape of " "predictions {}".format(label_shape, pred_shape)) +def _check_lengths_equal(labels, preds): + label_len, pred_len = len(labels), len(preds) + + if label_len != pred_len: + raise ValueError("Length of labels {} does not match length of " + "predictions {}".format(label_len, pred_len)) + class EvalMetric(object): """Base class for all evaluation metrics. @@ -368,7 +372,7 @@ def update(self, labels, preds): preds : list of `NDArray` Predicted values. """ - check_label_shapes(labels, preds) + _check_lengths_equal(labels, preds) for label, pred_label in zip(labels, preds): if pred_label.shape != label.shape: @@ -376,7 +380,7 @@ def update(self, labels, preds): pred_label = pred_label.asnumpy().astype('int32') label = label.asnumpy().astype('int32') - check_label_shapes(label, pred_label) + _check_shapes_equal(label, pred_label) self.sum_metric += (pred_label.flat == label.flat).sum() self.num_inst += len(pred_label.flat) @@ -438,13 +442,13 @@ def update(self, labels, preds): preds : list of `NDArray` Predicted values. """ - check_label_shapes(labels, preds) + _check_lengths_equal(labels, preds) for label, pred_label in zip(labels, preds): assert(len(pred_label.shape) <= 2), 'Predictions should be no more than 2 dims' pred_label = numpy.argsort(pred_label.asnumpy().astype('float32'), axis=1) label = label.asnumpy().astype('int32') - check_label_shapes(label, pred_label) + _check_shapes_equal(label, pred_label) num_samples = pred_label.shape[0] num_dims = len(pred_label.shape) if num_dims == 1: @@ -512,14 +516,14 @@ def update(self, labels, preds): preds : list of `NDArray` Predicted values. """ - check_label_shapes(labels, preds) + _check_lengths_equal(labels, preds) for label, pred in zip(labels, preds): pred = pred.asnumpy() label = label.asnumpy().astype('int32') pred_label = numpy.argmax(pred, axis=1) - check_label_shapes(label, pred) + _check_shapes_equal(label, pred) if len(numpy.unique(label)) > 2: raise ValueError("F1 currently only supports binary classification.") @@ -703,9 +707,10 @@ def update(self, labels, preds): preds : list of `NDArray` Predicted values. """ - check_label_shapes(labels, preds) + _check_lengths_equal(labels, preds) for label, pred in zip(labels, preds): + _check_shapes_equal(label, pred) label = label.asnumpy() pred = pred.asnumpy() @@ -761,9 +766,10 @@ def update(self, labels, preds): preds : list of `NDArray` Predicted values. """ - check_label_shapes(labels, preds) + _check_lengths_equal(labels, preds) for label, pred in zip(labels, preds): + _check_shapes_equal(label, pred) label = label.asnumpy() pred = pred.asnumpy() @@ -819,9 +825,10 @@ def update(self, labels, preds): preds : list of `NDArray` Predicted values. """ - check_label_shapes(labels, preds) + _check_lengths_equal(labels, preds) for label, pred in zip(labels, preds): + _check_shapes_equal(label, pred) label = label.asnumpy() pred = pred.asnumpy() @@ -883,9 +890,10 @@ def update(self, labels, preds): preds : list of `NDArray` Predicted values. """ - check_label_shapes(labels, preds) + _check_lengths_equal(labels, preds) for label, pred in zip(labels, preds): + _check_shapes_equal(label, pred) label = label.asnumpy() pred = pred.asnumpy() @@ -896,6 +904,60 @@ def update(self, labels, preds): self.sum_metric += (-numpy.log(prob + self.eps)).sum() self.num_inst += label.shape[0] +@register +@alias('pearsonr') +class PearsonCorrelation(EvalMetric): + """Computes Pearson correlation. + + The pearson correlation is given by + + .. math:: + \\frac{cov(y, \\hat{y})}{\\sigma{y}\\sigma{\\hat{y}}} + + Parameters + ---------- + name : str + Name of this metric instance for display. + output_names : list of str, or None + Name of predictions that should be used when updating with update_dict. + By default include all predictions. + label_names : list of str, or None + Name of labels that should be used when updating with update_dict. + By default include all labels. + + Examples + -------- + >>> predicts = [mx.nd.array([[0.3, 0.7], [0, 1.], [0.4, 0.6]])] + >>> labels = [mx.nd.array([[1, 0], [0, 1], [0, 1]])] + >>> pr = mx.metric.PearsonCorrelation() + >>> pr.update(labels, predicts) + >>> print pr.get() + ('pearson-correlation', 0.42163704544016178) + """ + def __init__(self, name='pearsonr', + output_names=None, label_names=None): + super(PearsonCorrelation, self).__init__( + name, output_names=output_names, label_names=label_names) + + def update(self, labels, preds): + """Updates the internal evaluation result. + + Parameters + ---------- + labels : list of `NDArray` + The labels of the data. + + preds : list of `NDArray` + Predicted values. + """ + _check_lengths_equal(labels, preds) + for label, pred in zip(labels, preds): + _check_shapes_equal(label, pred) + label = label.asnumpy() + pred = pred.asnumpy() + self.sum_metric += numpy.corrcoef(pred.ravel(), label.ravel())[0, 1] + self.num_inst += 1 + @register class Loss(EvalMetric): @@ -1002,7 +1064,7 @@ def update(self, labels, preds): Predicted values. """ if not self._allow_extra_outputs: - check_label_shapes(labels, preds) + _check_lengths_equal(labels, preds) for pred, label in zip(preds, labels): label = label.asnumpy() diff --git a/tests/python/unittest/test_metric.py b/tests/python/unittest/test_metric.py index 98740b05ee32..54b58b2b4d73 100644 --- a/tests/python/unittest/test_metric.py +++ b/tests/python/unittest/test_metric.py @@ -13,6 +13,7 @@ def test_metrics(): check_metric('acc', axis=0) check_metric('f1') check_metric('perplexity', -1) + check_metric('pearsonr') composite = mx.metric.create(['acc', 'f1']) check_metric(composite) From dad83816511ec04157d2743ee1f220e74a6b0c27 Mon Sep 17 00:00:00 2001 From: Eric Junyuan Xie Date: Thu, 13 Jul 2017 13:31:49 -0700 Subject: [PATCH 210/834] Some fixes for gluon (#7013) * fix * fix * fix * fix * more fixes & add ndarray slicing * fix * fix * fix * fix * fix --- docs/api/python/gluon.md | 2 +- example/gluon/resnet.py | 8 +- python/mxnet/gluon/block.py | 20 ++++- python/mxnet/gluon/nn/basic_layers.py | 58 +++++++++++--- python/mxnet/gluon/nn/conv_layers.py | 70 +++++++++-------- python/mxnet/gluon/parameter.py | 74 +++++++++++------- python/mxnet/gluon/rnn/rnn_cell.py | 60 +++++++++------ python/mxnet/gluon/rnn/rnn_layer.py | 102 ++++++++++++++++++++----- python/mxnet/gluon/trainer.py | 22 +++++- python/mxnet/ndarray.py | 43 +++++++++-- python/mxnet/operator.py | 23 +++--- src/c_api/c_api_ndarray.cc | 5 +- src/operator/custom/custom.cc | 10 +-- src/operator/tensor/matrix_op-inl.h | 6 +- tests/python/unittest/test_ndarray.py | 9 +++ tests/python/unittest/test_nn.py | 14 +++- tests/python/unittest/test_operator.py | 4 +- 17 files changed, 374 insertions(+), 156 deletions(-) diff --git a/docs/api/python/gluon.md b/docs/api/python/gluon.md index 3149deb50d53..4748a1a64bfb 100644 --- a/docs/api/python/gluon.md +++ b/docs/api/python/gluon.md @@ -63,7 +63,7 @@ in Python and then deploy with symbolic graph in C++ and Scala. .. automethod:: __call__ .. autoclass:: mxnet.gluon.nn.Sequential :members: -.. autoclass:: mxnet.gluon.nn.HSequential +.. autoclass:: mxnet.gluon.nn.HybridSequential :members: ``` diff --git a/example/gluon/resnet.py b/example/gluon/resnet.py index 06ec21dfd224..44517eaf15f2 100644 --- a/example/gluon/resnet.py +++ b/example/gluon/resnet.py @@ -134,7 +134,7 @@ def __init__(self, block, classes, layers, filters, thumbnail=False, **kwargs): self.bn0 = nn.BatchNorm(in_channels=filters[0]) self.pool0 = nn.MaxPool2D(3, 2, 1) - self.body = nn.HSequential() + self.body = nn.HybridSequential() in_channels = filters[0] for i in range(len(layers)): stride = 1 if i == 0 else 2 @@ -146,7 +146,7 @@ def __init__(self, block, classes, layers, filters, thumbnail=False, **kwargs): self.dense1 = nn.Dense(classes, in_units=filters[-1]) def _make_layer(self, block, layers, filters, stride, in_channels=0): - layer = nn.HSequential() + layer = nn.HybridSequential() layer.add(block(filters, stride, True, in_channels=in_channels)) for i in range(layers-1): layer.add(block(filters, 1, False, in_channels=filters)) @@ -248,7 +248,7 @@ def __init__(self, block, classes, layers, filters, thumbnail=False, **kwargs): self.bn0 = nn.BatchNorm(in_channels=filters[0]) self.pool0 = nn.MaxPool2D(3, 2, 1) - self.body = nn.HSequential() + self.body = nn.HybridSequential() in_channels = filters[0] for i in range(len(layers)): stride = 1 if i == 0 else 2 @@ -261,7 +261,7 @@ def __init__(self, block, classes, layers, filters, thumbnail=False, **kwargs): self.dense1 = nn.Dense(classes, in_units=in_channels) def _make_layer(self, block, layers, filters, stride, in_channels=0): - layer = nn.HSequential() + layer = nn.HybridSequential() layer.add(block(filters, stride, True, in_channels=in_channels)) for i in range(layers-1): layer.add(block(filters, 1, False, in_channels=filters)) diff --git a/python/mxnet/gluon/block.py b/python/mxnet/gluon/block.py index 5d13aa09029d..bd072e7f60f2 100644 --- a/python/mxnet/gluon/block.py +++ b/python/mxnet/gluon/block.py @@ -2,7 +2,7 @@ # pylint: disable= arguments-differ """Base container class for all neural network models.""" -from .. import symbol, ndarray +from .. import symbol, ndarray, initializer from ..symbol import Symbol from ..ndarray import NDArray from .. import name as _name @@ -99,7 +99,7 @@ class Block(object): class Model(Block): def __init__(self, **kwargs): - super(Net, self).__init__(**kwargs) + super(Model, self).__init__(**kwargs) # use name_scope to give child Blocks appropriate names. # It also allows sharing Parameters between Blocks recursively. with self.name_scope(): @@ -110,6 +110,11 @@ def forward(self, x): x = F.relu(self.dense0(x)) return F.relu(self.dense1(x)) + model = Model() + model.initialize(ctx=mx.cpu(0)) + model(F.zeros((10, 10), ctx=mx.cpu(0))) + + Child `Block`s assigned this way will be registered and `collect_params` will collect their Parameters recursively. @@ -124,7 +129,7 @@ def forward(self, x): if you want `dense1` to share `dense0`'s weights, you can do:: dense0 = nn.Dense(20) - dense1 = nn.Dense(20, params=dense1.collect_params()) + dense1 = nn.Dense(20, params=dense0.collect_params()) """ def __init__(self, prefix=None, params=None): self._prefix, self._params = _BlockScope.create(prefix, params, self._alias()) @@ -181,6 +186,13 @@ def register_child(self, block): attributes will be registered automatically.""" self._children.append(block) + def initialize(self, init=initializer.Uniform(), ctx=None, verbose=False): + """Initialize `Parameter`s of this Block and its children. + + Equivalent to `block.collect_params().initialize(...)` + """ + self.collect_params().initialize(init, ctx, verbose) + def hybridize(self, active=True): """Activates or deactivates `HybridBlock`s recursively. Has no effect on non-hybrid children. @@ -250,7 +262,7 @@ def register_child(self, block): if isinstance(block, Sequential): raise ValueError( "Children of HybridBlock must also be HybridBlock. " \ - "Please use HSequential instead of Sequential.") + "Please use HybridSequential instead of Sequential.") raise ValueError( "Children of HybridBlock must also be HybridBlock, " \ "but %s has type %s."%(str(block), str(type(block)))) diff --git a/python/mxnet/gluon/nn/basic_layers.py b/python/mxnet/gluon/nn/basic_layers.py index b1234f9d9e16..3bd590f1d5de 100644 --- a/python/mxnet/gluon/nn/basic_layers.py +++ b/python/mxnet/gluon/nn/basic_layers.py @@ -29,7 +29,7 @@ def forward(self, x): return x -class HSequential(HybridBlock): +class HybridSequential(HybridBlock): """Stack `HybridBlock`s sequentially. Example:: @@ -41,7 +41,7 @@ class HSequential(HybridBlock): net.add(Dense(20)) """ def __init__(self, prefix=None, params=None): - super(HSequential, self).__init__(prefix=prefix, params=params) + super(HybridSequential, self).__init__(prefix=prefix, params=params) def add(self, block): """Add block on top of the stack.""" @@ -97,16 +97,18 @@ class Dense(HybridBlock): the output would have shape `(batch_size, units)`. """ def __init__(self, units, activation=None, use_bias=True, - weight_initializer=None, bias_initializer=None, + weight_initializer=None, bias_initializer='zeros', in_units=0, **kwargs): super(Dense, self).__init__(**kwargs) with self.name_scope(): self._units = units self.weight = self.params.get('weight', shape=(units, in_units), - init=weight_initializer) + init=weight_initializer, + allow_deferred_init=True) if use_bias: self.bias = self.params.get('bias', shape=(units,), - init=bias_initializer) + init=bias_initializer, + allow_deferred_init=True) else: self.bias = None if activation is not None: @@ -133,6 +135,7 @@ class Activation(HybridBlock): name of activation function to use. See :func:`~mxnet.ndarray.Activation` for available choices. + Input shape: Arbitrary. @@ -210,6 +213,13 @@ class BatchNorm(HybridBlock): Number of channels (feature maps) in input data. If not specified, initialization will be defered to the first time `forward` is called and `in_channels` will be inferred from the shape of input data. + + + Input shape: + Arbitrary. + + Output shape: + Same shape as input. """ def __init__(self, axis=1, momentum=0.9, epsilon=1e-3, center=True, scale=True, beta_initializer='zeros', gamma_initializer='ones', @@ -220,15 +230,19 @@ def __init__(self, axis=1, momentum=0.9, epsilon=1e-3, center=True, scale=True, 'fix_gamma': not center} self.gamma = self.params.get('gamma', grad_req='write' if scale else 'null', - shape=(in_channels,), init=gamma_initializer) + shape=(in_channels,), init=gamma_initializer, + allow_deferred_init=True) self.beta = self.params.get('beta', grad_req='write' if center else 'null', - shape=(in_channels,), init=beta_initializer) + shape=(in_channels,), init=beta_initializer, + allow_deferred_init=True) self.running_mean = self.params.get('running_mean', grad_req='null', shape=(in_channels,), - init=running_mean_initializer) + init=running_mean_initializer, + allow_deferred_init=True) self.running_var = self.params.get('running_var', grad_req='null', shape=(in_channels,), - init=running_variance_initializer) + init=running_variance_initializer, + allow_deferred_init=True) def hybrid_forward(self, F, x, gamma, beta, running_mean, running_var): return F.BatchNorm(x, gamma, beta, running_mean, running_var, **self._kwargs) @@ -246,6 +260,13 @@ class LeakyReLU(HybridBlock): ---------- alpha : float slope coefficient for the negative half axis. Must be >= 0. + + + Input shape: + Arbitrary. + + Output shape: + Same shape as input. """ def __init__(self, alpha, **kwargs): super(LeakyReLU, self).__init__(**kwargs) @@ -284,7 +305,24 @@ def __init__(self, input_dim, output_dim, dtype='float32', self._kwargs = {'input_dim': input_dim, 'output_dim': output_dim, 'dtype': dtype} self.weight = self.params.get('weight', shape=(input_dim, output_dim), - init=weight_initializer) + init=weight_initializer, + allow_deferred_init=True) def hybrid_forward(self, F, x, weight): return F.Embedding(x, weight, **self._kwargs) + + +class Flatten(HybridBlock): + """Flattens the input to two dimensional. + + Input shape: + Arbitrary shape `(N, a, b, c, ...)` + + Output shape: + 2D tensor with shape: `(N, a*b*c...)` + """ + def __init__(self, **kwargs): + super(Flatten, self).__init__(**kwargs) + + def hybrid_forward(self, F, x): + return x.reshape((0, -1)) diff --git a/python/mxnet/gluon/nn/conv_layers.py b/python/mxnet/gluon/nn/conv_layers.py index 3449a160cee8..86ae302f9e31 100644 --- a/python/mxnet/gluon/nn/conv_layers.py +++ b/python/mxnet/gluon/nn/conv_layers.py @@ -4,6 +4,8 @@ from ..block import HybridBlock from ... import symbol from ...base import numeric_types +from .basic_layers import Activation + def _infer_weight_shape(op_name, data_shape, kwargs): op = getattr(symbol, op_name) @@ -62,7 +64,7 @@ class _Conv(HybridBlock): """ def __init__(self, channels, kernel_size, strides, padding, dilation, groups, layout, in_channels=0, activation=None, use_bias=True, - weight_initializer=None, bias_initializer=None, + weight_initializer=None, bias_initializer='zeros', op_name='Convolution', prefix=None, params=None, **kwargs): super(_Conv, self).__init__(prefix=prefix, params=params) with self.name_scope(): @@ -86,10 +88,12 @@ def __init__(self, channels, kernel_size, strides, padding, dilation, dshape[layout.find('C')] = in_channels wshapes = _infer_weight_shape(op_name, dshape, self._kwargs) self.weight = self.params.get('weight', shape=wshapes[1], - init=weight_initializer) + init=weight_initializer, + allow_deferred_init=True) if use_bias: self.bias = self.params.get('bias', shape=wshapes[2], - init=bias_initializer) + init=bias_initializer, + allow_deferred_init=True) else: self.bias = None @@ -163,11 +167,11 @@ class Conv1D(_Conv): Initializer for the bias vector. - Input Shape: + Input shape: This depends on the `layout` parameter. Input is 3D array of shape (batch_size, in_channels, width) if `layout` is `NCW`. - Output Shape: + Output shape: This depends on the `layout` parameter. Output is 3D array of shape (batch_size, channels, out_width) if `layout` is `NCW`. out_width is calculated as:: @@ -176,7 +180,7 @@ class Conv1D(_Conv): """ def __init__(self, channels, kernel_size, strides=1, padding=0, dilation=1, groups=1, layout='NCW', activation=None, use_bias=True, - weight_initializer=None, bias_initializer=None, + weight_initializer=None, bias_initializer='zeros', in_channels=0, **kwargs): if isinstance(kernel_size, numeric_types): kernel_size = (kernel_size,) @@ -240,11 +244,11 @@ class Conv2D(_Conv): Initializer for the bias vector. - Input Shape: + Input shape: This depends on the `layout` parameter. Input is 4D array of shape (batch_size, in_channels, height, width) if `layout` is `NCHW`. - Output Shape: + Output shape: This depends on the `layout` parameter. Output is 4D array of shape (batch_size, channels, out_height, out_width) if `layout` is `NCHW`. @@ -256,7 +260,7 @@ class Conv2D(_Conv): def __init__(self, channels, kernel_size, strides=(1, 1), padding=(0, 0), dilation=(1, 1), groups=1, layout='NCHW', activation=None, use_bias=True, weight_initializer=None, - bias_initializer=None, in_channels=0, **kwargs): + bias_initializer='zeros', in_channels=0, **kwargs): if isinstance(kernel_size, numeric_types): kernel_size = (kernel_size,)*2 assert len(kernel_size) == 2, "kernel_size must be a number or a list of 2 ints" @@ -319,11 +323,11 @@ class Conv3D(_Conv): Initializer for the bias vector. - Input Shape: + Input shape: This depends on the `layout` parameter. Input is 5D array of shape (batch_size, in_channels, depth, height, width) if `layout` is `NCDHW`. - Output Shape: + Output shape: This depends on the `layout` parameter. Output is 5D array of shape (batch_size, channels, out_depth, out_height, out_width) if `layout` is `NCDHW`. @@ -336,7 +340,7 @@ class Conv3D(_Conv): """ def __init__(self, channels, kernel_size, strides=(1, 1, 1), padding=(0, 0, 0), dilation=(1, 1, 1), groups=1, layout='NCDHW', activation=None, - use_bias=True, weight_initializer=None, bias_initializer=None, + use_bias=True, weight_initializer=None, bias_initializer='zeros', in_channels=0, **kwargs): if isinstance(kernel_size, numeric_types): kernel_size = (kernel_size,)*3 @@ -400,11 +404,11 @@ class Conv1DTranspose(_Conv): Initializer for the bias vector. - Input Shape: + Input shape: This depends on the `layout` parameter. Input is 3D array of shape (batch_size, in_channels, width) if `layout` is `NCW`. - Output Shape: + Output shape: This depends on the `layout` parameter. Output is 3D array of shape (batch_size, channels, out_width) if `layout` is `NCW`. @@ -414,7 +418,7 @@ class Conv1DTranspose(_Conv): """ def __init__(self, channels, kernel_size, strides=1, padding=0, output_padding=0, dilation=1, groups=1, layout='NCW', activation=None, use_bias=True, - weight_initializer=None, bias_initializer=None, + weight_initializer=None, bias_initializer='zeros', in_channels=0, **kwargs): if isinstance(kernel_size, numeric_types): kernel_size = (kernel_size,) @@ -484,11 +488,11 @@ class Conv2DTranspose(_Conv): Initializer for the bias vector. - Input Shape: + Input shape: This depends on the `layout` parameter. Input is 4D array of shape (batch_size, in_channels, height, width) if `layout` is `NCHW`. - Output Shape: + Output shape: This depends on the `layout` parameter. Output is 4D array of shape (batch_size, channels, out_height, out_width) if `layout` is `NCHW`. @@ -500,7 +504,7 @@ class Conv2DTranspose(_Conv): def __init__(self, channels, kernel_size, strides=(1, 1), padding=(0, 0), output_padding=(0, 0), dilation=(1, 1), groups=1, layout='NCHW', activation=None, use_bias=True, weight_initializer=None, - bias_initializer=None, in_channels=0, **kwargs): + bias_initializer='zeros', in_channels=0, **kwargs): if isinstance(kernel_size, numeric_types): kernel_size = (kernel_size,)*2 if isinstance(output_padding, numeric_types): @@ -569,11 +573,11 @@ class Conv3DTranspose(_Conv): Initializer for the bias vector. - Input Shape: + Input shape: This depends on the `layout` parameter. Input is 5D array of shape (batch_size, in_channels, depth, height, width) if `layout` is `NCDHW`. - Output Shape: + Output shape: This depends on the `layout` parameter. Output is 5D array of shape (batch_size, channels, out_depth, out_height, out_width) if `layout` is `NCDHW`. out_depth, out_height and out_width are calculated as:: @@ -585,7 +589,7 @@ class Conv3DTranspose(_Conv): def __init__(self, channels, kernel_size, strides=(1, 1, 1), padding=(0, 0, 0), output_padding=(0, 0, 0), dilation=(1, 1, 1), groups=1, layout='NCDHW', activation=None, use_bias=True, weight_initializer=None, - bias_initializer=None, in_channels=0, **kwargs): + bias_initializer='zeros', in_channels=0, **kwargs): if isinstance(kernel_size, numeric_types): kernel_size = (kernel_size,)*3 if isinstance(output_padding, numeric_types): @@ -640,11 +644,11 @@ class MaxPool1D(_Pooling): When True, will use ceil instead of floor to compute the output shape. - Input Shape: + Input shape: This depends on the `layout` parameter. Input is 3D array of shape (batch_size, channels, width) if `layout` is `NCW`. - Output Shape: + Output shape: This depends on the `layout` parameter. Output is 3D array of shape (batch_size, channels, out_width) if `layout` is `NCW`. @@ -687,11 +691,11 @@ class MaxPool2D(_Pooling): When True, will use ceil instead of floor to compute the output shape. - Input Shape: + Input shape: This depends on the `layout` parameter. Input is 4D array of shape (batch_size, channels, height, width) if `layout` is `NCHW`. - Output Shape: + Output shape: This depends on the `layout` parameter. Output is 4D array of shape (batch_size, channels, out_height, out_width) if `layout` is `NCHW`. @@ -736,11 +740,11 @@ class MaxPool3D(_Pooling): When True, will use ceil instead of floor to compute the output shape. - Input Shape: + Input shape: This depends on the `layout` parameter. Input is 5D array of shape (batch_size, channels, depth, height, width) if `layout` is `NCDHW`. - Output Shape: + Output shape: This depends on the `layout` parameter. Output is 5D array of shape (batch_size, channels, out_depth, out_height, out_width) if `layout` is `NCDHW`. @@ -785,11 +789,11 @@ class AvgPool1D(_Pooling): When True, will use ceil instead of floor to compute the output shape. - Input Shape: + Input shape: This depends on the `layout` parameter. Input is 3D array of shape (batch_size, channels, width) if `layout` is `NCW`. - Output Shape: + Output shape: This depends on the `layout` parameter. Output is 3D array of shape (batch_size, channels, out_width) if `layout` is `NCW`. @@ -831,11 +835,11 @@ class AvgPool2D(_Pooling): When True, will use ceil instead of floor to compute the output shape. - Input Shape: + Input shape: This depends on the `layout` parameter. Input is 4D array of shape (batch_size, channels, height, width) if `layout` is `NCHW`. - Output Shape: + Output shape: This depends on the `layout` parameter. Output is 4D array of shape (batch_size, channels, out_height, out_width) if `layout` is `NCHW`. @@ -879,11 +883,11 @@ class AvgPool3D(_Pooling): When True, will use ceil instead of floor to compute the output shape. - Input Shape: + Input shape: This depends on the `layout` parameter. Input is 5D array of shape (batch_size, channels, depth, height, width) if `layout` is `NCDHW`. - Output Shape: + Output shape: This depends on the `layout` parameter. Output is 5D array of shape (batch_size, channels, out_depth, out_height, out_width) if `layout` is `NCDHW`. diff --git a/python/mxnet/gluon/parameter.py b/python/mxnet/gluon/parameter.py index 083db134a65b..af51f399c018 100644 --- a/python/mxnet/gluon/parameter.py +++ b/python/mxnet/gluon/parameter.py @@ -21,21 +21,14 @@ class DeferredInitializationError(MXNetError): class Parameter(object): """A Container holding parameters (weights) of `Block`s. - `Parameter` can be used with both `Symbol` and `NDArray` API. For `Symbol` API, - `Parameter.var()` will return a `Symbol` representing this parameter. It - can then be used for composing networks:: - x = mx.sym.Variable('data') - w = mx.nn.Parameter('fc_weight', init=mx.init.Xavier()) - b = mx.nn.Parameter('fc_bias', init=mx.init.Zero()) - out = mx.sym.FullyConnected(x, w.var(), b.var(), num_hidden=64) - - For `NDArray` API, `Parameter` must be initialized with `Parameter.init`. It - will then hold a copy of the the parameter on each `Context`. If `grad_req` is + `Parameter` holds a copy of the the parameter on each `Context` after + it is initialized with `Parameter.initialize(...)`. If `grad_req` is not `null`, it will also hold a gradient array on each `Context`:: + ctx = mx.gpu(0) x = mx.nd.zeros((16, 100), ctx=ctx) - w = mx.nn.Parameter('fc_weight', shape=(64, 100), init=mx.init.Xavier()) - b = mx.nn.Parameter('fc_bias', shape(64,), init=mx.init.Zero()) + w = mx.gluon.Parameter('fc_weight', shape=(64, 100), init=mx.init.Xavier()) + b = mx.gluon.Parameter('fc_bias', shape=(64,), init=mx.init.Zero()) w.initialize(ctx=ctx) b.initialize(ctx=ctx) out = mx.nd.FullyConnected(x, w.data(ctx), b.data(ctx), num_hidden=64) @@ -66,9 +59,10 @@ class Parameter(object): Weight decay multiplier (L2 regulerizer coefficient). Works similarly to lr_mult. init : Initializer, default None Initializer of this parameter. Will use the global initializer by default. + """ def __init__(self, name, grad_req='write', shape=None, dtype=mx_real_t, - lr_mult=1.0, wd_mult=1.0, init=None): + lr_mult=1.0, wd_mult=1.0, init=None, allow_deferred_init=False): self.name = name self.shape = shape self.dtype = dtype @@ -76,13 +70,13 @@ def __init__(self, name, grad_req='write', shape=None, dtype=mx_real_t, self.wd_mult = wd_mult self.grad_req = grad_req self.init = init + self.allow_deferred_init = allow_deferred_init self._var = None self._data = None self._grad = None self._defered_init = () - def initialize(self, init=None, ctx=None, default_init=initializer.Xavier(), - allow_deferring=True): + def initialize(self, init=None, ctx=None, default_init=initializer.Uniform()): """Intialize parameter and gradient arrays. Only used for `NDArray` API. Parameters @@ -97,21 +91,41 @@ def initialize(self, init=None, ctx=None, default_init=initializer.Xavier(), their values consistent when updating. Normally nn.Trainer does this for you. default_init : Initializer Default initializer is used when both `init` and `Parameter.init` are None. + + Examples + -------- + >>> weight = mx.gluon.Parameter('weight', shape=(2, 2)) + >>> weight.initialize(ctx=mx.cpu(0)) + >>> weight.data() + [[-0.01068833 0.01729892] + [ 0.02042518 -0.01618656]] + + >>> weight.grad() + [[ 0. 0.] + [ 0. 0.]] + + >>> weight.initialize(ctx=[mx.gpu(0), mx.gpu(1)]) + >>> weight.data(mx.gpu(0)) + [[-0.00873779 -0.02834515] + [ 0.05484822 -0.06206018]] + + >>> weight.data(mx.gpu(1)) + [[-0.00873779 -0.02834515] + [ 0.05484822 -0.06206018]] + """ if ctx is None: ctx = [context.current_context()] if isinstance(ctx, Context): ctx = [ctx] - - if self.shape is None or np.prod(self.shape) <= 0: - if allow_deferring: + if init is None: + init = default_init if self.init is None else self.init + if not self.shape or np.prod(self.shape) <= 0: + if self.allow_deferred_init: self._defered_init = (init, ctx, default_init) return raise ValueError("Cannot initialize Parameter %s because it has " \ - "invalid shape: %s. Please specify in_units, " \ - "in_channels, etc for `Block`s or " \ - "set allow_deferring to True to defer initialization " \ - "to first forward pass."%(self.name, str(self.shape))) + "invalid shape: %s."%(self.name, str(self.shape))) self._defered_init = (init, ctx, default_init) self._finish_deferred_init() @@ -161,8 +175,6 @@ def _finish_deferred_init(self): with autograd.pause(): data = ndarray.zeros(shape=self.shape, dtype=self.dtype, ctx=context.cpu()) - if init is None: - init = self.init initializer.create(default_init)( initializer.InitDesc(self.name, {'__init__': init}), data) @@ -222,7 +234,11 @@ def data(self, ctx=None): NDArray on ctx """ if ctx is None: - ctx = context.current_context() + list_ctx = self.list_ctx() + if len(list_ctx) == 1: + ctx = list_ctx[0] + else: + ctx = context.current_context() self._check_initialized(ctx) return self._data[ctx] @@ -241,7 +257,11 @@ def grad(self, ctx=None): Desired context. """ if ctx is None: - ctx = context.current_context() + list_ctx = self.list_ctx() + if len(list_ctx) == 1: + ctx = list_ctx[0] + else: + ctx = context.current_context() self._check_initialized(ctx) if self._grad is None: raise RuntimeError( @@ -371,7 +391,7 @@ def update(self, other): else: self._params[k] = v - def initialize(self, init=initializer.Xavier(), ctx=None, verbose=False): + def initialize(self, init=initializer.Uniform(), ctx=None, verbose=False): """Intialize all Parameters manage by this dictionary to be used for `NDArray` API. Has no effect when using `Symbol` API. diff --git a/python/mxnet/gluon/rnn/rnn_cell.py b/python/mxnet/gluon/rnn/rnn_cell.py index 986d3cce363a..7333892da4f5 100644 --- a/python/mxnet/gluon/rnn/rnn_cell.py +++ b/python/mxnet/gluon/rnn/rnn_cell.py @@ -237,16 +237,16 @@ def forward(self, inputs, states): return super(RecurrentCell, self).forward(inputs, states) -class HRecurrentCell(RecurrentCell, HybridBlock): - """HRecurrentCell supports both Symbol and NDArray forwarding.""" +class HybridRecurrentCell(RecurrentCell, HybridBlock): + """HybridRecurrentCell supports hybridize.""" def __init__(self, prefix=None, params=None): - super(HRecurrentCell, self).__init__(prefix=prefix, params=params) + super(HybridRecurrentCell, self).__init__(prefix=prefix, params=params) def hybrid_forward(self, F, x, *args, **kwargs): raise NotImplementedError -class RNNCell(HRecurrentCell): +class RNNCell(HybridRecurrentCell): """Simple recurrent neural network cell. Parameters @@ -274,20 +274,24 @@ class RNNCell(HRecurrentCell): """ def __init__(self, hidden_size, activation='tanh', i2h_weight_initializer=None, h2h_weight_initializer=None, - i2h_bias_initializer=None, h2h_bias_initializer=None, + i2h_bias_initializer='zeros', h2h_bias_initializer='zeros', input_size=0, prefix=None, params=None): super(RNNCell, self).__init__(prefix=prefix, params=params) self._hidden_size = hidden_size self._activation = activation self._input_size = input_size self.i2h_weight = self.params.get('i2h_weight', shape=(hidden_size, input_size), - init=i2h_weight_initializer) + init=i2h_weight_initializer, + allow_deferred_init=True) self.h2h_weight = self.params.get('h2h_weight', shape=(hidden_size, hidden_size), - init=h2h_weight_initializer) + init=h2h_weight_initializer, + allow_deferred_init=True) self.i2h_bias = self.params.get('i2h_bias', shape=(hidden_size,), - init=i2h_bias_initializer) + init=i2h_bias_initializer, + allow_deferred_init=True) self.h2h_bias = self.params.get('h2h_bias', shape=(hidden_size,), - init=h2h_bias_initializer) + init=h2h_bias_initializer, + allow_deferred_init=True) def state_info(self, batch_size=0): return [{'shape': (batch_size, self._hidden_size), '__layout__': 'NC'}] @@ -310,7 +314,7 @@ def hybrid_forward(self, F, inputs, states, i2h_weight, return output, [output] -class LSTMCell(HRecurrentCell): +class LSTMCell(HybridRecurrentCell): """Long-Short Term Memory (LSTM) network cell. Parameters @@ -338,20 +342,24 @@ class LSTMCell(HRecurrentCell): """ def __init__(self, hidden_size, i2h_weight_initializer=None, h2h_weight_initializer=None, - i2h_bias_initializer='lstmbias', h2h_bias_initializer=None, + i2h_bias_initializer='zeros', h2h_bias_initializer='zeros', input_size=0, prefix=None, params=None): super(LSTMCell, self).__init__(prefix=prefix, params=params) self._hidden_size = hidden_size self._input_size = input_size self.i2h_weight = self.params.get('i2h_weight', shape=(4*hidden_size, input_size), - init=i2h_weight_initializer) + init=i2h_weight_initializer, + allow_deferred_init=True) self.h2h_weight = self.params.get('h2h_weight', shape=(4*hidden_size, hidden_size), - init=h2h_weight_initializer) + init=h2h_weight_initializer, + allow_deferred_init=True) self.i2h_bias = self.params.get('i2h_bias', shape=(4*hidden_size,), - init=i2h_bias_initializer) + init=i2h_bias_initializer, + allow_deferred_init=True) self.h2h_bias = self.params.get('h2h_bias', shape=(4*hidden_size,), - init=h2h_bias_initializer) + init=h2h_bias_initializer, + allow_deferred_init=True) def state_info(self, batch_size=0): return [{'shape': (batch_size, self._hidden_size), '__layout__': 'NC'}, @@ -388,7 +396,7 @@ def hybrid_forward(self, F, inputs, states, i2h_weight, return next_h, [next_h, next_c] -class GRUCell(HRecurrentCell): +class GRUCell(HybridRecurrentCell): """Gated Rectified Unit (GRU) network cell. Note: this is an implementation of the cuDNN version of GRUs (slight modification compared to Cho et al. 2014). @@ -416,18 +424,22 @@ class GRUCell(HRecurrentCell): """ def __init__(self, hidden_size, i2h_weight_initializer=None, h2h_weight_initializer=None, - i2h_bias_initializer=None, h2h_bias_initializer=None, + i2h_bias_initializer='zeros', h2h_bias_initializer='zeros', input_size=0, prefix=None, params=None): super(GRUCell, self).__init__(prefix=prefix, params=params) self._hidden_size = hidden_size self.i2h_weight = self.params.get('i2h_weight', shape=(3*hidden_size, input_size), - init=i2h_weight_initializer) + init=i2h_weight_initializer, + allow_deferred_init=True) self.h2h_weight = self.params.get('h2h_weight', shape=(3*hidden_size, hidden_size), - init=h2h_weight_initializer) + init=h2h_weight_initializer, + allow_deferred_init=True) self.i2h_bias = self.params.get('i2h_bias', shape=(3*hidden_size,), - init=i2h_bias_initializer) + init=i2h_bias_initializer, + allow_deferred_init=True) self.h2h_bias = self.params.get('h2h_bias', shape=(3*hidden_size,), - init=h2h_bias_initializer) + init=h2h_bias_initializer, + allow_deferred_init=True) def state_info(self, batch_size=0): return [{'shape': (batch_size, self._hidden_size), '__layout__': 'NC'}] @@ -527,7 +539,7 @@ def hybrid_forward(self, *args, **kwargs): raise NotImplementedError -class DropoutCell(HRecurrentCell): +class DropoutCell(HybridRecurrentCell): """Apply dropout on input. Parameters @@ -564,7 +576,7 @@ def unroll(self, length, inputs, begin_state=None, layout='NTC', merge_outputs=N merge_outputs=merge_outputs) -class ModifierCell(HRecurrentCell): +class ModifierCell(HybridRecurrentCell): """Base class for modifier cells. A modifier cell takes a base cell, apply modifications on it (e.g. Zoneout), and returns a new cell. @@ -673,7 +685,7 @@ def unroll(self, length, inputs, begin_state=None, layout='NTC', merge_outputs=N return outputs, states -class BidirectionalCell(HRecurrentCell): +class BidirectionalCell(HybridRecurrentCell): """Bidirectional RNN cell. Parameters diff --git a/python/mxnet/gluon/rnn/rnn_layer.py b/python/mxnet/gluon/rnn/rnn_layer.py index 166bbc4bb63b..8a2309841e12 100644 --- a/python/mxnet/gluon/rnn/rnn_layer.py +++ b/python/mxnet/gluon/rnn/rnn_layer.py @@ -44,16 +44,20 @@ def __init__(self, hidden_size, num_layers, layout, for j in (['l', 'r'] if self._dir == 2 else ['l']): self.i2h_weight.append( self.params.get('%s%d_i2h_weight'%(j, i), shape=(ng*nh, ni), - init=i2h_weight_initializer)) + init=i2h_weight_initializer, + allow_deferred_init=True)) self.h2h_weight.append( self.params.get('%s%d_h2h_weight'%(j, i), shape=(ng*nh, nh), - init=h2h_weight_initializer)) + init=h2h_weight_initializer, + allow_deferred_init=True)) self.i2h_bias.append( self.params.get('%s%d_i2h_bias'%(j, i), shape=(ng*nh,), - init=i2h_bias_initializer)) + init=i2h_bias_initializer, + allow_deferred_init=True)) self.h2h_bias.append( self.params.get('%s%d_h2h_bias'%(j, i), shape=(ng*nh,), - init=h2h_bias_initializer)) + init=h2h_bias_initializer, + allow_deferred_init=True)) ni = nh * self._dir self._unfused = self._unfuse() @@ -133,6 +137,14 @@ def begin_state(self, batch_size=0, func=ndarray.zeros, **kwargs): return states def forward(self, inputs, states): + if isinstance(states, ndarray.NDArray): + states = [states] + batch_size = states[0].shape[self._layout.find('N')] + for state, info in zip(states, self.state_info(batch_size)): + if state.shape != info['shape']: + raise ValueError( + "Invalid recurrent state shape. Expecting %s, got %s."%( + str(info['shape']), str(state.shape))) if self._input_size == 0: for i in range(self._dir): self.i2h_weight[i].shape = (self._gates*self._hidden_size, inputs.shape[2]) @@ -226,17 +238,36 @@ class RNN(_RNNLayer): params : ParameterDict or None Shared Parameters for this `Block`. + + Input shapes: + The input shape depends on `layout`. For `layout='TNC'`, the + input has shape `(sequence_length, batch_size, input_size)` + + + Output shape: + The output shape depends on `layout`. For `layout='TNC'`, the + output has shape `(sequence_length, batch_size, num_hidden)`. + If `bidirectional` is True, output shape will instead be + `(sequence_length, batch_size, 2*num_hidden)` + + Recurrent state shape: + The recurrent state's shape is `(num_layers, batch_size, num_hidden)`. + If `bidirectional` is True, state shape will instead be + `(num_layers, batch_size, 2*num_hidden)` + + Examples -------- - >>> rnn = nn.RNN(100, 3) + >>> layer = mx.gluon.rnn.RNN(100, 3) + >>> layer.initialize() >>> input = mx.nd.random_uniform(shape=(5, 3, 10)) - >>> h0 = mx.nd.random_uniform(shape=(2, 3, 100)) - >>> output, hn = rnn(input, h0) + >>> h0 = mx.nd.random_uniform(shape=(3, 3, 100)) + >>> output, hn = layer(input, h0) """ def __init__(self, hidden_size, num_layers=1, activation='relu', layout='TNC', dropout=0, bidirectional=False, i2h_weight_initializer=None, h2h_weight_initializer=None, - i2h_bias_initializer=None, h2h_bias_initializer=None, + i2h_bias_initializer='zeros', h2h_bias_initializer='zeros', input_size=0, **kwargs): super(RNN, self).__init__(hidden_size, num_layers, layout, dropout, bidirectional, input_size, @@ -305,18 +336,37 @@ class LSTM(_RNNLayer): params : ParameterDict or None Shared Parameters for this `Block`. + + Input shapes: + The input shape depends on `layout`. For `layout='TNC'`, the + input has shape `(sequence_length, batch_size, input_size)` + + Output shape: + The output shape depends on `layout`. For `layout='TNC'`, the + output has shape `(sequence_length, batch_size, num_hidden)`. + If `bidirectional` is True, output shape will instead be + `(sequence_length, batch_size, 2*num_hidden)` + + Recurrent state shape: + The recurrent state is a list of two NDArrays. Both has shape + `(num_layers, batch_size, num_hidden)`. + If `bidirectional` is True, state shape will instead be + `(num_layers, batch_size, 2*num_hidden)` + + Examples -------- - >>> rnn = nn.LSTM(100, 3) + >>> layer = mx.gluon.rnn.LSTM(100, 3) + >>> layer.initialize() >>> input = mx.nd.random_uniform(shape=(5, 3, 10)) - >>> h0 = mx.nd.random_uniform(shape=(2, 3, 100)) - >>> c0 = mx.nd.random_uniform(shape=(2, 3, 100)) - >>> output, hn = rnn(input, (h0, c0)) + >>> h0 = mx.nd.random_uniform(shape=(3, 3, 100)) + >>> c0 = mx.nd.random_uniform(shape=(3, 3, 100)) + >>> output, hn = layer(input, [h0, c0]) """ def __init__(self, hidden_size, num_layers=1, layout='TNC', dropout=0, bidirectional=False, input_size=0, i2h_weight_initializer=None, h2h_weight_initializer=None, - i2h_bias_initializer='lstmbias', h2h_bias_initializer=None, + i2h_bias_initializer='zeros', h2h_bias_initializer='zeros', **kwargs): super(LSTM, self).__init__(hidden_size, num_layers, layout, dropout, bidirectional, input_size, @@ -381,17 +431,35 @@ class GRU(_RNNLayer): params : ParameterDict or None Shared Parameters for this `Block`. + + Input shapes: + The input shape depends on `layout`. For `layout='TNC'`, the + input has shape `(sequence_length, batch_size, input_size)` + + Output shape: + The output shape depends on `layout`. For `layout='TNC'`, the + output has shape `(sequence_length, batch_size, num_hidden)`. + If `bidirectional` is True, output shape will instead be + `(sequence_length, batch_size, 2*num_hidden)` + + Recurrent state shape: + The recurrent state's shape is `(num_layers, batch_size, num_hidden)`. + If `bidirectional` is True, state shape will instead be + `(num_layers, batch_size, 2*num_hidden)` + + Examples -------- - >>> rnn = nn.GRU(100, 2) + >>> layer = mx.gluon.rnn.GRU(100, 3) + >>> layer.initialize() >>> input = mx.nd.random_uniform(shape=(5, 3, 10)) - >>> h0 = mx.nd.random_uniform(shape=(2, 3, 100)) - >>> output, hn = rnn(input, h0) + >>> h0 = mx.nd.random_uniform(shape=(3, 3, 100)) + >>> output, hn = layer(input, h0) """ def __init__(self, hidden_size, num_layers=1, layout='TNC', dropout=0, bidirectional=False, input_size=0, i2h_weight_initializer=None, h2h_weight_initializer=None, - i2h_bias_initializer=None, h2h_bias_initializer=None, + i2h_bias_initializer='zeros', h2h_bias_initializer='zeros', **kwargs): super(GRU, self).__init__(hidden_size, num_layers, layout, dropout, bidirectional, input_size, diff --git a/python/mxnet/gluon/trainer.py b/python/mxnet/gluon/trainer.py index 5d79f7342b1a..8f20bd1e698b 100644 --- a/python/mxnet/gluon/trainer.py +++ b/python/mxnet/gluon/trainer.py @@ -4,6 +4,7 @@ from .. import optimizer as opt from ..model import _create_kvstore +from .parameter import ParameterDict, Parameter class Trainer(object): """Applies an Optimizer on a set of Parameters. Trainer should @@ -16,15 +17,28 @@ class Trainer(object): optimizer : str or Optimizer The optimizer to use. optimizer_params : dict - key-word arguments to be passed to Optimizer.create_optimizer. For example, - {'learning_rate': 0.1} + key-word arguments to be passed to optimizer constructor. For example, + `{'learning_rate': 0.1}` kvstore : str or KVStore kvstore type for multi-gpu and distributed training. """ def __init__(self, params, optimizer, optimizer_params, kvstore='device'): - self._params = [param for param in params.values() if param.grad_req != 'null'] - self._scale = optimizer_params.get('rescale_grad', 1.0) + if isinstance(params, (dict, ParameterDict)): + params = list(params.values()) + if not isinstance(params, (list, tuple)): + raise ValueError( + "First argument must be a list or dict of Parameters, " \ + "got %s."%(type(params))) + self._params = [] + for param in params: + if not isinstance(param, Parameter): + raise ValueError( + "First argument must be a list or dict of Parameters, " \ + "got list of %s."%(type(param))) + if param.grad_req != 'null': + self._params.append(param) + self._scale = optimizer_params.get('rescale_grad', 1.0) self._contexts = self._check_contexts() self._init_optimizer(optimizer, optimizer_params) self._kv_initialized = False diff --git a/python/mxnet/ndarray.py b/python/mxnet/ndarray.py index 29f0f769ed63..537228a495fc 100644 --- a/python/mxnet/ndarray.py +++ b/python/mxnet/ndarray.py @@ -122,9 +122,9 @@ class NDArray(NDArrayBase): def __repr__(self): """Returns a string representation of the array.""" shape_info = 'x'.join(['%d' % x for x in self.shape]) - return '%s\n<%s %s @%s>' % (str(self.asnumpy()), - self.__class__.__name__, - shape_info, self.context) + return '\n%s\n<%s %s @%s>' % (str(self.asnumpy()), + self.__class__.__name__, + shape_info, self.context) def __add__(self, other): """x.__add__(y) <=> x+y <=> mx.nd.add(x, y) """ @@ -370,13 +370,18 @@ def __setitem__(self, key, value): assert slice_i < my_shape[i] begin[i] = slice_i end[i] = slice_i + 1 - if isinstance(slice_i, py_slice): + elif isinstance(slice_i, py_slice): # only support continuous slicing - assert slice_i.step is None + assert slice_i.step is None, \ + "NDArray only supports continuous slicing." begin[i] = slice_i.start or 0 end[i] = slice_i.stop or my_shape[i] assert begin[i] < end[i] assert end[i] <= my_shape[i] + else: + raise ValueError( + "NDArray does not support slicing with %s."%( + str(slice_i))) begin = tuple(begin) end = tuple(end) if isinstance(value, NDArray): @@ -434,8 +439,32 @@ def __getitem__(self, key): else: return self if isinstance(key, tuple): - raise ValueError('Multi-dimension indexing is not supported') - + shape = self.shape + oshape = [] + begin = [] + end = [] + assert len(shape) >= len(key), \ + "Slicing dimensions exceeds array dimensions, %d vs %d"%( + len(key), len(shape)) + i = -1 + for i, slice_i in enumerate(key): + if isinstance(slice_i, int): + begin.append(slice_i) + end.append(slice_i+1) + elif isinstance(slice_i, py_slice): + if slice_i.step is not None: + raise ValueError("NDArray only supports continuous slicing.") + begin.append(0 if slice_i.start is None else slice_i.start) + end.append(shape[i] if slice_i.stop is None else slice_i.stop) + oshape.append(end[i] - begin[i]) + else: + raise ValueError( + "NDArray does not support slicing with %s."%( + str(slice_i))) + oshape.extend(shape[i+1:]) + if len(oshape) == 0: + oshape.append(1) + return slice(self, begin, end).reshape(oshape) def _sync_copyfrom(self, source_array): """Performs a synchronized copy from the `source_array` to the current array. diff --git a/python/mxnet/operator.py b/python/mxnet/operator.py index d57ee717fcf6..884775d26317 100644 --- a/python/mxnet/operator.py +++ b/python/mxnet/operator.py @@ -11,7 +11,7 @@ from .base import _LIB, check_call from .base import c_array, c_str, mx_uint, mx_float, ctypes2numpy_shared, NDArrayHandle, py_str -from . import symbol +from . import symbol, context from .ndarray import NDArray, _DTYPE_NP_TO_MX, _DTYPE_MX_TO_NP c_int_p = POINTER(c_int) @@ -448,7 +448,7 @@ class CustomOpProp(object): The default declare_backward_dependency function. Use this value to determine whether this operator needs gradient input. """ - def __init__(self, need_top_grad=False): + def __init__(self, need_top_grad=True): self.need_top_grad_ = need_top_grad def infer_shape(self, in_shape): @@ -734,6 +734,9 @@ def declare_backward_dependency_entry(out_grad, in_data, out_data, num_dep, deps def create_operator_entry(ctx, num_inputs, shapes, ndims, dtypes, ret, _): """C Callback for CustomOpProp::CreateOperator""" try: + ctx = py_str(ctx) + sep = ctx.find('(') + ctx = context.Context(ctx[:sep], int(ctx[sep+1:-1])) ndims = [ndims[i] for i in range(num_inputs)] shapes = [[shapes[i][j] for j in range(ndims[i])] for i in range(num_inputs)] dtypes = [dtypes[i] for i in range(num_inputs)] @@ -753,9 +756,10 @@ def forward_entry(num_ndarray, ndarraies, tags, reqs, is_train, _): NDArrayHandle), writable=False)) reqs = [req_enum[reqs[i]] for i in range(len(tensors[1]))] - op.forward(is_train=is_train, req=reqs, - in_data=tensors[0], out_data=tensors[1], - aux=tensors[4]) + with ctx: + op.forward(is_train=is_train, req=reqs, + in_data=tensors[0], out_data=tensors[1], + aux=tensors[4]) except Exception: print('Error in CustomOp.forward: %s' % traceback.format_exc()) return False @@ -776,10 +780,11 @@ def backward_entry(num_ndarray, ndarraies, tags, reqs, is_train, _): NDArrayHandle), writable=False)) reqs = [req_enum[reqs[i]] for i in range(len(tensors[2]))] - op.backward(req=reqs, - in_data=tensors[0], out_data=tensors[1], - in_grad=tensors[2], out_grad=tensors[3], - aux=tensors[4]) + with ctx: + op.backward(req=reqs, + in_data=tensors[0], out_data=tensors[1], + in_grad=tensors[2], out_grad=tensors[3], + aux=tensors[4]) except Exception: print('Error in CustomOp.backward: %s' % traceback.format_exc()) return False diff --git a/src/c_api/c_api_ndarray.cc b/src/c_api/c_api_ndarray.cc index 63ed6c482c6a..818f263cb3b7 100644 --- a/src/c_api/c_api_ndarray.cc +++ b/src/c_api/c_api_ndarray.cc @@ -110,9 +110,8 @@ void SetContext(Context* p_ctx, CHECK_EQ(ndinputs[i].ctx().dev_mask(), ctx.dev_mask()) << "All inputs must live on the same context. " << "But the first argument is on " - << (ctx.dev_mask() == gpu::kDevMask ? "GPU" : "CPU") - << " while the " << i+1 << "-th argument is on " - << (ndinputs[i].ctx().dev_mask() == gpu::kDevMask ? "GPU" : "CPU"); + << ctx << " while the " << i+1 << "-th argument is on " + << ndinputs[i].ctx(); } } else if (ndoutputs.size() && !ndoutputs[0].is_none()) { ctx = ndoutputs[0].ctx(); diff --git a/src/operator/custom/custom.cc b/src/operator/custom/custom.cc index 1854bb7f05d0..ee420635f824 100644 --- a/src/operator/custom/custom.cc +++ b/src/operator/custom/custom.cc @@ -222,17 +222,13 @@ OpStatePtr CreateState(const NodeAttrs& attrs, Context ctx, } } - std::string str_ctx; - if (ctx.dev_mask() == cpu::kDevMask) { - str_ctx = "cpu"; - } else { - str_ctx = "gpu"; - } + std::ostringstream os; + os << ctx; MXCallbackList *op_info = new MXCallbackList; CHECK(reinterpret_cast( params.info->callbacks[kCustomOpPropCreateOperator])( - str_ctx.c_str(), shapes.size(), shapes.data(), ndims.data(), in_type.data(), + os.str().c_str(), shapes.size(), shapes.data(), ndims.data(), in_type.data(), op_info, params.info->contexts[kCustomOpPropCreateOperator])); CustomParam state = params; diff --git a/src/operator/tensor/matrix_op-inl.h b/src/operator/tensor/matrix_op-inl.h index 88e9d3095e24..10273c0a8c68 100644 --- a/src/operator/tensor/matrix_op-inl.h +++ b/src/operator/tensor/matrix_op-inl.h @@ -722,9 +722,11 @@ inline TShape GetSliceShape(const SliceParam& param, const TShape& dshape) { << "Slicing axis exceeds data dimensions"; CHECK_LE(param.end.ndim(), dshape.ndim()) << "Slicing axis exceeds data dimensions"; + CHECK_EQ(param.begin.ndim(), param.end.ndim()) + << "begin and end must have the same length"; - TShape oshape(dshape.ndim()); - for (index_t i = 0; i < dshape.ndim(); ++i) { + TShape oshape = dshape; + for (index_t i = 0; i < param.begin.ndim(); ++i) { int s = 0, e = dshape[i]; if (e != 0) { if (param.begin[i]) { diff --git a/tests/python/unittest/test_ndarray.py b/tests/python/unittest/test_ndarray.py index 8c58d3b47a69..8b7f8d6d7bf3 100644 --- a/tests/python/unittest/test_ndarray.py +++ b/tests/python/unittest/test_ndarray.py @@ -249,6 +249,14 @@ def test_ndarray_slice(): A[3:8] = A2[3:8] assert same(A[3:8].asnumpy(), A2[3:8]) + shape = (3,4,5,6,7) + A = mx.nd.random_uniform(shape=shape) + A2 = A.asnumpy() + + assert same(A[1,3:4,:,1:5].asnumpy(), A2[1,3:4,:,1:5]) + + assert A[1,2,3,4,5].asscalar() == A2[1,2,3,4,5] + def test_ndarray_crop(): # get crop @@ -653,6 +661,7 @@ def test_output(): mx.nd.full(shape, 2, out=out) assert_almost_equal(out.asnumpy(), ones.asnumpy() * 2) + if __name__ == '__main__': import nose nose.runmodule() diff --git a/tests/python/unittest/test_nn.py b/tests/python/unittest/test_nn.py index 6dc38b4b0ce9..cc1b2dd48553 100644 --- a/tests/python/unittest/test_nn.py +++ b/tests/python/unittest/test_nn.py @@ -54,7 +54,7 @@ def test_basic(): assert len(y.list_arguments()) == 7 # ndarray - model.collect_params().initialize() + model.collect_params().initialize(mx.init.Xavier(magnitude=2.24)) x = model(mx.nd.zeros((32, 10))) assert x.shape == (32, 32) x.wait_to_read() @@ -95,7 +95,7 @@ def test_conv(): layers3d = [ - nn.Conv3D(16, (1, 8, 4), in_channels=4), + nn.Conv3D(16, (1, 8, 4), in_channels=4, activation='relu'), nn.Conv3D(16, (5, 4, 3), in_channels=4), nn.Conv3D(16, (3, 3, 3), groups=2, in_channels=4), nn.Conv3D(16, 4, strides=4, in_channels=4), @@ -263,6 +263,16 @@ def test_split_data(): assert False, "Should have failed" +def test_flatten(): + flatten = nn.Flatten() + x = mx.nd.zeros((3,4,5,6)) + assert flatten(x).shape == (3, 4*5*6) + x = mx.nd.zeros((3,6)) + assert flatten(x).shape == (3, 6) + x = mx.nd.zeros((3,)) + assert flatten(x).shape == (3, 1) + + if __name__ == '__main__': import nose nose.runmodule() diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py index 14593f6ce5b2..e13c3c07f2fd 100644 --- a/tests/python/unittest/test_operator.py +++ b/tests/python/unittest/test_operator.py @@ -1972,7 +1972,7 @@ def check_instance_norm_with_shape(shape, xpu): exec1 = Y.bind(xpu, args = {'X':x, 'G':gamma, 'B':beta}) exec1.forward(is_train=False) out = exec1.outputs[0].asnumpy() - assert_almost_equal(out, np_out, rtol=1e-4) + assert_almost_equal(out, np_out, rtol=1e-4, atol=1e-5) check_numeric_gradient(Y, {'X':x.asnumpy(), 'G':gamma.asnumpy(), 'B':beta.asnumpy()}, numeric_eps=1e-2, rtol=1e-2, atol=1e-2) @@ -2010,7 +2010,7 @@ def check_l2_normalization(in_shape, mode, ctx=default_context(), norm_eps=1e-10 exe = out.simple_bind(ctx=ctx, data=in_data.shape) output = exe.forward(is_train=True, data=in_data) # compare numpy + mxnet - assert_almost_equal(exe.outputs[0].asnumpy(), np_out, rtol=1e-5) + assert_almost_equal(exe.outputs[0].asnumpy(), np_out, rtol=1e-4, atol=1e-5) # check gradient check_numeric_gradient(out, [in_data], numeric_eps=1e-3, rtol=1e-2, atol=1e-3) From 2f53cb01fd96677f4afea2e97e57855a49192ce8 Mon Sep 17 00:00:00 2001 From: Eric Junyuan Xie Date: Thu, 13 Jul 2017 16:10:44 -0700 Subject: [PATCH 211/834] fix mkl (#7030) --- src/operator/activation-inl.h | 2 +- src/operator/activation.cc | 8 ++++---- src/operator/activation.cu | 3 +-- src/operator/pooling.cc | 1 - 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/operator/activation-inl.h b/src/operator/activation-inl.h index 26ed07ce9c70..ead79be95865 100644 --- a/src/operator/activation-inl.h +++ b/src/operator/activation-inl.h @@ -84,7 +84,7 @@ class ActivationOp : public Operator { // Decalre Factory function, used for dispatch specialization template -Operator* CreateOp(ActivationParam type, int dtype); +Operator* CreateOp(ActivationParam type, int dtype, const TShape& dshape); #if DMLC_USE_CXX11 class ActivationProp : public OperatorProperty { diff --git a/src/operator/activation.cc b/src/operator/activation.cc index c8b8c3b5acb2..7ef452f4ec62 100644 --- a/src/operator/activation.cc +++ b/src/operator/activation.cc @@ -15,10 +15,10 @@ namespace mxnet { namespace op { template<> -Operator *CreateOp(ActivationParam param, int dtype) { +Operator *CreateOp(ActivationParam param, int dtype, const TShape& dshape) { Operator *op = NULL; #if MXNET_USE_MKL2017 == 1 - if (param.act_type == activation::kReLU) { + if (param.act_type == activation::kReLU && dshape.ndim() <= 4) { switch (dtype) { case mshadow::kFloat32: return new MKLReluOp(); @@ -54,8 +54,8 @@ Operator *CreateOp(ActivationParam param, int dtype) { // DO_BIND_DISPATCH comes from operator_common.h Operator *ActivationProp::CreateOperatorEx(Context ctx, std::vector *in_shape, - std::vector *in_type) const { - DO_BIND_DISPATCH(CreateOp, param_, (*in_type)[0]); + std::vector *in_type) const { + DO_BIND_DISPATCH(CreateOp, param_, (*in_type)[0], (*in_shape)[0]); } DMLC_REGISTER_PARAMETER(ActivationParam); diff --git a/src/operator/activation.cu b/src/operator/activation.cu index 66e734965784..9a55354de8b0 100644 --- a/src/operator/activation.cu +++ b/src/operator/activation.cu @@ -13,7 +13,7 @@ namespace mxnet { namespace op { template<> -Operator *CreateOp(ActivationParam param, int dtype) { +Operator *CreateOp(ActivationParam param, int dtype, const TShape& dshape) { Operator *op = NULL; // SoftReLU not supported by CUDNN yet if (param.act_type == activation::kSoftReLU) { @@ -48,4 +48,3 @@ Operator *CreateOp(ActivationParam param, int dtype) { } } // namespace op } // namespace mxnet - diff --git a/src/operator/pooling.cc b/src/operator/pooling.cc index f26c2e8b199e..72b17038af9f 100644 --- a/src/operator/pooling.cc +++ b/src/operator/pooling.cc @@ -35,7 +35,6 @@ Operator *CreateOp(PoolingParam param, int dtype) { break; } } - LOG(INFO) << MKLPoolingOp::getName() << " Skip MKL optimization"; #endif #if MXNET_USE_NNPACK == 1 // NNPACK only support max-pooling with kernel = 2, stride = 2, pooling_convention From 903ba3a98642bb88517b0c5b63bf6a1a8b51c04d Mon Sep 17 00:00:00 2001 From: minhtannguyen Date: Thu, 13 Jul 2017 20:23:18 -0700 Subject: [PATCH 212/834] Dcgan fix (#7032) * fix dc-gan * fix dc-gan * fix dc-gan * fix dc-gan * fix dc-gan * fix dc-gan --- example/gluon/dcgan.py | 321 +++++++++++++++++++++++++---------------- 1 file changed, 197 insertions(+), 124 deletions(-) diff --git a/example/gluon/dcgan.py b/example/gluon/dcgan.py index a66c11e7d8ee..17d02e7fbede 100644 --- a/example/gluon/dcgan.py +++ b/example/gluon/dcgan.py @@ -1,131 +1,204 @@ +import matplotlib as mpl +mpl.use('Agg') +from matplotlib import pyplot as plt + import argparse import mxnet as mx from mxnet import gluon from mxnet.gluon import nn from mxnet import autograd from data import cifar10_iterator +import numpy as np +import logging +import cv2 +from datetime import datetime +import os +import time + +def fill_buf(buf, i, img, shape): + n = buf.shape[0]/shape[1] + m = buf.shape[1]/shape[0] + + sx = (i%m)*shape[0] + sy = (i/m)*shape[1] + buf[sy:sy+shape[1], sx:sx+shape[0], :] = img + return None + +def visual(title, X, name): + assert len(X.shape) == 4 + X = X.transpose((0, 2, 3, 1)) + X = np.clip((X - np.min(X))*(255.0/(np.max(X) - np.min(X))), 0, 255).astype(np.uint8) + n = np.ceil(np.sqrt(X.shape[0])) + buff = np.zeros((int(n*X.shape[1]), int(n*X.shape[2]), int(X.shape[3])), dtype=np.uint8) + for i, img in enumerate(X): + fill_buf(buff, i, img, X.shape[1:3]) + buff = cv2.cvtColor(buff, cv2.COLOR_BGR2RGB) + plt.imshow(buff) + plt.title(title) + plt.savefig(name) + return None + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--dataset', type=str, default='cifar10', help='dataset to use. options are cifar10 and imagenet.') + parser.add_argument('--batchSize', type=int, default=64, help='input batch size') + parser.add_argument('--imageSize', type=int, default=64, help='the height / width of the input image to network') + parser.add_argument('--nz', type=int, default=100, help='size of the latent z vector') + parser.add_argument('--ngf', type=int, default=64) + parser.add_argument('--ndf', type=int, default=64) + parser.add_argument('--niter', type=int, default=25, help='number of epochs to train for') + parser.add_argument('--lr', type=float, default=0.0002, help='learning rate, default=0.0002') + parser.add_argument('--beta1', type=float, default=0.5, help='beta1 for adam. default=0.5') + parser.add_argument('--cuda', action='store_true', help='enables cuda') + parser.add_argument('--ngpu', type=int, default=1, help='number of GPUs to use') + parser.add_argument('--netG', default='', help="path to netG (to continue training)") + parser.add_argument('--netD', default='', help="path to netD (to continue training)") + parser.add_argument('--outf', default='./results', help='folder to output images and model checkpoints') + parser.add_argument('--manualSeed', type=int, help='manual seed') + parser.add_argument('--check_point', default=True, help="save results at each epoch or not") + + opt = parser.parse_args() + print(opt) + + logging.basicConfig(level=logging.DEBUG) + ngpu = int(opt.ngpu) + nz = int(opt.nz) + ngf = int(opt.ngf) + ndf = int(opt.ndf) + nc = 3 + ctx = mx.gpu(0) + check_point = bool(opt.check_point) + outf = opt.outf + + if not os.path.exists(outf): + os.makedirs(outf) + + if opt.dataset == 'cifar10': + train_iter, val_iter = cifar10_iterator(opt.batchSize, (3, 64, 64), 64) + + # build the generator + netG = nn.Sequential() + with netG.name_scope(): + # input is Z, going into a convolution + netG.add(nn.Conv2DTranspose(ngf * 8, 4, 1, 0, use_bias=False)) + netG.add(nn.BatchNorm()) + netG.add(nn.Activation('relu')) + # state size. (ngf*8) x 4 x 4 + netG.add(nn.Conv2DTranspose(ngf * 4, 4, 2, 1, use_bias=False)) + netG.add(nn.BatchNorm()) + netG.add(nn.Activation('relu')) + # state size. (ngf*8) x 8 x 8 + netG.add(nn.Conv2DTranspose(ngf * 2, 4, 2, 1, use_bias=False)) + netG.add(nn.BatchNorm()) + netG.add(nn.Activation('relu')) + # state size. (ngf*8) x 16 x 16 + netG.add(nn.Conv2DTranspose(ngf, 4, 2, 1, use_bias=False)) + netG.add(nn.BatchNorm()) + netG.add(nn.Activation('relu')) + # state size. (ngf*8) x 32 x 32 + netG.add(nn.Conv2DTranspose(nc, 4, 2, 1, use_bias=False)) + netG.add(nn.Activation('tanh')) + # state size. (nc) x 64 x 64 + + # build the discriminator + netD = nn.Sequential() + with netD.name_scope(): + # input is (nc) x 64 x 64 + netD.add(nn.Conv2D(ndf, 4, 2, 1, use_bias=False)) + netD.add(nn.LeakyReLU(0.2)) + # state size. (ndf) x 32 x 32 + netD.add(nn.Conv2D(ndf * 2, 4, 2, 1, use_bias=False)) + netD.add(nn.BatchNorm()) + netD.add(nn.LeakyReLU(0.2)) + # state size. (ndf) x 16 x 16 + netD.add(nn.Conv2D(ndf * 4, 4, 2, 1, use_bias=False)) + netD.add(nn.BatchNorm()) + netD.add(nn.LeakyReLU(0.2)) + # state size. (ndf) x 8 x 8 + netD.add(nn.Conv2D(ndf * 8, 4, 2, 1, use_bias=False)) + netD.add(nn.BatchNorm()) + netD.add(nn.LeakyReLU(0.2)) + # state size. (ndf) x 4 x 4 + netD.add(nn.Conv2D(2, 4, 1, 0, use_bias=False)) + + # loss + loss = gluon.loss.SoftmaxCrossEntropyLoss() + + # initialize the generator and the discriminator + netG.collect_params().initialize(mx.init.Normal(0.02), ctx=ctx) + netD.collect_params().initialize(mx.init.Normal(0.02), ctx=ctx) + + # trainer for the generator and the discriminator + trainerG = gluon.Trainer(netG.collect_params(), 'adam', {'learning_rate': opt.lr, 'beta1': opt.beta1}) + trainerD = gluon.Trainer(netD.collect_params(), 'adam', {'learning_rate': opt.lr, 'beta1': opt.beta1}) + + # ============printing============== + real_label = mx.nd.ones((opt.batchSize,), ctx=ctx) + fake_label = mx.nd.zeros((opt.batchSize,), ctx=ctx) + + metric = mx.metric.Accuracy() + print('Training... ') + stamp = datetime.now().strftime('%Y_%m_%d-%H_%M') + + iter = 0 + for epoch in range(opt.niter): + tic = time.time() + train_iter.reset() + btic = time.time() + for batch in train_iter: + ############################ + # (1) Update D network: maximize log(D(x)) + log(1 - D(G(z))) + ########################### + # train with real_t + data = batch.data[0].copyto(ctx) + noise = mx.nd.random_normal(0, 1, shape=(opt.batchSize, nz, 1, 1), ctx=ctx) + + with autograd.record(): + output = netD(data) + output = output.reshape((opt.batchSize, 2)) + errD_real = loss(output, real_label) + metric.update([real_label,], [output,]) + + fake = netG(noise) + output = netD(fake.detach()) + output = output.reshape((opt.batchSize, 2)) + errD_fake = loss(output, fake_label) + errD = errD_real + errD_fake + errD.backward() + metric.update([fake_label,], [output,]) + + trainerD.step(opt.batchSize) + + ############################ + # (2) Update G network: maximize log(D(G(z))) + ########################### + with autograd.record(): + output = netD(fake) + output = output.reshape((opt.batchSize, 2)) + errG = loss(output, real_label) + errG.backward() + + trainerG.step(opt.batchSize) + + name, acc = metric.get() + # logging.info('speed: {} samples/s'.format(opt.batchSize / (time.time() - btic))) + logging.info('discriminator loss = %f, generator loss = %f, binary training acc = %f at iter %d epoch %d' %(mx.nd.mean(errD).asscalar(), mx.nd.mean(errG).asscalar(), acc, iter, epoch)) + if iter % 200 == 0: + visual('gout', fake.asnumpy(), name=os.path.join(outf,'fake_img_iter_%d.png' %iter)) + visual('data', batch.data[0].asnumpy(), name=os.path.join(outf,'real_img_iter_%d.png' %iter)) + + iter = iter + 1 + btic = time.time() + + name, acc = metric.get() + metric.reset() + logging.info('\nbinary training acc at epoch %d: %s=%f' % (epoch, name, acc)) + logging.info('time: %f' % (time.time() - tic)) + + if check_point: + netG.collect_params().save(os.path.join(outf,'generator_epoch_%d.params' %epoch)) + netD.collect_params().save(os.path.join(outf,'discriminator_epoch_%d.params' % epoch)) + + netG.collect_params().save(os.path.join(outf, 'generator.params')) + netD.collect_params().save(os.path.join(outf, 'discriminator.params')) - -parser = argparse.ArgumentParser() -parser.add_argument('--batchSize', type=int, default=64, help='input batch size') -parser.add_argument('--imageSize', type=int, default=64, help='the height / width of the input image to network') -parser.add_argument('--nz', type=int, default=100, help='size of the latent z vector') -parser.add_argument('--ngf', type=int, default=64) -parser.add_argument('--ndf', type=int, default=64) -parser.add_argument('--niter', type=int, default=25, help='number of epochs to train for') -parser.add_argument('--lr', type=float, default=0.0002, help='learning rate, default=0.0002') -parser.add_argument('--beta1', type=float, default=0.5, help='beta1 for adam. default=0.5') -parser.add_argument('--cuda', action='store_true', help='enables cuda') -parser.add_argument('--ngpu', type=int, default=1, help='number of GPUs to use') -parser.add_argument('--netG', default='', help="path to netG (to continue training)") -parser.add_argument('--netD', default='', help="path to netD (to continue training)") -parser.add_argument('--outf', default='.', help='folder to output images and model checkpoints') -parser.add_argument('--manualSeed', type=int, help='manual seed') - -opt = parser.parse_args() -print(opt) - -ngpu = int(opt.ngpu) -nz = int(opt.nz) -ngf = int(opt.ngf) -ndf = int(opt.ndf) -nc = 3 -ctx = mx.gpu(0) - -train_iter, val_iter = cifar10_iterator(opt.batchSize, (3, 64, 64), 64) - - -netG = nn.Sequential() -with netG.name_scope(): - # input is Z, going into a convolution - netG.add(nn.Conv2DTranspose(ngf * 8, 4, 1, 0, use_bias=False)) - netG.add(nn.BatchNorm()) - netG.add(nn.Activation('relu')) - # state size. (ngf*8) x 4 x 4 - netG.add(nn.Conv2DTranspose(ngf * 4, 4, 2, 1, use_bias=False)) - netG.add(nn.BatchNorm()) - netG.add(nn.Activation('relu')) - # state size. (ngf*8) x 8 x 8 - netG.add(nn.Conv2DTranspose(ngf * 2, 4, 2, 1, use_bias=False)) - netG.add(nn.BatchNorm()) - netG.add(nn.Activation('relu')) - # state size. (ngf*8) x 16 x 16 - netG.add(nn.Conv2DTranspose(ngf, 4, 2, 1, use_bias=False)) - netG.add(nn.BatchNorm()) - netG.add(nn.Activation('relu')) - # state size. (ngf*8) x 32 x 32 - netG.add(nn.Conv2DTranspose(nc, 4, 2, 1, use_bias=False)) - netG.add(nn.Activation('tanh')) - # state size. (nc) x 64 x 64 - - -netD = nn.Sequential() -with netD.name_scope(): - # input is (nc) x 64 x 64 - netD.add(nn.Conv2D(ndf, 4, 2, 1, use_bias=False)) - netD.add(nn.LeakyReLU(0.2)) - # state size. (ndf) x 32 x 32 - netD.add(nn.Conv2D(ndf * 2, 4, 2, 1, use_bias=False)) - netD.add(nn.BatchNorm()) - netD.add(nn.LeakyReLU(0.2)) - # state size. (ndf) x 16 x 16 - netD.add(nn.Conv2D(ndf * 4, 4, 2, 1, use_bias=False)) - netD.add(nn.BatchNorm()) - netD.add(nn.LeakyReLU(0.2)) - # state size. (ndf) x 8 x 8 - netD.add(nn.Conv2D(ndf * 8, 4, 2, 1, use_bias=False)) - netD.add(nn.BatchNorm()) - netD.add(nn.LeakyReLU(0.2)) - # state size. (ndf) x 4 x 4 - netD.add(nn.Conv2D(2, 4, 1, 0, use_bias=False)) - # netD.add(nn.Activation('sigmoid')) - - -netG.collect_params().initialize(mx.init.Normal(0.02), ctx=ctx) -netD.collect_params().initialize(mx.init.Normal(0.02), ctx=ctx) - - -trainerG = gluon.Trainer(netG.collect_params(), 'adam', {'learning_rate': opt.lr, 'beta1': opt.beta1}) -trainerD = gluon.Trainer(netD.collect_params(), 'adam', {'learning_rate': opt.lr, 'beta1': opt.beta1}) - - -real_label = mx.nd.ones((opt.batchSize,), ctx=ctx) -fake_label = mx.nd.zeros((opt.batchSize,), ctx=ctx) - -loss = gluon.loss.SoftmaxCrossEntropyLoss() - -for epoch in range(opt.niter): - for batch in train_iter: - ############################ - # (1) Update D network: maximize log(D(x)) + log(1 - D(G(z))) - ########################### - # train with real_t - data = batch.data[0].copyto(ctx) - noise = mx.nd.random_normal(0, 1, shape=(opt.batchSize, nz, 1, 1), ctx=ctx) - - with autograd.record(): - output = netD(data) - output = output.reshape((opt.batchSize, 2)) - errD_real = loss(output, real_label) - - fake = netG(noise) - output = netD(fake.detach()) - output = output.reshape((opt.batchSize, 2)) - errD_fake = loss(output, fake_label) - errD = errD_real + errD_fake - errD.backward() - - trainerD.step(opt.batchSize) - - ############################ - # (2) Update G network: maximize log(D(G(z))) - ########################### - with autograd.record(): - output = netD(fake) - output = output.reshape((opt.batchSize, 2)) - errG = loss(output, real_label) - errG.backward() - - trainerG.step(opt.batchSize) - - print mx.nd.mean(errD).asscalar(), mx.nd.mean(errG).asscalar() From 54e4614590789204caa6ab7c9e91fcc8392ef0a0 Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Thu, 13 Jul 2017 20:25:12 -0700 Subject: [PATCH 213/834] fix (#7033) --- python/mxnet/metric.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python/mxnet/metric.py b/python/mxnet/metric.py index 4ab36f8f91e8..c4356a9d1840 100644 --- a/python/mxnet/metric.py +++ b/python/mxnet/metric.py @@ -380,7 +380,7 @@ def update(self, labels, preds): pred_label = pred_label.asnumpy().astype('int32') label = label.asnumpy().astype('int32') - _check_shapes_equal(label, pred_label) + _check_lengths_equal(label, pred_label) self.sum_metric += (pred_label.flat == label.flat).sum() self.num_inst += len(pred_label.flat) @@ -448,7 +448,7 @@ def update(self, labels, preds): assert(len(pred_label.shape) <= 2), 'Predictions should be no more than 2 dims' pred_label = numpy.argsort(pred_label.asnumpy().astype('float32'), axis=1) label = label.asnumpy().astype('int32') - _check_shapes_equal(label, pred_label) + _check_lengths_equal(label, pred_label) num_samples = pred_label.shape[0] num_dims = len(pred_label.shape) if num_dims == 1: @@ -523,7 +523,7 @@ def update(self, labels, preds): label = label.asnumpy().astype('int32') pred_label = numpy.argmax(pred, axis=1) - _check_shapes_equal(label, pred) + _check_lengths_equal(label, pred) if len(numpy.unique(label)) > 2: raise ValueError("F1 currently only supports binary classification.") @@ -629,7 +629,7 @@ def update(self, labels, preds): preds : list of `NDArray` Predicted values. """ - assert len(labels) == len(preds) + _check_lengths_equal(labels, preds) loss = 0. num = 0 for label, pred in zip(labels, preds): @@ -893,7 +893,7 @@ def update(self, labels, preds): _check_lengths_equal(labels, preds) for label, pred in zip(labels, preds): - _check_shapes_equal(label, pred) + _check_lengths_equal(label, pred) label = label.asnumpy() pred = pred.asnumpy() From 1529c5c43be0728d3d4017bd2318d447fb8f0818 Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Thu, 13 Jul 2017 21:27:23 -0700 Subject: [PATCH 214/834] kldiv loss (#7014) * kldiv loss * fix * fix * fix --- python/mxnet/gluon/loss.py | 42 ++++++++++++++++++++++++++++++ tests/python/unittest/test_loss.py | 18 +++++++++++++ 2 files changed, 60 insertions(+) diff --git a/python/mxnet/gluon/loss.py b/python/mxnet/gluon/loss.py index 98f108522d2c..0f8c38c06b00 100644 --- a/python/mxnet/gluon/loss.py +++ b/python/mxnet/gluon/loss.py @@ -171,3 +171,45 @@ def hybrid_forward(self, F, output, label, sample_weight=None): loss = -F.sum(output*label, axis=self._axis, keepdims=True) loss = _apply_weighting(F, loss, self._weight, sample_weight) return F.mean(loss, axis=self._batch_axis, exclude=True) + + +class KLDivLoss(HybridBlock): + """The Kullback-Leibler divergence loss. + + KL divergence is a useful distance measure for continuous distributions + and is often useful when performing direct regression over the space of + (discretely sampled) continuous output distributions. + + .. _Kullback-Leibler divergence: + https://en.wikipedia.org/wiki/Kullback-Leibler_divergence + .. math:: + L = 1/n \\sum_i (label_i * (log(label_i) - output_i)) + label's shape should be the same as output's. + + Parameters + ---------- + from_logits : bool, default True + whether input is log probability (usually from log_softmax) instead + of unnormalized numbers. + weight : float or None + global scalar weight for loss + sample_weight : Symbol or None + per sample weighting. Must be broadcastable to + the same shape as loss. For example, if loss has + shape (64, 10) and you want to weight each sample + in the batch, sample_weight should have shape (64, 1) + batch_axis : int, default 0 + The axis that represents mini-batch. + """ + def __init__(self, from_logits=True, weight=None, batch_axis=0, **kwargs): + super(KLDivLoss, self).__init__(**kwargs) + self._from_logits = from_logits + self._weight = weight + self._batch_axis = batch_axis + + def hybrid_forward(self, F, output, label, sample_weight=None): + if not self._from_logits: + output = F.log_softmax(output) + loss = label * (F.log(label+1e-8) - output) + loss = _apply_weighting(F, loss, self._weight, sample_weight) + return F.mean(loss, axis=self._batch_axis, exclude=True) diff --git a/tests/python/unittest/test_loss.py b/tests/python/unittest/test_loss.py index aaf46e5a57bc..7d4c586c2aa8 100644 --- a/tests/python/unittest/test_loss.py +++ b/tests/python/unittest/test_loss.py @@ -64,6 +64,24 @@ def test_ce_loss(): assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.01 +def test_kl_loss(): + mx.random.seed(1234) + np.random.seed(1234) + N = 20 + data = mx.random.uniform(-1, 1, shape=(N, 10)) + label = mx.nd.softmax(mx.random.uniform(0, 1, shape=(N, 2))) + data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label') + output = mx.sym.log_softmax(get_net(2)) + l = mx.symbol.Variable('label') + Loss = gluon.loss.KLDivLoss() + loss = Loss(output, l) + loss = mx.sym.make_loss(loss) + mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',)) + mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 1.}, + eval_metric=mx.metric.Loss()) + assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.05 + + def test_l2_loss(): mx.random.seed(1234) np.random.seed(1234) From 7e2475537df2c6cfd306c1d7966f8ed6f1c05dc6 Mon Sep 17 00:00:00 2001 From: Roshani Nagmote Date: Thu, 13 Jul 2017 23:34:03 -0700 Subject: [PATCH 215/834] gluon docs (#7038) * some fixes * loss not rendered * fix * fox --- docs/api/python/gluon.md | 18 ++-- python/mxnet/gluon/block.py | 39 ++++---- python/mxnet/gluon/loss.py | 67 +++++++------- python/mxnet/gluon/nn/basic_layers.py | 41 +++++---- python/mxnet/gluon/nn/conv_layers.py | 124 +++++++++++++------------- python/mxnet/gluon/parameter.py | 58 ++++++------ python/mxnet/gluon/rnn/rnn_cell.py | 94 +++++++++---------- python/mxnet/gluon/rnn/rnn_layer.py | 40 ++++----- python/mxnet/gluon/trainer.py | 12 +-- python/mxnet/gluon/utils.py | 17 ++-- 10 files changed, 260 insertions(+), 250 deletions(-) diff --git a/docs/api/python/gluon.md b/docs/api/python/gluon.md index 4748a1a64bfb..cda4a07f85dd 100644 --- a/docs/api/python/gluon.md +++ b/docs/api/python/gluon.md @@ -196,18 +196,16 @@ in Python and then deploy with symbolic graph in C++ and Scala. ```eval_rst .. currentmodule:: mxnet.gluon.loss +.. autoclass:: mxnet.gluon.loss.L2Loss + :members: +.. autoclass:: mxnet.gluon.loss.L1Loss + :members: +.. autoclass:: mxnet.gluon.loss.SoftmaxCrossEntropyLoss + :members: +.. autoclass:: mxnet.gluon.loss.KLDivLoss + :members: ``` - -```eval_rst -.. automethod:: mxnet.gluon.loss.custom_loss -.. automethod:: mxnet.gluon.loss.multitask_loss -.. automethod:: mxnet.gluon.loss.l1_loss -.. automethod:: mxnet.gluon.loss.l2_loss -.. automethod:: mxnet.gluon.loss.softmax_cross_entropy_loss -``` - - ## Utilities ```eval_rst diff --git a/python/mxnet/gluon/block.py b/python/mxnet/gluon/block.py index bd072e7f60f2..d284e954d87b 100644 --- a/python/mxnet/gluon/block.py +++ b/python/mxnet/gluon/block.py @@ -20,7 +20,7 @@ def __init__(self, block): @staticmethod def create(prefix, params, hint): - """Create prefix and params for new `Block`.""" + """Creates prefix and params for new `Block`.""" current = _BlockScope._current if current is None: if prefix is None: @@ -91,8 +91,8 @@ class Block(object): """Base class for all neural network layers and models. Your models should subclass this class. - `Block`s can be nested recursively in a tree structure. You can create and - assign child `Block`s as regular attributes:: + `Block` can be nested recursively in a tree structure. You can create and + assign child `Block` as regular attributes:: from mxnet.gluon import Block, nn from mxnet import ndarray as F @@ -115,7 +115,7 @@ def forward(self, x): model(F.zeros((10, 10), ctx=mx.cpu(0))) - Child `Block`s assigned this way will be registered and `collect_params` + Child `Block` assigned this way will be registered and `collect_params` will collect their Parameters recursively. Parameters @@ -125,7 +125,7 @@ def forward(self, x): Parameters and child `Block`s in this `Block`'s `name_scope`. Prefix should be unique within one model to prevent name collisions. params : ParameterDict or None - ParameterDict for sharing weights with the new `Block`. For example, + `ParameterDict` for sharing weights with the new `Block`. For example, if you want `dense1` to share `dense0`'s weights, you can do:: dense0 = nn.Dense(20) @@ -152,7 +152,7 @@ def params(self): return self._params def collect_params(self): - """Returns a ParameterDict containing this `Block` and all of its + """Returns a `ParameterDict` containing this `Block` and all of its children's Parameters.""" ret = ParameterDict(self._params.prefix) ret.update(self.params) @@ -162,19 +162,19 @@ def collect_params(self): @property def prefix(self): - """Prefix of this Block.""" + """Prefix of this `Block`.""" return self._prefix @property def name(self): - """Name of this Block, without '_' in the end.""" + """Name of this `Block`, without '_' in the end.""" if self.prefix.endswith('_'): return self.prefix[:-1] return self.prefix def name_scope(self): - """Returns a name space object managing child `Block` and parameter - names. Should be used by a `with` statement:: + """Returns a name space object managing a child `Block` and parameter + names. Should be used within a `with` statement:: with self.name_scope(): self.dense = nn.Dense(20) @@ -182,12 +182,12 @@ def name_scope(self): return self._scope def register_child(self, block): - """Register block as a child of self. `Block`s assigned to self as + """Registers block as a child of self. `Block`s assigned to self as attributes will be registered automatically.""" self._children.append(block) def initialize(self, init=initializer.Uniform(), ctx=None, verbose=False): - """Initialize `Parameter`s of this Block and its children. + """Initializes `Parameter`s of this `Block` and its children. Equivalent to `block.collect_params().initialize(...)` """ @@ -210,7 +210,7 @@ def __call__(self, *args): return self.forward(*args) def forward(self, *args): - """Override to implement forward computation using NDArray. Only + """Overrides to implement forward computation using `NDArray`. Only accepts positional arguments. Parameters @@ -231,10 +231,13 @@ class HybridBlock(Block): expressions like random numbers or intermediate results, since they change the graph structure for each iteration. - Before activated with `hybridize()`, `HybridBlock` works just like normal + Before activating with `hybridize()`, `HybridBlock` works just like normal `Block`. After activation, `HybridBlock` will create a symbolic graph - representing the forward computation and cache it. On subsequent forwards + representing the forward computation and cache it. On subsequent forwards, the cached graph will be used instead of `hybrid_forward`. + + Refer `Hybrid tutorial `_ to see + the end-to-end usage. """ def __init__(self, prefix=None, params=None): super(HybridBlock, self).__init__(prefix=prefix, params=params) @@ -288,7 +291,7 @@ def _get_graph(self, *args): return self._cached_graph def infer_shape(self, *args): - """Infer shape of Parameters from inputs.""" + """Infers shape of Parameters from inputs.""" syms, out = self._get_graph(*args) args, _, = _flatten(args) arg_shapes, _, aux_shapes = out.infer_shape( @@ -324,7 +327,7 @@ def _call_cached_op(self, *args): def forward(self, x, *args): """Defines the forward computation. Arguments can be either - NDArray or Symbol.""" + `NDArray` or `Symbol`.""" if isinstance(x, NDArray): if self._active and self._cached_op is None: self._build_cache(x, *args) @@ -348,7 +351,7 @@ def forward(self, x, *args): return self.hybrid_forward(symbol, x, *args, **params) def hybrid_forward(self, F, x, *args, **kwargs): - """Override to construct symbolic graph for this `Block`. + """Overrides to construct symbolic graph for this `Block`. Parameters ---------- diff --git a/python/mxnet/gluon/loss.py b/python/mxnet/gluon/loss.py index 0f8c38c06b00..7cc1bcfbdd7f 100644 --- a/python/mxnet/gluon/loss.py +++ b/python/mxnet/gluon/loss.py @@ -13,20 +13,20 @@ def _apply_weighting(F, loss, weight=None, sample_weight=None): Parameters ---------- loss : Symbol - the loss to be weighted. + The loss to be weighted. weight : float or None - global scalar weight for loss + Global scalar weight for loss. sample_weight : Symbol or None - per sample weighting. Must be broadcastable to + Per sample weighting. Must be broadcastable to the same shape as loss. For example, if loss has shape (64, 10) and you want to weight each sample - in the batch separately, sample_weight should have - shape (64, 1) + in the batch separately, `sample_weight` should have + shape (64, 1). Returns ------- loss : Symbol - weighted loss + Weighted loss """ if sample_weight is not None: loss = F.broadcast_mul(loss, sample_weight) @@ -39,23 +39,23 @@ def _apply_weighting(F, loss, weight=None, sample_weight=None): class L2Loss(HybridBlock): - """Calculate the mean squared error between output and label: + """Calculates the mean squared error between output and label: .. math:: L = \\frac{1}{2}\\sum_i \\Vert {output}_i - {label}_i \\Vert^2. - output and label can have arbitrary shape as long as they have the same + Output and label can have arbitrary shape as long as they have the same number of elements. Parameters ---------- weight : float or None - global scalar weight for loss + Global scalar weight for loss. sample_weight : Symbol or None - per sample weighting. Must be broadcastable to + Per sample weighting. Must be broadcastable to the same shape as loss. For example, if loss has shape (64, 10) and you want to weight each sample - in the batch, sample_weight should have shape (64, 1) + in the batch, `sample_weight` should have shape (64, 1). batch_axis : int, default 0 The axis that represents mini-batch. """ @@ -77,22 +77,22 @@ def hybrid_forward(self, F, output, label, sample_weight=None): class L1Loss(HybridBlock): - """Calculate the mean absolute error between output and label: + """Calculates the mean absolute error between output and label: .. math:: L = \\frac{1}{2}\\sum_i \\vert {output}_i - {label}_i \\vert. - output and label must have the same shape. + Output and label must have the same shape. Parameters ---------- weight : float or None - global scalar weight for loss + Global scalar weight for loss. sample_weight : Symbol or None - per sample weighting. Must be broadcastable to + Per sample weighting. Must be broadcastable to the same shape as loss. For example, if loss has shape (64, 10) and you want to weight each sample - in the batch, sample_weight should have shape (64, 1) + in the batch, `sample_weight` should have shape (64, 1). batch_axis : int, default 0 The axis that represents mini-batch. """ @@ -114,19 +114,19 @@ def hybrid_forward(self, F, output, label, sample_weight=None): class SoftmaxCrossEntropyLoss(HybridBlock): - """Compute the softmax cross entropy loss. + """Computes the softmax cross entropy loss. - If sparse_label is True, label should contain integer category indicators: + If `sparse_label` is `True`, label should contain integer category indicators: .. math:: p = {softmax}({output}) L = -\\sum_i {log}(p_{i,{label}_i}) - label's shape should be output's shape without the `axis` dimension. i.e. for - output.shape = (1,2,3,4) and axis = 2, label.shape should be (1,2,4) + Label's shape should be output's shape without the `axis` dimension. i.e. for + `output.shape` = (1,2,3,4) and axis = 2, `label.shape` should be (1,2,4). - If sparse_label is False, label should cantain probability distribution + If `sparse_label` is `False`, label should contain probability distribution with the same shape as output: .. math:: @@ -137,19 +137,19 @@ class SoftmaxCrossEntropyLoss(HybridBlock): Parameters ---------- axis : int, default -1 - The axis to sum over when computing softmax and entropy + The axis to sum over when computing softmax and entropy. sparse_label : bool, default True - whether label is a integer array instead of probability distribution + Whether label is an integer array instead of probability distribution. from_logits : bool, default False - whether input is log probability (usually from log_softmax) instead + Whether input is a log probability (usually from log_softmax) instead of unnormalized numbers. weight : float or None - global scalar weight for loss + Global scalar weight for loss. sample_weight : Symbol or None - per sample weighting. Must be broadcastable to + Per sample weighting. Must be broadcastable to the same shape as loss. For example, if loss has shape (64, 10) and you want to weight each sample - in the batch, sample_weight should have shape (64, 1) + in the batch, `sample_weight` should have shape (64, 1). batch_axis : int, default 0 The axis that represents mini-batch. """ @@ -184,20 +184,21 @@ class KLDivLoss(HybridBlock): https://en.wikipedia.org/wiki/Kullback-Leibler_divergence .. math:: L = 1/n \\sum_i (label_i * (log(label_i) - output_i)) - label's shape should be the same as output's. + + Label's shape should be the same as output's. Parameters ---------- - from_logits : bool, default True - whether input is log probability (usually from log_softmax) instead + from_logits : bool, default is `True` + Whether the input is log probability (usually from log_softmax) instead of unnormalized numbers. weight : float or None - global scalar weight for loss + Global scalar weight for loss. sample_weight : Symbol or None - per sample weighting. Must be broadcastable to + Per sample weighting. Must be broadcastable to the same shape as loss. For example, if loss has shape (64, 10) and you want to weight each sample - in the batch, sample_weight should have shape (64, 1) + in the batch, `sample_weight` should have shape (64, 1). batch_axis : int, default 0 The axis that represents mini-batch. """ diff --git a/python/mxnet/gluon/nn/basic_layers.py b/python/mxnet/gluon/nn/basic_layers.py index 3bd590f1d5de..a0617caf0215 100644 --- a/python/mxnet/gluon/nn/basic_layers.py +++ b/python/mxnet/gluon/nn/basic_layers.py @@ -13,14 +13,14 @@ class Sequential(Block): net = nn.Sequential() # use net's name_scope to give child Blocks appropriate names. with net.name_scope(): - net.add(Dense(10, activation='relu')) - net.add(Dense(20)) + net.add(nn.Dense(10, activation='relu')) + net.add(nn.Dense(20)) """ def __init__(self, prefix=None, params=None): super(Sequential, self).__init__(prefix=prefix, params=params) def add(self, block): - """Add block on top of the stack.""" + """Adds block on top of the stack.""" self.register_child(block) def forward(self, x): @@ -30,21 +30,21 @@ def forward(self, x): class HybridSequential(HybridBlock): - """Stack `HybridBlock`s sequentially. + """Stacks `HybridBlock`s sequentially. Example:: net = nn.Sequential() # use net's name_scope to give child Blocks appropriate names. with net.name_scope(): - net.add(Dense(10, activation='relu')) - net.add(Dense(20)) + net.add(nn.Dense(10, activation='relu')) + net.add(nn.Dense(20)) """ def __init__(self, prefix=None, params=None): super(HybridSequential, self).__init__(prefix=prefix, params=params) def add(self, block): - """Add block on top of the stack.""" + """Adds block on top of the stack.""" self.register_child(block) def hybrid_forward(self, F, x): @@ -63,7 +63,7 @@ class Dense(HybridBlock): created by the layer, and `bias` is a bias vector created by the layer (only applicable if `use_bias` is `True`). - Note: the input must be a tensor with rank 2. Use flatten to convert it + Note: the input must be a tensor with rank 2. Use `flatten` to convert it to rank 2 manually if necessary. Parameters @@ -81,8 +81,8 @@ class Dense(HybridBlock): bias_initializer: str or `Initializer` Initializer for the bias vector. in_units : int, optional - Size of input data. If not specified, initialization will be - defered to the first time `forward` is called and `in_units` + Size of the input data. If not specified, initialization will be + deferred to the first time `forward` is called and `in_units` will be inferred from the shape of input data. prefix : str or None See document of `Block`. @@ -91,10 +91,10 @@ class Dense(HybridBlock): Input shape: - a 2D input with shape `(batch_size, in_units)`. + A 2D input with shape `(batch_size, in_units)`. Output shape: - the output would have shape `(batch_size, units)`. + The output would have shape `(batch_size, units)`. """ def __init__(self, units, activation=None, use_bias=True, weight_initializer=None, bias_initializer='zeros', @@ -132,7 +132,7 @@ class Activation(HybridBlock): Parameters ---------- activation : str - name of activation function to use. + Name of activation function to use. See :func:`~mxnet.ndarray.Activation` for available choices. @@ -164,6 +164,13 @@ class Dropout(HybridBlock): rate : float Fraction of the input units to drop. Must be a number between 0 and 1. + + Input shape: + Arbitrary. + + Output shape: + Same shape as input. + References ---------- `Dropout: A Simple Way to Prevent Neural Networks from Overfitting @@ -179,14 +186,14 @@ def hybrid_forward(self, F, x): class BatchNorm(HybridBlock): """Batch normalization layer (Ioffe and Szegedy, 2014). - Normalize the input at each batch, i.e. applies a transformation + Normalizes the input at each batch, i.e. applies a transformation that maintains the mean activation close to 0 and the activation standard deviation close to 1. Parameters ---------- axis : int, default 1 - The axis that should be normalized. This is ypically the channels + The axis that should be normalized. This is typically the channels (C) axis. For instance, after a `Conv2D` layer with `layout='NCHW'`, set `axis=1` in `BatchNorm`. If `layout='NHWC'`, then set `axis=3`. momentum: float, default 0.9 @@ -211,7 +218,7 @@ class BatchNorm(HybridBlock): Initializer for the moving variance. in_channels : int, default 0 Number of channels (feature maps) in input data. If not specified, - initialization will be defered to the first time `forward` is called + initialization will be deferred to the first time `forward` is called and `in_channels` will be inferred from the shape of input data. @@ -290,7 +297,7 @@ class Embedding(HybridBlock): dtype : str or np.dtype, default 'float32' Data type of output embeddings. weight_initializer : Initializer - Initializer for the `embeddings` matrix + Initializer for the `embeddings` matrix. Input shape: diff --git a/python/mxnet/gluon/nn/conv_layers.py b/python/mxnet/gluon/nn/conv_layers.py index 86ae302f9e31..60fd848c9a14 100644 --- a/python/mxnet/gluon/nn/conv_layers.py +++ b/python/mxnet/gluon/nn/conv_layers.py @@ -18,7 +18,7 @@ class _Conv(HybridBlock): This layer creates a convolution kernel that is convolved with the layer input to produce a tensor of outputs. - If `use_bias` is True, a bias vector is created and added to the outputs. + If `use_bias` is `True`, a bias vector is created and added to the outputs. Finally, if `activation` is not `None`, it is applied to the outputs as well. @@ -28,31 +28,31 @@ class _Conv(HybridBlock): The dimensionality of the output space i.e. the number of output channels in the convolution. kernel_size : int or tuple/list of n ints - Specifys the dimensions of the convolution window. + Specifies the dimensions of the convolution window. strides: int or tuple/list of n ints, - Specifys the strides of the convolution. + Specifies the strides of the convolution. padding : int or tuple/list of n ints, If padding is non-zero, then the input is implicitly zero-padded on both sides for padding number of points dilation: int or tuple/list of n ints, - Specifys the dilation rate to use for dilated convolution. + Specifies the dilation rate to use for dilated convolution. groups : int - controls the connections between inputs and outputs. + Controls the connections between inputs and outputs. At groups=1, all inputs are convolved to all outputs. - At groups=2, the operation becomes equivalent to having two conv + At groups=2, the operation becomes equivalent to having two convolution layers side by side, each seeing half the input channels, and producing half the output channels, and both subsequently concatenated. layout : str, Dimension ordering of data and weight. Can be 'NCW', 'NWC', 'NCHW', 'NHWC', 'NCDHW', 'NDHWC', etc. 'N', 'C', 'H', 'W', 'D' stands for batch, channel, height, width and depth dimensions respectively. - Convolution is perform over 'D', 'H', and 'W' dimensions. + Convolution is performed over 'D', 'H', and 'W' dimensions. in_channels : int, default 0 The number of input channels to this layer. If not specified, - initialization will be defered to the first time `forward` is called + initialization will be deferred to the first time `forward` is called and `in_channels` will be inferred from the shape of input data. activation : str - Activation function to use. See :func:`~mxnet.nd.Activation`. + Activation function to use. See :func:`~mxnet.ndarray.Activation`. If you don't specify anything, no activation is applied (ie. "linear" activation: `a(x) = x`). use_bias: bool @@ -123,7 +123,7 @@ class Conv1D(_Conv): it is applied to the outputs as well. If `in_channels` is not specified, `Parameter` initialization will be - defered to the first time `forward` is called and `in_channels` will be + deferred to the first time `forward` is called and `in_channels` will be inferred from the shape of input data. @@ -133,16 +133,16 @@ class Conv1D(_Conv): The dimensionality of the output space, i.e. the number of output channels (filters) in the convolution. kernel_size :int or tuple/list of 1 int - Specifys the dimensions of the convolution window. + Specifies the dimensions of the convolution window. strides : int or tuple/list of 1 int, Specify the strides of the convolution. padding : int or a tuple/list of 1 int, If padding is non-zero, then the input is implicitly zero-padded on both sides for padding number of points dilation : int or tuple/list of 1 int - Specifys the dilation rate to use for dilated convolution. + Specifies the dilation rate to use for dilated convolution. groups : int - controls the connections between inputs and outputs. + Controls the connections between inputs and outputs. At groups=1, all inputs are convolved to all outputs. At groups=2, the operation becomes equivalent to having two conv layers side by side, each seeing half the input channels, and producing @@ -153,10 +153,10 @@ class Conv1D(_Conv): respectively. Convolution is applied on the 'W' dimension. in_channels : int, default 0 The number of input channels to this layer. If not specified, - initialization will be defered to the first time `forward` is called + initialization will be deferred to the first time `forward` is called and `in_channels` will be inferred from the shape of input data. activation : str - Activation function to use. See :func:`mx.nd.Activation`. + Activation function to use. See :func:`~mxnet.ndarray.Activation`. If you don't specify anything, no activation is applied (ie. "linear" activation: `a(x) = x`). use_bias : bool @@ -200,7 +200,7 @@ class Conv2D(_Conv): `activation` is not `None`, it is applied to the outputs as well. If `in_channels` is not specified, `Parameter` initialization will be - defered to the first time `forward` is called and `in_channels` will be + deferred to the first time `forward` is called and `in_channels` will be inferred from the shape of input data. Parameters @@ -209,16 +209,16 @@ class Conv2D(_Conv): The dimensionality of the output space, i.e. the number of output channels (filters) in the convolution. kernel_size :int or tuple/list of 2 int - Specifys the dimensions of the convolution window. + Specifies the dimensions of the convolution window. strides : int or tuple/list of 2 int, Specify the strides of the convolution. padding : int or a tuple/list of 2 int, If padding is non-zero, then the input is implicitly zero-padded on both sides for padding number of points dilation : int or tuple/list of 2 int - Specifys the dilation rate to use for dilated convolution. + Specifies the dilation rate to use for dilated convolution. groups : int - controls the connections between inputs and outputs. + Controls the connections between inputs and outputs. At groups=1, all inputs are convolved to all outputs. At groups=2, the operation becomes equivalent to having two conv layers side by side, each seeing half the input channels, and producing @@ -230,10 +230,10 @@ class Conv2D(_Conv): 'W' dimensions. in_channels : int, default 0 The number of input channels to this layer. If not specified, - initialization will be defered to the first time `forward` is called + initialization will be deferred to the first time `forward` is called and `in_channels` will be inferred from the shape of input data. activation : str - Activation function to use. See :func:`mx.nd.Activation`. + Activation function to use. See :func:`~mxnet.ndarray.Activation`. If you don't specify anything, no activation is applied (ie. "linear" activation: `a(x) = x`). use_bias : bool @@ -274,12 +274,12 @@ class Conv3D(_Conv): This layer creates a convolution kernel that is convolved with the layer input to produce a tensor of - outputs. If `use_bias` is True, + outputs. If `use_bias` is `True`, a bias vector is created and added to the outputs. Finally, if `activation` is not `None`, it is applied to the outputs as well. If `in_channels` is not specified, `Parameter` initialization will be - defered to the first time `forward` is called and `in_channels` will be + deferred to the first time `forward` is called and `in_channels` will be inferred from the shape of input data. Parameters @@ -288,16 +288,16 @@ class Conv3D(_Conv): The dimensionality of the output space, i.e. the number of output channels (filters) in the convolution. kernel_size :int or tuple/list of 3 int - Specifys the dimensions of the convolution window. + Specifies the dimensions of the convolution window. strides : int or tuple/list of 3 int, Specify the strides of the convolution. padding : int or a tuple/list of 3 int, If padding is non-zero, then the input is implicitly zero-padded on both sides for padding number of points dilation : int or tuple/list of 3 int - Specifys the dilation rate to use for dilated convolution. + Specifies the dilation rate to use for dilated convolution. groups : int - controls the connections between inputs and outputs. + Controls the connections between inputs and outputs. At groups=1, all inputs are convolved to all outputs. At groups=2, the operation becomes equivalent to having two conv layers side by side, each seeing half the input channels, and producing @@ -309,10 +309,10 @@ class Conv3D(_Conv): 'H' and 'W' dimensions. in_channels : int, default 0 The number of input channels to this layer. If not specified, - initialization will be defered to the first time `forward` is called + initialization will be deferred to the first time `forward` is called and `in_channels` will be inferred from the shape of input data. activation : str - Activation function to use. See :func:`mx.nd.Activation`. + Activation function to use. See :func:`~mxnet.ndarray.Activation`. If you don't specify anything, no activation is applied (ie. "linear" activation: `a(x) = x`). use_bias : bool @@ -361,7 +361,7 @@ class Conv1DTranspose(_Conv): said convolution. If `in_channels` is not specified, `Parameter` initialization will be - defered to the first time `forward` is called and `in_channels` will be + deferred to the first time `forward` is called and `in_channels` will be inferred from the shape of input data. Parameters @@ -370,16 +370,16 @@ class Conv1DTranspose(_Conv): The dimensionality of the output space, i.e. the number of output channels (filters) in the convolution. kernel_size :int or tuple/list of 3 int - Specifys the dimensions of the convolution window. + Specifies the dimensions of the convolution window. strides : int or tuple/list of 3 int, Specify the strides of the convolution. padding : int or a tuple/list of 3 int, If padding is non-zero, then the input is implicitly zero-padded on both sides for padding number of points dilation : int or tuple/list of 3 int - Specifys the dilation rate to use for dilated convolution. + Specifies the dilation rate to use for dilated convolution. groups : int - controls the connections between inputs and outputs. + Controls the connections between inputs and outputs. At groups=1, all inputs are convolved to all outputs. At groups=2, the operation becomes equivalent to having two conv layers side by side, each seeing half the input channels, and producing @@ -390,10 +390,10 @@ class Conv1DTranspose(_Conv): respectively. Convolution is applied on the 'W' dimension. in_channels : int, default 0 The number of input channels to this layer. If not specified, - initialization will be defered to the first time `forward` is called + initialization will be deferred to the first time `forward` is called and `in_channels` will be inferred from the shape of input data. activation : str - Activation function to use. See :func:`mx.nd.Activation`. + Activation function to use. See :func:`~mxnet.ndarray.Activation`. If you don't specify anything, no activation is applied (ie. "linear" activation: `a(x) = x`). use_bias : bool @@ -443,7 +443,7 @@ class Conv2DTranspose(_Conv): said convolution. If `in_channels` is not specified, `Parameter` initialization will be - defered to the first time `forward` is called and `in_channels` will be + deferred to the first time `forward` is called and `in_channels` will be inferred from the shape of input data. @@ -453,16 +453,16 @@ class Conv2DTranspose(_Conv): The dimensionality of the output space, i.e. the number of output channels (filters) in the convolution. kernel_size :int or tuple/list of 3 int - Specifys the dimensions of the convolution window. + Specifies the dimensions of the convolution window. strides : int or tuple/list of 3 int, Specify the strides of the convolution. padding : int or a tuple/list of 3 int, If padding is non-zero, then the input is implicitly zero-padded on both sides for padding number of points dilation : int or tuple/list of 3 int - Specifys the dilation rate to use for dilated convolution. + Specifies the dilation rate to use for dilated convolution. groups : int - controls the connections between inputs and outputs. + Controls the connections between inputs and outputs. At groups=1, all inputs are convolved to all outputs. At groups=2, the operation becomes equivalent to having two conv layers side by side, each seeing half the input channels, and producing @@ -474,10 +474,10 @@ class Conv2DTranspose(_Conv): 'W' dimensions. in_channels : int, default 0 The number of input channels to this layer. If not specified, - initialization will be defered to the first time `forward` is called + initialization will be deferred to the first time `forward` is called and `in_channels` will be inferred from the shape of input data. activation : str - Activation function to use. See :func:`mx.nd.Activation`. + Activation function to use. See :func:`~mxnet.ndarray.Activation`. If you don't specify anything, no activation is applied (ie. "linear" activation: `a(x) = x`). use_bias : bool @@ -528,7 +528,7 @@ class Conv3DTranspose(_Conv): said convolution. If `in_channels` is not specified, `Parameter` initialization will be - defered to the first time `forward` is called and `in_channels` will be + deferred to the first time `forward` is called and `in_channels` will be inferred from the shape of input data. @@ -538,16 +538,16 @@ class Conv3DTranspose(_Conv): The dimensionality of the output space, i.e. the number of output channels (filters) in the convolution. kernel_size :int or tuple/list of 3 int - Specifys the dimensions of the convolution window. + Specifies the dimensions of the convolution window. strides : int or tuple/list of 3 int, Specify the strides of the convolution. padding : int or a tuple/list of 3 int, If padding is non-zero, then the input is implicitly zero-padded on both sides for padding number of points dilation : int or tuple/list of 3 int - Specifys the dilation rate to use for dilated convolution. + Specifies the dilation rate to use for dilated convolution. groups : int - controls the connections between inputs and outputs. + Controls the connections between inputs and outputs. At groups=1, all inputs are convolved to all outputs. At groups=2, the operation becomes equivalent to having two conv layers side by side, each seeing half the input channels, and producing @@ -559,10 +559,10 @@ class Conv3DTranspose(_Conv): 'H', and 'W' dimensions. in_channels : int, default 0 The number of input channels to this layer. If not specified, - initialization will be defered to the first time `forward` is called + initialization will be deferred to the first time `forward` is called and `in_channels` will be inferred from the shape of input data. activation : str - Activation function to use. See :func:`mx.nd.Activation`. + Activation function to use. See :func:`~mxnet.ndarray.Activation`. If you don't specify anything, no activation is applied (ie. "linear" activation: `a(x) = x`). use_bias : bool @@ -632,7 +632,7 @@ class MaxPool1D(_Pooling): Size of the max pooling windows. strides: int, or None Factor by which to downscale. E.g. 2 will halve the input size. - If None, it will default to `pool_size`. + If `None`, it will default to `pool_size`. padding: int If padding is non-zero, then the input is implicitly zero-padded on both sides for padding number of points. @@ -641,7 +641,7 @@ class MaxPool1D(_Pooling): 'N', 'C', 'W' stands for batch, channel, and width (time) dimensions respectively. Pooling is applied on the W dimension. ceil_mode : bool, default False - When True, will use ceil instead of floor to compute the output shape. + When `True`, will use ceil instead of floor to compute the output shape. Input shape: @@ -656,7 +656,7 @@ class MaxPool1D(_Pooling): out_width = floor((width+2*padding-pool_size)/strides)+1 - When ceil_mode is True, ceil will be used instead of floor in this + When `ceil_mode` is `True`, ceil will be used instead of floor in this equation. """ def __init__(self, pool_size=2, strides=None, padding=0, layout='NCW', @@ -679,7 +679,7 @@ class MaxPool2D(_Pooling): Size of the max pooling windows. strides: int, list/tuple of 2 ints, or None. Factor by which to downscale. E.g. 2 will halve the input size. - If None, it will default to `pool_size`. + If `None`, it will default to `pool_size`. padding: int or list/tuple of 2 ints, If padding is non-zero, then the input is implicitly zero-padded on both sides for padding number of points. @@ -688,7 +688,7 @@ class MaxPool2D(_Pooling): 'N', 'C', 'H', 'W' stands for batch, channel, height, and width dimensions respectively. padding is applied on 'H' and 'W' dimension. ceil_mode : bool, default False - When True, will use ceil instead of floor to compute the output shape. + When `True`, will use ceil instead of floor to compute the output shape. Input shape: @@ -704,7 +704,7 @@ class MaxPool2D(_Pooling): out_height = floor((height+2*padding[0]-pool_size[0])/strides[0])+1 out_width = floor((width+2*padding[1]-pool_size[1])/strides[1])+1 - When ceil_mode is True, ceil will be used instead of floor in this + When `ceil_mode` is `True`, ceil will be used instead of floor in this equation. """ def __init__(self, pool_size=(2, 2), strides=None, padding=0, layout='NCHW', @@ -727,7 +727,7 @@ class MaxPool3D(_Pooling): Size of the max pooling windows. strides: int, list/tuple of 3 ints, or None. Factor by which to downscale. E.g. 2 will halve the input size. - If None, it will default to `pool_size`. + If `None`, it will default to `pool_size`. padding: int or list/tuple of 3 ints, If padding is non-zero, then the input is implicitly zero-padded on both sides for padding number of points. @@ -737,7 +737,7 @@ class MaxPool3D(_Pooling): depth dimensions respectively. padding is applied on 'D', 'H' and 'W' dimension. ceil_mode : bool, default False - When True, will use ceil instead of floor to compute the output shape. + When `True`, will use ceil instead of floor to compute the output shape. Input shape: @@ -755,7 +755,7 @@ class MaxPool3D(_Pooling): out_height = floor((height+2*padding[1]-pool_size[1])/strides[1])+1 out_width = floor((width+2*padding[2]-pool_size[2])/strides[2])+1 - When ceil_mode is True, ceil will be used instead of floor in this + When `ceil_mode` is `True`, ceil will be used instead of floor in this equation. """ def __init__(self, pool_size=(2, 2, 2), strides=None, padding=0, @@ -777,7 +777,7 @@ class AvgPool1D(_Pooling): Size of the max pooling windows. strides: int, or None Factor by which to downscale. E.g. 2 will halve the input size. - If None, it will default to `pool_size`. + If `None`, it will default to `pool_size`. padding: int If padding is non-zero, then the input is implicitly zero-padded on both sides for padding number of points. @@ -786,7 +786,7 @@ class AvgPool1D(_Pooling): 'N', 'C', 'W' stands for batch, channel, and width (time) dimensions respectively. padding is applied on 'W' dimension. ceil_mode : bool, default False - When True, will use ceil instead of floor to compute the output shape. + When `True`, will use ceil instead of floor to compute the output shape. Input shape: @@ -801,7 +801,7 @@ class AvgPool1D(_Pooling): out_width = floor((width+2*padding-pool_size)/strides)+1 - When ceil_mode is True, ceil will be used instead of floor in this + When `ceil_mode` is `True`, ceil will be used instead of floor in this equation. """ def __init__(self, pool_size=2, strides=None, padding=0, layout='NCW', @@ -823,7 +823,7 @@ class AvgPool2D(_Pooling): Size of the max pooling windows. strides: int, list/tuple of 2 ints, or None. Factor by which to downscale. E.g. 2 will halve the input size. - If None, it will default to `pool_size`. + If `None`, it will default to `pool_size`. padding: int or list/tuple of 2 ints, If padding is non-zero, then the input is implicitly zero-padded on both sides for padding number of points. @@ -848,7 +848,7 @@ class AvgPool2D(_Pooling): out_height = floor((height+2*padding[0]-pool_size[0])/strides[0])+1 out_width = floor((width+2*padding[1]-pool_size[1])/strides[1])+1 - When ceil_mode is True, ceil will be used instead of floor in this + When `ceil_mode` is `True`, ceil will be used instead of floor in this equation. """ def __init__(self, pool_size=(2, 2), strides=None, padding=0, @@ -870,7 +870,7 @@ class AvgPool3D(_Pooling): Size of the max pooling windows. strides: int, list/tuple of 3 ints, or None. Factor by which to downscale. E.g. 2 will halve the input size. - If None, it will default to `pool_size`. + If `None`, it will default to `pool_size`. padding: int or list/tuple of 3 ints, If padding is non-zero, then the input is implicitly zero-padded on both sides for padding number of points. @@ -898,7 +898,7 @@ class AvgPool3D(_Pooling): out_height = floor((height+2*padding[1]-pool_size[1])/strides[1])+1 out_width = floor((width+2*padding[2]-pool_size[2])/strides[2])+1 - When ceil_mode is True, ceil will be used instead of floor in this + When `ceil_mode` is `True,` ceil will be used instead of floor in this equation. """ def __init__(self, pool_size=(2, 2, 2), strides=None, padding=0, diff --git a/python/mxnet/gluon/parameter.py b/python/mxnet/gluon/parameter.py index af51f399c018..2b3cb05ec1fc 100644 --- a/python/mxnet/gluon/parameter.py +++ b/python/mxnet/gluon/parameter.py @@ -44,11 +44,11 @@ class Parameter(object): - 'add' means everytime gradient is added to the grad `NDArray`. You need to manually call `zero_grad()` to clear the gradient buffer before each iteration when using this option. - - 'null' means gradient is not reqested for this parameter. gradient arrays + - 'null' means gradient is not requested for this parameter. gradient arrays will not be allocated. shape : tuple of int, default None Shape of this parameter. By default shape is not specified. Parameter with - unknown shaped can be used for `Symbol` API, but `init` will throw an error + unknown shape can be used for `Symbol` API, but `init` will throw an error when using `NDArray` API. dtype : numpy.dtype or str, default 'float32' Data type of this parameter. For example, numpy.float32 or 'float32'. @@ -56,7 +56,7 @@ class Parameter(object): Learning rate multiplier. Learning rate will be multiplied by lr_mult when updating this parameter with optimizer. wd_mult : float, default 1.0 - Weight decay multiplier (L2 regulerizer coefficient). Works similarly to lr_mult. + Weight decay multiplier (L2 regularizer coefficient). Works similar to lr_mult. init : Initializer, default None Initializer of this parameter. Will use the global initializer by default. @@ -77,7 +77,7 @@ def __init__(self, name, grad_req='write', shape=None, dtype=mx_real_t, self._defered_init = () def initialize(self, init=None, ctx=None, default_init=initializer.Uniform()): - """Intialize parameter and gradient arrays. Only used for `NDArray` API. + """Initializes parameter and gradient arrays. Only used for `NDArray` API. Parameters ---------- @@ -88,9 +88,9 @@ def initialize(self, init=None, ctx=None, default_init=initializer.Uniform()): copy will be made for each context. .. note:: Copies are independent arrays. User is responsible for keeping - their values consistent when updating. Normally nn.Trainer does this for you. + their values consistent when updating. Normally `gluon.Trainer` does this for you. default_init : Initializer - Default initializer is used when both `init` and `Parameter.init` are None. + Default initializer is used when both `init` and `Parameter.init` are `None`. Examples -------- @@ -131,7 +131,7 @@ def initialize(self, init=None, ctx=None, default_init=initializer.Uniform()): self._finish_deferred_init() def _load_init(self, data, ctx): - """(Re)init by loading from data.""" + """(Re)initializes by loading from data.""" if self.shape: for i, j in zip(self.shape, data.shape): assert i == 0 or i == j, \ @@ -161,7 +161,7 @@ def _load_init(self, data, ctx): self._defered_init = () def _finish_deferred_init(self): - """Finish deferred initialization.""" + """Finishes deferred initialization.""" if not self._defered_init: return init, ctx, default_init = self._defered_init @@ -181,7 +181,7 @@ def _finish_deferred_init(self): self._init_impl(data, ctx) def _init_impl(self, data, ctx): - """Set data and grad.""" + """Sets data and grad.""" self._data = OrderedDict() for i in ctx: self._data[i] = data.copyto(i) @@ -197,7 +197,7 @@ def _init_impl(self, data, ctx): autograd.mark_variables(self.list_data(), self.list_grad(), self.grad_req) def set_data(self, data): - """Set this parameter's value on all contexts to data.""" + """Sets this parameter's value on all contexts to data.""" assert self._data is not None, \ "Parameter %s has not been initialized"%self.name for arr in self.list_data(): @@ -222,7 +222,7 @@ def _check_initialized(self, ctx=None): def data(self, ctx=None): """Returns a copy of this parameter on one context. Must have been - intialized on this context before. + initialized on this context before. Parameters ---------- @@ -278,7 +278,7 @@ def list_grad(self): return list(self._grad.values()) def list_ctx(self): - """Returns a list of contexts this parameter is initialized on""" + """Returns a list of contexts this parameter is initialized on.""" if self._data is None: if self._defered_init: return self._defered_init[1] @@ -286,7 +286,7 @@ def list_ctx(self): return list(self._data.keys()) def zero_grad(self): - """Set gradient buffer on all contexts to 0. No action is taken if + """Sets gradient buffer on all contexts to 0. No action is taken if parameter is uninitialized or doesn't require gradient.""" if self._grad is None: return @@ -310,9 +310,9 @@ class ParameterDict(object): prefix : str, default '' The prefix to be prepended to all Parameters' name created by this dict. shared : ParameterDict or None - If not None, when this dict's get method creates a new parameter, will + If not `None`, when this dict's `get` method creates a new parameter, will first try to retrieve it from `shared` dict. Usually used for sharing - parameters with another Block. + parameters with another `Block`. """ def __init__(self, prefix='', shared=None): self._prefix = prefix @@ -334,7 +334,7 @@ def values(self): @property def prefix(self): """Prefix of this dict. It will be prepended to Parameters' name created - with `get`""" + with `get`.""" return self._prefix def _get_impl(self, name): @@ -346,23 +346,23 @@ def _get_impl(self, name): return None def get(self, name, **kwargs): - """Retrieve a Parameter with name `self.prefix+name`. If not found, - `get` will first try to retrive it from `shared` dict. If still not - found, `get` will create a new Parameter with key-word arguments and + """Retrieves a `Parameter` with name `self.prefix+name`. If not found, + `get` will first try to retrieve it from `shared` dict. If still not + found, `get` will create a new `Parameter` with key-word arguments and insert it to self. Parameters ---------- name : str - name of the desired Parameter. It will be prepended with this dictionary's + Name of the desired Parameter. It will be prepended with this dictionary's prefix. **kwargs : dict - The rest of key-word arguments for the created Parameter. + The rest of key-word arguments for the created `Parameter`. Returns ------- Parameter - The created or retrieved Parameter. + The created or retrieved `Parameter`. """ name = self.prefix + name param = self._get_impl(name) @@ -382,7 +382,7 @@ def get(self, name, **kwargs): return param def update(self, other): - """Copy all Parameters in `other` to self.""" + """Copies all Parameters in `other` to self.""" for k, v in other.items(): if k in self._params: assert self._params[k] is v, \ @@ -392,16 +392,16 @@ def update(self, other): self._params[k] = v def initialize(self, init=initializer.Uniform(), ctx=None, verbose=False): - """Intialize all Parameters manage by this dictionary to be used for `NDArray` - API. Has no effect when using `Symbol` API. + """Initializes all Parameters managed by this dictionary to be used for `NDArray` + API. It has no effect when using `Symbol` API. Parameters ---------- init : Initializer - Global default Initializer to be used when `Parameter.init` is None. - Otherwise `Parameter.init` takes precedence. + Global default Initializer to be used when `Parameter.init` is `None`. + Otherwise, `Parameter.init` takes precedence. ctx : Context or list of Context - Keep a copy of Parameters on one or many context(s). + Keeps a copy of Parameters on one or many context(s). """ if verbose: init.set_verbosity(verbose=verbose) @@ -409,7 +409,7 @@ def initialize(self, init=initializer.Uniform(), ctx=None, verbose=False): v.initialize(None, ctx, init) def zero_grad(self): - """Set all Parameters' gradient buffer to 0.""" + """Sets all Parameters' gradient buffer to 0.""" for i in self.values(): i.zero_grad() diff --git a/python/mxnet/gluon/rnn/rnn_cell.py b/python/mxnet/gluon/rnn/rnn_cell.py index 7333892da4f5..a90039febcf3 100644 --- a/python/mxnet/gluon/rnn/rnn_cell.py +++ b/python/mxnet/gluon/rnn/rnn_cell.py @@ -76,11 +76,11 @@ class RecurrentCell(Block): ---------- prefix : str, optional Prefix for names of `Block`s - (this prefix is also used for names of weights if `params` is None + (this prefix is also used for names of weights if `params` is `None` i.e. if `params` are being created and not reused) params : Parameter or None, optional Container for weight sharing between cells. - A new Parameter container is created if `params` is None. + A new Parameter container is created if `params` is `None`. """ def __init__(self, prefix=None, params=None): super(RecurrentCell, self).__init__(prefix=prefix, params=params) @@ -108,18 +108,18 @@ def begin_state(self, batch_size=0, func=ndarray.zeros, **kwargs): func : callable, default symbol.zeros Function for creating initial state. - For Symbol API, func can be symbol.zeros, symbol.uniform, - symbol.var etc. Use symbol.var if you want to directly + For Symbol API, func can be `symbol.zeros`, `symbol.uniform`, + `symbol.var etc`. Use `symbol.var` if you want to directly feed input as states. - For NDArray API, func can be ndarray.zeros, ndarray.ones, etc. + For NDArray API, func can be `ndarray.zeros`, `ndarray.ones`, etc. batch_size: int, default 0 Only required for NDArray API. Size of the batch ('N' in layout) dimension of input. **kwargs : - additional keyword arguments passed to func. For example - mean, std, dtype, etc. + Additional keyword arguments passed to func. For example + `mean`, `std`, `dtype`, etc. Returns ------- @@ -142,17 +142,17 @@ def begin_state(self, batch_size=0, func=ndarray.zeros, **kwargs): return states def unroll(self, length, inputs, begin_state=None, layout='NTC', merge_outputs=None): - """Unroll an RNN cell across time steps. + """Unrolls an RNN cell across time steps. Parameters ---------- length : int - number of steps to unroll + Number of steps to unroll. inputs : Symbol, list of Symbol, or None If `inputs` is a single Symbol (usually the output of Embedding symbol), it should have shape - (batch_size, length, ...) if layout == 'NTC', - or (length, batch_size, ...) if layout == 'TNC'. + (batch_size, length, ...) if `layout` is 'NTC', + or (length, batch_size, ...) if `layout` is 'TNC'. If `inputs` is a list of symbols (usually output of previous unroll), they should all have shape @@ -160,17 +160,17 @@ def unroll(self, length, inputs, begin_state=None, layout='NTC', merge_outputs=N begin_state : nested list of Symbol, optional Input states created by `begin_state()` or output state of another cell. - Created from `begin_state()` if None. + Created from `begin_state()` if `None`. layout : str, optional `layout` of input symbol. Only used if inputs is a single Symbol. merge_outputs : bool, optional - If False, return outputs as a list of Symbols. - If True, concatenate output across time steps - and return a single symbol with shape - (batch_size, length, ...) if layout == 'NTC', - or (length, batch_size, ...) if layout == 'TNC'. - If None, output whatever is faster + If `False`, returns outputs as a list of Symbols. + If `True`, concatenates output across time steps + and returns a single symbol with shape + (batch_size, length, ...) if layout is 'NTC', + or (length, batch_size, ...) if layout is 'TNC'. + If `None`, output whatever is faster. Returns ------- @@ -181,7 +181,7 @@ def unroll(self, length, inputs, begin_state=None, layout='NTC', merge_outputs=N states : list of Symbol The new state of this RNN after this unrolling. - The type of this symbol is same as the output of begin_state(). + The type of this symbol is same as the output of `begin_state()`. """ self.reset() @@ -207,12 +207,12 @@ def _get_activation(self, F, inputs, activation, **kwargs): return activation(inputs, **kwargs) def forward(self, inputs, states): - """Unroll the recurrent cell for one time step. + """Unrolls the recurrent cell for one time step. Parameters ---------- inputs : sym.Variable - input symbol, 2D, batch_size * num_units + Input symbol, 2D, of shape (batch_size * num_units). states : list of sym.Variable RNN state from previous step or the output of begin_state(). @@ -223,8 +223,8 @@ def forward(self, inputs, states): for a single time step. states : list of Symbol The new state of this RNN after this unrolling. - The type of this symbol is same as the output of begin_state(). - This can be used as input state to the next time step + The type of this symbol is same as the output of `begin_state()`. + This can be used as an input state to the next time step of this RNN. See Also @@ -252,9 +252,9 @@ class RNNCell(HybridRecurrentCell): Parameters ---------- hidden_size : int - number of units in output symbol + Number of units in output symbol activation : str or Symbol, default 'tanh' - type of activation function. + Type of activation function. i2h_weight_initializer : str or Initializer Initializer for the input weights matrix, used for the linear transformation of the inputs. @@ -266,11 +266,11 @@ class RNNCell(HybridRecurrentCell): h2h_bias_initializer : str or Initializer Initializer for the bias vector. prefix : str, default 'rnn_' - prefix for name of `Block`s - (and name of weight if params is None) + Prefix for name of `Block`s + (and name of weight if params is `None`). params : Parameter or None - container for weight sharing between cells. - created if None. + Container for weight sharing between cells. + Created if `None`. """ def __init__(self, hidden_size, activation='tanh', i2h_weight_initializer=None, h2h_weight_initializer=None, @@ -320,7 +320,7 @@ class LSTMCell(HybridRecurrentCell): Parameters ---------- hidden_size : int - number of units in output symbol. + Number of units in output symbol. i2h_weight_initializer : str or Initializer Initializer for the input weights matrix, used for the linear transformation of the inputs. @@ -328,17 +328,17 @@ class LSTMCell(HybridRecurrentCell): Initializer for the recurrent weights matrix, used for the linear transformation of the recurrent state. i2h_bias_initializer : str or Initializer, default 'lstmbias' - Initializer for the bias vector. By default bias for the forget + Initializer for the bias vector. By default, bias for the forget gate is initialized to 1 while all other biases are initialized to zero. h2h_bias_initializer : str or Initializer Initializer for the bias vector. prefix : str, default 'lstm_' - prefix for name of `Block`s - (and name of weight if params is None) + Prefix for name of `Block`s + (and name of weight if params is `None`). params : Parameter or None - container for weight sharing between cells. - created if None. + Container for weight sharing between cells. + Created if `None`. """ def __init__(self, hidden_size, i2h_weight_initializer=None, h2h_weight_initializer=None, @@ -404,7 +404,7 @@ class GRUCell(HybridRecurrentCell): Parameters ---------- hidden_size : int - number of units in output symbol. + Number of units in output symbol. i2h_weight_initializer : str or Initializer Initializer for the input weights matrix, used for the linear transformation of the inputs. @@ -417,10 +417,10 @@ class GRUCell(HybridRecurrentCell): Initializer for the bias vector. prefix : str, default 'gru_' prefix for name of `Block`s - (and name of weight if params is None) + (and name of weight if params is `None`). params : Parameter or None - container for weight sharing between cells. - created if None. + Container for weight sharing between cells. + Created if `None`. """ def __init__(self, hidden_size, i2h_weight_initializer=None, h2h_weight_initializer=None, @@ -481,12 +481,12 @@ def hybrid_forward(self, F, inputs, states, i2h_weight, class SequentialRNNCell(RecurrentCell): - """Sequantially stacking multiple RNN cells.""" + """Sequentially stacking multiple RNN cells.""" def __init__(self, prefix=None, params=None): super(SequentialRNNCell, self).__init__(prefix=prefix, params=params) def add(self, cell): - """Append a cell into the stack. + """Appends a cell into the stack. Parameters ---------- @@ -540,12 +540,12 @@ def hybrid_forward(self, *args, **kwargs): class DropoutCell(HybridRecurrentCell): - """Apply dropout on input. + """Applies dropout on input. Parameters ---------- dropout : float - percentage of elements to drop out, which + Percentage of elements to drop out, which is 1 - percentage to retain. """ def __init__(self, dropout, prefix=None, params=None): @@ -582,7 +582,7 @@ class ModifierCell(HybridRecurrentCell): on it (e.g. Zoneout), and returns a new cell. After applying modifiers the base cell should - no longer be called directly. The modifer cell + no longer be called directly. The modifier cell should be used instead. """ def __init__(self, base_cell): @@ -612,7 +612,7 @@ def hybrid_forward(self, F, inputs, states): class ZoneoutCell(ModifierCell): - """Apply Zoneout on base cell.""" + """Applies Zoneout on base cell.""" def __init__(self, base_cell, zoneout_outputs=0., zoneout_states=0.): assert not isinstance(base_cell, BidirectionalCell), \ "BidirectionalCell doesn't support zoneout since it doesn't support step. " \ @@ -691,9 +691,9 @@ class BidirectionalCell(HybridRecurrentCell): Parameters ---------- l_cell : RecurrentCell - cell for forward unrolling + Cell for forward unrolling r_cell : RecurrentCell - cell for backward unrolling + Cell for backward unrolling """ def __init__(self, l_cell, r_cell, output_prefix='bi_'): super(BidirectionalCell, self).__init__(prefix='', params=None) diff --git a/python/mxnet/gluon/rnn/rnn_layer.py b/python/mxnet/gluon/rnn/rnn_layer.py index 8a2309841e12..0e7efff6639a 100644 --- a/python/mxnet/gluon/rnn/rnn_layer.py +++ b/python/mxnet/gluon/rnn/rnn_layer.py @@ -11,7 +11,7 @@ class _RNNLayer(Block): - """implementation of recurrent layers.""" + """Implementation of recurrent layers.""" def __init__(self, hidden_size, num_layers, layout, dropout, bidirectional, input_size, i2h_weight_initializer, h2h_weight_initializer, @@ -66,7 +66,7 @@ def state_info(self, batch_size=0): raise NotImplementedError def _unfuse(self): - """Unfuse the fused RNN in to a stack of rnn cells.""" + """Unfuses the fused RNN in to a stack of rnn cells.""" get_cell = {'rnn_relu': lambda **kwargs: rnn_cell.RNNCell(self._hidden_size, activation='relu', **kwargs), @@ -107,20 +107,20 @@ def begin_state(self, batch_size=0, func=ndarray.zeros, **kwargs): Parameters ---------- batch_size: int - Only required for NDArray API. Size of the batch ('N' in layout) - dimension of input. - func : callable, default symbol.zeros + Only required for `NDArray` API. Size of the batch ('N' in layout). + Dimension of the input. + func : callable, default `symbol.zeros` Function for creating initial state. - For Symbol API, func can be symbol.zeros, symbol.uniform, - symbol.var etc. Use symbol.var if you want to directly + For Symbol API, func can be `symbol.zeros`, `symbol.uniform`, + `symbol.var` etc. Use `symbol.var` if you want to directly feed input as states. - For NDArray API, func can be ndarray.zeros, ndarray.ones, etc. + For NDArray API, func can be `ndarray.zeros`, `ndarray.ones`, etc. **kwargs : - additional keyword arguments passed to func. For example - mean, std, dtype, etc. + Additional keyword arguments passed to func. For example + `mean`, `std`, `dtype`, etc. Returns ------- @@ -192,7 +192,7 @@ def _forward_gpu(self, inputs, states): class RNN(_RNNLayer): - r"""Applies a multi-layer Elman RNN with tanh or ReLU non-linearity to an input sequence. + r"""Applies a multi-layer Elman RNN with `tanh` or `ReLU` non-linearity to an input sequence. For each element in the input sequence, each layer computes the following function: @@ -207,7 +207,7 @@ class RNN(_RNNLayer): Parameters ---------- hidden_size: int - The number of features in the hidden state h + The number of features in the hidden state h. num_layers: int, default 1 Number of recurrent layers. activation: {'relu' or 'tanh'}, default 'tanh' @@ -217,9 +217,9 @@ class RNN(_RNNLayer): sequence length, batch size, and feature dimensions respectively. dropout: float, default 0 If non-zero, introduces a dropout layer on the outputs of each - RNN layer except the last layer + RNN layer except the last layer. bidirectional: bool, default False - If True, becomes a bidirectional RNN. + If `True`, becomes a bidirectional RNN. i2h_weight_initializer : str or Initializer Initializer for the input weights matrix, used for the linear transformation of the inputs. @@ -305,7 +305,7 @@ class LSTM(_RNNLayer): Parameters ---------- hidden_size: int - The number of features in the hidden state h + The number of features in the hidden state h. num_layers: int, default 1 Number of recurrent layers. layout : str, default 'TNC' @@ -313,9 +313,9 @@ class LSTM(_RNNLayer): sequence length, batch size, and feature dimensions respectively. dropout: float, default 0 If non-zero, introduces a dropout layer on the outputs of each - RNN layer except the last layer + RNN layer except the last layer. bidirectional: bool, default False - If True, becomes a bidirectional RNN. + If `True`, becomes a bidirectional RNN. i2h_weight_initializer : str or Initializer Initializer for the input weights matrix, used for the linear transformation of the inputs. @@ -323,7 +323,7 @@ class LSTM(_RNNLayer): Initializer for the recurrent weights matrix, used for the linear transformation of the recurrent state. i2h_bias_initializer : str or Initializer, default 'lstmbias' - Initializer for the bias vector. By default bias for the forget + Initializer for the bias vector. By default, bias for the forget gate is initialized to 1 while all other biases are initialized to zero. h2h_bias_initializer : str or Initializer @@ -333,7 +333,7 @@ class LSTM(_RNNLayer): If not specified, it will be inferred from input. prefix : str or None Prefix of this `Block`. - params : ParameterDict or None + params : `ParameterDict` or `None` Shared Parameters for this `Block`. @@ -351,7 +351,7 @@ class LSTM(_RNNLayer): The recurrent state is a list of two NDArrays. Both has shape `(num_layers, batch_size, num_hidden)`. If `bidirectional` is True, state shape will instead be - `(num_layers, batch_size, 2*num_hidden)` + `(num_layers, batch_size, 2*num_hidden)`. Examples diff --git a/python/mxnet/gluon/trainer.py b/python/mxnet/gluon/trainer.py index 8f20bd1e698b..5483f6bc7d9c 100644 --- a/python/mxnet/gluon/trainer.py +++ b/python/mxnet/gluon/trainer.py @@ -7,8 +7,8 @@ from .parameter import ParameterDict, Parameter class Trainer(object): - """Applies an Optimizer on a set of Parameters. Trainer should - be used together with autograd. + """Applies an `Optimizer` on a set of Parameters. Trainer should + be used together with `autograd`. Parameters ---------- @@ -17,7 +17,7 @@ class Trainer(object): optimizer : str or Optimizer The optimizer to use. optimizer_params : dict - key-word arguments to be passed to optimizer constructor. For example, + Key-word arguments to be passed to optimizer constructor. For example, `{'learning_rate': 0.1}` kvstore : str or KVStore kvstore type for multi-gpu and distributed training. @@ -85,13 +85,13 @@ def _init_kvstore(self): self._kv_initialized = True def step(self, batch_size, ignore_stale_grad=False): - """Make one step of parameter update. Should be called after - autograd.compute_gradient and outside of record() scope. + """Makes one step of parameter update. Should be called after + `autograd.compute_gradient` and outside of `record()` scope. Parameters ---------- batch_size : int - Batch size of data processed. Gradient will be normalized by 1/batch_size. + Batch size of data processed. Gradient will be normalized by `1/batch_size`. Set this to 1 if you normalized loss manually with `loss = mean(loss)`. ignore_stale_grad : bool, optional, default=False If true, ignores Parameters with stale gradient (gradient that has not diff --git a/python/mxnet/gluon/utils.py b/python/mxnet/gluon/utils.py index 18703cda4058..842f260763d2 100644 --- a/python/mxnet/gluon/utils.py +++ b/python/mxnet/gluon/utils.py @@ -6,7 +6,7 @@ from .. import ndarray def split_data(data, num_slice, batch_axis=0, even_split=True): - """Split a NDArray into num_slice slices along batch_axis. + """Splits an NDArray into `num_slice` slices along `batch_axis`. Usually used for data parallelism where each slices is sent to one device (i.e. GPU). @@ -20,13 +20,13 @@ def split_data(data, num_slice, batch_axis=0, even_split=True): The axis along which to slice. even_split : bool, default True Whether to force all slices to have the same number of elements. - If True, An error will be raised when `num_slice` does not evenly + If `True`, an error will be raised when `num_slice` does not evenly divide `data.shape[batch_axis]`. Returns ------- list of NDArray - Return value is a list even if num_slice is 1. + Return value is a list even if `num_slice` is 1. """ size = data.shape[batch_axis] if size < num_slice: @@ -55,15 +55,15 @@ def split_data(data, num_slice, batch_axis=0, even_split=True): def split_and_load(data, ctx_list, batch_axis=0, even_split=True): - """Split a NDArray into `len(ctx_list)` slices along `batch_axis` and load - each slice to one context in ctx_list. + """Splits an NDArray into `len(ctx_list)` slices along `batch_axis` and loads + each slice to one context in `ctx_list`. Parameters ---------- data : NDArray A batch of data. ctx_list : list of Context - A list of Contexts + A list of Contexts. batch_axis : int, default 0 The axis along which to slice. even_split : bool, default True @@ -71,7 +71,8 @@ def split_and_load(data, ctx_list, batch_axis=0, even_split=True): Returns ------- - list of NDArray, each corresponds to a context in ctx_list. + list of NDArray + Each corresponds to a context in `ctx_list`. """ if not isinstance(data, ndarray.NDArray): data = ndarray.array(data, ctx=ctx_list[0]) @@ -83,7 +84,7 @@ def split_and_load(data, ctx_list, batch_axis=0, even_split=True): def clip_global_norm(arrays, max_norm): - """Rescales NDArrays so that the sum of their 2-norm is smaller than max_norm. + """Rescales NDArrays so that the sum of their 2-norm is smaller than `max_norm`. """ assert len(arrays) > 0 total_norm = 0 From 1bb9e45f5df2b4a283bb930a25c985b1a9b5cd82 Mon Sep 17 00:00:00 2001 From: Eric Junyuan Xie Date: Fri, 14 Jul 2017 00:04:53 -0700 Subject: [PATCH 216/834] fix ndarray slicing (#7035) * fix ndarray slicing * add warning --- python/mxnet/base.py | 2 + python/mxnet/gluon/block.py | 8 ++-- python/mxnet/ndarray.py | 63 +++++++++++++++++------------ src/ndarray/autograd.cc | 15 +++++++ src/operator/tensor/matrix_op-inl.h | 2 +- 5 files changed, 58 insertions(+), 32 deletions(-) diff --git a/python/mxnet/base.py b/python/mxnet/base.py index 731cc392413c..f58429980eab 100644 --- a/python/mxnet/base.py +++ b/python/mxnet/base.py @@ -19,12 +19,14 @@ if sys.version_info[0] == 3: string_types = str, numeric_types = (float, int, np.float32, np.int32) + integer_types = int # this function is needed for python3 # to convert ctypes.char_p .value back to python str py_str = lambda x: x.decode('utf-8') else: string_types = basestring, numeric_types = (float, int, long, np.float32, np.int32) + integer_types = (int, long) py_str = lambda x: x class _NullType(object): diff --git a/python/mxnet/gluon/block.py b/python/mxnet/gluon/block.py index d284e954d87b..4a0060a03345 100644 --- a/python/mxnet/gluon/block.py +++ b/python/mxnet/gluon/block.py @@ -262,13 +262,11 @@ def __setattr__(self, name, value): def register_child(self, block): if not isinstance(block, HybridBlock): - if isinstance(block, Sequential): - raise ValueError( - "Children of HybridBlock must also be HybridBlock. " \ - "Please use HybridSequential instead of Sequential.") raise ValueError( "Children of HybridBlock must also be HybridBlock, " \ - "but %s has type %s."%(str(block), str(type(block)))) + "but %s has type %s. If you are using Sequential, " \ + "please try HybridSequential instead"%( + str(block), str(type(block)))) super(HybridBlock, self).register_child(block) def hybridize(self, active=True): diff --git a/python/mxnet/ndarray.py b/python/mxnet/ndarray.py index 537228a495fc..88a4b0569ef5 100644 --- a/python/mxnet/ndarray.py +++ b/python/mxnet/ndarray.py @@ -17,7 +17,7 @@ import operator import numpy as np -from .base import _LIB, string_types, numeric_types +from .base import _LIB, string_types, numeric_types, integer_types from .base import c_array, py_str, c_str, mx_real_t, _Null # pylint: disable=unused-import from .base import mx_uint, NDArrayHandle, check_call, OpHandle from .base import ctypes2buffer @@ -336,14 +336,14 @@ def __setitem__(self, key, value): """ # pylint: disable=too-many-branches if not self.writable: - raise ValueError('Failed to assign to a readonly NDArray') - if isinstance(key, int): + raise ValueError('Cannot assign to readonly NDArray') + if isinstance(key, integer_types): sliced_arr = self._at(key) sliced_arr[:] = value return - if isinstance(key, py_slice): + elif isinstance(key, py_slice): if key.step is not None: - raise ValueError('NDArray only supports continuous slicing on axis 0') + raise ValueError('NDArray only supports slicing with step size 1') if key.start is not None or key.stop is not None: sliced_arr = self._slice(key.start, key.stop) sliced_arr[:] = value @@ -356,34 +356,35 @@ def __setitem__(self, key, value): elif isinstance(value, (np.ndarray, np.generic)): self._sync_copyfrom(value) else: - raise TypeError('type %s not supported' % str(type(value))) - if isinstance(key, tuple): + raise TypeError( + 'NDArray does not support assignment with %s of type %s'%( + str(value), str(type(value)))) + elif isinstance(key, tuple): # multi-dimension indexing my_shape = self.shape - assert len(key) == len(my_shape) - for slice_i in key: - assert isinstance(slice_i, (py_slice, int)) + assert len(key) <= len(my_shape), \ + "Indexing dimensions exceed array dimensions, %d vs %d"%( + len(key), len(my_shape)) begin = [0 for _ in my_shape] end = [x for x in my_shape] for i, slice_i in enumerate(key): - if isinstance(slice_i, int): + if isinstance(slice_i, integer_types): assert slice_i < my_shape[i] begin[i] = slice_i end[i] = slice_i + 1 elif isinstance(slice_i, py_slice): # only support continuous slicing assert slice_i.step is None, \ - "NDArray only supports continuous slicing." + "NDArray only supports slicing with step size 1." begin[i] = slice_i.start or 0 end[i] = slice_i.stop or my_shape[i] assert begin[i] < end[i] assert end[i] <= my_shape[i] else: raise ValueError( - "NDArray does not support slicing with %s."%( - str(slice_i))) - begin = tuple(begin) - end = tuple(end) + "NDArray does not support slicing with key %s of type %s."%( + str(slice_i), str(type(slice_i)))) + if isinstance(value, NDArray): value = value.as_in_context(self.context) _internal._crop_assign(self, value, out=self, @@ -397,7 +398,13 @@ def __setitem__(self, key, value): _internal._crop_assign(self, value, out=self, begin=begin, end=end) else: - raise TypeError('type %s not supported' % str(type(value))) + raise TypeError( + 'NDArray does not support assignment with %s of type %s'%( + str(value), str(type(value)))) + else: + raise ValueError( + "NDArray does not support slicing with key %s of type %s."%( + str(key), str(type(key)))) # pylint: enable=too-many-branches def __getitem__(self, key): @@ -425,20 +432,20 @@ def __getitem__(self, key): [ 3., 4., 5.]], dtype=float32) """ # multi-dimensional slicing is not supported yet - if isinstance(key, int): + if isinstance(key, integer_types): if key > self.shape[0] - 1: raise IndexError( 'index {} is out of bounds for axis 0 with size {}'.format( key, self.shape[0])) return self._at(key) - if isinstance(key, py_slice): + elif isinstance(key, py_slice): if key.step is not None: - raise ValueError('NDArray only supports continuous slicing on axis 0') + raise ValueError("NDArray only supports slicing with step size 1.") if key.start is not None or key.stop is not None: return self._slice(key.start, key.stop) else: return self - if isinstance(key, tuple): + elif isinstance(key, tuple): shape = self.shape oshape = [] begin = [] @@ -448,23 +455,27 @@ def __getitem__(self, key): len(key), len(shape)) i = -1 for i, slice_i in enumerate(key): - if isinstance(slice_i, int): + if isinstance(slice_i, integer_types): begin.append(slice_i) end.append(slice_i+1) elif isinstance(slice_i, py_slice): if slice_i.step is not None: - raise ValueError("NDArray only supports continuous slicing.") + raise ValueError("NDArray only supports slicing with step size 1.") begin.append(0 if slice_i.start is None else slice_i.start) end.append(shape[i] if slice_i.stop is None else slice_i.stop) oshape.append(end[i] - begin[i]) else: raise ValueError( - "NDArray does not support slicing with %s."%( - str(slice_i))) + "NDArray does not support slicing with key %s of type %s."%( + str(slice_i), str(type(slice_i)))) oshape.extend(shape[i+1:]) if len(oshape) == 0: oshape.append(1) return slice(self, begin, end).reshape(oshape) + else: + raise ValueError( + "NDArray does not support slicing with key %s of type %s."%( + str(key), str(type(key)))) def _sync_copyfrom(self, source_array): """Performs a synchronized copy from the `source_array` to the current array. @@ -1072,7 +1083,7 @@ def empty(shape, ctx=None, dtype=mx_real_t): >>> mx.nd.empty((1,2), mx.gpu(0), 'float16') """ - if isinstance(shape, int): + if isinstance(shape, integer_types): shape = (shape, ) if ctx is None: ctx = Context.default_ctx diff --git a/src/ndarray/autograd.cc b/src/ndarray/autograd.cc index 4875419b5973..b606a4dcdaa8 100644 --- a/src/ndarray/autograd.cc +++ b/src/ndarray/autograd.cc @@ -170,6 +170,12 @@ void AutogradRuntime::ComputeGradient(const std::vector& outputs, std::vector grad_reqs; std::unordered_map saved_states; AGDFSVisit(heads, [&](const AGNodePtr& n) { + CHECK(n->nn_node != nullptr) + << "Node is differentiated twice without retaining graph the first time. " + << "This usually happens when you want to differentiate a graph twice but " + << "forgot to set retain_graph=True the first time. If you are training " + << "recurrent model (like LSTMs) maybe you forgot to detach the hidden " + << "state from the previous iteration before feeding it to the next iteration."; if (n->nn_node->is_variable()) { vlist.push_back(n); } else { @@ -187,6 +193,7 @@ void AutogradRuntime::ComputeGradient(const std::vector& outputs, } }); + bool has_writeto = false; for (const auto& n : vlist) { if (mutable_set.count(n.get())) { aux_states.push_back(n->outputs[0]); @@ -197,6 +204,7 @@ void AutogradRuntime::ComputeGradient(const std::vector& outputs, args.push_back(n->outputs[0]); args_grad.push_back(n->out_grads[0]); grad_reqs.push_back(n->grad_req); + has_writeto = has_writeto || n->grad_req == kWriteTo; } } @@ -232,6 +240,13 @@ void AutogradRuntime::ComputeGradient(const std::vector& outputs, for (auto& i : heads) { i.ag_node->clear_history(); } + } else if (has_writeto) { + LOG(INFO) + << "Warning: when calling backward with retain_graph=True, grad_req for " + << "Parameters should be set to 'add'. Otherwise the second backward " + << "will over-write gradients from the first backward. Also remember " + << "to manually set gradients to zero with zero_grad before starting the " + << "next iteration."; } } diff --git a/src/operator/tensor/matrix_op-inl.h b/src/operator/tensor/matrix_op-inl.h index 10273c0a8c68..72fd2773c8f8 100644 --- a/src/operator/tensor/matrix_op-inl.h +++ b/src/operator/tensor/matrix_op-inl.h @@ -820,7 +820,7 @@ void Slice(const nnvm::NodeAttrs& attrs, break; } default: - LOG(FATAL) << "crop supports at most 5 dimensions"; + LOG(FATAL) << "slice supports at most 5 dimensions"; break; } }); From cac919b1cfe70d213058aaed363fe84a73a14ab3 Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Fri, 14 Jul 2017 00:07:26 -0700 Subject: [PATCH 217/834] similarity tree lstm model (childsum) (#7022) --- example/gluon/tree_lstm/LICENSE | 21 ++ example/gluon/tree_lstm/dataset.py | 210 ++++++++++++++++ .../gluon/tree_lstm/fetch_and_preprocess.sh | 7 + .../lib/CollapseUnaryTransformer.java | 34 +++ .../tree_lstm/lib/ConstituencyParse.java | 234 ++++++++++++++++++ .../gluon/tree_lstm/lib/DependencyParse.java | 140 +++++++++++ example/gluon/tree_lstm/main.py | 171 +++++++++++++ example/gluon/tree_lstm/scripts/download.py | 90 +++++++ .../tree_lstm/scripts/preprocess-sick.py | 105 ++++++++ example/gluon/tree_lstm/tree_lstm.py | 137 ++++++++++ python/mxnet/initializer.py | 5 +- 11 files changed, 1153 insertions(+), 1 deletion(-) create mode 100644 example/gluon/tree_lstm/LICENSE create mode 100644 example/gluon/tree_lstm/dataset.py create mode 100755 example/gluon/tree_lstm/fetch_and_preprocess.sh create mode 100644 example/gluon/tree_lstm/lib/CollapseUnaryTransformer.java create mode 100644 example/gluon/tree_lstm/lib/ConstituencyParse.java create mode 100644 example/gluon/tree_lstm/lib/DependencyParse.java create mode 100644 example/gluon/tree_lstm/main.py create mode 100644 example/gluon/tree_lstm/scripts/download.py create mode 100644 example/gluon/tree_lstm/scripts/preprocess-sick.py create mode 100644 example/gluon/tree_lstm/tree_lstm.py diff --git a/example/gluon/tree_lstm/LICENSE b/example/gluon/tree_lstm/LICENSE new file mode 100644 index 000000000000..441cb8a1d7de --- /dev/null +++ b/example/gluon/tree_lstm/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017 Riddhiman Dasgupta, Sheng Zha + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/example/gluon/tree_lstm/dataset.py b/example/gluon/tree_lstm/dataset.py new file mode 100644 index 000000000000..f9cfce5c2bff --- /dev/null +++ b/example/gluon/tree_lstm/dataset.py @@ -0,0 +1,210 @@ +import os +import logging +logging.basicConfig(level=logging.INFO) +import numpy as np +import random +from tqdm import tqdm + +import mxnet as mx + +class Vocab(object): + # constants for special tokens: padding, unknown, and beginning/end of sentence. + PAD = 0 + UNK = 1 + BOS = 2 + EOS = 3 + PAD_WORD = '' + UNK_WORD = '' + BOS_WORD = '' + EOS_WORD = '' + def __init__(self, filepaths=[], embedpath=None, include_unseen=False, lower=False): + self.idx2tok = [] + self.tok2idx = {} + self.lower = lower + self.include_unseen = include_unseen + + self.add(Vocab.PAD_WORD) + self.add(Vocab.UNK_WORD) + self.add(Vocab.BOS_WORD) + self.add(Vocab.EOS_WORD) + + self.embed = None + + for filename in filepaths: + logging.info('loading %s'%filename) + with open(filename, 'r') as f: + self.load_file(f) + if embedpath is not None: + logging.info('loading %s'%embedpath) + with open(embedpath, 'r') as f: + self.load_embedding(f, reset=set([Vocab.PAD_WORD, Vocab.UNK_WORD, Vocab.BOS_WORD, + Vocab.EOS_WORD])) + + @property + def size(self): + return len(self.idx2tok) + + def get_index(self, key): + return self.tok2idx.get(key.lower() if self.lower else key, + Vocab.UNK) + + def get_token(self, idx): + if idx < self.size: + return self.idx2tok[idx] + else: + return Vocab.UNK_WORD + + def add(self, token): + token = token.lower() if self.lower else token + if token in self.tok2idx: + idx = self.tok2idx[token] + else: + idx = len(self.idx2tok) + self.idx2tok.append(token) + self.tok2idx[token] = idx + return idx + + def to_indices(self, tokens, add_bos=False, add_eos=False): + vec = [BOS] if add_bos else [] + vec += [self.get_index(token) for token in tokens] + if add_eos: + vec.append(EOS) + return vec + + def to_tokens(self, indices, stop): + tokens = [] + for i in indices: + tokens += [self.get_token(i)] + if i == stop: + break + return tokens + + def load_file(self, f): + for line in f: + tokens = line.rstrip('\n').split() + for token in tokens: + self.add(token) + + def load_embedding(self, f, reset=[]): + vectors = {} + for line in tqdm(f.readlines(), desc='Loading embeddings'): + tokens = line.rstrip('\n').split(' ') + word = tokens[0].lower() if self.lower else tokens[0] + if self.include_unseen: + self.add(word) + if word in self.tok2idx: + vectors[word] = [float(x) for x in tokens[1:]] + dim = len(vectors.values()[0]) + def to_vector(tok): + if tok in vectors and tok not in reset: + return vectors[tok] + elif tok not in vectors: + return np.random.normal(-0.05, 0.05, size=dim) + else: + return [0.0]*dim + self.embed = mx.nd.array([vectors[tok] if tok in vectors and tok not in reset + else [0.0]*dim for tok in self.idx2tok]) + +class Tree(object): + def __init__(self, idx): + self.children = [] + self.idx = idx + + def __repr__(self): + if self.children: + return '{0}: {1}'.format(self.idx, str(self.children)) + else: + return str(self.idx) + +# Dataset class for SICK dataset +class SICKDataIter(object): + def __init__(self, path, vocab, num_classes, shuffle=True): + super(SICKDataIter, self).__init__() + self.vocab = vocab + self.num_classes = num_classes + self.l_sentences = self.read_sentences(os.path.join(path,'a.toks')) + self.r_sentences = self.read_sentences(os.path.join(path,'b.toks')) + self.l_trees = self.read_trees(os.path.join(path,'a.parents')) + self.r_trees = self.read_trees(os.path.join(path,'b.parents')) + self.labels = self.read_labels(os.path.join(path,'sim.txt')) + self.size = len(self.labels) + self.shuffle = shuffle + self.reset() + + def reset(self): + if self.shuffle: + mask = range(self.size) + random.shuffle(mask) + self.l_sentences = [self.l_sentences[i] for i in mask] + self.r_sentences = [self.r_sentences[i] for i in mask] + self.l_trees = [self.l_trees[i] for i in mask] + self.r_trees = [self.r_trees[i] for i in mask] + self.labels = [self.labels[i] for i in mask] + self.index = 0 + + def next(self): + out = self[self.index] + self.index += 1 + return out + + def set_context(self, context): + self.l_sentences = [a.as_in_context(context) for a in self.l_sentences] + self.r_sentences = [a.as_in_context(context) for a in self.r_sentences] + + def __len__(self): + return self.size + + def __getitem__(self, index): + l_tree = self.l_trees[index] + r_tree = self.r_trees[index] + l_sent = self.l_sentences[index] + r_sent = self.r_sentences[index] + label = self.labels[index] + return (l_tree,l_sent,r_tree,r_sent,label) + + def read_sentence(self, line): + indices = self.vocab.to_indices(line.split()) + return mx.nd.array(indices) + + def read_sentences(self, filename): + with open(filename,'r') as f: + sentences = [self.read_sentence(line) for line in f.readlines()] + return sentences + + def read_tree(self, line): + parents = [int(x) for x in line.split()] + nodes = {} + root = None + for i in range(1,len(parents)+1): + if i-1 not in nodes and parents[i-1]!=-1: + idx = i + prev = None + while True: + parent = parents[idx-1] + if parent == -1: + break + tree = Tree(idx) + if prev is not None: + tree.children.append(prev) + nodes[idx-1] = tree + tree.idx = idx-1 + if parent-1 in nodes: + nodes[parent-1].children.append(tree) + break + elif parent==0: + root = tree + break + else: + prev = tree + idx = parent + return root + + def read_trees(self, filename): + with open(filename,'r') as f: + trees = [self.read_tree(line) for line in tqdm(f.readlines(), 'Parsing trees')] + return trees + + def read_labels(self, filename): + with open(filename,'r') as f: + labels = [float(x) for x in f.readlines()] + return labels diff --git a/example/gluon/tree_lstm/fetch_and_preprocess.sh b/example/gluon/tree_lstm/fetch_and_preprocess.sh new file mode 100755 index 000000000000..dfbf82a872c1 --- /dev/null +++ b/example/gluon/tree_lstm/fetch_and_preprocess.sh @@ -0,0 +1,7 @@ +#!/bin/bash +set -e +python2.7 scripts/download.py + +CLASSPATH="lib:lib/stanford-parser/stanford-parser.jar:lib/stanford-parser/stanford-parser-3.5.1-models.jar" +javac -cp $CLASSPATH lib/*.java +python2.7 scripts/preprocess-sick.py \ No newline at end of file diff --git a/example/gluon/tree_lstm/lib/CollapseUnaryTransformer.java b/example/gluon/tree_lstm/lib/CollapseUnaryTransformer.java new file mode 100644 index 000000000000..590dcb3dfa05 --- /dev/null +++ b/example/gluon/tree_lstm/lib/CollapseUnaryTransformer.java @@ -0,0 +1,34 @@ +import java.util.List; + +import edu.stanford.nlp.ling.Label; +import edu.stanford.nlp.trees.Tree; +import edu.stanford.nlp.trees.TreeTransformer; +import edu.stanford.nlp.util.Generics; + +/** + * This transformer collapses chains of unary nodes so that the top + * node is the only node left. The Sentiment model does not handle + * unary nodes, so this simplifies them to make a binary tree consist + * entirely of binary nodes and preterminals. A new tree with new + * nodes and labels is returned; the original tree is unchanged. + * + * @author John Bauer + */ +public class CollapseUnaryTransformer implements TreeTransformer { + public Tree transformTree(Tree tree) { + if (tree.isPreTerminal() || tree.isLeaf()) { + return tree.deepCopy(); + } + + Label label = tree.label().labelFactory().newLabel(tree.label()); + Tree[] children = tree.children(); + while (children.length == 1 && !children[0].isLeaf()) { + children = children[0].children(); + } + List processedChildren = Generics.newArrayList(); + for (Tree child : children) { + processedChildren.add(transformTree(child)); + } + return tree.treeFactory().newTreeNode(label, processedChildren); + } +} diff --git a/example/gluon/tree_lstm/lib/ConstituencyParse.java b/example/gluon/tree_lstm/lib/ConstituencyParse.java new file mode 100644 index 000000000000..7100eccde7f0 --- /dev/null +++ b/example/gluon/tree_lstm/lib/ConstituencyParse.java @@ -0,0 +1,234 @@ +import edu.stanford.nlp.process.WordTokenFactory; +import edu.stanford.nlp.ling.HasWord; +import edu.stanford.nlp.ling.Word; +import edu.stanford.nlp.ling.CoreLabel; +import edu.stanford.nlp.process.PTBTokenizer; +import edu.stanford.nlp.util.StringUtils; +import edu.stanford.nlp.parser.lexparser.LexicalizedParser; +import edu.stanford.nlp.parser.lexparser.TreeBinarizer; +import edu.stanford.nlp.trees.GrammaticalStructure; +import edu.stanford.nlp.trees.GrammaticalStructureFactory; +import edu.stanford.nlp.trees.PennTreebankLanguagePack; +import edu.stanford.nlp.trees.Tree; +import edu.stanford.nlp.trees.Trees; +import edu.stanford.nlp.trees.TreebankLanguagePack; +import edu.stanford.nlp.trees.TypedDependency; + +import java.io.BufferedWriter; +import java.io.FileWriter; +import java.io.StringReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.HashMap; +import java.util.Properties; +import java.util.Scanner; + +public class ConstituencyParse { + + private boolean tokenize; + private BufferedWriter tokWriter, parentWriter; + private LexicalizedParser parser; + private TreeBinarizer binarizer; + private CollapseUnaryTransformer transformer; + private GrammaticalStructureFactory gsf; + + private static final String PCFG_PATH = "edu/stanford/nlp/models/lexparser/englishPCFG.ser.gz"; + + public ConstituencyParse(String tokPath, String parentPath, boolean tokenize) throws IOException { + this.tokenize = tokenize; + if (tokPath != null) { + tokWriter = new BufferedWriter(new FileWriter(tokPath)); + } + parentWriter = new BufferedWriter(new FileWriter(parentPath)); + parser = LexicalizedParser.loadModel(PCFG_PATH); + binarizer = TreeBinarizer.simpleTreeBinarizer( + parser.getTLPParams().headFinder(), parser.treebankLanguagePack()); + transformer = new CollapseUnaryTransformer(); + + // set up to produce dependency representations from constituency trees + TreebankLanguagePack tlp = new PennTreebankLanguagePack(); + gsf = tlp.grammaticalStructureFactory(); + } + + public List sentenceToTokens(String line) { + List tokens = new ArrayList<>(); + if (tokenize) { + PTBTokenizer tokenizer = new PTBTokenizer(new StringReader(line), new WordTokenFactory(), ""); + for (Word label; tokenizer.hasNext(); ) { + tokens.add(tokenizer.next()); + } + } else { + for (String word : line.split(" ")) { + tokens.add(new Word(word)); + } + } + + return tokens; + } + + public Tree parse(List tokens) { + Tree tree = parser.apply(tokens); + return tree; + } + + public int[] constTreeParents(Tree tree) { + Tree binarized = binarizer.transformTree(tree); + Tree collapsedUnary = transformer.transformTree(binarized); + Trees.convertToCoreLabels(collapsedUnary); + collapsedUnary.indexSpans(); + List leaves = collapsedUnary.getLeaves(); + int size = collapsedUnary.size() - leaves.size(); + int[] parents = new int[size]; + HashMap index = new HashMap(); + + int idx = leaves.size(); + int leafIdx = 0; + for (Tree leaf : leaves) { + Tree cur = leaf.parent(collapsedUnary); // go to preterminal + int curIdx = leafIdx++; + boolean done = false; + while (!done) { + Tree parent = cur.parent(collapsedUnary); + if (parent == null) { + parents[curIdx] = 0; + break; + } + + int parentIdx; + int parentNumber = parent.nodeNumber(collapsedUnary); + if (!index.containsKey(parentNumber)) { + parentIdx = idx++; + index.put(parentNumber, parentIdx); + } else { + parentIdx = index.get(parentNumber); + done = true; + } + + parents[curIdx] = parentIdx + 1; + cur = parent; + curIdx = parentIdx; + } + } + + return parents; + } + + // convert constituency parse to a dependency representation and return the + // parent pointer representation of the tree + public int[] depTreeParents(Tree tree, List tokens) { + GrammaticalStructure gs = gsf.newGrammaticalStructure(tree); + Collection tdl = gs.typedDependencies(); + int len = tokens.size(); + int[] parents = new int[len]; + for (int i = 0; i < len; i++) { + // if a node has a parent of -1 at the end of parsing, then the node + // has no parent. + parents[i] = -1; + } + + for (TypedDependency td : tdl) { + // let root have index 0 + int child = td.dep().index(); + int parent = td.gov().index(); + parents[child - 1] = parent; + } + + return parents; + } + + public void printTokens(List tokens) throws IOException { + int len = tokens.size(); + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < len - 1; i++) { + if (tokenize) { + sb.append(PTBTokenizer.ptbToken2Text(tokens.get(i).word())); + } else { + sb.append(tokens.get(i).word()); + } + sb.append(' '); + } + + if (tokenize) { + sb.append(PTBTokenizer.ptbToken2Text(tokens.get(len - 1).word())); + } else { + sb.append(tokens.get(len - 1).word()); + } + + sb.append('\n'); + tokWriter.write(sb.toString()); + } + + public void printParents(int[] parents) throws IOException { + StringBuilder sb = new StringBuilder(); + int size = parents.length; + for (int i = 0; i < size - 1; i++) { + sb.append(parents[i]); + sb.append(' '); + } + sb.append(parents[size - 1]); + sb.append('\n'); + parentWriter.write(sb.toString()); + } + + public void close() throws IOException { + if (tokWriter != null) tokWriter.close(); + parentWriter.close(); + } + + public static void main(String[] args) throws Exception { + Properties props = StringUtils.argsToProperties(args); + if (!props.containsKey("parentpath")) { + System.err.println( + "usage: java ConstituencyParse -deps - -tokenize - -tokpath -parentpath "); + System.exit(1); + } + + // whether to tokenize input sentences + boolean tokenize = false; + if (props.containsKey("tokenize")) { + tokenize = true; + } + + // whether to produce dependency trees from the constituency parse + boolean deps = false; + if (props.containsKey("deps")) { + deps = true; + } + + String tokPath = props.containsKey("tokpath") ? props.getProperty("tokpath") : null; + String parentPath = props.getProperty("parentpath"); + ConstituencyParse processor = new ConstituencyParse(tokPath, parentPath, tokenize); + + Scanner stdin = new Scanner(System.in); + int count = 0; + long start = System.currentTimeMillis(); + while (stdin.hasNextLine()) { + String line = stdin.nextLine(); + List tokens = processor.sentenceToTokens(line); + Tree parse = processor.parse(tokens); + + // produce parent pointer representation + int[] parents = deps ? processor.depTreeParents(parse, tokens) + : processor.constTreeParents(parse); + + // print + if (tokPath != null) { + processor.printTokens(tokens); + } + processor.printParents(parents); + + count++; + if (count % 1000 == 0) { + double elapsed = (System.currentTimeMillis() - start) / 1000.0; + System.err.printf("Parsed %d lines (%.2fs)\n", count, elapsed); + } + } + + long totalTimeMillis = System.currentTimeMillis() - start; + System.err.printf("Done: %d lines in %.2fs (%.1fms per line)\n", + count, totalTimeMillis / 1000.0, totalTimeMillis / (double) count); + processor.close(); + } +} diff --git a/example/gluon/tree_lstm/lib/DependencyParse.java b/example/gluon/tree_lstm/lib/DependencyParse.java new file mode 100644 index 000000000000..e94de7764e3c --- /dev/null +++ b/example/gluon/tree_lstm/lib/DependencyParse.java @@ -0,0 +1,140 @@ +import edu.stanford.nlp.process.WordTokenFactory; +import edu.stanford.nlp.ling.HasWord; +import edu.stanford.nlp.ling.Word; +import edu.stanford.nlp.ling.TaggedWord; +import edu.stanford.nlp.parser.nndep.DependencyParser; +import edu.stanford.nlp.process.PTBTokenizer; +import edu.stanford.nlp.trees.TypedDependency; +import edu.stanford.nlp.util.StringUtils; +import edu.stanford.nlp.tagger.maxent.MaxentTagger; + +import java.io.BufferedWriter; +import java.io.FileWriter; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Properties; +import java.util.Scanner; + +public class DependencyParse { + + public static final String TAGGER_MODEL = "stanford-tagger/models/english-left3words-distsim.tagger"; + public static final String PARSER_MODEL = "edu/stanford/nlp/models/parser/nndep/english_SD.gz"; + + public static void main(String[] args) throws Exception { + Properties props = StringUtils.argsToProperties(args); + if (!props.containsKey("tokpath") || + !props.containsKey("parentpath") || + !props.containsKey("relpath")) { + System.err.println( + "usage: java DependencyParse -tokenize - -tokpath -parentpath -relpath "); + System.exit(1); + } + + boolean tokenize = false; + if (props.containsKey("tokenize")) { + tokenize = true; + } + + String tokPath = props.getProperty("tokpath"); + String parentPath = props.getProperty("parentpath"); + String relPath = props.getProperty("relpath"); + + BufferedWriter tokWriter = new BufferedWriter(new FileWriter(tokPath)); + BufferedWriter parentWriter = new BufferedWriter(new FileWriter(parentPath)); + BufferedWriter relWriter = new BufferedWriter(new FileWriter(relPath)); + + MaxentTagger tagger = new MaxentTagger(TAGGER_MODEL); + DependencyParser parser = DependencyParser.loadFromModelFile(PARSER_MODEL); + Scanner stdin = new Scanner(System.in); + int count = 0; + long start = System.currentTimeMillis(); + while (stdin.hasNextLine()) { + String line = stdin.nextLine(); + List tokens = new ArrayList<>(); + if (tokenize) { + PTBTokenizer tokenizer = new PTBTokenizer( + new StringReader(line), new WordTokenFactory(), ""); + for (Word label; tokenizer.hasNext(); ) { + tokens.add(tokenizer.next()); + } + } else { + for (String word : line.split(" ")) { + tokens.add(new Word(word)); + } + } + + List tagged = tagger.tagSentence(tokens); + + int len = tagged.size(); + Collection tdl = parser.predict(tagged).typedDependencies(); + int[] parents = new int[len]; + for (int i = 0; i < len; i++) { + // if a node has a parent of -1 at the end of parsing, then the node + // has no parent. + parents[i] = -1; + } + + String[] relns = new String[len]; + for (TypedDependency td : tdl) { + // let root have index 0 + int child = td.dep().index(); + int parent = td.gov().index(); + relns[child - 1] = td.reln().toString(); + parents[child - 1] = parent; + } + + // print tokens + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < len - 1; i++) { + if (tokenize) { + sb.append(PTBTokenizer.ptbToken2Text(tokens.get(i).word())); + } else { + sb.append(tokens.get(i).word()); + } + sb.append(' '); + } + if (tokenize) { + sb.append(PTBTokenizer.ptbToken2Text(tokens.get(len - 1).word())); + } else { + sb.append(tokens.get(len - 1).word()); + } + sb.append('\n'); + tokWriter.write(sb.toString()); + + // print parent pointers + sb = new StringBuilder(); + for (int i = 0; i < len - 1; i++) { + sb.append(parents[i]); + sb.append(' '); + } + sb.append(parents[len - 1]); + sb.append('\n'); + parentWriter.write(sb.toString()); + + // print relations + sb = new StringBuilder(); + for (int i = 0; i < len - 1; i++) { + sb.append(relns[i]); + sb.append(' '); + } + sb.append(relns[len - 1]); + sb.append('\n'); + relWriter.write(sb.toString()); + + count++; + if (count % 1000 == 0) { + double elapsed = (System.currentTimeMillis() - start) / 1000.0; + System.err.printf("Parsed %d lines (%.2fs)\n", count, elapsed); + } + } + + long totalTimeMillis = System.currentTimeMillis() - start; + System.err.printf("Done: %d lines in %.2fs (%.1fms per line)\n", + count, totalTimeMillis / 1000.0, totalTimeMillis / (double) count); + tokWriter.close(); + parentWriter.close(); + relWriter.close(); + } +} diff --git a/example/gluon/tree_lstm/main.py b/example/gluon/tree_lstm/main.py new file mode 100644 index 000000000000..7903e67a1441 --- /dev/null +++ b/example/gluon/tree_lstm/main.py @@ -0,0 +1,171 @@ +# This example is inspired by https://github.com/dasguptar/treelstm.pytorch +import argparse, cPickle, math, os, random +import logging +logging.basicConfig(level=logging.INFO) +import numpy as np +from tqdm import tqdm + +import mxnet as mx +from mxnet import gluon +from mxnet.gluon import nn +from mxnet import autograd as ag + +from tree_lstm import SimilarityTreeLSTM +from dataset import Vocab, SICKDataIter + +parser = argparse.ArgumentParser(description='TreeLSTM for Sentence Similarity on Dependency Trees') +parser.add_argument('--data', default='data/sick/', + help='path to raw dataset. required when preprocessed dataset is not available.') +parser.add_argument('--word_embed', default='data/glove/glove.840B.300d.txt', + help='directory with word embeddings. required when preprocessed dataset is not available.') +parser.add_argument('--batch_size', type=int, default=25, + help='training batch size per device (CPU/GPU).') +parser.add_argument('--epochs', default=50, type=int, + help='number of total epochs to run') +parser.add_argument('--lr', default=0.02, type=float, + help='initial learning rate') +parser.add_argument('--wd', default=0.0001, type=float, + help='weight decay factor') +parser.add_argument('--optimizer', default='adagrad', + help='optimizer (default: adagrad)') +parser.add_argument('--seed', default=123, type=int, + help='random seed (default: 123)') +parser.add_argument('--use-gpu', action='store_true', + help='whether to use GPU.') + +opt = parser.parse_args() + +logging.info(opt) + +context = [mx.gpu(0) if opt.use_gpu else mx.cpu()] + +rnn_hidden_size, sim_hidden_size, num_classes = 150, 50, 5 +optimizer = opt.optimizer.lower() + +mx.random.seed(opt.seed) +np.random.seed(opt.seed) +random.seed(opt.seed) + +batch_size = opt.batch_size + +# read dataset +if os.path.exists('dataset.cPickle'): + with open('dataset.cPickle', 'rb') as f: + train_iter, dev_iter, test_iter, vocab = cPickle.load(f) +else: + root_dir = opt.data + segments = ['train', 'dev', 'test'] + token_files = [os.path.join(root_dir, seg, '%s.toks'%tok) + for tok in ['a', 'b'] + for seg in segments] + + vocab = Vocab(filepaths=token_files, embedpath=opt.word_embed) + + train_iter, dev_iter, test_iter = [SICKDataIter(os.path.join(root_dir, segment), vocab, num_classes) + for segment in segments] + with open('dataset.cPickle', 'wb') as f: + cPickle.dump([train_iter, dev_iter, test_iter, vocab], f) + +logging.info('==> SICK vocabulary size : %d ' % vocab.size) +logging.info('==> Size of train data : %d ' % len(train_iter)) +logging.info('==> Size of dev data : %d ' % len(dev_iter)) +logging.info('==> Size of test data : %d ' % len(test_iter)) + +# get network +net = SimilarityTreeLSTM(sim_hidden_size, rnn_hidden_size, vocab.size, vocab.embed.shape[1], num_classes) + +# use pearson correlation and mean-square error for evaluation +metric = mx.metric.create(['pearsonr', 'mse']) + +def to_target(x): + target = np.zeros((1, num_classes)) + ceil = int(math.ceil(x)) + floor = int(math.floor(x)) + if ceil==floor: + target[0][floor-1] = 1 + else: + target[0][floor-1] = ceil - x + target[0][ceil-1] = x - floor + return mx.nd.array(target) + +def to_score(x): + levels = mx.nd.arange(1, 6, ctx=x.context) + return [mx.nd.sum(levels*mx.nd.exp(x), axis=1).reshape((-1,1))] + +# when evaluating in validation mode, check and see if pearson-r is improved +# if so, checkpoint and run evaluation on test dataset +def test(ctx, data_iter, best, mode='validation', num_iter=-1): + data_iter.reset() + batches = len(data_iter) + data_iter.set_context(ctx[0]) + preds = [] + labels = [mx.nd.array(data_iter.labels, ctx=ctx[0]).reshape((-1,1))] + for _ in tqdm(range(batches), desc='Testing in {} mode'.format(mode)): + l_tree, l_sent, r_tree, r_sent, label = data_iter.next() + z = net(mx.nd, l_sent, r_sent, l_tree, r_tree) + preds.append(z) + + preds = to_score(mx.nd.concat(*preds, dim=0)) + metric.update(preds, labels) + names, values = metric.get() + metric.reset() + for name, acc in zip(names, values): + logging.info(mode+' acc: %s=%f'%(name, acc)) + if name == 'pearsonr': + test_r = acc + if mode == 'validation' and num_iter >= 0: + if test_r >= best: + best = test_r + logging.info('New optimum found: {}. Checkpointing.'.format(best)) + net.collect_params().save('childsum_tree_lstm_{}.params'.format(num_iter)) + test(ctx, test_iter, -1, 'test') + return best + + +def train(epoch, ctx, train_data, dev_data): + + # initialization with context + if isinstance(ctx, mx.Context): + ctx = [ctx] + net.collect_params().initialize(mx.init.Xavier(magnitude=2.24), ctx=ctx[0]) + net.embed.weight.set_data(vocab.embed.as_in_context(ctx[0])) + train_data.set_context(ctx[0]) + dev_data.set_context(ctx[0]) + + # set up trainer for optimizing the network. + trainer = gluon.Trainer(net.collect_params(), optimizer, {'learning_rate': opt.lr, 'wd': opt.wd}) + + best_r = -1 + Loss = gluon.loss.KLDivLoss() + for i in range(epoch): + train_data.reset() + num_batches = len(train_data) + # collect predictions and labels for evaluation metrics + preds = [] + labels = [mx.nd.array(train_data.labels, ctx=ctx[0]).reshape((-1,1))] + for j in tqdm(range(num_batches), desc='Training epoch {}'.format(i)): + # get next batch + l_tree, l_sent, r_tree, r_sent, label = train_data.next() + # use autograd to record the forward calculation + with ag.record(): + # forward calculation. the output is log probability + z = net(mx.nd, l_sent, r_sent, l_tree, r_tree) + # calculate loss + loss = Loss(z, to_target(label).as_in_context(ctx[0])) + # backward calculation for gradients. + loss.backward() + preds.append(z) + # update weight after every batch_size samples + if (j+1) % batch_size == 0: + trainer.step(batch_size) + + # translate log-probability to scores, and evaluate + preds = to_score(mx.nd.concat(*preds, dim=0)) + metric.update(preds, labels) + names, values = metric.get() + metric.reset() + for name, acc in zip(names, values): + logging.info('training acc at epoch %d: %s=%f'%(i, name, acc)) + best_r = test(ctx, dev_data, best_r, num_iter=i) + +train(opt.epochs, context, train_iter, dev_iter) diff --git a/example/gluon/tree_lstm/scripts/download.py b/example/gluon/tree_lstm/scripts/download.py new file mode 100644 index 000000000000..d38b3a46ecd8 --- /dev/null +++ b/example/gluon/tree_lstm/scripts/download.py @@ -0,0 +1,90 @@ +""" +Downloads the following: +- Stanford parser +- Stanford POS tagger +- Glove vectors +- SICK dataset (semantic relatedness task) +""" + +from __future__ import print_function +import urllib2 +import sys +import os +import shutil +import zipfile +import gzip +from mxnet.test_utils import download + +def unzip(filepath): + print("Extracting: " + filepath) + dirpath = os.path.dirname(filepath) + with zipfile.ZipFile(filepath) as zf: + zf.extractall(dirpath) + os.remove(filepath) + +def download_tagger(dirpath): + tagger_dir = 'stanford-tagger' + if os.path.exists(os.path.join(dirpath, tagger_dir)): + print('Found Stanford POS Tagger - skip') + return + url = 'http://nlp.stanford.edu/software/stanford-postagger-2015-01-29.zip' + filepath = download(url, dirname=dirpath) + zip_dir = '' + with zipfile.ZipFile(filepath) as zf: + zip_dir = zf.namelist()[0] + zf.extractall(dirpath) + os.remove(filepath) + os.rename(os.path.join(dirpath, zip_dir), os.path.join(dirpath, tagger_dir)) + +def download_parser(dirpath): + parser_dir = 'stanford-parser' + if os.path.exists(os.path.join(dirpath, parser_dir)): + print('Found Stanford Parser - skip') + return + url = 'http://nlp.stanford.edu/software/stanford-parser-full-2015-01-29.zip' + filepath = download(url, dirname=dirpath) + zip_dir = '' + with zipfile.ZipFile(filepath) as zf: + zip_dir = zf.namelist()[0] + zf.extractall(dirpath) + os.remove(filepath) + os.rename(os.path.join(dirpath, zip_dir), os.path.join(dirpath, parser_dir)) + +def download_wordvecs(dirpath): + if os.path.exists(dirpath): + print('Found Glove vectors - skip') + return + else: + os.makedirs(dirpath) + url = 'http://www-nlp.stanford.edu/data/glove.840B.300d.zip' + unzip(download(url, dirname=dirpath)) + +def download_sick(dirpath): + if os.path.exists(dirpath): + print('Found SICK dataset - skip') + return + else: + os.makedirs(dirpath) + train_url = 'http://alt.qcri.org/semeval2014/task1/data/uploads/sick_train.zip' + trial_url = 'http://alt.qcri.org/semeval2014/task1/data/uploads/sick_trial.zip' + test_url = 'http://alt.qcri.org/semeval2014/task1/data/uploads/sick_test_annotated.zip' + unzip(download(train_url, dirname=dirpath)) + unzip(download(trial_url, dirname=dirpath)) + unzip(download(test_url, dirname=dirpath)) + +if __name__ == '__main__': + base_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + + # data + data_dir = os.path.join(base_dir, 'data') + wordvec_dir = os.path.join(data_dir, 'glove') + sick_dir = os.path.join(data_dir, 'sick') + + # libraries + lib_dir = os.path.join(base_dir, 'lib') + + # download dependencies + download_tagger(lib_dir) + download_parser(lib_dir) + download_wordvecs(wordvec_dir) + download_sick(sick_dir) diff --git a/example/gluon/tree_lstm/scripts/preprocess-sick.py b/example/gluon/tree_lstm/scripts/preprocess-sick.py new file mode 100644 index 000000000000..fd28b58a5f7f --- /dev/null +++ b/example/gluon/tree_lstm/scripts/preprocess-sick.py @@ -0,0 +1,105 @@ +""" +Preprocessing script for SICK data. + +""" + +import os +import glob + +def make_dirs(dirs): + for d in dirs: + if not os.path.exists(d): + os.makedirs(d) + +def dependency_parse(filepath, cp='', tokenize=True): + print('\nDependency parsing ' + filepath) + dirpath = os.path.dirname(filepath) + filepre = os.path.splitext(os.path.basename(filepath))[0] + tokpath = os.path.join(dirpath, filepre + '.toks') + parentpath = os.path.join(dirpath, filepre + '.parents') + relpath = os.path.join(dirpath, filepre + '.rels') + tokenize_flag = '-tokenize - ' if tokenize else '' + cmd = ('java -cp %s DependencyParse -tokpath %s -parentpath %s -relpath %s %s < %s' + % (cp, tokpath, parentpath, relpath, tokenize_flag, filepath)) + os.system(cmd) + +def constituency_parse(filepath, cp='', tokenize=True): + dirpath = os.path.dirname(filepath) + filepre = os.path.splitext(os.path.basename(filepath))[0] + tokpath = os.path.join(dirpath, filepre + '.toks') + parentpath = os.path.join(dirpath, filepre + '.cparents') + tokenize_flag = '-tokenize - ' if tokenize else '' + cmd = ('java -cp %s ConstituencyParse -tokpath %s -parentpath %s %s < %s' + % (cp, tokpath, parentpath, tokenize_flag, filepath)) + os.system(cmd) + +def build_vocab(filepaths, dst_path, lowercase=True): + vocab = set() + for filepath in filepaths: + with open(filepath) as f: + for line in f: + if lowercase: + line = line.lower() + vocab |= set(line.split()) + with open(dst_path, 'w') as f: + for w in sorted(vocab): + f.write(w + '\n') + +def split(filepath, dst_dir): + with open(filepath) as datafile, \ + open(os.path.join(dst_dir, 'a.txt'), 'w') as afile, \ + open(os.path.join(dst_dir, 'b.txt'), 'w') as bfile, \ + open(os.path.join(dst_dir, 'id.txt'), 'w') as idfile, \ + open(os.path.join(dst_dir, 'sim.txt'), 'w') as simfile: + datafile.readline() + for line in datafile: + i, a, b, sim, ent = line.strip().split('\t') + idfile.write(i + '\n') + afile.write(a + '\n') + bfile.write(b + '\n') + simfile.write(sim + '\n') + +def parse(dirpath, cp=''): + dependency_parse(os.path.join(dirpath, 'a.txt'), cp=cp, tokenize=True) + dependency_parse(os.path.join(dirpath, 'b.txt'), cp=cp, tokenize=True) + constituency_parse(os.path.join(dirpath, 'a.txt'), cp=cp, tokenize=True) + constituency_parse(os.path.join(dirpath, 'b.txt'), cp=cp, tokenize=True) + +if __name__ == '__main__': + print('=' * 80) + print('Preprocessing SICK dataset') + print('=' * 80) + + base_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + data_dir = os.path.join(base_dir, 'data') + sick_dir = os.path.join(data_dir, 'sick') + lib_dir = os.path.join(base_dir, 'lib') + train_dir = os.path.join(sick_dir, 'train') + dev_dir = os.path.join(sick_dir, 'dev') + test_dir = os.path.join(sick_dir, 'test') + make_dirs([train_dir, dev_dir, test_dir]) + + # java classpath for calling Stanford parser + classpath = ':'.join([ + lib_dir, + os.path.join(lib_dir, 'stanford-parser/stanford-parser.jar'), + os.path.join(lib_dir, 'stanford-parser/stanford-parser-3.5.1-models.jar')]) + + # split into separate files + split(os.path.join(sick_dir, 'SICK_train.txt'), train_dir) + split(os.path.join(sick_dir, 'SICK_trial.txt'), dev_dir) + split(os.path.join(sick_dir, 'SICK_test_annotated.txt'), test_dir) + + # parse sentences + parse(train_dir, cp=classpath) + parse(dev_dir, cp=classpath) + parse(test_dir, cp=classpath) + + # get vocabulary + build_vocab( + glob.glob(os.path.join(sick_dir, '*/*.toks')), + os.path.join(sick_dir, 'vocab.txt')) + build_vocab( + glob.glob(os.path.join(sick_dir, '*/*.toks')), + os.path.join(sick_dir, 'vocab-cased.txt'), + lowercase=False) diff --git a/example/gluon/tree_lstm/tree_lstm.py b/example/gluon/tree_lstm/tree_lstm.py new file mode 100644 index 000000000000..ced7f7eeadf5 --- /dev/null +++ b/example/gluon/tree_lstm/tree_lstm.py @@ -0,0 +1,137 @@ +import mxnet as mx +from mxnet.gluon import Block, nn +from mxnet.gluon.parameter import Parameter + +class ChildSumLSTMCell(Block): + def __init__(self, hidden_size, + i2h_weight_initializer=None, + hs2h_weight_initializer=None, + hc2h_weight_initializer=None, + i2h_bias_initializer='zeros', + hs2h_bias_initializer='zeros', + hc2h_bias_initializer='zeros', + input_size=0, prefix=None, params=None): + super(ChildSumLSTMCell, self).__init__(prefix=prefix, params=params) + with self.name_scope(): + self._hidden_size = hidden_size + self._input_size = input_size + self.i2h_weight = self.params.get('i2h_weight', shape=(4*hidden_size, input_size), + init=i2h_weight_initializer) + self.hs2h_weight = self.params.get('hs2h_weight', shape=(3*hidden_size, hidden_size), + init=hs2h_weight_initializer) + self.hc2h_weight = self.params.get('hc2h_weight', shape=(hidden_size, hidden_size), + init=hc2h_weight_initializer) + self.i2h_bias = self.params.get('i2h_bias', shape=(4*hidden_size,), + init=i2h_bias_initializer) + self.hs2h_bias = self.params.get('hs2h_bias', shape=(3*hidden_size,), + init=hs2h_bias_initializer) + self.hc2h_bias = self.params.get('hc2h_bias', shape=(hidden_size,), + init=hc2h_bias_initializer) + + def _alias(self): + return 'childsum_lstm' + + def forward(self, F, inputs, tree): + children_outputs = [self.forward(F, inputs, child) + for child in tree.children] + if children_outputs: + _, children_states = zip(*children_outputs) # unzip + else: + children_states = None + + with inputs.context as ctx: + return self.node_forward(F, F.expand_dims(inputs[tree.idx], axis=0), children_states, + self.i2h_weight.data(ctx), + self.hs2h_weight.data(ctx), + self.hc2h_weight.data(ctx), + self.i2h_bias.data(ctx), + self.hs2h_bias.data(ctx), + self.hc2h_bias.data(ctx)) + + def node_forward(self, F, inputs, children_states, + i2h_weight, hs2h_weight, hc2h_weight, + i2h_bias, hs2h_bias, hc2h_bias): + name = '{0}{1}_'.format(self.prefix, self._alias) + # notation: N for batch size, C for hidden state dimensions, K for number of children. + + # FC for i, f, u, o gates (N, 4*C), from input to hidden + i2h = F.FullyConnected(data=inputs, weight=i2h_weight, bias=i2h_bias, + num_hidden=self._hidden_size*4, + name='%si2h'%name) + i2h_slices = F.split(i2h, num_outputs=4, name='%siuo_slice'%name) # (N, C)*4 + i2h_iuo = F.concat(*[i2h_slices[i] for i in [0, 2, 3]], dim=1) # (N, C*3) + if children_states: + # sum of children states + hs = F.add_n(*[state[0] for state in children_states], name='%shs'%name) # (N, C) + # concatenation of children hidden states + hc = F.concat(*[F.expand_dims(state[0], axis=1) for state in children_states], dim=1, + name='%shc') # (N, K, C) + # concatenation of children cell states + cs = F.concat(*[F.expand_dims(state[1], axis=1) for state in children_states], dim=1, + name='%scs') # (N, K, C) + + # calculate activation for forget gate. addition in f_act is done with broadcast + i2h_f_slice = i2h_slices[1] + f_act = i2h_f_slice + hc2h_bias + F.dot(hc, hc2h_weight) # (N, K, C) + forget_gates = F.Activation(f_act, act_type='sigmoid', name='%sf'%name) # (N, K, C) + else: + # for leaf nodes, summation of children hidden states are zeros. + hs = F.zeros_like(i2h_slices[0]) + + # FC for i, u, o gates, from summation of children states to hidden state + hs2h_iuo = F.FullyConnected(data=hs, weight=hs2h_weight, bias=hs2h_bias, + num_hidden=self._hidden_size*3, + name='%shs2h'%name) + i2h_iuo = i2h_iuo + hs2h_iuo + + iuo_act_slices = F.SliceChannel(i2h_iuo, num_outputs=3, + name='%sslice'%name) # (N, C)*3 + i_act, u_act, o_act = iuo_act_slices[0], iuo_act_slices[1], iuo_act_slices[2] # (N, C) each + + # calculate gate outputs + in_gate = F.Activation(i_act, act_type='sigmoid', name='%si'%name) + in_transform = F.Activation(u_act, act_type='tanh', name='%sc'%name) + out_gate = F.Activation(o_act, act_type='sigmoid', name='%so'%name) + + # calculate cell state and hidden state + next_c = in_gate * in_transform + if children_states: + next_c = F._internal._plus(F.sum(forget_gates * cs, axis=1), next_c, + name='%sstate'%name) + next_h = F._internal._mul(out_gate, F.Activation(next_c, act_type='tanh'), + name='%sout'%name) + + return next_h, [next_h, next_c] + +# module for distance-angle similarity +class Similarity(nn.Block): + def __init__(self, sim_hidden_size, rnn_hidden_size, num_classes): + super(Similarity, self).__init__() + with self.name_scope(): + self.wh = nn.Dense(sim_hidden_size, in_units=2*rnn_hidden_size, prefix='sim_embed_') + self.wp = nn.Dense(num_classes, in_units=sim_hidden_size, prefix='sim_out_') + + def forward(self, F, lvec, rvec): + # lvec and rvec will be tree_lstm cell states at roots + mult_dist = F.broadcast_mul(lvec, rvec) + abs_dist = F.abs(F.add(lvec,-rvec)) + vec_dist = F.concat(*[mult_dist, abs_dist],dim=1) + out = F.log_softmax(self.wp(F.sigmoid(self.wh(vec_dist)))) + return out + +# putting the whole model together +class SimilarityTreeLSTM(nn.Block): + def __init__(self, sim_hidden_size, rnn_hidden_size, embed_in_size, embed_dim, num_classes): + super(SimilarityTreeLSTM, self).__init__() + with self.name_scope(): + self.embed = nn.Embedding(embed_in_size, embed_dim, prefix='word_embed_') + self.childsumtreelstm = ChildSumLSTMCell(rnn_hidden_size, input_size=embed_dim) + self.similarity = Similarity(sim_hidden_size, rnn_hidden_size, num_classes) + + def forward(self, F, l_inputs, r_inputs, l_tree, r_tree): + l_inputs = self.embed(l_inputs) + r_inputs = self.embed(r_inputs) + lstate = self.childsumtreelstm(F, l_inputs, l_tree)[1][1] + rstate = self.childsumtreelstm(F, r_inputs, r_tree)[1][1] + output = self.similarity(F, lstate, rstate) + return output diff --git a/python/mxnet/initializer.py b/python/mxnet/initializer.py index 64921b9a5796..a5962b664e6b 100755 --- a/python/mxnet/initializer.py +++ b/python/mxnet/initializer.py @@ -564,9 +564,12 @@ def __init__(self, rnd_type="uniform", factor_type="avg", magnitude=3): self.magnitude = float(magnitude) - def _init_weight(self, _, arr): + def _init_weight(self, name, arr): shape = arr.shape hw_scale = 1. + if len(shape) < 2: + raise ValueError('Xavier initializer cannot be applied to vector {0}. It requires at' + ' least 2D.'.format(name)) if len(shape) > 2: hw_scale = np.prod(shape[2:]) fan_in, fan_out = shape[1] * hw_scale, shape[0] * hw_scale From 897cc55a0ffd53317628bbccadea34b072228937 Mon Sep 17 00:00:00 2001 From: Eric Junyuan Xie Date: Fri, 14 Jul 2017 07:10:27 -0700 Subject: [PATCH 218/834] revert (#7043) --- python/mxnet/metric.py | 48 ++++++++++++++++++------------------------ 1 file changed, 20 insertions(+), 28 deletions(-) diff --git a/python/mxnet/metric.py b/python/mxnet/metric.py index c4356a9d1840..2fe38ab751e0 100644 --- a/python/mxnet/metric.py +++ b/python/mxnet/metric.py @@ -13,20 +13,16 @@ from . import registry -def _check_shapes_equal(labels, preds): - label_shape, pred_shape = labels.shape, preds.shape +def check_label_shapes(labels, preds, shape=0): + if shape == 0: + label_shape, pred_shape = len(labels), len(preds) + else: + label_shape, pred_shape = labels.shape, preds.shape if label_shape != pred_shape: raise ValueError("Shape of labels {} does not match shape of " "predictions {}".format(label_shape, pred_shape)) -def _check_lengths_equal(labels, preds): - label_len, pred_len = len(labels), len(preds) - - if label_len != pred_len: - raise ValueError("Length of labels {} does not match length of " - "predictions {}".format(label_len, pred_len)) - class EvalMetric(object): """Base class for all evaluation metrics. @@ -372,7 +368,7 @@ def update(self, labels, preds): preds : list of `NDArray` Predicted values. """ - _check_lengths_equal(labels, preds) + check_label_shapes(labels, preds) for label, pred_label in zip(labels, preds): if pred_label.shape != label.shape: @@ -380,7 +376,7 @@ def update(self, labels, preds): pred_label = pred_label.asnumpy().astype('int32') label = label.asnumpy().astype('int32') - _check_lengths_equal(label, pred_label) + check_label_shapes(label, pred_label) self.sum_metric += (pred_label.flat == label.flat).sum() self.num_inst += len(pred_label.flat) @@ -442,13 +438,13 @@ def update(self, labels, preds): preds : list of `NDArray` Predicted values. """ - _check_lengths_equal(labels, preds) + check_label_shapes(labels, preds) for label, pred_label in zip(labels, preds): assert(len(pred_label.shape) <= 2), 'Predictions should be no more than 2 dims' pred_label = numpy.argsort(pred_label.asnumpy().astype('float32'), axis=1) label = label.asnumpy().astype('int32') - _check_lengths_equal(label, pred_label) + check_label_shapes(label, pred_label) num_samples = pred_label.shape[0] num_dims = len(pred_label.shape) if num_dims == 1: @@ -516,14 +512,14 @@ def update(self, labels, preds): preds : list of `NDArray` Predicted values. """ - _check_lengths_equal(labels, preds) + check_label_shapes(labels, preds) for label, pred in zip(labels, preds): pred = pred.asnumpy() label = label.asnumpy().astype('int32') pred_label = numpy.argmax(pred, axis=1) - _check_lengths_equal(label, pred) + check_label_shapes(label, pred) if len(numpy.unique(label)) > 2: raise ValueError("F1 currently only supports binary classification.") @@ -629,7 +625,7 @@ def update(self, labels, preds): preds : list of `NDArray` Predicted values. """ - _check_lengths_equal(labels, preds) + assert len(labels) == len(preds) loss = 0. num = 0 for label, pred in zip(labels, preds): @@ -707,10 +703,9 @@ def update(self, labels, preds): preds : list of `NDArray` Predicted values. """ - _check_lengths_equal(labels, preds) + check_label_shapes(labels, preds) for label, pred in zip(labels, preds): - _check_shapes_equal(label, pred) label = label.asnumpy() pred = pred.asnumpy() @@ -766,10 +761,9 @@ def update(self, labels, preds): preds : list of `NDArray` Predicted values. """ - _check_lengths_equal(labels, preds) + check_label_shapes(labels, preds) for label, pred in zip(labels, preds): - _check_shapes_equal(label, pred) label = label.asnumpy() pred = pred.asnumpy() @@ -825,10 +819,9 @@ def update(self, labels, preds): preds : list of `NDArray` Predicted values. """ - _check_lengths_equal(labels, preds) + check_label_shapes(labels, preds) for label, pred in zip(labels, preds): - _check_shapes_equal(label, pred) label = label.asnumpy() pred = pred.asnumpy() @@ -890,10 +883,9 @@ def update(self, labels, preds): preds : list of `NDArray` Predicted values. """ - _check_lengths_equal(labels, preds) + check_label_shapes(labels, preds) for label, pred in zip(labels, preds): - _check_lengths_equal(label, pred) label = label.asnumpy() pred = pred.asnumpy() @@ -904,6 +896,7 @@ def update(self, labels, preds): self.sum_metric += (-numpy.log(prob + self.eps)).sum() self.num_inst += label.shape[0] + @register @alias('pearsonr') class PearsonCorrelation(EvalMetric): @@ -946,13 +939,12 @@ def update(self, labels, preds): ---------- labels : list of `NDArray` The labels of the data. - preds : list of `NDArray` Predicted values. """ - _check_lengths_equal(labels, preds) + check_label_shapes(labels, preds) for label, pred in zip(labels, preds): - _check_shapes_equal(label, pred) + check_label_shapes(label, pred, 1) label = label.asnumpy() pred = pred.asnumpy() self.sum_metric += numpy.corrcoef(pred.ravel(), label.ravel())[0, 1] @@ -1064,7 +1056,7 @@ def update(self, labels, preds): Predicted values. """ if not self._allow_extra_outputs: - _check_lengths_equal(labels, preds) + check_label_shapes(labels, preds) for pred, label in zip(preds, labels): label = label.asnumpy() From 4a16fa75ce01f91b830c73b275e39609ca82c33b Mon Sep 17 00:00:00 2001 From: Yao Wang Date: Fri, 14 Jul 2017 14:55:10 -0700 Subject: [PATCH 219/834] Fix tutorial notebook names (#7050) --- tests/nightly/test_tutorial.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/nightly/test_tutorial.py b/tests/nightly/test_tutorial.py index 19f297878ab3..304642c73a23 100644 --- a/tests/nightly/test_tutorial.py +++ b/tests/nightly/test_tutorial.py @@ -47,7 +47,7 @@ def test_tutorial_nb(file_path): path of tutorial markdown file """ tutorial_name = os.path.basename(file_path) - notebook = nbformat.read(file_path + '_python.ipynb', as_version=4) + notebook = nbformat.read(file_path + '.ipynb', as_version=4) eprocessor = ExecutePreprocessor(timeout=1800) try: eprocessor.preprocess(notebook, {'metadata': {}}) From 3410e493bfcdfb7b648f4a1f786977fed44c25a6 Mon Sep 17 00:00:00 2001 From: lxn2 Date: Fri, 14 Jul 2017 14:55:57 -0700 Subject: [PATCH 220/834] Remove empty commit file (#7049) --- emptycommit | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 emptycommit diff --git a/emptycommit b/emptycommit deleted file mode 100644 index e69de29bb2d1..000000000000 From 845b62f4eb05153c6f09b2bef1404e2c062be69f Mon Sep 17 00:00:00 2001 From: "Qiang Kou (KK)" Date: Fri, 14 Jul 2017 22:18:29 -0700 Subject: [PATCH 221/834] [R]fix optimizer with multi-GPU. close #5296 (#7056) --- R-package/src/kvstore.cc | 50 +++++++++++++++++++++++++++------------- R-package/src/kvstore.h | 6 ++--- 2 files changed, 37 insertions(+), 19 deletions(-) diff --git a/R-package/src/kvstore.cc b/R-package/src/kvstore.cc index 9896a8762b94..b15106b1dda6 100644 --- a/R-package/src/kvstore.cc +++ b/R-package/src/kvstore.cc @@ -112,33 +112,51 @@ void KVStore::SetOptimizer(const Rcpp::List& optimizer) { this)); } -NDArray KVStore::CreateState(int index, const NDArray& weight) const { +Rcpp::List KVStore::CreateState(int index, const NDArray& weight) const { RCHECK(optimizer_set_) << "Need to call set.optimizer for KVStore " << type(); - // TODO(KK) review this // Use R Internal API here Rcpp::Shield call(Rf_lang3(fcreate_state_, Rcpp::wrap(index), weight.RObject())); - return NDArray(Rcpp_eval(call)); + SEXP ret = Rcpp_eval(call); + if (Rf_isNull(ret)) { + return Rcpp::List::create(); + } else if (TYPEOF(ret) == EXTPTRSXP) { + return Rcpp::List::create(Rcpp::Named("state") = ret); + } else { + return ret; + } } void KVStore::Update(int index, const NDArray& grad, NDArray *weight) { RCHECK(optimizer_set_) << "Need to call set.optimizer for KVStore " << type(); - std::map::iterator it = states_.find(index); + std::map::iterator it = states_.find(index); + Rcpp::List state_lst = this->CreateState(index, *weight); if (it == states_.end()) { - NDArray nd = this->CreateState(index, *weight); - states_.insert(std::make_pair(index, nd)); - it = states_.find(index); + if (state_lst.size() != 0) { + states_.insert(std::make_pair(index, state_lst)); + it = states_.find(index); + } + } + + Rcpp::List rlist; + if (state_lst.size() == 0) { + Rcpp::Shield call(Rf_lang5(fupdate_, Rcpp::wrap(index), + weight->RObject(), grad.RObject(), + R_NilValue)); + rlist = Rcpp_eval(call); + } else if (state_lst.size() == 1) { + Rcpp::Shield call(Rf_lang5(fupdate_, Rcpp::wrap(index), + weight->RObject(), grad.RObject(), + it->second[0])); + rlist = Rcpp_eval(call); + } else { + // Use R Internal API here + Rcpp::Shield call(Rf_lang5(fupdate_, Rcpp::wrap(index), + weight->RObject(), grad.RObject(), + it->second)); + rlist = Rcpp_eval(call); } - NDArray& state = it->second; - // TODO(KK) review this - // Use R Internal API here - Rcpp::Shield call(Rf_lang5(fupdate_, Rcpp::wrap(index), - weight->RObject(), grad.RObject(), - state.RObject())); - Rcpp::List rlist(Rcpp_eval(call)); - // update the state, and eight - state = rlist["state"]; NDArray::CopyFromTo(NDArray::FromRObject(rlist["weight"]), weight); } diff --git a/R-package/src/kvstore.h b/R-package/src/kvstore.h index d4a92dfb7dad..f93613042825 100644 --- a/R-package/src/kvstore.h +++ b/R-package/src/kvstore.h @@ -76,14 +76,14 @@ class KVStore { private: explicit KVStore(KVStoreHandle handle) : handle_(handle), optimizer_set_(false) {} - // the internal callback to kvstore. - NDArray CreateState(int index, const NDArray& weight) const; + // the internal callback to kvstore. This might return NULL + Rcpp::List CreateState(int index, const NDArray& weight) const; /*! \brief internal KVStore handle */ KVStoreHandle handle_; /*! \brief Whether optimizer is setted*/ bool optimizer_set_; /*! \brief The internal state */ - std::map states_; + std::map states_; /*! \brief Function to create state */ Rcpp::RObject fcreate_state_; /*! \brief Function to perform update */ From 4747c6b5678cf4519576e4a246ab68017ea5c304 Mon Sep 17 00:00:00 2001 From: Eric Junyuan Xie Date: Fri, 14 Jul 2017 23:11:23 -0700 Subject: [PATCH 222/834] revert module changes (#7055) --- python/mxnet/module/base_module.py | 51 ++++----------------------- python/mxnet/module/executor_group.py | 14 ++++---- python/mxnet/module/module.py | 25 ++++++------- 3 files changed, 23 insertions(+), 67 deletions(-) diff --git a/python/mxnet/module/base_module.py b/python/mxnet/module/base_module.py index fa7434bb2dc5..cb6cfccb2759 100644 --- a/python/mxnet/module/base_module.py +++ b/python/mxnet/module/base_module.py @@ -1,5 +1,4 @@ # pylint: disable=fixme, too-many-arguments, too-many-locals, too-many-public-methods, too-many-branches -# pylint: disable=too-many-lines """`BaseModule` defines an API for modules.""" import time @@ -77,43 +76,6 @@ def _parse_data_desc(data_names, label_names, data_shapes, label_shapes): return data_shapes, label_shapes -def _parse_metric(sym, metrics): - output_names = [] - if not metrics: - metrics = [] - elif isinstance(metrics, (str, metric.EvalMetric)): - metrics = [metric.create(metrics)] - else: - metrics = [metric.create(i) for i in metrics] - - sym_metrics = [] - loss_metrics = [] - for i in sym: - tag = i.attr('__output__') - if tag is None or tag == 'pred': - output_names.append(i.list_outputs()[0]) - elif tag == 'loss': - name = i.list_outputs()[0] - loss_metrics.append( - metric.Loss(name=name, output_names=[name], - label_names=[])) - - str_metric = i.attr('__metric__') - if str_metric: - sym_metrics.append(metric.create(str_metric)) - - for m in metrics: - m.output_names = output_names - metrics += sym_metrics - metrics += loss_metrics - if len(metrics) > 1: - return metric.CompositeEvalMetric(metrics) - elif len(metrics) == 1: - return metrics[0] - else: - return None - - class BaseModule(object): """The base class of a module. @@ -228,7 +190,7 @@ def forward_backward(self, data_batch): self.forward(data_batch, is_train=True) self.backward() - def score(self, eval_data, eval_metric=None, num_batch=None, batch_end_callback=None, + def score(self, eval_data, eval_metric, num_batch=None, batch_end_callback=None, score_end_callback=None, reset=True, epoch=0): """Runs prediction on ``eval_data`` and evaluates the performance according to @@ -268,7 +230,8 @@ def score(self, eval_data, eval_metric=None, num_batch=None, batch_end_callback= if reset: eval_data.reset() - eval_metric = _parse_metric(self.symbol, eval_metric) + if not isinstance(eval_metric, metric.EvalMetric): + eval_metric = metric.create(eval_metric) eval_metric.reset() actual_num_batch = 0 @@ -409,7 +372,7 @@ def predict(self, eval_data, num_batch=None, merge_batches=True, reset=True, return output_list - def fit(self, train_data, eval_data=None, eval_metric=None, + def fit(self, train_data, eval_data=None, eval_metric='acc', epoch_end_callback=None, batch_end_callback=None, kvstore='local', optimizer='sgd', optimizer_params=(('learning_rate', 0.01),), eval_end_callback=None, @@ -503,10 +466,8 @@ def fit(self, train_data, eval_data=None, eval_metric=None, if validation_metric is None: validation_metric = eval_metric - eval_metric = _parse_metric(self.symbol, eval_metric) - if eval_metric is None: - eval_metric = metric.create('acc') - validation_metric = 'acc' + if not isinstance(eval_metric, metric.EvalMetric): + eval_metric = metric.create(eval_metric) ################################################################################ # training loop diff --git a/python/mxnet/module/executor_group.py b/python/mxnet/module/executor_group.py index 063b00732b81..169e81ee326e 100755 --- a/python/mxnet/module/executor_group.py +++ b/python/mxnet/module/executor_group.py @@ -152,7 +152,7 @@ def __init__(self, symbol, contexts, workload, data_shapes, label_shapes, param_ grad_req = 'null' data_shapes = [x if isinstance(x, DataDesc) else DataDesc(*x) for x in data_shapes] - if label_shapes: + if label_shapes is not None: label_shapes = [x if isinstance(x, DataDesc) else DataDesc(*x) for x in label_shapes] data_names = [x.name for x in data_shapes] @@ -248,7 +248,7 @@ def _collect_arrays(self): self.state_arrays = [[e.arg_dict[name] for e in self.execs] for name in self.state_names] - if self.label_shapes: + if self.label_shapes is not None: self.label_arrays = [[(self.slices[i], e.arg_dict[name]) for i, e in enumerate(self.execs)] for name, _ in self.label_shapes] @@ -291,13 +291,13 @@ def bind_exec(self, data_shapes, label_shapes, shared_group=None, reshape=False) # calculate workload and bind executors self.data_layouts = self.decide_slices(data_shapes) - if label_shapes: + if label_shapes is not None: # call it to make sure labels has the same batch size as data self.label_layouts = self.decide_slices(label_shapes) for i in range(len(self.contexts)): data_shapes_i = self._sliced_shape(data_shapes, i, self.data_layouts) - if label_shapes: + if label_shapes is not None: label_shapes_i = self._sliced_shape(label_shapes, i, self.label_layouts) else: label_shapes_i = [] @@ -312,7 +312,7 @@ def bind_exec(self, data_shapes, label_shapes, shared_group=None, reshape=False) self.data_shapes = data_shapes self.label_shapes = label_shapes self.data_names = [i.name for i in self.data_shapes] - if label_shapes: + if label_shapes is not None: self.label_names = [i.name for i in self.label_shapes] self._collect_arrays() @@ -573,11 +573,11 @@ def _bind_ith_exec(self, i, data_shapes, label_shapes, shared_group): shared_data_arrays = self.shared_data_arrays[i] input_shapes = dict(data_shapes) - if label_shapes: + if label_shapes is not None: input_shapes.update(dict(label_shapes)) input_types = {x.name: x.dtype for x in data_shapes} - if label_shapes: + if label_shapes is not None: input_types.update({x.name: x.dtype for x in label_shapes}) executor = self.symbol.simple_bind(ctx=context, grad_req=self.grad_req, diff --git a/python/mxnet/module/module.py b/python/mxnet/module/module.py index 65c277c7f54b..2a36c6ad7e7e 100644 --- a/python/mxnet/module/module.py +++ b/python/mxnet/module/module.py @@ -9,7 +9,6 @@ from .. import context as ctx from .. import ndarray as nd -from .. import symbol as _sym from .. import optimizer as opt from .executor_group import DataParallelExecutorGroup @@ -58,7 +57,6 @@ def __init__(self, symbol, data_names=('data',), label_names=('softmax_label',), self._work_load_list = work_load_list self._symbol = symbol - self._pred_symbol = _sym.Group([i for i in symbol if i.attr('__output__') != 'loss']) data_names = list(data_names) if data_names is not None else [] label_names = list(label_names) if label_names is not None else [] @@ -373,14 +371,16 @@ def bind(self, data_shapes, label_shapes=None, for_training=True, self.binded = True self._grad_req = grad_req - if not for_training and self._label_names and not label_shapes: - symbol = self._pred_symbol - self._data_shapes, self._label_shapes = _parse_data_desc( - self.data_names, [], data_shapes, []) + if not for_training: + assert not inputs_need_grad else: - symbol = self._symbol - self._data_shapes, self._label_shapes = _parse_data_desc( - self.data_names, self.label_names, data_shapes, label_shapes) + pass + # this is not True, as some module might not contains a loss function + # that consumes the labels + # assert label_shapes is not None + + self._data_shapes, self._label_shapes = _parse_data_desc( + self.data_names, self.label_names, data_shapes, label_shapes) if shared_module is not None: assert isinstance(shared_module, Module) and \ @@ -389,7 +389,7 @@ def bind(self, data_shapes, label_shapes=None, for_training=True, else: shared_group = None - self._exec_group = DataParallelExecutorGroup(symbol, self._context, + self._exec_group = DataParallelExecutorGroup(self._symbol, self._context, self._work_load_list, self._data_shapes, self._label_shapes, self._param_names, for_training, inputs_need_grad, @@ -555,11 +555,6 @@ def forward(self, data_batch, is_train=None): """ assert self.binded and self.params_initialized - # If start to inference, force rebind module. - if self._label_shapes and not data_batch.label: - raise RuntimeError("If you are trying to do inference, rebind module " - "with 'force_rebind=True' and 'for_training=False'") - curr_data_shapes = tuple(i.shape for i in self._data_shapes) new_data_shapes = tuple(i.shape for i in data_batch.data) From c70d549ac434981046415aa3898fc2a8ac4959fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=A2=81=E5=BE=B7=E6=BE=8E?= Date: Sun, 16 Jul 2017 13:08:31 +0800 Subject: [PATCH 223/834] [Scala] add arange to Symbol and NDArray (#6904) * [Scala] add arange to Symbol and NDArray * make emun DType self-explanatory --- .../src/main/scala/ml/dmlc/mxnet/DType.scala | 10 +-- .../main/scala/ml/dmlc/mxnet/NDArray.scala | 21 ++++++ .../src/main/scala/ml/dmlc/mxnet/Symbol.scala | 67 ++++++++++++++++++- .../scala/ml/dmlc/mxnet/NDArraySuite.scala | 13 ++++ .../scala/ml/dmlc/mxnet/OperatorSuite.scala | 29 ++++++++ 5 files changed, 132 insertions(+), 8 deletions(-) diff --git a/scala-package/core/src/main/scala/ml/dmlc/mxnet/DType.scala b/scala-package/core/src/main/scala/ml/dmlc/mxnet/DType.scala index 2ea09f4bee83..bfe757d5cfad 100644 --- a/scala-package/core/src/main/scala/ml/dmlc/mxnet/DType.scala +++ b/scala-package/core/src/main/scala/ml/dmlc/mxnet/DType.scala @@ -19,11 +19,11 @@ package ml.dmlc.mxnet object DType extends Enumeration { type DType = Value - val Float32 = Value(0) - val Float64 = Value(1) - val Float16 = Value(2) - val UInt8 = Value(3) - val Int32 = Value(4) + val Float32 = Value(0, "float32") + val Float64 = Value(1, "float64") + val Float16 = Value(2, "float16") + val UInt8 = Value(3, "uint8") + val Int32 = Value(4, "int32") private[mxnet] def numOfBytes(dtype: DType): Int = { dtype match { case DType.UInt8 => 1 diff --git a/scala-package/core/src/main/scala/ml/dmlc/mxnet/NDArray.scala b/scala-package/core/src/main/scala/ml/dmlc/mxnet/NDArray.scala index 94ce0086f6e4..5314dc4a1896 100644 --- a/scala-package/core/src/main/scala/ml/dmlc/mxnet/NDArray.scala +++ b/scala-package/core/src/main/scala/ml/dmlc/mxnet/NDArray.scala @@ -387,6 +387,27 @@ object NDArray { arr } + /** + * Returns evenly spaced values within a given interval. + * Values are generated within the half-open interval [`start`, `stop`). In other + * words, the interval includes `start` but excludes `stop`. + * @param start Start of interval. The default start value is 0. + * @param stop End of interval. + * @param step Spacing between values. The default step size is 1. + * @param repeat Number of times to repeat each element. The default repeat count is 1. + * @param ctx Device context. Default context is the current default context. + * @param dType The data type of the `NDArray`. The default datatype is `DType.Float32`. + * @return NDArray of evenly spaced values in the specified range. + */ + def arange(start: Float, stop: Option[Float] = None, step: Float = 1.0f, + repeat: Int = 1, ctx: Context = Context.defaultCtx, + dType: DType = Base.MX_REAL_TYPE): NDArray = { + val params = Map("start" -> start, "step" -> step, + "repeat" -> repeat, "ctx" -> ctx.toString, "dtype" -> dType.toString()) + val fParams = if (stop == None) params else params ++ Map("stop" -> stop.get) + NDArray.genericNDArrayFunctionInvoke("_arange", Seq(), fParams)(0) + } + /** * Concatenate a list of NDArrays along the specified dimension. * @param arrays Arrays to be concatenate. diff --git a/scala-package/core/src/main/scala/ml/dmlc/mxnet/Symbol.scala b/scala-package/core/src/main/scala/ml/dmlc/mxnet/Symbol.scala index 4e8d4c2bd9f9..d8da1c67c252 100644 --- a/scala-package/core/src/main/scala/ml/dmlc/mxnet/Symbol.scala +++ b/scala-package/core/src/main/scala/ml/dmlc/mxnet/Symbol.scala @@ -928,17 +928,78 @@ object Symbol { createFromListedSymbols("_lesser_equal_scalar")(Array(left), Map("scalar" -> right.toString)) } + /** + * Returns a new symbol of given shape and type, filled with zeros. + */ + def zeros(shape: Shape, dType: DType = Base.MX_REAL_TYPE, ctx: Context = null): Symbol = { + val params = Map("shape" -> shape.toString, "dtype" -> dType.toString()) + val fParams = if (ctx == null) params else params ++ Map("ctx" -> ctx.toString) + createSymbolGeneral("_zeros", null, null, Array.empty[Symbol], fParams) + } + + /** + * Returns a new symbol of given shape and type, filled with ones. + */ + def ones(shape: Shape, dType: DType = Base.MX_REAL_TYPE, ctx: Context = null): Symbol = { + val params = Map("shape" -> shape.toString, "dtype" -> dType.toString()) + val fParams = if (ctx == null) params else params ++ Map("ctx" -> ctx.toString) + createSymbolGeneral("_ones", null, null, Array.empty[Symbol], fParams) + } + + /** + * Returns evenly spaced values within a given interval. + * @param start Start of interval. The default start value is 0. + * @param stop End of interval. + * @param step Spacing between values. The default step size is 1. + * @param repeat Number of times to repeat each element. The default repeat count is 1. + * @param dType The data type of the `NDArray`. The default datatype is `DType.Float32`. + * @return Symbol The created Symbol. + */ + def arange(start: Float, stop: Option[Float] = None, step: Float = 1.0f, + repeat: Int = 1, name: String = null, dType: DType = Base.MX_REAL_TYPE): Symbol = { + val params = Map("start" -> start, "step" -> step, + "repeat" -> repeat, "dtype" -> dType.toString()) + val fParams = if (stop == None) params else params ++ Map("stop" -> stop.get) + createSymbolGeneral("_arange", name, null, Array.empty[Symbol], fParams) + } + + // TODO(depeng) support setting initialization pattern /** * Create a symbolic variable with specified name. * @param name Name of the variable. * @param attr Additional attributes to set on the variable. - * @return The created variable symbol. + * @param shape + * The shape of a variable. If specified, this will be used during the shape inference. + * If one has specified a different shape for this variable using a keyword argument + * when calling shape inference, this shape information will be ignored. + * @param lrMult The learning rate multiplier for input variable. + * @param wdMult Weight decay multiplier for input variable. + * @param dType The dtype for input variable. If not specified, this value will be inferred. + * @param init Initializer for this variable to (optionally) override the default initializer. + * @param kwargs Additional attributes which must start and end with double underscores. + * @return A symbol corresponding to an input to the computation graph. */ - def Variable(name: String, attr: Map[String, String] = null): Symbol = { + def Variable(name: String, attr: Map[String, String] = null, shape: Shape = null, + lrMult: Option[Float] = None, wdMult: Option[Float] = None, dType: DType = null, + kwargs: Map[String, String] = Map.empty[String, String]): Symbol = { val handle = new SymbolHandleRef checkCall(_LIB.mxSymbolCreateVariable(name, handle)) val sym = new Symbol(handle.value) - sym.setAttr(AttrScope.current.get(Option(attr))) + val tmpAttr = scala.collection.mutable.Map[String, String]() + if (shape != null) tmpAttr += "__shape__" -> shape.toString + if (lrMult != None) tmpAttr += "__lr_mult__" -> lrMult.get.toString + if (wdMult != None) tmpAttr += "__wd_mult__" -> wdMult.get.toString + if (dType != null) tmpAttr += "__dtype__" -> dType.id.toString + for ((k, v) <- kwargs) { + require(k.startsWith("__") && k.endsWith("__"), + s"Attribute name=$k is not supported. " + + "Additional attributes must start and end with double underscores, e.g, __yourattr__") + tmpAttr += k -> v + } + if (attr != null) { + attr.foreach { case (k, v) => tmpAttr += k -> v } + } + sym.setAttr(AttrScope.current.get(Option(tmpAttr.toMap))) sym } diff --git a/scala-package/core/src/test/scala/ml/dmlc/mxnet/NDArraySuite.scala b/scala-package/core/src/test/scala/ml/dmlc/mxnet/NDArraySuite.scala index f30bab88aaa7..e1d091d1cd01 100644 --- a/scala-package/core/src/test/scala/ml/dmlc/mxnet/NDArraySuite.scala +++ b/scala-package/core/src/test/scala/ml/dmlc/mxnet/NDArraySuite.scala @@ -161,6 +161,19 @@ class NDArraySuite extends FunSuite with BeforeAndAfterAll with Matchers { assert(res.toArray === Array(11f)) } + test("arange") { + for (i <- 0 until 5) { + val start = scala.util.Random.nextFloat() * 5 + val stop = start + scala.util.Random.nextFloat() * 100 + val step = scala.util.Random.nextFloat() * 4 + val repeat = (scala.util.Random.nextFloat() * 5).toInt + 1 + val result = (start until stop by step).flatMap(x => Array.fill[Float](repeat)(x)) + val range = NDArray.arange(start = start, stop = Some(stop), step = step, + repeat = repeat, ctx = Context.cpu(), dType = DType.Float32) + assert(CheckUtils.reldiff(result.toArray, range.toArray) <= 1e-5f) + } + } + test("power") { val arr = NDArray.array(Array(3f, 5f), shape = Shape(2, 1)) diff --git a/scala-package/core/src/test/scala/ml/dmlc/mxnet/OperatorSuite.scala b/scala-package/core/src/test/scala/ml/dmlc/mxnet/OperatorSuite.scala index dfbc864785f1..187869c3af21 100644 --- a/scala-package/core/src/test/scala/ml/dmlc/mxnet/OperatorSuite.scala +++ b/scala-package/core/src/test/scala/ml/dmlc/mxnet/OperatorSuite.scala @@ -214,6 +214,35 @@ class OperatorSuite extends FunSuite with BeforeAndAfterAll checkSymbolicBackward(test, Array(dataTmp), Array(NDArray.ones(shape) * 2), Array(npoutGrad)) } + test("ones") { + val ones = Symbol.ones(shape = Shape(2, 2)) + val exe = ones.simpleBind(ctx = Context.cpu(), gradReq = "write", shapeDict = Map()) + exe.forward(isTrain = false) + assert(CheckUtils.reldiff(Array(1f, 1f, 1f, 1f), exe.outputs.head.toArray) <= 1e-5f) + } + + test("zeros") { + val zeros = Symbol.zeros(shape = Shape(2, 2)) + val exe = zeros.simpleBind(ctx = Context.cpu(), gradReq = "write", shapeDict = Map()) + exe.forward(isTrain = false) + assert(Array(0f, 0f, 0f, 0f) === exe.outputs.head.toArray) + } + + test("arange") { + for (i <- 0 until 5) { + val start = scala.util.Random.nextFloat() * 5 + val stop = start + scala.util.Random.nextFloat() * 100 + val step = scala.util.Random.nextFloat() * 4 + val repeat = (scala.util.Random.nextFloat() * 5).toInt + 1 + val result = (start until stop by step).flatMap(x => Array.fill[Float](repeat)(x)) + val x = Symbol.arange(start = start, stop = Some(stop), step = step, repeat = repeat) + var exe = x.simpleBind(ctx = Context.cpu(), gradReq = "write", shapeDict = Map()) + exe.forward(isTrain = false) + assert(exe.gradArrays.length == 0) + assert(CheckUtils.reldiff(result.toArray, exe.outputs.head.toArray) <= 1e-5f) + } + } + test("scalar pow") { val data = Symbol.Variable("data") val shape = Shape(1, 1) From 1ae1fbebd8873f85f2089efbb76f3255658aaeae Mon Sep 17 00:00:00 2001 From: Yan Huang Date: Mon, 17 Jul 2017 05:37:58 +0800 Subject: [PATCH 224/834] avoid runtime crash caused by error context of gradient storage when using context group (#7042) * fix runtime crash caused by error context of gradient storage * fix unittest failure in scala package --- .../ml/dmlc/mxnet/ModelParallelSuite.scala | 34 ++++++++--------- src/executor/graph_executor.cc | 15 ++++++++ tests/python/unittest/test_model_parallel.py | 38 ++++++++++++------- 3 files changed, 56 insertions(+), 31 deletions(-) diff --git a/scala-package/core/src/test/scala/ml/dmlc/mxnet/ModelParallelSuite.scala b/scala-package/core/src/test/scala/ml/dmlc/mxnet/ModelParallelSuite.scala index 6fb30731fdd5..e95ab09b5bd2 100644 --- a/scala-package/core/src/test/scala/ml/dmlc/mxnet/ModelParallelSuite.scala +++ b/scala-package/core/src/test/scala/ml/dmlc/mxnet/ModelParallelSuite.scala @@ -23,8 +23,11 @@ import org.scalatest.{BeforeAndAfterAll, FunSuite} class ModelParallelSuite extends FunSuite with BeforeAndAfterAll { test("chain") { val n = 2 + val ctx1 = Context.cpu(0) + val ctx2 = Context.cpu(1) val data1 = Symbol.Variable("data1") val data2 = Symbol.Variable("data2") + val data3 = Symbol.Variable("data3") var net: Symbol = null new AttrScope(Map("ctx_group" -> "dev1")).withScope { @@ -32,31 +35,28 @@ class ModelParallelSuite extends FunSuite with BeforeAndAfterAll { } new AttrScope(Map("ctx_group" -> "dev2")).withScope { - net = net + data1 + net = net + data3 } val shape = Shape(4, 5) - val (arr, arrGrad) = - new Context(Context.cpu(0)).withScope { - val arr = (0 until n).map(_ => NDArray.empty(shape)) - val arrGrad = (0 until n).map(_ => NDArray.empty(shape)) - (arr, arrGrad) - } + val arr = (0 until n + 1).map(_ => NDArray.empty(shape, ctx1)) + val arrGrad = (0 until n).map(_ => NDArray.empty(shape, ctx1)) :+ NDArray.empty(shape, ctx2) - val exec1 = net.bind(Context.cpu(), + val exec1 = net.bind(ctx1, args = arr, argsGrad = arrGrad, gradReq = "write", auxStates = Nil, - group2ctx = Map("dev1" -> Context.cpu(0), "dev2" -> Context.cpu(1)), + group2ctx = Map("dev1" -> ctx1, "dev2" -> ctx2), sharedExec = null) arr(0).set(1f) arr(1).set(2f) + arr(2).set(3f) - val arr2 = arr.map(_.copyTo(Context.cpu())) - val arrGrad2 = arrGrad.map(_.copyTo(Context.cpu())) - val exec2 = net.bind(Context.cpu(), args = arr2, argsGrad = arrGrad2) + val arr2 = arr.map(_.copyTo(ctx1)) + val arrGrad2 = arrGrad.map(_.copyTo(ctx1)) + val exec2 = net.bind(ctx1, args = arr2, argsGrad = arrGrad2) // Show the execution plan that involves copynode // scalastyle:off println @@ -65,14 +65,14 @@ class ModelParallelSuite extends FunSuite with BeforeAndAfterAll { exec1.forward() exec2.forward() - assert(reldiff(exec1.outputs(0).copyTo(Context.cpu()), - exec2.outputs(0).copyTo(Context.cpu())) < 1e-6f) + assert(reldiff(exec1.outputs(0).copyTo(ctx1), + exec2.outputs(0).copyTo(ctx1)) < 1e-6f) - val outGrad = NDArray.ones(shape, Context.cpu(1)) + val outGrad = NDArray.ones(shape, ctx2) exec1.backward(Array(outGrad)) - exec2.backward(Array(outGrad.copyTo(Context.cpu()))) + exec2.backward(Array(outGrad.copyTo(ctx1))) (arrGrad zip arrGrad2) foreach { case (a, b) => - assert(reldiff(a, b) < 1e-6f) + assert(reldiff(a.copyTo(ctx1), b) < 1e-6f) } } } diff --git a/src/executor/graph_executor.cc b/src/executor/graph_executor.cc index add1d36434a8..af5ec7f492dd 100644 --- a/src/executor/graph_executor.cc +++ b/src/executor/graph_executor.cc @@ -355,6 +355,21 @@ Graph AssignContext(Graph g, vcontext.push_back(ctx_list[assigned_device[i]]); } } + + // after device planning, we should check again + // if the assigned device of gradient node + // corresponds to storage of grads + auto &new_idx = g.indexed_graph(); + for (size_t i = num_forward_outputs; i < g.outputs.size(); ++i) { + const uint32_t nid = new_idx.outputs()[i].node_id; + Context ctx = arg_grad_ctxes[i - num_forward_outputs]; + CHECK(ctx == vcontext[nid]) + << "Trying to save gradient to " << ctx + << " while its source node \"" << new_idx[nid].source->attrs.name + << "\" computes it on " << vcontext[nid] + << ". Check your ctx in NDArray allocation."; + } + g.attrs["context"] = std::make_shared(std::move(vcontext)); return g; } diff --git a/tests/python/unittest/test_model_parallel.py b/tests/python/unittest/test_model_parallel.py index a531c5064551..96990e72075e 100644 --- a/tests/python/unittest/test_model_parallel.py +++ b/tests/python/unittest/test_model_parallel.py @@ -10,42 +10,52 @@ def reldiff(a, b): return reldiff def test_chain(): + ctx1 = mx.cpu(0) + ctx2 = mx.cpu(1) n = 2 data1 = mx.sym.Variable('data1') data2 = mx.sym.Variable('data2') + data3 = mx.sym.Variable('data3') with mx.AttrScope(ctx_group='dev1'): net = data1 + data2 net = net * 3 with mx.AttrScope(ctx_group='dev2'): - net = net + data1 + net = net + data3 - with mx.Context(mx.cpu(0)): - shape = (4, 5) - arr = [mx.nd.empty(shape) for i in range(n)] - arr_grad = [mx.nd.empty(shape) for i in range(n)] + arr = [] + arr_grad = [] + shape = (4, 5) + with mx.Context(ctx1): + for i in range(n): + arr.append(mx.nd.empty(shape)) + arr_grad.append(mx.nd.empty(shape)) + with mx.Context(ctx2): + arr.append(mx.nd.empty(shape)) + arr_grad.append(mx.nd.empty(shape)) - exec1 = net.bind(mx.cpu(), + exec1 = net.bind(ctx1, args=arr, args_grad=arr_grad, - group2ctx={'dev1': mx.cpu(0), 'dev2': mx.cpu(1)}) + group2ctx={'dev1': ctx1, 'dev2': ctx2}) arr[0][:] = 1.0 arr[1][:] = 2.0 - arr2 = [a.copyto(mx.cpu()) for a in arr] - arr_grad2 = [a.copyto(mx.cpu()) for a in arr_grad] - exec2 = net.bind(mx.cpu(), + arr[2][:] = 3.0 + arr2 = [a.copyto(ctx1) for a in arr] + arr_grad2 = [a.copyto(ctx1) for a in arr_grad] + exec2 = net.bind(ctx1, args=arr2, args_grad=arr_grad2) # Show the execution plan that involves copynode print(exec1.debug_str()) - exec1.forward() - exec2.forward() + exec1.forward(is_train=True) + exec2.forward(is_train=True) assert reldiff(exec1.outputs[0].asnumpy(), exec2.outputs[0].asnumpy()) < 1e-6 - out_grad = mx.nd.empty(shape, mx.cpu(1)) + out_grad = mx.nd.empty(shape, ctx1) out_grad[:] = 1.0 exec1.backward([out_grad]) - exec2.backward([out_grad.copyto(mx.cpu())]) + exec2.backward([out_grad.copyto(ctx1)]) for a, b in zip(arr_grad, arr_grad2): assert reldiff(a.asnumpy(), b.asnumpy()) < 1e-6 From 5af56bbc81ff1aaa26efe38f7436664d59a18d1d Mon Sep 17 00:00:00 2001 From: Yu Zhang Date: Sun, 16 Jul 2017 17:41:53 -0400 Subject: [PATCH 225/834] Improve PTB results (#7059) * Fix speech demo. * Using a random seed for cudnn dropout. Previously, the fixed seed will generate the same mask for each iteration in imperative mode. * PTB LM example now has far btter PPL: 1) forget_bias=0 2) clipping range 3) lr anealing 4) initliazation. * (1) Remove mean for loss function (good for multi-gpu). (2) Change clip and lr to sample based. (3) Change hyperparameters, now we get slightly better results than pytorch. * Remove the lstmbias init in model.py since it already been set to 0. --- example/gluon/word_language_model/README.md | 49 ++++++++++++++++++++ example/gluon/word_language_model/model.py | 6 ++- example/gluon/word_language_model/train.py | 51 ++++++++++++--------- example/speech-demo/decode_mxnet.py | 2 +- src/operator/cudnn_rnn-inl.h | 2 +- 5 files changed, 84 insertions(+), 26 deletions(-) create mode 100644 example/gluon/word_language_model/README.md diff --git a/example/gluon/word_language_model/README.md b/example/gluon/word_language_model/README.md new file mode 100644 index 000000000000..f200c164a78a --- /dev/null +++ b/example/gluon/word_language_model/README.md @@ -0,0 +1,49 @@ +# Word-level language modeling RNN + +This example trains a multi-layer RNN (Elman, GRU, or LSTM) on Penn Treebank (PTB) language modeling benchmark. + +The model obtains the state-of-the-art result on PTB using LSTM, getting a test perplexity of ~72. + +The following techniques have been adopted for SOTA results: +- [LSTM for LM](https://arxiv.org/pdf/1409.2329.pdf) +- [Weight tying](https://arxiv.org/abs/1608.05859) between word vectors and softmax output embeddings + +## Data + +The PTB data is the processed version from [(Mikolov et al, 2010)](http://www.fit.vutbr.cz/research/groups/speech/publi/2010/mikolov_interspeech2010_IS100722.pdf): + +```bash +python data.py +``` + +## Usage + +Example runs and the results: + +``` +python train.py --cuda --tied --nhid 650 --emsize 650 --dropout 0.5 # Test ppl of 75.3 +python train.py --cuda --tied --nhid 1500 --emsize 1500 --dropout 0.65 # Test ppl of 72.0 +``` + +
    + +`python train.py --help` gives the following arguments: +``` +Optional arguments: + -h, --help show this help message and exit + --data DATA location of the data corpus + --model MODEL type of recurrent net (rnn_tanh, rnn_relu, lstm, gru) + --emsize EMSIZE size of word embeddings + --nhid NHID number of hidden units per layer + --nlayers NLAYERS number of layers + --lr LR initial learning rate + --clip CLIP gradient clipping + --epochs EPOCHS upper epoch limit + --batch_size N batch size + --bptt BPTT sequence length + --dropout DROPOUT dropout applied to layers (0 = no dropout) + --tied tie the word embedding and softmax weights + --cuda Whether to use gpu + --log-interval N report interval + --save SAVE path to save the final model +``` diff --git a/example/gluon/word_language_model/model.py b/example/gluon/word_language_model/model.py index 29a170975674..91378cee3cb4 100644 --- a/example/gluon/word_language_model/model.py +++ b/example/gluon/word_language_model/model.py @@ -1,15 +1,17 @@ import mxnet as mx -import mxnet.ndarray as F from mxnet import gluon from mxnet.gluon import nn, rnn class RNNModel(gluon.Block): + """A model with an encoder, recurrent layer, and a decoder.""" + def __init__(self, mode, vocab_size, num_embed, num_hidden, num_layers, dropout=0.5, tie_weights=False, **kwargs): super(RNNModel, self).__init__(**kwargs) with self.name_scope(): self.drop = nn.Dropout(dropout) - self.encoder = nn.Embedding(vocab_size, num_embed) + self.encoder = nn.Embedding(vocab_size, num_embed, + weight_initializer=mx.init.Uniform(0.1)) if mode == 'rnn_relu': self.rnn = rnn.RNN(num_hidden, 'relu', num_layers, dropout=dropout, input_size=num_embed) diff --git a/example/gluon/word_language_model/train.py b/example/gluon/word_language_model/train.py index 1e7e5f36e271..5b34c00ecea0 100644 --- a/example/gluon/word_language_model/train.py +++ b/example/gluon/word_language_model/train.py @@ -3,7 +3,6 @@ import math import mxnet as mx from mxnet import gluon, autograd -from mxnet.gluon import nn, rnn import model import data @@ -18,13 +17,13 @@ help='number of hidden units per layer') parser.add_argument('--nlayers', type=int, default=2, help='number of layers') -parser.add_argument('--lr', type=float, default=20, +parser.add_argument('--lr', type=float, default=1.0, help='initial learning rate') -parser.add_argument('--clip', type=float, default=0.25, +parser.add_argument('--clip', type=float, default=0.2, help='gradient clipping') parser.add_argument('--epochs', type=int, default=40, help='upper epoch limit') -parser.add_argument('--batch_size', type=int, default=20, metavar='N', +parser.add_argument('--batch_size', type=int, default=32, metavar='N', help='batch size') parser.add_argument('--bptt', type=int, default=35, help='sequence length') @@ -32,13 +31,11 @@ help='dropout applied to layers (0 = no dropout)') parser.add_argument('--tied', action='store_true', help='tie the word embedding and softmax weights') -parser.add_argument('--seed', type=int, default=1111, - help='random seed') parser.add_argument('--cuda', action='store_true', help='Whether to use gpu') parser.add_argument('--log-interval', type=int, default=200, metavar='N', help='report interval') -parser.add_argument('--save', type=str, default='model.params', +parser.add_argument('--save', type=str, default='model.params', help='path to save the final model') args = parser.parse_args() @@ -73,26 +70,25 @@ def batchify(data, batch_size): ntokens = len(corpus.dictionary) -model = model.RNNModel(args.model, ntokens, args.emsize, args.nhid, args.nlayers, args.dropout, args.tied) +model = model.RNNModel(args.model, ntokens, args.emsize, args.nhid, + args.nlayers, args.dropout, args.tied) model.collect_params().initialize(mx.init.Xavier(), ctx=context) trainer = gluon.Trainer(model.collect_params(), 'sgd', - {'learning_rate': args.lr, - 'momentum': 0, - 'wd': 0}) + {'learning_rate': args.lr, + 'momentum': 0, + 'wd': 0}) loss = gluon.loss.SoftmaxCrossEntropyLoss() ############################################################################### # Training code ############################################################################### - def get_batch(source, i): seq_len = min(args.bptt, source.shape[0] - 1 - i) data = source[i:i+seq_len] target = source[i+1:i+1+seq_len] return data, target.reshape((-1,)) - def detach(hidden): if isinstance(hidden, (tuple, list)): hidden = [i.detach() for i in hidden] @@ -100,12 +96,11 @@ def detach(hidden): hidden = hidden.detach() return hidden - def eval(data_source): total_L = 0.0 ntotal = 0 hidden = model.begin_state(func=mx.nd.zeros, batch_size=args.batch_size, ctx=context) - for ibatch, i in enumerate(range(0, data_source.shape[0] - 1, args.bptt)): + for i in range(0, data_source.shape[0] - 1, args.bptt): data, target = get_batch(data_source, i) output, hidden = model(data, hidden) L = loss(output, target) @@ -113,9 +108,8 @@ def eval(data_source): ntotal += L.size return total_L / ntotal - def train(): - best_val = None + best_val = float("Inf") for epoch in range(args.epochs): total_L = 0.0 start_time = time.time() @@ -129,15 +123,15 @@ def train(): L.backward() grads = [i.grad(context) for i in model.collect_params().values()] - # Here gradient is not divided by batch_size yet. - # So we multiply max_norm by batch_size to balance it. - gluon.utils.clip_global_norm(grads, args.clip * args.batch_size) + # Here gradient is for the whole batch. + # So we multiply max_norm by batch_size and bptt size to balance it. + gluon.utils.clip_global_norm(grads, args.clip * args.bptt * args.batch_size) trainer.step(args.batch_size) total_L += mx.nd.sum(L).asscalar() if ibatch % args.log_interval == 0 and ibatch > 0: - cur_L = total_L / args.batch_size / args.bptt / args.log_interval + cur_L = total_L / args.bptt / args.batch_size / args.log_interval print('[Epoch %d Batch %d] loss %.2f, ppl %.2f'%( epoch, ibatch, cur_L, math.exp(cur_L))) total_L = 0.0 @@ -147,8 +141,21 @@ def train(): print('[Epoch %d] time cost %.2fs, valid loss %.2f, valid ppl %.2f'%( epoch, time.time()-start_time, val_L, math.exp(val_L))) + if val_L < best_val: + best_val = val_L + test_L = eval(test_data) + model.collect_params().save(args.save) + print('test loss %.2f, test ppl %.2f'%(test_L, math.exp(test_L))) + else: + args.lr = args.lr*0.25 + trainer._init_optimizer('sgd', + {'learning_rate': args.lr, + 'momentum': 0, + 'wd': 0}) + model.collect_params().load(args.save, context) if __name__ == '__main__': train() + model.collect_params().load(args.save, context) test_L = eval(test_data) - print('test loss %.2f, test ppl %.2f'%(test_L, math.exp(test_L))) + print('Best test loss %.2f, test ppl %.2f'%(test_L, math.exp(test_L))) diff --git a/example/speech-demo/decode_mxnet.py b/example/speech-demo/decode_mxnet.py index 1826e1265de7..4680fbe904df 100644 --- a/example/speech-demo/decode_mxnet.py +++ b/example/speech-demo/decode_mxnet.py @@ -142,7 +142,7 @@ def sym_gen(seq_len): elif decoding_method == METHOD_SIMPLE: for (ind, utt) in enumerate(batch.utt_id): if utt != "GAP_UTT": - posteriors = posteriors[:batch.utt_len,1:] - np.log(data_test.label_mean[1:]).T + posteriors = posteriors[:batch.utt_len[0],1:] - np.log(data_test.label_mean[1:]).T kaldiWriter.write(utt, posteriors) else: outputs = module.get_outputs() diff --git a/src/operator/cudnn_rnn-inl.h b/src/operator/cudnn_rnn-inl.h index a4ce10edd886..17acf4a5b35f 100644 --- a/src/operator/cudnn_rnn-inl.h +++ b/src/operator/cudnn_rnn-inl.h @@ -526,7 +526,7 @@ class CuDNNRNNOp : public Operator { cudnnRNNInputMode_t input_mode_; cudnnDropoutDescriptor_t dropout_desc_; Storage::Handle dropout_states_, reserve_space_; - uint64_t seed_ = 1337ull; + uint64_t seed_ = 17 + rand() % 4096; // NOLINT(runtime/threadsafe_fn) size_t workspace_byte_, reserve_space_byte_, dropout_byte_; int workspace_size_, dropout_size_; std::vector x_desc_vec_, y_desc_vec_, dx_desc_vec_, dy_desc_vec_; From 8144361741ca70d0f95a731c503a9e8292add6f7 Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Sun, 16 Jul 2017 14:43:58 -0700 Subject: [PATCH 226/834] pretty-print structure/parameters of gluon layers (#7060) --- python/mxnet/gluon/block.py | 9 ++++ python/mxnet/gluon/loss.py | 57 ++++++++++++++++++------- python/mxnet/gluon/nn/basic_layers.py | 61 +++++++++++++++++++++++++++ python/mxnet/gluon/nn/conv_layers.py | 29 +++++++++++++ python/mxnet/gluon/parameter.py | 12 ++++++ python/mxnet/gluon/rnn/rnn_cell.py | 40 ++++++++++++++++++ python/mxnet/gluon/rnn/rnn_layer.py | 15 +++++++ python/mxnet/gluon/utils.py | 12 ++++++ 8 files changed, 219 insertions(+), 16 deletions(-) diff --git a/python/mxnet/gluon/block.py b/python/mxnet/gluon/block.py index 4a0060a03345..0916e2345fe4 100644 --- a/python/mxnet/gluon/block.py +++ b/python/mxnet/gluon/block.py @@ -7,6 +7,7 @@ from ..ndarray import NDArray from .. import name as _name from .parameter import Parameter, ParameterDict, DeferredInitializationError +from .utils import _indent class _BlockScope(object): @@ -145,6 +146,14 @@ def __setattr__(self, name, value): def _alias(self): return self.__class__.__name__.lower() + def __repr__(self): + s = '{name}(\n{modstr}\n)' + modstr = '\n'.join([' ({key}): {block}'.format(key=key, + block=_indent(block.__repr__(), 2)) + for key, block in self.__dict__.items() if isinstance(block, Block)]) + return s.format(name=self.__class__.__name__, + modstr=modstr) + @property def params(self): """Returns this `Block`'s parameter dictionary (does not include its diff --git a/python/mxnet/gluon/loss.py b/python/mxnet/gluon/loss.py index 7cc1bcfbdd7f..38fdcb820919 100644 --- a/python/mxnet/gluon/loss.py +++ b/python/mxnet/gluon/loss.py @@ -38,7 +38,40 @@ def _apply_weighting(F, loss, weight=None, sample_weight=None): return loss -class L2Loss(HybridBlock): +class Loss(HybridBlock): + """Base class for loss. + + Parameters + ---------- + weight : float or None + Global scalar weight for loss. + batch_axis : int, default 0 + The axis that represents mini-batch. + """ + def __init__(self, weight, batch_axis, **kwargs): + super(Loss, self).__init__(**kwargs) + self._weight = weight + self._batch_axis = batch_axis + + def __repr__(self): + s = '{name}(batch_axis={_batch_axis}, w={_weight})' + return s.format(name=self.__class__.__name__, **self.__dict__) + + def hybrid_forward(self, F, x, *args, **kwargs): + """Overrides to construct symbolic graph for this `Block`. + + Parameters + ---------- + x : Symbol or NDArray + The first input tensor. + *args : list of Symbol or list of NDArray + Additional input tensors. + """ + # pylint: disable= invalid-name + raise NotImplementedError + + +class L2Loss(Loss): """Calculates the mean squared error between output and label: .. math:: @@ -60,9 +93,7 @@ class L2Loss(HybridBlock): The axis that represents mini-batch. """ def __init__(self, weight=1., batch_axis=0, **kwargs): - super(L2Loss, self).__init__(**kwargs) - self._weight = weight - self._batch_axis = batch_axis + super(L2Loss, self).__init__(weight, batch_axis, **kwargs) def hybrid_forward(self, F, output, label, sample_weight=None): if F is ndarray: @@ -76,7 +107,7 @@ def hybrid_forward(self, F, output, label, sample_weight=None): return F.mean(loss, axis=self._batch_axis, exclude=True) -class L1Loss(HybridBlock): +class L1Loss(Loss): """Calculates the mean absolute error between output and label: .. math:: @@ -97,9 +128,7 @@ class L1Loss(HybridBlock): The axis that represents mini-batch. """ def __init__(self, weight=None, batch_axis=0, **kwargs): - super(L1Loss, self).__init__(**kwargs) - self._weight = weight - self._batch_axis = batch_axis + super(L1Loss, self).__init__(weight, batch_axis, **kwargs) def hybrid_forward(self, F, output, label, sample_weight=None): if F is ndarray: @@ -113,7 +142,7 @@ def hybrid_forward(self, F, output, label, sample_weight=None): return F.mean(loss, axis=self._batch_axis, exclude=True) -class SoftmaxCrossEntropyLoss(HybridBlock): +class SoftmaxCrossEntropyLoss(Loss): """Computes the softmax cross entropy loss. If `sparse_label` is `True`, label should contain integer category indicators: @@ -155,12 +184,10 @@ class SoftmaxCrossEntropyLoss(HybridBlock): """ def __init__(self, axis=-1, sparse_label=True, from_logits=False, weight=None, batch_axis=0, **kwargs): - super(SoftmaxCrossEntropyLoss, self).__init__(**kwargs) + super(SoftmaxCrossEntropyLoss, self).__init__(weight, batch_axis, **kwargs) self._axis = axis self._sparse_label = sparse_label self._from_logits = from_logits - self._weight = weight - self._batch_axis = batch_axis def hybrid_forward(self, F, output, label, sample_weight=None): if not self._from_logits: @@ -173,7 +200,7 @@ def hybrid_forward(self, F, output, label, sample_weight=None): return F.mean(loss, axis=self._batch_axis, exclude=True) -class KLDivLoss(HybridBlock): +class KLDivLoss(Loss): """The Kullback-Leibler divergence loss. KL divergence is a useful distance measure for continuous distributions @@ -203,10 +230,8 @@ class KLDivLoss(HybridBlock): The axis that represents mini-batch. """ def __init__(self, from_logits=True, weight=None, batch_axis=0, **kwargs): - super(KLDivLoss, self).__init__(**kwargs) + super(KLDivLoss, self).__init__(weight, batch_axis, **kwargs) self._from_logits = from_logits - self._weight = weight - self._batch_axis = batch_axis def hybrid_forward(self, F, output, label, sample_weight=None): if not self._from_logits: diff --git a/python/mxnet/gluon/nn/basic_layers.py b/python/mxnet/gluon/nn/basic_layers.py index a0617caf0215..069baf94079d 100644 --- a/python/mxnet/gluon/nn/basic_layers.py +++ b/python/mxnet/gluon/nn/basic_layers.py @@ -3,6 +3,7 @@ """Basic neural network layers.""" from ..block import Block, HybridBlock +from ..utils import _indent class Sequential(Block): @@ -28,6 +29,15 @@ def forward(self, x): x = block(x) return x + def __repr__(self): + s = '{name}(\n{modstr}\n)' + modstr = '\n'.join([' ({key}): {block}'.format(key=key, + block=_indent(block.__repr__(), 2)) + for key, block in enumerate(self._children) + if isinstance(block, Block)]) + return s.format(name=self.__class__.__name__, + modstr=modstr) + class HybridSequential(HybridBlock): """Stacks `HybridBlock`s sequentially. @@ -52,6 +62,15 @@ def hybrid_forward(self, F, x): x = block(x) return x + def __repr__(self): + s = '{name}(\n{modstr}\n)' + modstr = '\n'.join([' ({key}): {block}'.format(key=key, + block=_indent(block.__repr__(), 2)) + for key, block in enumerate(self._children) + if isinstance(block, Block)]) + return s.format(name=self.__class__.__name__, + modstr=modstr) + class Dense(HybridBlock): """Just your regular densely-connected NN layer. @@ -102,6 +121,7 @@ def __init__(self, units, activation=None, use_bias=True, super(Dense, self).__init__(**kwargs) with self.name_scope(): self._units = units + self._in_units = in_units self.weight = self.params.get('weight', shape=(units, in_units), init=weight_initializer, allow_deferred_init=True) @@ -125,6 +145,13 @@ def hybrid_forward(self, F, x, weight, bias=None): act = self.act(act) return act + def __repr__(self): + s = '{name}({layout}, {act})' + return s.format(name=self.__class__.__name__, + act=self.act if self.act else 'linear', + layout='{0} -> {1}'.format(self._in_units, self._units) if self._in_units + else self._units) + class Activation(HybridBlock): """Applies an activation function to input. @@ -152,6 +179,11 @@ def _alias(self): def hybrid_forward(self, F, x): return F.Activation(x, act_type=self._act_type) + def __repr__(self): + s = '{name}({_act_type})' + return s.format(name=self.__class__.__name__, + **self.__dict__) + class Dropout(HybridBlock): """Applies Dropout to the input. @@ -183,6 +215,11 @@ def __init__(self, rate, **kwargs): def hybrid_forward(self, F, x): return F.Dropout(x, p=self._rate) + def __repr__(self): + s = '{name}(p = {_rate})' + return s.format(name=self.__class__.__name__, + **self.__dict__) + class BatchNorm(HybridBlock): """Batch normalization layer (Ioffe and Szegedy, 2014). @@ -235,6 +272,8 @@ def __init__(self, axis=1, momentum=0.9, epsilon=1e-3, center=True, scale=True, super(BatchNorm, self).__init__(**kwargs) self._kwargs = {'axis': axis, 'eps': epsilon, 'momentum': momentum, 'fix_gamma': not center} + if in_channels != 0: + self.in_channels = in_channels self.gamma = self.params.get('gamma', grad_req='write' if scale else 'null', shape=(in_channels,), init=gamma_initializer, @@ -254,6 +293,15 @@ def __init__(self, axis=1, momentum=0.9, epsilon=1e-3, center=True, scale=True, def hybrid_forward(self, F, x, gamma, beta, running_mean, running_var): return F.BatchNorm(x, gamma, beta, running_mean, running_var, **self._kwargs) + def __repr__(self): + s = '{name}({content}' + if hasattr(self, 'in_channels'): + s += ', in_channels={0}'.format(self.in_channels) + s += ')' + return s.format(name=self.__class__.__name__, + content=', '.join(['='.join([k, v.__repr__()]) + for k, v in self._kwargs.items()])) + class LeakyReLU(HybridBlock): """Leaky version of a Rectified Linear Unit. @@ -282,6 +330,11 @@ def __init__(self, alpha, **kwargs): def hybrid_forward(self, F, x): return F.LeakyReLU(x, act_type='leaky', slope=self._alpha) + def __repr__(self): + s = '{name}({alpha})' + return s.format(name=self.__class__.__name__, + alpha=self._alpha) + class Embedding(HybridBlock): """Turns non-negative integers (indexes/tokens) into dense vectors @@ -318,6 +371,11 @@ def __init__(self, input_dim, output_dim, dtype='float32', def hybrid_forward(self, F, x, weight): return F.Embedding(x, weight, **self._kwargs) + def __repr__(self): + s = '{name}({input_dim} -> {output_dim}, {dtype})' + return s.format(name=self.__class__.__name__, + **self._kwargs) + class Flatten(HybridBlock): """Flattens the input to two dimensional. @@ -333,3 +391,6 @@ def __init__(self, **kwargs): def hybrid_forward(self, F, x): return x.reshape((0, -1)) + + def __repr__(self): + return self.__class__.__name__ diff --git a/python/mxnet/gluon/nn/conv_layers.py b/python/mxnet/gluon/nn/conv_layers.py index 60fd848c9a14..caa2dd87eb5f 100644 --- a/python/mxnet/gluon/nn/conv_layers.py +++ b/python/mxnet/gluon/nn/conv_layers.py @@ -111,6 +111,26 @@ def hybrid_forward(self, F, x, weight, bias=None): act = self.act(act) return act + def __repr__(self): + s = '{name}({mapping}, kernel_size={kernel}, stride={stride}' + len_kernel_size = len(self._kwargs['kernel']) + if self._kwargs['pad'] != (0,) * len_kernel_size: + s += ', padding={pad}' + if self._kwargs['dilate'] != (1,) * len_kernel_size: + s += ', dilation={dilate}' + if hasattr(self, 'out_pad') and self.out_pad != (0,) * len_kernel_size: + s += ', output_padding={out_pad}'.format(out_pad=self.out_pad) + if self._kwargs['num_group'] != 1: + s += ', groups={num_group}' + if self.bias is None: + s += ', bias=False' + s += ')' + return s.format(name=self.__class__.__name__, + mapping=self._channels if not self._in_channels + else '{0} -> {1}'.format(self._in_channels, + self._channels), + **self._kwargs) + class Conv1D(_Conv): """1D convolution layer (e.g. temporal convolution). @@ -430,6 +450,7 @@ def __init__(self, channels, kernel_size, strides=1, padding=0, output_padding=0 channels, kernel_size, strides, padding, dilation, groups, layout, in_channels, activation, use_bias, weight_initializer, bias_initializer, op_name='Deconvolution', adj=output_padding, **kwargs) + self.outpad = output_padding class Conv2DTranspose(_Conv): @@ -515,6 +536,7 @@ def __init__(self, channels, kernel_size, strides=(1, 1), padding=(0, 0), channels, kernel_size, strides, padding, dilation, groups, layout, in_channels, activation, use_bias, weight_initializer, bias_initializer, op_name='Deconvolution', adj=output_padding, **kwargs) + self.outpad = output_padding class Conv3DTranspose(_Conv): @@ -600,6 +622,7 @@ def __init__(self, channels, kernel_size, strides=(1, 1, 1), padding=(0, 0, 0), channels, kernel_size, strides, padding, dilation, groups, layout, in_channels, activation, use_bias, weight_initializer, bias_initializer, op_name='Deconvolution', adj=output_padding, **kwargs) + self.outpad = output_padding class _Pooling(HybridBlock): @@ -621,6 +644,12 @@ def __init__(self, pool_size, strides, padding, ceil_mode, global_pool, def hybrid_forward(self, F, x): return F.Pooling(x, **self._kwargs) + def __repr__(self): + s = '{name}(size={kernel}, stride={stride}, padding={pad}, ceil_mode={ceil_mode})' + return s.format(name=self.__class__.__name__, + ceil_mode=self._kwargs['pooling_convention'] == 'full', + **self._kwargs) + class MaxPool1D(_Pooling): """Max pooling operation for one dimensional data. diff --git a/python/mxnet/gluon/parameter.py b/python/mxnet/gluon/parameter.py index 2b3cb05ec1fc..1bf48f93a6b8 100644 --- a/python/mxnet/gluon/parameter.py +++ b/python/mxnet/gluon/parameter.py @@ -9,6 +9,7 @@ from .. import symbol, ndarray, initializer, context from ..context import Context from .. import autograd +from .utils import _indent # pylint: disable= invalid-name tensor_types = (symbol.Symbol, ndarray.NDArray) @@ -76,6 +77,10 @@ def __init__(self, name, grad_req='write', shape=None, dtype=mx_real_t, self._grad = None self._defered_init = () + def __repr__(self): + s = 'Parameter {name} (shape={shape}, dtype={dtype})' + return s.format(**self.__dict__) + def initialize(self, init=None, ctx=None, default_init=initializer.Uniform()): """Initializes parameter and gradient arrays. Only used for `NDArray` API. @@ -322,6 +327,13 @@ def __init__(self, prefix='', shared=None): def __getitem__(self, key): return self._params[key] + def __repr__(self): + s = '{name}(\n{content}\n)' + name = self._prefix+' ' if self._prefix else '' + return s.format(name=name, + content='\n'.join([_indent(' {0}'.format(v), 2) + for v in self.values()])) + def items(self): return self._params.items() diff --git a/python/mxnet/gluon/rnn/rnn_cell.py b/python/mxnet/gluon/rnn/rnn_cell.py index a90039febcf3..e06599cc89d3 100644 --- a/python/mxnet/gluon/rnn/rnn_cell.py +++ b/python/mxnet/gluon/rnn/rnn_cell.py @@ -8,6 +8,7 @@ from ... import symbol, ndarray from ...base import string_types, numeric_types from ..block import Block, HybridBlock +from ..utils import _indent from .. import tensor_types @@ -87,6 +88,17 @@ def __init__(self, prefix=None, params=None): self._modified = False self.reset() + def __repr__(self): + s = '{name}({mapping}' + if hasattr(self, '_activation'): + s += ', {_activation}' + s += ')' + mapping = ('{_input_size} -> {_hidden_size}'.format(**self.__dict__) if self._input_size + else self._hidden_size) + return s.format(name=self.__class__.__name__, + mapping=mapping, + **self.__dict__) + def reset(self): """Reset before re-using the cell for another graph.""" self._init_counter = -1 @@ -428,6 +440,7 @@ def __init__(self, hidden_size, input_size=0, prefix=None, params=None): super(GRUCell, self).__init__(prefix=prefix, params=params) self._hidden_size = hidden_size + self._input_size = input_size self.i2h_weight = self.params.get('i2h_weight', shape=(3*hidden_size, input_size), init=i2h_weight_initializer, allow_deferred_init=True) @@ -485,6 +498,12 @@ class SequentialRNNCell(RecurrentCell): def __init__(self, prefix=None, params=None): super(SequentialRNNCell, self).__init__(prefix=prefix, params=params) + def __repr__(self): + s = '{name}(\n{modstr}\n)' + return s.format(name=self.__class__.__name__, + modstr='\n'.join(['({i}): {m}'.format(i=i, m=_indent(m.__repr__(), 2)) + for i, m in enumerate(self._children)])) + def add(self, cell): """Appends a cell into the stack. @@ -553,6 +572,11 @@ def __init__(self, dropout, prefix=None, params=None): assert isinstance(dropout, numeric_types), "dropout probability must be a number" self.dropout = dropout + def __repr__(self): + s = '{name}(p = {dropout})' + return s.format(name=self.__class__.__name__, + **self.__dict__) + def state_info(self, batch_size=0): return [] @@ -610,6 +634,11 @@ def begin_state(self, func=symbol.zeros, **kwargs): def hybrid_forward(self, F, inputs, states): raise NotImplementedError + def __repr__(self): + s = '{name}({base_cell})' + return s.format(name=self.__class__.__name__, + **self.__dict__) + class ZoneoutCell(ModifierCell): """Applies Zoneout on base cell.""" @@ -625,6 +654,11 @@ def __init__(self, base_cell, zoneout_outputs=0., zoneout_states=0.): self.zoneout_states = zoneout_states self.prev_output = None + def __repr__(self): + s = '{name}(p_out={zoneout_outputs}, p_state={zoneout_states}, {base_cell})' + return s.format(name=self.__class__.__name__, + **self.__dict__) + def _alias(self): return 'zoneout' @@ -704,6 +738,12 @@ def __init__(self, l_cell, r_cell, output_prefix='bi_'): def __call__(self, inputs, states): raise NotImplementedError("Bidirectional cannot be stepped. Please use unroll") + def __repr__(self): + s = '{name}(forward={l_cell}, backward={r_cell})' + return s.format(name=self.__class__.__name__, + l_cell=self._children[0], + r_cell=self._children[1]) + def state_info(self, batch_size=0): return _cells_state_info(self._children, batch_size) diff --git a/python/mxnet/gluon/rnn/rnn_layer.py b/python/mxnet/gluon/rnn/rnn_layer.py index 0e7efff6639a..a22cb0138bcb 100644 --- a/python/mxnet/gluon/rnn/rnn_layer.py +++ b/python/mxnet/gluon/rnn/rnn_layer.py @@ -62,6 +62,21 @@ def __init__(self, hidden_size, num_layers, layout, self._unfused = self._unfuse() + def __repr__(self): + s = '{name}({mapping}, {_layout}' + if self._num_layers != 1: + s += ', num_layers={_num_layers}' + if self._dropout != 0: + s += ', dropout={_dropout}' + if self._dir == 2: + s += ', bidirectional' + s += ')' + mapping = ('{_input_size} -> {_hidden_size}'.format(**self.__dict__) if self._input_size + else self._hidden_size) + return s.format(name=self.__class__.__name__, + mapping=mapping, + **self.__dict__) + def state_info(self, batch_size=0): raise NotImplementedError diff --git a/python/mxnet/gluon/utils.py b/python/mxnet/gluon/utils.py index 842f260763d2..27576b55f7f9 100644 --- a/python/mxnet/gluon/utils.py +++ b/python/mxnet/gluon/utils.py @@ -97,3 +97,15 @@ def clip_global_norm(arrays, max_norm): for arr in arrays: arr *= scale return total_norm + + +def _indent(s_, numSpaces): + """Indent string + """ + s = s_.split('\n') + if len(s) == 1: + return s_ + first = s.pop(0) + s = [first] + [(numSpaces * ' ') + line for line in s] + s = '\n'.join(s) + return s From a8804a1b7dd3793176a4ffbf38d0f8e97b010237 Mon Sep 17 00:00:00 2001 From: Marek Kolodziej Date: Sun, 16 Jul 2017 17:41:32 -0700 Subject: [PATCH 227/834] Optimized sequence reverse operator (#6946) --- src/operator/sequence_reverse-inl.h | 105 ++++++++++++++++--------- tests/python/gpu/test_operator_gpu.py | 5 +- tests/python/unittest/test_operator.py | 72 ++++++++++++++++- 3 files changed, 140 insertions(+), 42 deletions(-) diff --git a/src/operator/sequence_reverse-inl.h b/src/operator/sequence_reverse-inl.h index 048eb3e2eb78..44a1bd7ceaad 100644 --- a/src/operator/sequence_reverse-inl.h +++ b/src/operator/sequence_reverse-inl.h @@ -1,8 +1,9 @@ -/*! +/* * Copyright (c) 2016 by Contributors * \file sequence_reverse-inl.h * \brief * \author Sebastian Bodenstien + * \author Marek Kolodziej */ #ifndef MXNET_OPERATOR_SEQUENCE_REVERSE_INL_H_ @@ -13,12 +14,13 @@ #include #include #include -#include #include #include +#include +#include "./mshadow_op.h" +#include "./mxnet_op.h" #include "./operator_common.h" #include "./sequence_op_common.h" -#include "./mshadow_op.h" namespace mxnet { namespace op { @@ -34,35 +36,68 @@ struct SequenceReverseParam : public dmlc::Parameter { DMLC_DECLARE_FIELD(use_sequence_length) .set_default(false) .describe( - "If set to true, this layer takes in an extra input parameter `sequence_length` " + "If set to true, this layer takes in an extra input parameter " + "`sequence_length` " "to specify variable length sequence"); } }; +struct ReverseKernel { + template + MSHADOW_XINLINE static void Map( + const int i, DType *const out_data, const DType *const in_data, + const OpReqType req, const index_t max_seq_len, const index_t batch_size, + const index_t other_dim, const index_t numel, const DType *const indices + ) { + for (index_t batch = 0; batch < batch_size; ++batch) { + const index_t num_seq = indices + ? static_cast(indices[batch]) + : max_seq_len; + const index_t padded_periods = max_seq_len - num_seq; + // padded part + if (padded_periods > 0 && i < padded_periods) { + const int padded_in_offset = + (i + num_seq) * batch_size * other_dim + batch * other_dim; + + for (index_t j = 0; j < other_dim; ++j) { + KERNEL_ASSIGN(out_data[padded_in_offset + j], req, + in_data[padded_in_offset + j]); + } + } + // unpadded part + if (i < num_seq) { + const int in_offset = i * batch_size * other_dim + batch * other_dim; + const int out_offset = + numel - (i + 1 + padded_periods) * batch_size * other_dim + + batch * other_dim; + + for (index_t j = 0; j < other_dim; ++j) { + KERNEL_ASSIGN(out_data[out_offset + j], req, in_data[in_offset + j]); + } + } + } + } +}; + template class SequenceReverseOp : public Operator { public: explicit SequenceReverseOp(SequenceReverseParam p) { this->param_ = p; } - void sequence_reverse(const mshadow::Tensor data, + void sequence_reverse(const mshadow::Tensor &data, const mshadow::Tensor &out, - std::vector indices, OpReqType req) { + const OpReqType req, const DType *const indices, + mshadow::Stream *const s) { using namespace mshadow; using namespace mshadow::expr; - index_t seq_length; - index_t max_seq_len = data.size(0); - index_t batch_size = data.size(1); - for (index_t b = 0; b < batch_size; ++b) { - seq_length = indices[b]; - for (index_t s = 0; s < max_seq_len; ++s) { - if (s < seq_length) - Assign( - out[s][b], req, - F( - data[seq_length - s - 1][b])) - else // preserve padding type - Assign(out[s][b], req, F(data[s][b])) - } - } + + const index_t max_seq_len = data.size(0); + const index_t batch_size = data.size(1); + const index_t other_dim = data.size(2); + const index_t tensor_numel = data.shape_.Size(); + + mxnet_op::Kernel::Launch( + s, max_seq_len, out.dptr_, data.dptr_, req, max_seq_len, batch_size, + other_dim, tensor_numel, indices); } virtual void Forward(const OpContext &ctx, const std::vector &in_data, @@ -73,7 +108,7 @@ class SequenceReverseOp : public Operator { using namespace mshadow::expr; CHECK_EQ(in_data.size(), param_.use_sequence_length ? 2U : 1U); CHECK_EQ(out_data.size(), 1U); - Stream *s = ctx.get_stream(); + Stream *const s = ctx.get_stream(); // Get any size input + output into required form int max_seq_len = in_data[seq_reverse::kData].size(0); @@ -87,14 +122,12 @@ class SequenceReverseOp : public Operator { Tensor out = out_data[seq_reverse::kOut].get_with_shape(s3, s); - // copy indices to vector - std::vector indices_vec(n, max_seq_len); - if (param_.use_sequence_length) - IndexTensorToVector( - in_data[seq_reverse::kSequenceLength].get(s), - &indices_vec); + const DType *const indices = + param_.use_sequence_length + ? in_data[seq_reverse::kSequenceLength].dptr() + : nullptr; - sequence_reverse(data, out, indices_vec, req[seq_reverse::kOut]); + sequence_reverse(data, out, req[seq_reverse::kOut], indices, s); } virtual void Backward(const OpContext &ctx, @@ -122,15 +155,13 @@ class SequenceReverseOp : public Operator { in_grad[seq_reverse::kData].get_with_shape(s3, s); Tensor output_grad = out_grad[seq_reverse::kOut].get_with_shape(s3, s); - // copy indices to vector - std::vector indices_vec(n, max_seq_len); - if (param_.use_sequence_length) - IndexTensorToVector( - in_data[seq_reverse::kSequenceLength].get(s), - &indices_vec); - sequence_reverse(output_grad, data_grad, indices_vec, - req[seq_reverse::kData]); + const DType *const indices = + param_.use_sequence_length + ? in_data[seq_reverse::kSequenceLength].dptr() + : nullptr; + + sequence_reverse(output_grad, data_grad, req[seq_reverse::kData], indices, s); } private: diff --git a/tests/python/gpu/test_operator_gpu.py b/tests/python/gpu/test_operator_gpu.py index 9f3f8a3b4f9d..487197f2ad7e 100644 --- a/tests/python/gpu/test_operator_gpu.py +++ b/tests/python/gpu/test_operator_gpu.py @@ -1277,7 +1277,6 @@ def test_residual_fused(): expected_outputs = np.ones((10, 2, 50))+5 assert np.array_equal(outputs[0].asnumpy(), expected_outputs) - def check_rnn_layer(layer): layer.collect_params().initialize(ctx=[mx.cpu(0), mx.gpu(0)]) with mx.gpu(0): @@ -1304,6 +1303,10 @@ def test_rnn_layer(): check_rnn_layer(gluon.rnn.LSTM(100, num_layers=3, bidirectional=True)) +def test_sequence_reverse(): + check_sequence_reverse(mx.gpu(0)) + + if __name__ == '__main__': import nose nose.runmodule() diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py index e13c3c07f2fd..2a4c8068db65 100644 --- a/tests/python/unittest/test_operator.py +++ b/tests/python/unittest/test_operator.py @@ -1,9 +1,10 @@ # pylint: skip-file +from __future__ import print_function import numpy as np import mxnet as mx import random import itertools -from numpy.testing import assert_allclose +from numpy.testing import assert_allclose, assert_array_equal from mxnet.test_utils import * def np_softmax(x, axis=-1): @@ -1314,7 +1315,7 @@ def test_reduce_inner(numpy_reduce_func, numpy_reduce_grad_func, mx_reduce_sym, ndim = np.random.randint(1, 6) shape = np.random.randint(1, 6, size=(ndim,)) axis_num = np.random.randint(0, ndim, size=1) - axis_flags = np.random.randint(-5, 6, size=ndim) + axis_flags = np.random.randint(0, 2, size=ndim) exclude = np.random.randint(0, 2) axes = [] for (axis, flag) in enumerate(axis_flags): @@ -1972,7 +1973,7 @@ def check_instance_norm_with_shape(shape, xpu): exec1 = Y.bind(xpu, args = {'X':x, 'G':gamma, 'B':beta}) exec1.forward(is_train=False) out = exec1.outputs[0].asnumpy() - assert_almost_equal(out, np_out, rtol=1e-4, atol=1e-5) + assert_almost_equal(out, np_out, rtol=1e-4) check_numeric_gradient(Y, {'X':x.asnumpy(), 'G':gamma.asnumpy(), 'B':beta.asnumpy()}, numeric_eps=1e-2, rtol=1e-2, atol=1e-2) @@ -2010,7 +2011,7 @@ def check_l2_normalization(in_shape, mode, ctx=default_context(), norm_eps=1e-10 exe = out.simple_bind(ctx=ctx, data=in_data.shape) output = exe.forward(is_train=True, data=in_data) # compare numpy + mxnet - assert_almost_equal(exe.outputs[0].asnumpy(), np_out, rtol=1e-4, atol=1e-5) + assert_almost_equal(exe.outputs[0].asnumpy(), np_out, rtol=1e-5) # check gradient check_numeric_gradient(out, [in_data], numeric_eps=1e-3, rtol=1e-2, atol=1e-3) @@ -2059,6 +2060,69 @@ def test_sequence_mask(): check_sequence_mask(shape1, default_context(), 2.1) check_sequence_mask(shape2, default_context(), 0.1) +def check_sequence_reverse(xpu): + + # sample data + arr = np.array( + [[[ 1., 2., 3.], + [ 4., 5., 6.]], + [[ 7., 8., 9.], + [ 10., 11., 12.]], + [[ 13., 14., 15.], + [ 16., 17., 18.]]]) + + arr1 = np.array( + [[[ 13., 14., 15.], + [ 16., 17., 18.]], + [[ 7., 8., 9.], + [ 10., 11., 12.]], + [[ 1., 2., 3.], + [ 4., 5., 6.]]]) + + arr2 = np.array( + [[[ 7., 8., 9.], + [ 10., 11., 12.]], + [[ 1., 2., 3.], + [ 4., 5., 6.]], + [[ 13., 14., 15.], + [ 16., 17., 18.]]]) + + arr3 = np.array( + [[[ 7., 8., 9.], + [ 16., 17., 18.]], + [[ 1., 2., 3.], + [ 10., 11., 12.]], + [[ 13., 14., 15.], + [ 4., 5., 6.]]]) + + def test_wrapper(arr, xpu, sequence_length=None, use_sequence_length=False): + # MxNet symbol creation + seq = mx.sym.Variable('seq') + if sequence_length and use_sequence_length: + seq_len = mx.sym.Variable('seq_len') + else: + # ensure that both are disabled, not just one + seq_len=None + use_sequence_length=False + rev = mx.sym.SequenceReverse(data=seq, sequence_length=seq_len, use_sequence_length=use_sequence_length) + # MxNet symbol execution + if sequence_length: + bound = rev.bind(xpu, {'seq': mx.nd.array(arr), 'seq_len': mx.nd.array(sequence_length)}) + else: + bound = rev.bind(xpu, {'seq': mx.nd.array(arr)}) + fwd = bound.forward() + return fwd[0].asnumpy() + + # test cases + assert_array_equal(test_wrapper(arr, xpu, use_sequence_length=False), arr1) + assert_array_equal(test_wrapper(arr, xpu, sequence_length=[3, 3], use_sequence_length=True), arr1) + assert_array_equal(test_wrapper(arr, xpu, sequence_length=[2, 2], use_sequence_length=True), arr2) + assert_array_equal(test_wrapper(arr, xpu, sequence_length=[2, 3], use_sequence_length=True), arr3) + + +def test_sequence_reverse(): + check_sequence_reverse(mx.cpu()) + def mathematical_core_binary(name, forward_mxnet_call, forward_numpy_call, From 70a781b62a0dcaf782cf5cf96394b1e87091ed5f Mon Sep 17 00:00:00 2001 From: Xu Dong Date: Tue, 18 Jul 2017 00:31:30 +0800 Subject: [PATCH 228/834] Update documentation for correlation operation. (#7071) * CorrelationOp document modified * Fix --- src/operator/correlation.cc | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/src/operator/correlation.cc b/src/operator/correlation.cc index 7b762af78149..5160d1f245ad 100644 --- a/src/operator/correlation.cc +++ b/src/operator/correlation.cc @@ -136,9 +136,39 @@ Operator* CorrelationProp::CreateOperator(Context ctx) const { } DMLC_REGISTER_PARAMETER(CorrelationParam); MXNET_REGISTER_OP_PROPERTY(Correlation, CorrelationProp) -.describe("Applies correlation to inputs.") .add_argument("data1", "NDArray-or-Symbol", "Input data1 to the correlation.") .add_argument("data2", "NDArray-or-Symbol", "Input data2 to the correlation.") -.add_arguments(CorrelationParam::__FIELDS__()); +.add_arguments(CorrelationParam::__FIELDS__()) +.describe(R"code(Applies correlation to inputs. + +The correlation layer performs multiplicative patch comparisons between two feature maps. + +Given two multi-channel feature maps :math:`f_{1}, f_{2}`, with :math:`w`, :math:`h`, and :math:`c` being their width, height, and number of channels, +the correlation layer lets the network compare each patch from :math:`f_{1}` with each patch from :math:`f_{2}`. + +For now we consider only a single comparison of two patches. The 'correlation' of two patches centered at :math:`x_{1}` in the first map and +:math:`x_{2}` in the second map is then defined as: + +.. math:: + c(x_{1}, x_{2}) = \sum_{o \in [-k,k] \times [-k,k]} + +for a square patch of size :math:`K:=2k+1`. + +Note that the equation above is identical to one step of a convolution in neural networks, but instead of convolving data with a filter, it convolves data with other +data. For this reason, it has no training weights. + +Computing :math:`c(x_{1}, x_{2})` involves :math:`c * K^{2}` multiplications. Comparing all patch combinations involves :math:`w^{2}*h^{2}` such computations. + +Given a maximum displacement :math:`d`, for each location :math:`x_{1}` it computes correlations :math:`c(x_{1}, x_{2})` only in a neighborhood of size :math:`D:=2d+1`, +by limiting the range of :math:`x_{2}`. We use strides :math:`s_{1}, s_{2}`, to quantize :math:`x_{1}` globally and to quantize :math:`x_{2}` within the neighborhood +centered around :math:`x_{1}`. + +The final output is defined by the following expression: + +.. math:: + out[n, q, i, j] = c(x_{i, j}, x_{q}) + +where :math:`i` and :math:`j` enumerate spatial locations in :math:`f_{1}`, and :math:`q` denotes the :math:`q^{th}` neighborhood of :math:`x_{i,j}`. +)code" ADD_FILELINE); } // namespace op } // namespace mxnet From e9aa0118b1f6f15bc060908034280a0c88b3bd97 Mon Sep 17 00:00:00 2001 From: Terence Wu <2326428753@qq.com> Date: Wed, 19 Jul 2017 00:29:10 +0800 Subject: [PATCH 229/834] Fix a spelling mistake (#7085) --- python/mxnet/ndarray.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/mxnet/ndarray.py b/python/mxnet/ndarray.py index 88a4b0569ef5..90d890d38968 100644 --- a/python/mxnet/ndarray.py +++ b/python/mxnet/ndarray.py @@ -26,7 +26,7 @@ from .ndarray_doc import _build_doc -# Use different verison of SymbolBase +# Use different version of SymbolBase # When possible, use cython to speedup part of computation. # pylint: disable=unused-import try: From add7e437e43c99a6b10a5639bf624e3272b7fd5b Mon Sep 17 00:00:00 2001 From: Eric Junyuan Xie Date: Tue, 18 Jul 2017 09:30:23 -0700 Subject: [PATCH 230/834] fix custom op and add tutorial (#7076) --- docs/tutorials/gluon/customop.md | 198 +++++++++++++++++++++++++++++++ python/mxnet/ndarray.py | 2 + python/mxnet/operator.py | 24 +++- 3 files changed, 218 insertions(+), 6 deletions(-) create mode 100644 docs/tutorials/gluon/customop.md diff --git a/docs/tutorials/gluon/customop.md b/docs/tutorials/gluon/customop.md new file mode 100644 index 000000000000..dbb1907badb1 --- /dev/null +++ b/docs/tutorials/gluon/customop.md @@ -0,0 +1,198 @@ + +# Creating custom operators with numpy + +In this tutorial, we will learn how to build custom operators with numpy in python. We will go through two examples: +- Custom operator without any `Parameter`s +- Custom operator with `Parameter`s + +Custom operator in python is easy to develop and good for prototyping, but may hurt performance. If you find it to be a bottleneck, please consider moving to a C++ based implementation in the backend. + + + +```python +import numpy as np +import mxnet as mx +from mxnet import gluon, autograd +``` + +## Parameter-less operators + +This operator implements the standard sigmoid activation function. This is only for illustration purposes, in real life you would use the built-in operator `mx.nd.relu`. + +### Forward & backward implementation + +First we implement the forward and backward computation by sub-classing `mx.operator.CustomOp`: + + +```python +class Sigmoid(mx.operator.CustomOp): + def forward(self, is_train, req, in_data, out_data, aux): + """Implements forward computation. + + is_train : bool, whether forwarding for training or testing. + req : list of {'null', 'write', 'inplace', 'add'}, how to assign to out_data. 'null' means skip assignment, etc. + in_data : list of NDArray, input data. + out_data : list of NDArray, pre-allocated output buffers. + aux : list of NDArray, mutable auxiliary states. Usually not used. + """ + x = in_data[0].asnumpy() + y = 1.0 / (1.0 + np.exp(-x)) + self.assign(out_data[0], req[0], mx.nd.array(y)) + + def backward(self, req, out_grad, in_data, out_data, in_grad, aux): + """Implements backward computation + + req : list of {'null', 'write', 'inplace', 'add'}, how to assign to in_grad + out_grad : list of NDArray, gradient w.r.t. output data. + in_grad : list of NDArray, gradient w.r.t. input data. This is the output buffer. + """ + y = out_data[0].asnumpy() + dy = out_grad[0].asnumpy() + dx = dy*(1.0 - y)*y + self.assign(in_grad[0], req[0], mx.nd.array(dx)) +``` + +### Register custom operator + +Then we need to register the custom op and describe it's properties like input and output shapes so that mxnet can recognize it. This is done by sub-classing `mx.operator.CustomOpProp`: + + +```python +@mx.operator.register("sigmoid") # register with name "sigmoid" +class SigmoidProp(mx.operator.CustomOpProp): + def __init__(self): + super(SigmoidProp, self).__init__(True) + + def list_arguments(self): + # this can be omitted if you only have 1 input. + return ['data'] + + def list_outputs(self): + # this can be omitted if you only have 1 output. + return ['output'] + + def infer_shape(self, in_shapes): + """Calculate output shapes from input shapes. This can be + omited if all your inputs and outputs have the same shape. + + in_shapes : list of shape. Shape is described by a tuple of int. + """ + data_shape = in_shapes[0] + output_shape = data_shape + # return 3 lists representing inputs shapes, outputs shapes, and aux data shapes. + return (data_shape,), (output_shape,), () + + def create_operator(self, ctx, in_shapes, in_dtypes): + # create and return the CustomOp class. + return Sigmoid() +``` + +### Example Usage + +We can now use this operator by calling `mx.nd.Custom`: + + +```python +x = mx.nd.array([0, 1, 2, 3]) +# attach gradient buffer to x for autograd +x.attach_grad() +# forward in a record() section to save computation graph for backward +# see autograd tutorial to learn more. +with autograd.record(): + y = mx.nd.Custom(x, op_type='sigmoid') +print(y) +``` + +```python +# call backward computation +y.backward() +# gradient is now saved to the grad buffer we attached previously +print(x.grad) +``` + +## Parametrized Operator + +In the second use case we implement an operator with learnable weights. We implement the dense (or fully connected) layer that has one input, one output, and two learnable parameters: weight and bias. + +The dense operator performs a dot product between data and weight, then add bias to it. + +### Forward & backward implementation + + +```python +class Dense(mx.operator.CustomOp): + def __init__(self, bias): + self._bias = bias + + def forward(self, is_train, req, in_data, out_data, aux): + x = in_data[0].asnumpy() + weight = in_data[1].asnumpy() + y = x.dot(weight.T) + self._bias + self.assign(out_data[0], req[0], mx.nd.array(y)) + + def backward(self, req, out_grad, in_data, out_data, in_grad, aux): + x = in_data[0].asnumpy() + dy = out_grad[0].asnumpy() + dx = dy.T.dot(x) + self.assign(in_grad[0], req[0], mx.nd.array(dx)) +``` + +### Registration + + +```python +@mx.operator.register("dense") # register with name "sigmoid" +class DenseProp(mx.operator.CustomOpProp): + def __init__(self, bias): + super(DenseProp, self).__init__(True) + # we use constant bias here to illustrate how to pass arguments + # to operators. All arguments are in string format so you need + # to convert them back to the type you want. + self._bias = float(bias) + + def list_arguments(self): + return ['data', 'weight'] + + def list_outputs(self): + # this can be omitted if you only have 1 output. + return ['output'] + + def infer_shape(self, in_shapes): + data_shape = in_shapes[0] + weight_shape = in_shapes[1] + output_shape = (data_shape[0], weight_shape[0]) + # return 3 lists representing inputs shapes, outputs shapes, and aux data shapes. + return (data_shape, weight_shape), (output_shape,), () + + def create_operator(self, ctx, in_shapes, in_dtypes): + # create and return the CustomOp class. + return Dense(self._bias) +``` + +### Use CustomOp together with Block + +Parameterized CustomOp are ususally used together with Blocks, which holds the parameter. + + +```python +class DenseBlock(mx.gluon.Block): + def __init__(self, in_channels, channels, bias, **kwargs): + super(DenseBlock, self).__init__(**kwargs) + self._bias = bias + self.weight = self.params.get('weight', shape=(channels, in_channels)) + + def forward(self, x): + ctx = x.context + return mx.nd.Custom(x, self.weight.data(ctx), bias=self._bias, op_type='dense') +``` + +### Example usage + + +```python +dense = DenseBlock(3, 5, 0.1) +dense.initialize() +x = mx.nd.uniform(shape=(4, 3)) +y = dense(x) +print(y) +``` diff --git a/python/mxnet/ndarray.py b/python/mxnet/ndarray.py index 90d890d38968..4939b6c221a5 100644 --- a/python/mxnet/ndarray.py +++ b/python/mxnet/ndarray.py @@ -48,6 +48,7 @@ # pylint: disable= no-member _DTYPE_NP_TO_MX = { + None : -1, np.float32 : 0, np.float64 : 1, np.float16 : 2, @@ -56,6 +57,7 @@ } _DTYPE_MX_TO_NP = { + -1 : None, 0 : np.float32, 1 : np.float64, 2 : np.float16, diff --git a/python/mxnet/operator.py b/python/mxnet/operator.py index 884775d26317..8274838a1f83 100644 --- a/python/mxnet/operator.py +++ b/python/mxnet/operator.py @@ -626,9 +626,15 @@ def infer_shape_entry(num_tensor, tensor_dims, ishape, oshape, ashape = ret else: raise AssertionError("infer_shape must return 2 or 3 lists") - assert len(oshape) == n_out - assert len(ishape) == n_in - assert len(ashape) == n_aux + assert len(oshape) == n_out, \ + "InferShape Error: expecting %d entries in returned output " \ + "shapes, got %d."%(n_out, len(oshape)) + assert len(ishape) == n_in, \ + "InferShape Error: expecting %d entries in returned input " \ + "shapes, got %d."%(n_in, len(ishape)) + assert len(ashape) == n_aux, \ + "InferShape Error: expecting %d entries in returned aux state " \ + "shapes, got %d."%(n_aux, len(ashape)) rshape = list(ishape) + list(oshape) + list(ashape) for i in range(n_in+n_out+n_aux): tensor_shapes[i] = cast(c_array(mx_uint, rshape[i]), POINTER(mx_uint)) @@ -657,9 +663,15 @@ def infer_type_entry(num_tensor, tensor_types, _): itype, otype, atype = ret else: raise AssertionError("infer_type must return 2 or 3 lists") - assert len(otype) == n_out - assert len(itype) == n_in - assert len(atype) == n_aux + assert len(otype) == n_out, \ + "InferType Error: expecting %d entries in returned output " \ + "shapes, got %d."%(n_out, len(otype)) + assert len(itype) == n_in, \ + "InferType Error: expecting %d entries in returned input " \ + "shapes, got %d."%(n_in, len(itype)) + assert len(atype) == n_aux, \ + "InferType Error: expecting %d entries in returned aux state " \ + "shapes, got %d."%(n_aux, len(atype)) rtype = list(itype) + list(otype) + list(atype) for i, dtype in enumerate(rtype): tensor_types[i] = _DTYPE_NP_TO_MX[dtype] From bd5df7ce0f52065ed813cc6b97e94e0f75e9b5e6 Mon Sep 17 00:00:00 2001 From: Leonard Lausen Date: Wed, 19 Jul 2017 04:41:56 +0900 Subject: [PATCH 231/834] Add h5py support to NDArrayIter (#6790) * Support h5py groups as input to NDArrayIter * Support shuffling indices for h5py data in NDArrayIter * Make h5py optional * Install h5py on linux based CI systems Tests are not run on Windows. I couldn't find the Windows CI system configuration / a place to define h5py test dependency on Windows. --- .travis.yml | 2 + python/mxnet/io.py | 74 +++++++++++++------ .../ci_build/install/ubuntu_install_python.sh | 4 +- tests/python/unittest/test_io.py | 62 ++++++++++++++-- 4 files changed, 113 insertions(+), 29 deletions(-) diff --git a/.travis.yml b/.travis.yml index c8ba0b1e645b..ca5d03b5008d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -68,6 +68,8 @@ addons: - python3-numpy - python3-dev - python3-nose + - python-h5py + - python3-h5py - graphviz - libmouse-perl - pdl diff --git a/python/mxnet/io.py b/python/mxnet/io.py index ec3c25f54d30..bb791cef035e 100644 --- a/python/mxnet/io.py +++ b/python/mxnet/io.py @@ -6,6 +6,10 @@ import ctypes import logging import threading +try: + import h5py +except ImportError: + h5py = None import numpy as np from .base import _LIB from .base import c_array, c_str, mx_uint, py_str @@ -465,7 +469,8 @@ def _init_data(data, allow_empty, default_name): if data is None: data = [] - if isinstance(data, (np.ndarray, NDArray)): + if isinstance(data, (np.ndarray, NDArray, h5py.Dataset) + if h5py else (np.ndarray, NDArray)): data = [data] if isinstance(data, list): if not allow_empty: @@ -476,20 +481,20 @@ def _init_data(data, allow_empty, default_name): data = OrderedDict( # pylint: disable=redefined-variable-type [('_%d_%s' % (i, default_name), d) for i, d in enumerate(data)]) if not isinstance(data, dict): - raise TypeError("Input must be NDArray, numpy.ndarray, " + \ + raise TypeError("Input must be NDArray, numpy.ndarray, h5py.Dataset " + \ "a list of them or dict with them as values") for k, v in data.items(): - if not isinstance(v, NDArray): + if not isinstance(v, (NDArray, h5py.Dataset) if h5py else NDArray): try: data[k] = array(v) except: raise TypeError(("Invalid type '%s' for %s, " % (type(v), k)) + \ - "should be NDArray or numpy.ndarray") + "should be NDArray, numpy.ndarray or h5py.Dataset") return list(data.items()) class NDArrayIter(DataIter): - """Returns an iterator for ``mx.nd.NDArray`` or ``numpy.ndarray``. + """Returns an iterator for ``mx.nd.NDArray``, ``numpy.ndarray`` or ``h5py.Dataset``. Example usage: ---------- @@ -562,6 +567,7 @@ class NDArrayIter(DataIter): Batch size of data. shuffle: bool, optional Whether to shuffle the data. + Only supported if no h5py.Dataset inputs are used. last_batch_handle : str, optional How to handle the last batch. This parameter can be 'pad', 'discard' or 'roll_over'. 'roll_over' is intended for training and can cause problems @@ -579,30 +585,29 @@ def __init__(self, data, label=None, batch_size=1, shuffle=False, self.data = _init_data(data, allow_empty=False, default_name=data_name) self.label = _init_data(label, allow_empty=True, default_name=label_name) + self.idx = np.arange(self.data[0][1].shape[0]) # shuffle data if shuffle: - idx = np.arange(self.data[0][1].shape[0]) - np.random.shuffle(idx) - self.data = [(k, array(v.asnumpy()[idx], v.context)) for k, v in self.data] - self.label = [(k, array(v.asnumpy()[idx], v.context)) for k, v in self.label] + np.random.shuffle(self.idx) + self.data = [(k, array(v.asnumpy()[self.idx], v.context)) + if not (isinstance(v, h5py.Dataset) + if h5py else False) else (k, v) + for k, v in self.data] + self.label = [(k, array(v.asnumpy()[self.idx], v.context)) + if not (isinstance(v, h5py.Dataset) + if h5py else False) else (k, v) + for k, v in self.label] # batching if last_batch_handle == 'discard': new_n = self.data[0][1].shape[0] - self.data[0][1].shape[0] % batch_size - data_dict = OrderedDict(self.data) - label_dict = OrderedDict(self.label) - for k, _ in self.data: - data_dict[k] = data_dict[k][:new_n] - for k, _ in self.label: - label_dict[k] = label_dict[k][:new_n] - self.data = data_dict.items() - self.label = label_dict.items() + self.idx = self.idx[:new_n] self.data_list = [x[1] for x in self.data] + [x[1] for x in self.label] self.num_source = len(self.data_list) - self.num_data = self.data_list[0].shape[0] + self.num_data = self.idx.shape[0] assert self.num_data >= batch_size, \ - "batch_size need to be smaller than data size." + "batch_size needs to be smaller than data size." self.cursor = -batch_size self.batch_size = batch_size self.last_batch_handle = last_batch_handle @@ -648,10 +653,37 @@ def _getdata(self, data_source): """Load data from underlying arrays, internal use only.""" assert(self.cursor < self.num_data), "DataIter needs reset." if self.cursor + self.batch_size <= self.num_data: - return [x[1][self.cursor:self.cursor+self.batch_size] for x in data_source] + return [ + # np.ndarray or NDArray case + x[1][self.cursor:self.cursor + self.batch_size] + if isinstance(x[1], (np.ndarray, NDArray)) else + # h5py (only supports indices in increasing order) + array(x[1][sorted(self.idx[ + self.cursor:self.cursor + self.batch_size])][[ + list(self.idx[self.cursor: + self.cursor + self.batch_size]).index(i) + for i in sorted(self.idx[ + self.cursor:self.cursor + self.batch_size]) + ]]) for x in data_source + ] else: pad = self.batch_size - self.num_data + self.cursor - return [concatenate([x[1][self.cursor:], x[1][:pad]]) for x in data_source] + return [ + # np.ndarray or NDArray case + concatenate([x[1][self.cursor:], x[1][:pad]]) + if isinstance(x[1], (np.ndarray, NDArray)) else + # h5py (only supports indices in increasing order) + concatenate([ + array(x[1][sorted(self.idx[self.cursor:])][[ + list(self.idx[self.cursor:]).index(i) + for i in sorted(self.idx[self.cursor:]) + ]]), + array(x[1][sorted(self.idx[:pad])][[ + list(self.idx[:pad]).index(i) + for i in sorted(self.idx[:pad]) + ]]) + ]) for x in data_source + ] def getdata(self): return self._getdata(self.data) diff --git a/tests/ci_build/install/ubuntu_install_python.sh b/tests/ci_build/install/ubuntu_install_python.sh index 0459bb9198c4..973523d0c8f3 100755 --- a/tests/ci_build/install/ubuntu_install_python.sh +++ b/tests/ci_build/install/ubuntu_install_python.sh @@ -6,5 +6,5 @@ apt-get update && apt-get install -y python-dev python3-dev # the version of the pip shipped with ubuntu may be too lower, install a recent version here cd /tmp && wget https://bootstrap.pypa.io/get-pip.py && python3 get-pip.py && python2 get-pip.py -pip2 install nose pylint numpy nose-timer requests -pip3 install nose pylint numpy nose-timer requests +pip2 install nose pylint numpy nose-timer requests h5py +pip3 install nose pylint numpy nose-timer requests h5py diff --git a/tests/python/unittest/test_io.py b/tests/python/unittest/test_io.py index 5fe61b185041..18326754c851 100644 --- a/tests/python/unittest/test_io.py +++ b/tests/python/unittest/test_io.py @@ -4,6 +4,10 @@ import os, gzip import pickle as pickle import time +try: + import h5py +except ImportError: + h5py = None import sys from common import get_data @@ -63,17 +67,17 @@ def test_Cifar10Rec(): assert(labelcount[i] == 5000) def test_NDArrayIter(): - datas = np.ones([1000, 2, 2]) - labels = np.ones([1000, 1]) + data = np.ones([1000, 2, 2]) + label = np.ones([1000, 1]) for i in range(1000): - datas[i] = i / 100 - labels[i] = i / 100 - dataiter = mx.io.NDArrayIter(datas, labels, 128, True, last_batch_handle='pad') + data[i] = i / 100 + label[i] = i / 100 + dataiter = mx.io.NDArrayIter(data, label, 128, True, last_batch_handle='pad') batchidx = 0 for batch in dataiter: batchidx += 1 assert(batchidx == 8) - dataiter = mx.io.NDArrayIter(datas, labels, 128, False, last_batch_handle='pad') + dataiter = mx.io.NDArrayIter(data, label, 128, False, last_batch_handle='pad') batchidx = 0 labelcount = [0 for i in range(10)] for batch in dataiter: @@ -88,7 +92,53 @@ def test_NDArrayIter(): else: assert(labelcount[i] == 100) +def test_NDArrayIter_h5py(): + if not h5py: + return + + data = np.ones([1000, 2, 2]) + label = np.ones([1000, 1]) + for i in range(1000): + data[i] = i / 100 + label[i] = i / 100 + + try: + os.remove("ndarraytest.h5") + except OSError: + pass + with h5py.File("ndarraytest.h5") as f: + f.create_dataset("data", data=data) + f.create_dataset("label", data=label) + + dataiter = mx.io.NDArrayIter(f["data"], f["label"], 128, True, last_batch_handle='pad') + batchidx = 0 + for batch in dataiter: + batchidx += 1 + assert(batchidx == 8) + + dataiter = mx.io.NDArrayIter(f["data"], f["label"], 128, False, last_batch_handle='pad') + labelcount = [0 for i in range(10)] + for batch in dataiter: + label = batch.label[0].asnumpy().flatten() + assert((batch.data[0].asnumpy()[:,0,0] == label).all()) + for i in range(label.shape[0]): + labelcount[int(label[i])] += 1 + + try: + os.remove("ndarraytest.h5") + except OSError: + pass + + for i in range(10): + if i == 0: + assert(labelcount[i] == 124) + else: + assert(labelcount[i] == 100) + + if __name__ == "__main__": test_NDArrayIter() + if h5py: + test_NDArrayIter_h5py() test_MNISTIter() test_Cifar10Rec() From 4fd0891187f399e5ae45c49dac1b5fad0eb604ff Mon Sep 17 00:00:00 2001 From: Joern Kottmann Date: Wed, 19 Jul 2017 17:26:00 +0200 Subject: [PATCH 232/834] Fix argmax_channel method and add preqs in scala mnist tutorial (#7101) (#7102) --- docs/tutorials/scala/mnist.md | 8 +++++++- scala-package/spark/README.md | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/tutorials/scala/mnist.md b/docs/tutorials/scala/mnist.md index e01ac49ed0c1..ad55ee4c0257 100644 --- a/docs/tutorials/scala/mnist.md +++ b/docs/tutorials/scala/mnist.md @@ -4,6 +4,12 @@ This Scala tutorial guides you through a classic computer vision application: id Let's train a 3-layer network (i.e multilayer perceptron network) on the MNIST dataset to classify handwritten digits. +## Prerequisites +To complete this tutorial, we need: + +- to compile the latest MXNet version. See the MXNet installation instructions for your operating system in [Setup and Installation](http://mxnet.io/get_started/install.html). +- to compile the Scala API. See Scala API build instructions in [Build](https://github.com/dmlc/mxnet/tree/master/scala-package). + ## Define the Network First, define the neural network's architecture using the Symbol API: @@ -87,7 +93,7 @@ while (valDataIter.hasNext) { val y = NDArray.concatenate(labels) // get predicted labels -val predictedY = NDArray.argmaxChannel(prob) +val predictedY = NDArray.argmax_channel(prob) require(y.shape == predictedY.shape) // calculate accuracy diff --git a/scala-package/spark/README.md b/scala-package/spark/README.md index 08077f7548d7..974691650ff4 100644 --- a/scala-package/spark/README.md +++ b/scala-package/spark/README.md @@ -71,7 +71,7 @@ val res = valData.mapPartitions { data => val probArrays = brModel.value.predict(points.toIterator) require(probArrays.length == 1) val prob = probArrays(0) - val py = NDArray.argmaxChannel(prob.get) + val py = NDArray.argmax_channel(prob.get) val labels = py.toArray.mkString(",") py.dispose() prob.get.dispose() From bfbff25835fbf9b476416bdb2c5ae368e0de2a72 Mon Sep 17 00:00:00 2001 From: Terence Wu <2326428753@qq.com> Date: Thu, 20 Jul 2017 00:45:32 +0800 Subject: [PATCH 233/834] Fix a spelling mistake (#7100) --- python/mxnet/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/mxnet/base.py b/python/mxnet/base.py index f58429980eab..f714924a2eb8 100644 --- a/python/mxnet/base.py +++ b/python/mxnet/base.py @@ -56,7 +56,7 @@ def __str__(self): return msg def _load_lib(): - """Load libary by searching possible path.""" + """Load library by searching possible path.""" lib_path = libinfo.find_lib_path() lib = ctypes.CDLL(lib_path[0], ctypes.RTLD_LOCAL) # DMatrix functions From e3fd434ed51c66afe173941c4800639b7467ff7c Mon Sep 17 00:00:00 2001 From: Soonhwan-Kwon Date: Thu, 20 Jul 2017 02:27:51 +0900 Subject: [PATCH 234/834] add missing files for speech_recognition example (#7099) --- example/speech_recognition/flac_to_wav.sh | 8 + example/speech_recognition/main.py | 703 +++++++++--------- .../stt_bucketing_module.py | 13 + .../stt_io_bucketingiter.py | 148 ++++ 4 files changed, 520 insertions(+), 352 deletions(-) create mode 100644 example/speech_recognition/flac_to_wav.sh create mode 100644 example/speech_recognition/stt_bucketing_module.py create mode 100644 example/speech_recognition/stt_io_bucketingiter.py diff --git a/example/speech_recognition/flac_to_wav.sh b/example/speech_recognition/flac_to_wav.sh new file mode 100644 index 000000000000..a622b60963e5 --- /dev/null +++ b/example/speech_recognition/flac_to_wav.sh @@ -0,0 +1,8 @@ +# Convert all .flac files within this folder to .wav files + +find . -iname "*.flac" | wc + +for flacfile in `find . -iname "*.flac"` +do + sox "${flacfile%.*}.flac" -e signed -b 16 -c 1 -r 16000 "${flacfile%.*}.wav" +done diff --git a/example/speech_recognition/main.py b/example/speech_recognition/main.py index a425e0a8ab40..4ecb1168b51f 100644 --- a/example/speech_recognition/main.py +++ b/example/speech_recognition/main.py @@ -1,352 +1,351 @@ -import json -import os -import sys -from collections import namedtuple -from datetime import datetime -from config_util import parse_args, parse_contexts, generate_file_path -from train import do_training -import mxnet as mx -from stt_io_iter import STTIter -from label_util import LabelUtil -from log_util import LogUtil -import numpy as np -from stt_datagenerator import DataGenerator -from stt_metric import STTMetric -from stt_bi_graphemes_util import generate_bi_graphemes_dictionary -from stt_bucketing_module import STTBucketingModule -from stt_io_bucketingiter import BucketSTTIter -sys.path.insert(0, "../../python") - -# os.environ['MXNET_ENGINE_TYPE'] = "NaiveEngine" -os.environ['MXNET_ENGINE_TYPE'] = "ThreadedEnginePerDevice" -os.environ['MXNET_ENABLE_GPU_P2P'] = "0" - -class WHCS: - width = 0 - height = 0 - channel = 0 - stride = 0 - -class ConfigLogger(object): - def __init__(self, log): - self.__log = log - - def __call__(self, config): - self.__log.info("Config:") - config.write(self) - - def write(self, data): - # stripping the data makes the output nicer and avoids empty lines - line = data.strip() - self.__log.info(line) - -def load_labelutil(labelUtil, is_bi_graphemes, language="en"): - if language == "en": - if is_bi_graphemes: - try: - labelUtil.load_unicode_set("resources/unicodemap_en_baidu_bi_graphemes.csv") - except: - raise Exception("There is no resources/unicodemap_en_baidu_bi_graphemes.csv." + - " Please set overwrite_bi_graphemes_dictionary True at train section") - else: - labelUtil.load_unicode_set("resources/unicodemap_en_baidu.csv") - else: - raise Exception("Error: Language Type: %s" % language) - - - -def load_data(args): - mode = args.config.get('common', 'mode') - if mode not in ['train', 'predict', 'load']: - raise Exception('mode must be the one of the followings - train,predict,load') - batch_size = args.config.getint('common', 'batch_size') - - whcs = WHCS() - whcs.width = args.config.getint('data', 'width') - whcs.height = args.config.getint('data', 'height') - whcs.channel = args.config.getint('data', 'channel') - whcs.stride = args.config.getint('data', 'stride') - save_dir = 'checkpoints' - model_name = args.config.get('common', 'prefix') - is_bi_graphemes = args.config.getboolean('common', 'is_bi_graphemes') - overwrite_meta_files = args.config.getboolean('train', 'overwrite_meta_files') - overwrite_bi_graphemes_dictionary = args.config.getboolean('train', 'overwrite_bi_graphemes_dictionary') - max_duration = args.config.getfloat('data', 'max_duration') - language = args.config.get('data', 'language') - - log = LogUtil().getlogger() - labelUtil = LabelUtil.getInstance() - if mode == "train" or mode == "load": - data_json = args.config.get('data', 'train_json') - val_json = args.config.get('data', 'val_json') - datagen = DataGenerator(save_dir=save_dir, model_name=model_name) - datagen.load_train_data(data_json, max_duration=max_duration) - if is_bi_graphemes: - if not os.path.isfile("resources/unicodemap_en_baidu_bi_graphemes.csv") or overwrite_bi_graphemes_dictionary: - load_labelutil(labelUtil=labelUtil, is_bi_graphemes=False, language=language) - generate_bi_graphemes_dictionary(datagen.train_texts) - load_labelutil(labelUtil=labelUtil, is_bi_graphemes=is_bi_graphemes, language=language) - args.config.set('arch', 'n_classes', str(labelUtil.get_count())) - - if mode == "train": - if overwrite_meta_files: - log.info("Generate mean and std from samples") - normalize_target_k = args.config.getint('train', 'normalize_target_k') - datagen.sample_normalize(normalize_target_k, True) - else: - log.info("Read mean and std from meta files") - datagen.get_meta_from_file( - np.loadtxt(generate_file_path(save_dir, model_name, 'feats_mean')), - np.loadtxt(generate_file_path(save_dir, model_name, 'feats_std'))) - datagen.load_validation_data(val_json, max_duration=max_duration) - - elif mode == "load": - # get feat_mean and feat_std to normalize dataset - datagen.get_meta_from_file( - np.loadtxt(generate_file_path(save_dir, model_name, 'feats_mean')), - np.loadtxt(generate_file_path(save_dir, model_name, 'feats_std'))) - datagen.load_validation_data(val_json, max_duration=max_duration) - elif mode == 'predict': - test_json = args.config.get('data', 'test_json') - datagen = DataGenerator(save_dir=save_dir, model_name=model_name) - datagen.load_train_data(test_json, max_duration=max_duration) - labelutil = load_labelutil(labelUtil, is_bi_graphemes, language="en") - args.config.set('arch', 'n_classes', str(labelUtil.get_count())) - datagen.get_meta_from_file( - np.loadtxt(generate_file_path(save_dir, model_name, 'feats_mean')), - np.loadtxt(generate_file_path(save_dir, model_name, 'feats_std'))) - - is_batchnorm = args.config.getboolean('arch', 'is_batchnorm') - if batch_size == 1 and is_batchnorm and (mode == 'train' or mode == 'load'): - raise Warning('batch size 1 is too small for is_batchnorm') - - # sort file paths by its duration in ascending order to implement sortaGrad - if mode == "train" or mode == "load": - max_t_count = datagen.get_max_seq_length(partition="train") - max_label_length = \ - datagen.get_max_label_length(partition="train", is_bi_graphemes=is_bi_graphemes) - elif mode == "predict": - max_t_count = datagen.get_max_seq_length(partition="test") - max_label_length = \ - datagen.get_max_label_length(partition="test", is_bi_graphemes=is_bi_graphemes) - - args.config.set('arch', 'max_t_count', str(max_t_count)) - args.config.set('arch', 'max_label_length', str(max_label_length)) - from importlib import import_module - prepare_data_template = import_module(args.config.get('arch', 'arch_file')) - init_states = prepare_data_template.prepare_data(args) - sort_by_duration = (mode == "train") - is_bucketing = args.config.getboolean('arch', 'is_bucketing') - save_feature_as_csvfile = args.config.getboolean('train', 'save_feature_as_csvfile') - if is_bucketing: - buckets = json.loads(args.config.get('arch', 'buckets')) - data_loaded = BucketSTTIter(partition="train", - count=datagen.count, - datagen=datagen, - batch_size=batch_size, - num_label=max_label_length, - init_states=init_states, - seq_length=max_t_count, - width=whcs.width, - height=whcs.height, - sort_by_duration=sort_by_duration, - is_bi_graphemes=is_bi_graphemes, - buckets=buckets, - save_feature_as_csvfile=save_feature_as_csvfile) - else: - data_loaded = STTIter(partition="train", - count=datagen.count, - datagen=datagen, - batch_size=batch_size, - num_label=max_label_length, - init_states=init_states, - seq_length=max_t_count, - width=whcs.width, - height=whcs.height, - sort_by_duration=sort_by_duration, - is_bi_graphemes=is_bi_graphemes, - save_feature_as_csvfile=save_feature_as_csvfile) - - if mode == 'train' or mode == 'load': - if is_bucketing: - validation_loaded = BucketSTTIter(partition="validation", - count=datagen.val_count, - datagen=datagen, - batch_size=batch_size, - num_label=max_label_length, - init_states=init_states, - seq_length=max_t_count, - width=whcs.width, - height=whcs.height, - sort_by_duration=False, - is_bi_graphemes=is_bi_graphemes, - buckets=buckets, - save_feature_as_csvfile=save_feature_as_csvfile) - else: - validation_loaded = STTIter(partition="validation", - count=datagen.val_count, - datagen=datagen, - batch_size=batch_size, - num_label=max_label_length, - init_states=init_states, - seq_length=max_t_count, - width=whcs.width, - height=whcs.height, - sort_by_duration=False, - is_bi_graphemes=is_bi_graphemes, - save_feature_as_csvfile=save_feature_as_csvfile) - return data_loaded, validation_loaded, args - elif mode == 'predict': - return data_loaded, args - - -def load_model(args, contexts, data_train): - # load model from model_name prefix and epoch of model_num_epoch with gpu contexts of contexts - mode = args.config.get('common', 'mode') - load_optimizer_states = args.config.getboolean('load', 'load_optimizer_states') - is_start_from_batch = args.config.getboolean('load', 'is_start_from_batch') - - from importlib import import_module - symbol_template = import_module(args.config.get('arch', 'arch_file')) - is_bucketing = args.config.getboolean('arch', 'is_bucketing') - - if mode == 'train': - if is_bucketing: - bucketing_arch = symbol_template.BucketingArch(args) - model_loaded = bucketing_arch.get_sym_gen() - else: - model_loaded = symbol_template.arch(args) - model_num_epoch = None - elif mode == 'load' or mode == 'predict': - model_file = args.config.get('common', 'model_file') - model_name = os.path.splitext(model_file)[0] - model_num_epoch = int(model_name[-4:]) - if is_bucketing: - bucketing_arch = symbol_template.BucketingArch(args) - model_loaded = bucketing_arch.get_sym_gen() - else: - model_path = 'checkpoints/' + str(model_name[:-5]) - - data_names = [x[0] for x in data_train.provide_data] - label_names = [x[0] for x in data_train.provide_label] - - model_loaded = mx.module.Module.load( - prefix=model_path, epoch=model_num_epoch, context=contexts, - data_names=data_names, label_names=label_names, - load_optimizer_states=load_optimizer_states) - if is_start_from_batch: - import re - model_num_epoch = int(re.findall('\d+', model_file)[0]) - - return model_loaded, model_num_epoch - - -if __name__ == '__main__': - if len(sys.argv) <= 1: - raise Exception('cfg file path must be provided. ' + - 'ex)python main.py --configfile examplecfg.cfg') - args = parse_args(sys.argv[1]) - # set parameters from cfg file - # give random seed - random_seed = args.config.getint('common', 'random_seed') - mx_random_seed = args.config.getint('common', 'mx_random_seed') - # random seed for shuffling data list - if random_seed != -1: - np.random.seed(random_seed) - # set mx.random.seed to give seed for parameter initialization - if mx_random_seed != -1: - mx.random.seed(mx_random_seed) - else: - mx.random.seed(hash(datetime.now())) - # set log file name - log_filename = args.config.get('common', 'log_filename') - log = LogUtil(filename=log_filename).getlogger() - - # set parameters from data section(common) - mode = args.config.get('common', 'mode') - if mode not in ['train', 'predict', 'load']: - raise Exception( - 'Define mode in the cfg file first. ' + - 'train or predict or load can be the candidate for the mode.') - - # get meta file where character to number conversions are defined - - contexts = parse_contexts(args) - num_gpu = len(contexts) - batch_size = args.config.getint('common', 'batch_size') - # check the number of gpus is positive divisor of the batch size for data parallel - if batch_size % num_gpu != 0: - raise Exception('num_gpu should be positive divisor of batch_size') - if mode == "train" or mode == "load": - data_train, data_val, args = load_data(args) - elif mode == "predict": - data_train, args = load_data(args) - is_batchnorm = args.config.getboolean('arch', 'is_batchnorm') - is_bucketing = args.config.getboolean('arch', 'is_bucketing') - - # log current config - config_logger = ConfigLogger(log) - config_logger(args.config) - - # load model - model_loaded, model_num_epoch = load_model(args, contexts, data_train) - # if mode is 'train', it trains the model - if mode == 'train': - if is_bucketing: - module = STTBucketingModule( - sym_gen=model_loaded, - default_bucket_key=data_train.default_bucket_key, - context=contexts - ) - else: - data_names = [x[0] for x in data_train.provide_data] - label_names = [x[0] for x in data_train.provide_label] - module = mx.mod.Module(model_loaded, context=contexts, - data_names=data_names, label_names=label_names) - do_training(args=args, module=module, data_train=data_train, data_val=data_val) - # if mode is 'load', it loads model from the checkpoint and continues the training. - elif mode == 'load': - do_training(args=args, module=model_loaded, data_train=data_train, data_val=data_val, - begin_epoch=model_num_epoch + 1) - # if mode is 'predict', it predict label from the input by the input model - elif mode == 'predict': - # predict through data - if is_bucketing: - max_t_count = args.config.getint('arch', 'max_t_count') - load_optimizer_states = args.config.getboolean('load', 'load_optimizer_states') - model_file = args.config.get('common', 'model_file') - model_name = os.path.splitext(model_file)[0] - model_num_epoch = int(model_name[-4:]) - - model_path = 'checkpoints/' + str(model_name[:-5]) - model = STTBucketingModule( - sym_gen=model_loaded, - default_bucket_key=data_train.default_bucket_key, - context=contexts - ) - - model.bind(data_shapes=data_train.provide_data, - label_shapes=data_train.provide_label, - for_training=True) - _, arg_params, aux_params = mx.model.load_checkpoint(model_path, model_num_epoch) - model.set_params(arg_params, aux_params) - model_loaded = model - else: - model_loaded.bind(for_training=False, data_shapes=data_train.provide_data, - label_shapes=data_train.provide_label) - max_t_count = args.config.getint('arch', 'max_t_count') - eval_metric = STTMetric(batch_size=batch_size, num_gpu=num_gpu) - if is_batchnorm: - for nbatch, data_batch in enumerate(data_train): - model_loaded.forward(data_batch, is_train=False) - model_loaded.update_metric(eval_metric, data_batch.label) - else: - #model_loaded.score(eval_data=data_train, num_batch=None, - # eval_metric=eval_metric, reset=True) - for nbatch, data_batch in enumerate(data_train): - model_loaded.forward(data_batch, is_train=False) - model_loaded.update_metric(eval_metric, data_batch.label) - else: - raise Exception( - 'Define mode in the cfg file first. ' + - 'train or predict or load can be the candidate for the mode') +import json +import os +import sys +from collections import namedtuple +from datetime import datetime +from config_util import parse_args, parse_contexts, generate_file_path +from train import do_training +import mxnet as mx +from stt_io_iter import STTIter +from label_util import LabelUtil +from log_util import LogUtil +import numpy as np +from stt_datagenerator import DataGenerator +from stt_metric import STTMetric +from stt_bi_graphemes_util import generate_bi_graphemes_dictionary +from stt_bucketing_module import STTBucketingModule +from stt_io_bucketingiter import BucketSTTIter +sys.path.insert(0, "../../python") + +# os.environ['MXNET_ENGINE_TYPE'] = "NaiveEngine" +os.environ['MXNET_ENGINE_TYPE'] = "ThreadedEnginePerDevice" +os.environ['MXNET_ENABLE_GPU_P2P'] = "0" + +class WHCS: + width = 0 + height = 0 + channel = 0 + stride = 0 + +class ConfigLogger(object): + def __init__(self, log): + self.__log = log + + def __call__(self, config): + self.__log.info("Config:") + config.write(self) + + def write(self, data): + # stripping the data makes the output nicer and avoids empty lines + line = data.strip() + self.__log.info(line) + +def load_labelutil(labelUtil, is_bi_graphemes, language="en"): + if language == "en": + if is_bi_graphemes: + try: + labelUtil.load_unicode_set("resources/unicodemap_en_baidu_bi_graphemes.csv") + except: + raise Exception("There is no resources/unicodemap_en_baidu_bi_graphemes.csv." + + " Please set overwrite_bi_graphemes_dictionary True at train section") + else: + labelUtil.load_unicode_set("resources/unicodemap_en_baidu.csv") + else: + raise Exception("Error: Language Type: %s" % language) + + + +def load_data(args): + mode = args.config.get('common', 'mode') + if mode not in ['train', 'predict', 'load']: + raise Exception('mode must be the one of the followings - train,predict,load') + batch_size = args.config.getint('common', 'batch_size') + + whcs = WHCS() + whcs.width = args.config.getint('data', 'width') + whcs.height = args.config.getint('data', 'height') + whcs.channel = args.config.getint('data', 'channel') + whcs.stride = args.config.getint('data', 'stride') + save_dir = 'checkpoints' + model_name = args.config.get('common', 'prefix') + is_bi_graphemes = args.config.getboolean('common', 'is_bi_graphemes') + overwrite_meta_files = args.config.getboolean('train', 'overwrite_meta_files') + overwrite_bi_graphemes_dictionary = args.config.getboolean('train', 'overwrite_bi_graphemes_dictionary') + max_duration = args.config.getfloat('data', 'max_duration') + language = args.config.get('data', 'language') + + log = LogUtil().getlogger() + labelUtil = LabelUtil.getInstance() + if mode == "train" or mode == "load": + data_json = args.config.get('data', 'train_json') + val_json = args.config.get('data', 'val_json') + datagen = DataGenerator(save_dir=save_dir, model_name=model_name) + datagen.load_train_data(data_json, max_duration=max_duration) + datagen.load_validation_data(val_json, max_duration=max_duration) + if is_bi_graphemes: + if not os.path.isfile("resources/unicodemap_en_baidu_bi_graphemes.csv") or overwrite_bi_graphemes_dictionary: + load_labelutil(labelUtil=labelUtil, is_bi_graphemes=False, language=language) + generate_bi_graphemes_dictionary(datagen.train_texts+datagen.val_texts) + load_labelutil(labelUtil=labelUtil, is_bi_graphemes=is_bi_graphemes, language=language) + args.config.set('arch', 'n_classes', str(labelUtil.get_count())) + + if mode == "train": + if overwrite_meta_files: + log.info("Generate mean and std from samples") + normalize_target_k = args.config.getint('train', 'normalize_target_k') + datagen.sample_normalize(normalize_target_k, True) + else: + log.info("Read mean and std from meta files") + datagen.get_meta_from_file( + np.loadtxt(generate_file_path(save_dir, model_name, 'feats_mean')), + np.loadtxt(generate_file_path(save_dir, model_name, 'feats_std'))) + elif mode == "load": + # get feat_mean and feat_std to normalize dataset + datagen.get_meta_from_file( + np.loadtxt(generate_file_path(save_dir, model_name, 'feats_mean')), + np.loadtxt(generate_file_path(save_dir, model_name, 'feats_std'))) + + elif mode == 'predict': + test_json = args.config.get('data', 'test_json') + datagen = DataGenerator(save_dir=save_dir, model_name=model_name) + datagen.load_train_data(test_json, max_duration=max_duration) + labelutil = load_labelutil(labelUtil, is_bi_graphemes, language="en") + args.config.set('arch', 'n_classes', str(labelUtil.get_count())) + datagen.get_meta_from_file( + np.loadtxt(generate_file_path(save_dir, model_name, 'feats_mean')), + np.loadtxt(generate_file_path(save_dir, model_name, 'feats_std'))) + + is_batchnorm = args.config.getboolean('arch', 'is_batchnorm') + if batch_size == 1 and is_batchnorm and (mode == 'train' or mode == 'load'): + raise Warning('batch size 1 is too small for is_batchnorm') + + # sort file paths by its duration in ascending order to implement sortaGrad + if mode == "train" or mode == "load": + max_t_count = datagen.get_max_seq_length(partition="train") + max_label_length = \ + datagen.get_max_label_length(partition="train", is_bi_graphemes=is_bi_graphemes) + elif mode == "predict": + max_t_count = datagen.get_max_seq_length(partition="test") + max_label_length = \ + datagen.get_max_label_length(partition="test", is_bi_graphemes=is_bi_graphemes) + + args.config.set('arch', 'max_t_count', str(max_t_count)) + args.config.set('arch', 'max_label_length', str(max_label_length)) + from importlib import import_module + prepare_data_template = import_module(args.config.get('arch', 'arch_file')) + init_states = prepare_data_template.prepare_data(args) + sort_by_duration = (mode == "train") + is_bucketing = args.config.getboolean('arch', 'is_bucketing') + save_feature_as_csvfile = args.config.getboolean('train', 'save_feature_as_csvfile') + if is_bucketing: + buckets = json.loads(args.config.get('arch', 'buckets')) + data_loaded = BucketSTTIter(partition="train", + count=datagen.count, + datagen=datagen, + batch_size=batch_size, + num_label=max_label_length, + init_states=init_states, + seq_length=max_t_count, + width=whcs.width, + height=whcs.height, + sort_by_duration=sort_by_duration, + is_bi_graphemes=is_bi_graphemes, + buckets=buckets, + save_feature_as_csvfile=save_feature_as_csvfile) + else: + data_loaded = STTIter(partition="train", + count=datagen.count, + datagen=datagen, + batch_size=batch_size, + num_label=max_label_length, + init_states=init_states, + seq_length=max_t_count, + width=whcs.width, + height=whcs.height, + sort_by_duration=sort_by_duration, + is_bi_graphemes=is_bi_graphemes, + save_feature_as_csvfile=save_feature_as_csvfile) + + if mode == 'train' or mode == 'load': + if is_bucketing: + validation_loaded = BucketSTTIter(partition="validation", + count=datagen.val_count, + datagen=datagen, + batch_size=batch_size, + num_label=max_label_length, + init_states=init_states, + seq_length=max_t_count, + width=whcs.width, + height=whcs.height, + sort_by_duration=False, + is_bi_graphemes=is_bi_graphemes, + buckets=buckets, + save_feature_as_csvfile=save_feature_as_csvfile) + else: + validation_loaded = STTIter(partition="validation", + count=datagen.val_count, + datagen=datagen, + batch_size=batch_size, + num_label=max_label_length, + init_states=init_states, + seq_length=max_t_count, + width=whcs.width, + height=whcs.height, + sort_by_duration=False, + is_bi_graphemes=is_bi_graphemes, + save_feature_as_csvfile=save_feature_as_csvfile) + return data_loaded, validation_loaded, args + elif mode == 'predict': + return data_loaded, args + + +def load_model(args, contexts, data_train): + # load model from model_name prefix and epoch of model_num_epoch with gpu contexts of contexts + mode = args.config.get('common', 'mode') + load_optimizer_states = args.config.getboolean('load', 'load_optimizer_states') + is_start_from_batch = args.config.getboolean('load', 'is_start_from_batch') + + from importlib import import_module + symbol_template = import_module(args.config.get('arch', 'arch_file')) + is_bucketing = args.config.getboolean('arch', 'is_bucketing') + + if mode == 'train': + if is_bucketing: + bucketing_arch = symbol_template.BucketingArch(args) + model_loaded = bucketing_arch.get_sym_gen() + else: + model_loaded = symbol_template.arch(args) + model_num_epoch = None + elif mode == 'load' or mode == 'predict': + model_file = args.config.get('common', 'model_file') + model_name = os.path.splitext(model_file)[0] + model_num_epoch = int(model_name[-4:]) + if is_bucketing: + bucketing_arch = symbol_template.BucketingArch(args) + model_loaded = bucketing_arch.get_sym_gen() + else: + model_path = 'checkpoints/' + str(model_name[:-5]) + + data_names = [x[0] for x in data_train.provide_data] + label_names = [x[0] for x in data_train.provide_label] + + model_loaded = mx.module.Module.load( + prefix=model_path, epoch=model_num_epoch, context=contexts, + data_names=data_names, label_names=label_names, + load_optimizer_states=load_optimizer_states) + if is_start_from_batch: + import re + model_num_epoch = int(re.findall('\d+', model_file)[0]) + + return model_loaded, model_num_epoch + + +if __name__ == '__main__': + if len(sys.argv) <= 1: + raise Exception('cfg file path must be provided. ' + + 'ex)python main.py --configfile examplecfg.cfg') + args = parse_args(sys.argv[1]) + # set parameters from cfg file + # give random seed + random_seed = args.config.getint('common', 'random_seed') + mx_random_seed = args.config.getint('common', 'mx_random_seed') + # random seed for shuffling data list + if random_seed != -1: + np.random.seed(random_seed) + # set mx.random.seed to give seed for parameter initialization + if mx_random_seed != -1: + mx.random.seed(mx_random_seed) + else: + mx.random.seed(hash(datetime.now())) + # set log file name + log_filename = args.config.get('common', 'log_filename') + log = LogUtil(filename=log_filename).getlogger() + + # set parameters from data section(common) + mode = args.config.get('common', 'mode') + if mode not in ['train', 'predict', 'load']: + raise Exception( + 'Define mode in the cfg file first. ' + + 'train or predict or load can be the candidate for the mode.') + + # get meta file where character to number conversions are defined + + contexts = parse_contexts(args) + num_gpu = len(contexts) + batch_size = args.config.getint('common', 'batch_size') + # check the number of gpus is positive divisor of the batch size for data parallel + if batch_size % num_gpu != 0: + raise Exception('num_gpu should be positive divisor of batch_size') + if mode == "train" or mode == "load": + data_train, data_val, args = load_data(args) + elif mode == "predict": + data_train, args = load_data(args) + is_batchnorm = args.config.getboolean('arch', 'is_batchnorm') + is_bucketing = args.config.getboolean('arch', 'is_bucketing') + + # log current config + config_logger = ConfigLogger(log) + config_logger(args.config) + + # load model + model_loaded, model_num_epoch = load_model(args, contexts, data_train) + # if mode is 'train', it trains the model + if mode == 'train': + if is_bucketing: + module = STTBucketingModule( + sym_gen=model_loaded, + default_bucket_key=data_train.default_bucket_key, + context=contexts + ) + else: + data_names = [x[0] for x in data_train.provide_data] + label_names = [x[0] for x in data_train.provide_label] + module = mx.mod.Module(model_loaded, context=contexts, + data_names=data_names, label_names=label_names) + do_training(args=args, module=module, data_train=data_train, data_val=data_val) + # if mode is 'load', it loads model from the checkpoint and continues the training. + elif mode == 'load': + do_training(args=args, module=model_loaded, data_train=data_train, data_val=data_val, + begin_epoch=model_num_epoch + 1) + # if mode is 'predict', it predict label from the input by the input model + elif mode == 'predict': + # predict through data + if is_bucketing: + max_t_count = args.config.getint('arch', 'max_t_count') + load_optimizer_states = args.config.getboolean('load', 'load_optimizer_states') + model_file = args.config.get('common', 'model_file') + model_name = os.path.splitext(model_file)[0] + model_num_epoch = int(model_name[-4:]) + + model_path = 'checkpoints/' + str(model_name[:-5]) + model = STTBucketingModule( + sym_gen=model_loaded, + default_bucket_key=data_train.default_bucket_key, + context=contexts + ) + + model.bind(data_shapes=data_train.provide_data, + label_shapes=data_train.provide_label, + for_training=True) + _, arg_params, aux_params = mx.model.load_checkpoint(model_path, model_num_epoch) + model.set_params(arg_params, aux_params) + model_loaded = model + else: + model_loaded.bind(for_training=False, data_shapes=data_train.provide_data, + label_shapes=data_train.provide_label) + max_t_count = args.config.getint('arch', 'max_t_count') + eval_metric = STTMetric(batch_size=batch_size, num_gpu=num_gpu) + if is_batchnorm: + for nbatch, data_batch in enumerate(data_train): + model_loaded.forward(data_batch, is_train=False) + model_loaded.update_metric(eval_metric, data_batch.label) + else: + #model_loaded.score(eval_data=data_train, num_batch=None, + # eval_metric=eval_metric, reset=True) + for nbatch, data_batch in enumerate(data_train): + model_loaded.forward(data_batch, is_train=False) + model_loaded.update_metric(eval_metric, data_batch.label) + else: + raise Exception( + 'Define mode in the cfg file first. ' + + 'train or predict or load can be the candidate for the mode') diff --git a/example/speech_recognition/stt_bucketing_module.py b/example/speech_recognition/stt_bucketing_module.py new file mode 100644 index 000000000000..796a3368a6ad --- /dev/null +++ b/example/speech_recognition/stt_bucketing_module.py @@ -0,0 +1,13 @@ +import mxnet as mx + + +class STTBucketingModule(mx.mod.BucketingModule): + + def save_checkpoint(self, prefix, epoch, save_optimizer_states=False): + symbol, data_names, label_names = self._sym_gen(self._default_bucket_key) + symbol.save('%s-symbol.json' % prefix) + param_name = '%s-%04d.params' % (prefix, epoch) + self.save_params(param_name) + if save_optimizer_states: + state_name = '%s-%04d.states' % (prefix, epoch) + self._curr_module.save_optimizer_states(state_name) \ No newline at end of file diff --git a/example/speech_recognition/stt_io_bucketingiter.py b/example/speech_recognition/stt_io_bucketingiter.py new file mode 100644 index 000000000000..9655688f214f --- /dev/null +++ b/example/speech_recognition/stt_io_bucketingiter.py @@ -0,0 +1,148 @@ +from __future__ import print_function +import mxnet as mx +import sys +sys.path.insert(0, "../../python") + +import bisect +import random +import numpy as np + +BATCH_SIZE = 1 +SEQ_LENGTH = 0 +NUM_GPU = 1 + + +def get_label(buf, num_lable): + ret = np.zeros(num_lable) + for i in range(len(buf)): + ret[i] = int(buf[i]) + return ret + + +class BucketSTTIter(mx.io.DataIter): + def __init__(self, count, datagen, batch_size, num_label, init_states, seq_length, width, height, + sort_by_duration=True, + is_bi_graphemes=False, + partition="train", + buckets=[], + save_feature_as_csvfile=False + ): + super(BucketSTTIter, self).__init__() + + self.maxLabelLength = num_label + # global param + self.batch_size = batch_size + self.count = count + self.num_label = num_label + self.init_states = init_states + self.init_state_arrays = [mx.nd.zeros(x[1]) for x in init_states] + self.width = width + self.height = height + self.datagen = datagen + self.label = None + self.is_bi_graphemes = is_bi_graphemes + # self.partition = datagen.partition + if partition == 'train': + durations = datagen.train_durations + audio_paths = datagen.train_audio_paths + texts = datagen.train_texts + elif partition == 'validation': + durations = datagen.val_durations + audio_paths = datagen.val_audio_paths + texts = datagen.val_texts + elif partition == 'test': + durations = datagen.test_durations + audio_paths = datagen.test_audio_paths + texts = datagen.test_texts + else: + raise Exception("Invalid partition to load metadata. " + "Must be train/validation/test") + # if sortagrad + if sort_by_duration: + durations, audio_paths, texts = datagen.sort_by_duration(durations, + audio_paths, + texts) + else: + durations = durations + audio_paths = audio_paths + texts = texts + self.trainDataList = zip(durations, audio_paths, texts) + + self.trainDataIter = iter(self.trainDataList) + self.is_first_epoch = True + + data_lengths = [int(d*100) for d in durations] + if len(buckets) == 0: + buckets = [i for i, j in enumerate(np.bincount(data_lengths)) + if j >= batch_size] + if len(buckets) == 0: + raise Exception('There is no valid buckets. It may occured by large batch_size for each buckets. max bincount:%d batch_size:%d' % (max(np.bincount(data_lengths)), batch_size)) + buckets.sort() + ndiscard = 0 + self.data = [[] for _ in buckets] + for i, sent in enumerate(data_lengths): + buck = bisect.bisect_left(buckets, sent) + if buck == len(buckets): + ndiscard += 1 + continue + self.data[buck].append(self.trainDataList[i]) + if ndiscard != 0: + print("WARNING: discarded %d sentences longer than the largest bucket."% ndiscard) + + self.buckets = buckets + self.nddata = [] + self.ndlabel = [] + self.default_bucket_key = max(buckets) + + self.idx = [] + for i, buck in enumerate(self.data): + self.idx.extend([(i, j) for j in range(0, len(buck) - batch_size + 1, batch_size)]) + self.curr_idx = 0 + + self.provide_data = [('data', (self.batch_size, self.default_bucket_key , width * height))] + init_states + self.provide_label = [('label', (self.batch_size, self.maxLabelLength))] + self.save_feature_as_csvfile=save_feature_as_csvfile + + #self.reset() + + def reset(self): + """Resets the iterator to the beginning of the data.""" + self.curr_idx = 0 + random.shuffle(self.idx) + for buck in self.data: + np.random.shuffle(buck) + + def next(self): + """Returns the next batch of data.""" + if self.curr_idx == len(self.idx): + raise StopIteration + i, j = self.idx[self.curr_idx] + self.curr_idx += 1 + + audio_paths = [] + texts = [] + for duration, audio_path, text in self.data[i][j:j+self.batch_size]: + audio_paths.append(audio_path) + texts.append(text) + + if self.is_first_epoch: + data_set = self.datagen.prepare_minibatch(audio_paths, texts, overwrite=True, + is_bi_graphemes=self.is_bi_graphemes, + seq_length=self.buckets[i], + save_feature_as_csvfile=self.save_feature_as_csvfile) + else: + data_set = self.datagen.prepare_minibatch(audio_paths, texts, overwrite=False, + is_bi_graphemes=self.is_bi_graphemes, + seq_length=self.buckets[i], + save_feature_as_csvfile=self.save_feature_as_csvfile) + + data_all = [mx.nd.array(data_set['x'])] + self.init_state_arrays + label_all = [mx.nd.array(data_set['y'])] + + self.label = label_all + provide_data = [('data', (self.batch_size, self.buckets[i], self.width * self.height))] + self.init_states + + return mx.io.DataBatch(data_all, label_all, pad=0, + bucket_key=self.buckets[i], + provide_data=provide_data, + provide_label=self.provide_label) From c40442a2911fee2dbc204c8211768ae048c57fee Mon Sep 17 00:00:00 2001 From: "Qiang Kou (KK)" Date: Wed, 19 Jul 2017 17:02:32 -0700 Subject: [PATCH 235/834] [R] Image segmentation example and test. close #5003 (#7096) --- R-package/tests/testthat/get_data.R | 18 +++ R-package/tests/testthat/test_img_seg.R | 130 +++++++++++++++++++++ R-package/tests/testthat/test_model.R | 6 +- example/image-classification/symbol_unet.R | 81 ------------- 4 files changed, 151 insertions(+), 84 deletions(-) create mode 100644 R-package/tests/testthat/test_img_seg.R delete mode 100644 example/image-classification/symbol_unet.R diff --git a/R-package/tests/testthat/get_data.R b/R-package/tests/testthat/get_data.R index 89b04476011e..555e5e9b77b2 100644 --- a/R-package/tests/testthat/get_data.R +++ b/R-package/tests/testthat/get_data.R @@ -9,6 +9,7 @@ GetMNIST_ubyte <- function() { !file.exists('data/t10k-labels-idx1-ubyte')) { download.file('http://data.mxnet.io/mxnet/data/mnist.zip', destfile = 'data/mnist.zip') unzip('data/mnist.zip', exdir = 'data/') + file.remove('data/mnist.zip') } } @@ -21,6 +22,7 @@ GetMNIST_csv <- function() { download.file('https://s3-us-west-2.amazonaws.com/apache-mxnet/R/data/mnist_csv.zip', destfile = 'data/mnist_csv.zip') unzip('data/mnist_csv.zip', exdir = 'data/') + file.remove('data/mnist_csv.zip') } } @@ -35,6 +37,7 @@ GetCifar10 <- function() { download.file('http://data.mxnet.io/mxnet/data/cifar10.zip', destfile = 'data/cifar10.zip') unzip('data/cifar10.zip', exdir = 'data/') + file.remove('data/cifar10.zip') } } @@ -61,6 +64,7 @@ GetCatDog <- function() { download.file('https://s3-us-west-2.amazonaws.com/apache-mxnet/R/data/cats_dogs.zip', destfile = 'data/cats_dogs.zip') unzip('data/cats_dogs.zip', exdir = 'data/') + file.remove('data/cats_dogs.zip') } } @@ -72,5 +76,19 @@ GetMovieLens <- function() { download.file('http://files.grouplens.org/datasets/movielens/ml-100k.zip', destfile = 'data/ml-100k.zip') unzip('data/ml-100k.zip', exdir = 'data/') + file.remove('data/ml-100k.zip') + } +} + +GetISBI_data <- function() { + if (!dir.exists("data")) { + dir.create("data/") + } + if (!file.exists('data/ISBI/train-volume.tif') | + !file.exists('data/ISBI/train-labels.tif')) { + download.file('https://s3-us-west-2.amazonaws.com/apache-mxnet/R/data/ISBI.zip', + destfile = 'data/ISBI.zip') + unzip('data/ISBI.zip', exdir = 'data/') + file.remove('data/ISBI.zip') } } diff --git a/R-package/tests/testthat/test_img_seg.R b/R-package/tests/testthat/test_img_seg.R new file mode 100644 index 000000000000..ba5c9cd8369b --- /dev/null +++ b/R-package/tests/testthat/test_img_seg.R @@ -0,0 +1,130 @@ +require(mxnet) + +source("get_data.R") + +print_inferred_shape <- function(net) { + slist <- mx.symbol.infer.shape(symbol = net, data = c(168, 168, 1, 2)) + print(slist$out.shapes) +} + +convolution_module <- function(net, kernel_size, pad_size, filter_count, + stride = c(1, 1), work_space = 2048, batch_norm = TRUE, + down_pool = FALSE, up_pool = FALSE, act_type = "relu", + convolution = TRUE) { + if (up_pool) { + net = mx.symbol.Deconvolution(net, kernel = c(2, 2), pad = c(0, 0), + stride = c(2, 2), num_filter = filter_count, + workspace = work_space) + net = mx.symbol.BatchNorm(net) + if (act_type != "") { + net = mx.symbol.Activation(net, act_type = act_type) + } + } + if (convolution) { + conv = mx.symbol.Convolution(data = net, kernel = kernel_size, stride = stride, + pad = pad_size, num_filter = filter_count, + workspace = work_space) + net = conv + } + if (batch_norm) { + net = mx.symbol.BatchNorm(net) + } + + if (act_type != "") { + net = mx.symbol.Activation(net, act_type = act_type) + } + + if (down_pool) { + pool = mx.symbol.Pooling(net, pool_type = "max", kernel = c(2, 2), stride = c(2, 2)) + net = pool + } + print_inferred_shape(net) + return(net) +} + +get_unet <- function() { + data = mx.symbol.Variable('data') + kernel_size = c(3, 3) + pad_size = c(1, 1) + filter_count = 32 + pool1 = convolution_module(data, kernel_size, pad_size, filter_count = filter_count, down_pool = TRUE) + net = pool1 + pool2 = convolution_module(net, kernel_size, pad_size, filter_count = filter_count * 2, down_pool = TRUE) + net = pool2 + pool3 = convolution_module(net, kernel_size, pad_size, filter_count = filter_count * 4, down_pool = TRUE) + net = pool3 + pool4 = convolution_module(net, kernel_size, pad_size, filter_count = filter_count * 4, down_pool = TRUE) + net = pool4 + net = mx.symbol.Dropout(net) + pool5 = convolution_module(net, kernel_size, pad_size, filter_count = filter_count * 8, down_pool = TRUE) + net = pool5 + net = convolution_module(net, kernel_size, pad_size, filter_count = filter_count * 4, up_pool = TRUE) + net = convolution_module(net, kernel_size, pad_size = c(2, 2), filter_count = filter_count * 4, up_pool = TRUE) + net = mx.symbol.Crop(net, pool3, num.args = 2) + net = mx.symbol.concat(c(pool3, net), num.args = 2) + net = mx.symbol.Dropout(net) + net = convolution_module(net, kernel_size, pad_size, filter_count = filter_count * 4) + net = convolution_module(net, kernel_size, pad_size, filter_count = filter_count * 4, up_pool = TRUE) + + net = mx.symbol.Concat(c(pool2, net), num.args = 2) + net = mx.symbol.Dropout(net) + net = convolution_module(net, kernel_size, pad_size, filter_count = filter_count * 4) + net = convolution_module(net, kernel_size, pad_size, filter_count = filter_count * 4, up_pool = TRUE) + convolution_module(net, kernel_size, pad_size, filter_count = filter_count * 4) + net = mx.symbol.Concat(c(pool1, net), num.args = 2) + net = mx.symbol.Dropout(net) + net = convolution_module(net, kernel_size, pad_size, filter_count = filter_count * 2) + net = convolution_module(net, kernel_size, pad_size, filter_count = filter_count * 2, up_pool = TRUE) + net = convolution_module(net, kernel_size, pad_size, filter_count = 1, batch_norm = FALSE, act_type = "") + net = mx.symbol.SoftmaxOutput(data = net, name = 'sm') + return(net) +} + +context("Image segmentation") + +test_that("UNET", { + list.of.packages <- c("imager") + new.packages <- list.of.packages[!(list.of.packages %in% installed.packages()[,"Package"])] + if(length(new.packages)) install.packages(new.packages) + GetISBI_data() + library(imager) + IMG_SIZE <- 168 + files <- list.files(path = "data/ISBI/train-volume/") + a = 'data/ISBI/train-volume/' + filess = paste(a, files, sep = '') + list_of_images = lapply(filess, function(x) { + x <- load.image(x) + y <- resize(x, size_x = IMG_SIZE, size_y = IMG_SIZE) + }) + + train.x = do.call('cbind', lapply(list_of_images, as.vector)) + train.array <- train.x + dim(train.array) <- c(IMG_SIZE, IMG_SIZE, 1, 30) + + files <- list.files(path = "data/ISBI/train-labels") + b = 'data/ISBI/train-labels/' + filess = paste(b, files, sep = '') + list_of_images = lapply(filess, function(x) { + x <- load.image(x) + y <- resize(x, size_x = IMG_SIZE, size_y = IMG_SIZE) + }) + + train.y = do.call('cbind', lapply(list_of_images, as.vector)) + + train.y[which(train.y < 0.5)] = 0 + train.y[which(train.y > 0.5)] = 1 + train.y.array = train.y + dim(train.y.array) = c(IMG_SIZE, IMG_SIZE, 1, 30) + + devices <- mx.cpu() + mx.set.seed(0) + + net <- get_unet() + + model <- mx.model.FeedForward.create(net, X = train.array, y = train.y.array, + ctx = devices, num.round = 2, + initializer = mx.init.normal(sqrt(2 / 576)), + learning.rate = 0.05, + momentum = 0.99, + array.batch.size = 2) +}) \ No newline at end of file diff --git a/R-package/tests/testthat/test_model.R b/R-package/tests/testthat/test_model.R index fcf8daee70a4..7c5b04ac27b1 100644 --- a/R-package/tests/testthat/test_model.R +++ b/R-package/tests/testthat/test_model.R @@ -83,7 +83,7 @@ test_that("Regression", { }) mx.set.seed(0) model <- mx.model.FeedForward.create(lro, X = train.x, y = train.y, - ctx = mx.cpu(), num.round = 50, + ctx = mx.cpu(), num.round = 5, array.batch.size = 20, learning.rate = 2e-6, momentum = 0.9, @@ -103,7 +103,7 @@ test_that("Classification", { mx.set.seed(0) model <- mx.mlp(train.x, train.y, hidden_node = 10, out_node = 2, out_activation = "softmax", - num.round = 20, array.batch.size = 15, + num.round = 5, array.batch.size = 15, learning.rate = 0.07, momentum = 0.9, eval.metric = mx.metric.accuracy) @@ -218,7 +218,7 @@ test_that("Matrix Factorization", { train_iter <- CustomIter$new(user_iter, item_iter) model <- mx.model.FeedForward.create(pred3, X = train_iter, ctx = devices, - num.round = 10, initializer = mx.init.uniform(0.07), + num.round = 5, initializer = mx.init.uniform(0.07), learning.rate = 0.07, eval.metric = mx.metric.rmse, momentum = 0.9, diff --git a/example/image-classification/symbol_unet.R b/example/image-classification/symbol_unet.R deleted file mode 100644 index e15b48a4a005..000000000000 --- a/example/image-classification/symbol_unet.R +++ /dev/null @@ -1,81 +0,0 @@ -library(mxnet) - -convolution_module <- function(net, kernel_size, pad_size, - filter_count, stride = c(1, 1), work_space = 2048, - batch_norm = TRUE, down_pool = FALSE, up_pool = FALSE, - act_type = "relu", convolution = TRUE) { - if (up_pool) { - net = mx.symbol.Deconvolution(net, kernel = c(2, 2), pad = c(0, 0), - stride = c(2, 2), num_filter = filter_count, workspace = work_space) - net = mx.symbol.BatchNorm(net) - if (act_type != "") { - net = mx.symbol.Activation(net, act_type = act_type) - } - } - if (convolution) { - conv = mx.symbol.Convolution(data = net, kernel = kernel_size, stride = stride, - pad = pad_size, num_filter = filter_count, workspace = work_space) - net = conv - } - - if (batch_norm) { - net = mx.symbol.BatchNorm(net) - } - - if (act_type != "") { - net = mx.symbol.Activation(net, act_type = act_type) - } - - if (down_pool) { - pool = mx.symbol.Pooling(net, pool_type = "max", kernel = c(2, 2), stride = c(2, 2)) - net = pool - } - return(net) -} - -get_symbol <- function(num_classes = 10) { - data = mx.symbol.Variable('data') - kernel_size = c(3, 3) - pad_size = c(1, 1) - filter_count = 32 - pool1 = convolution_module(data, kernel_size, pad_size, filter_count = filter_count, down_pool = TRUE) - net = pool1 - pool2 = convolution_module(net, kernel_size, pad_size, filter_count = filter_count * 2, down_pool = TRUE) - net = pool2 - pool3 = convolution_module(net, kernel_size, pad_size, filter_count = filter_count * 4, down_pool = TRUE) - net = pool3 - pool4 = convolution_module(net, - kernel_size, - pad_size, - filter_count = filter_count * 4, - down_pool = TRUE) - net = pool4 - net = mx.symbol.Dropout(net) - pool5 = convolution_module(net, kernel_size, pad_size, filter_count = filter_count * 8, down_pool = TRUE) - net = pool5 - net = convolution_module(net, kernel_size, pad_size, filter_count = filter_count * 4, up_pool = TRUE) - net = convolution_module(net, kernel_size, pad_size, filter_count = filter_count * 4, up_pool = TRUE) - - # dirty "CROP" to wanted size... I was on old MxNet branch so used conv instead of crop for cropping - net = convolution_module(net, c(4, 4), c(0, 0), filter_count = filter_count * 4) - - net = mx.symbol.Concat(c(pool3, net), num.args = 2) - net = mx.symbol.Dropout(net) - net = convolution_module(net, kernel_size, pad_size, filter_count = filter_count * 4) - net = convolution_module(net, kernel_size, pad_size, filter_count = filter_count * 4, up_pool = TRUE) - - net = mx.symbol.Concat(c(pool2, net), num.args = 2) - net = mx.symbol.Dropout(net) - net = convolution_module(net, kernel_size, pad_size, filter_count = filter_count * 4) - net = convolution_module(net, kernel_size, pad_size, - filter_count = filter_count * 4, up_pool = TRUE) - convolution_module(net, kernel_size, pad_size, filter_count = filter_count * 4) - net = mx.symbol.Concat(c(pool1, net), num.args = 2) - net = mx.symbol.Dropout(net) - net = convolution_module(net, kernel_size, pad_size, filter_count = filter_count * 2) - net = convolution_module(net, kernel_size, pad_size, filter_count = filter_count * 2, up_pool = TRUE) - net = mx.symbol.Flatten(net) - net = mx.symbol.FullyConnected(data = net, num_hidden = num_classes) - net = mx.symbol.SoftmaxOutput(data = net, name = 'softmax') - return(net) -} \ No newline at end of file From ef85876b75a2cc4dcfe7cbfc22fbc6ee7ed2c42f Mon Sep 17 00:00:00 2001 From: Rahul Date: Wed, 19 Jul 2017 21:35:03 -0700 Subject: [PATCH 236/834] added casts and overrides to fix compiler warnings (#7109) * added casts and overrides to fix compiler warnings * fix for index_t not working with openmp on windows * fix lint error * fix omp for loop init error * fixing build errors --- src/executor/attach_op_execs_pass.cc | 4 +- src/operator/batch_norm.cc | 2 +- src/operator/bilinear_sampler.cc | 16 ++--- src/operator/contrib/proposal.cc | 6 +- src/operator/correlation.cc | 15 ++--- src/operator/deconvolution-inl.h | 6 +- .../identity_attach_KL_sparse_reg-inl.h | 2 +- src/operator/nn/softmax-inl.h | 4 +- src/operator/pad.cc | 59 +++++++++++++------ src/operator/sequence_reverse-inl.h | 4 +- src/operator/spatial_transformer.cc | 4 +- src/operator/tensor/broadcast_reduce_op.h | 2 +- src/operator/tensor/matrix_op-inl.h | 2 +- src/operator/tensor/sort_op.h | 2 +- 14 files changed, 76 insertions(+), 52 deletions(-) diff --git a/src/executor/attach_op_execs_pass.cc b/src/executor/attach_op_execs_pass.cc index 6a0c489a1ec5..c4b3a1895ad8 100644 --- a/src/executor/attach_op_execs_pass.cc +++ b/src/executor/attach_op_execs_pass.cc @@ -49,7 +49,7 @@ class StatefulComputeExecutor : public OpExecutor { return exec_type_; } - virtual engine::VarHandle var() const { + engine::VarHandle var() const override { return state_.get_var(); } @@ -81,7 +81,7 @@ class StatefulComputeExExecutor : public OpExecutor { return exec_type_; } - virtual engine::VarHandle var() const { + engine::VarHandle var() const override { return state_.get_var(); } diff --git a/src/operator/batch_norm.cc b/src/operator/batch_norm.cc index 1bc6fd08e2ea..e56b30671e3e 100644 --- a/src/operator/batch_norm.cc +++ b/src/operator/batch_norm.cc @@ -98,7 +98,7 @@ void BatchNormOp::DoForward(mshadow::Stream *, const size_t itemCountPerChannel = inputData.Size() / channelCount; #pragma omp parallel for - for (int channel = 0; channel < channelCount; ++channel) { + for (int channel = 0; channel < static_cast(channelCount); ++channel) { if (is_train_and_not_global_stats) { // compute mean per input mean[channel] = 0; diff --git a/src/operator/bilinear_sampler.cc b/src/operator/bilinear_sampler.cc index fd2bff824fd7..f76e987440fa 100644 --- a/src/operator/bilinear_sampler.cc +++ b/src/operator/bilinear_sampler.cc @@ -21,10 +21,10 @@ inline void BilinearSamplerForward(const Tensor &output, const DType *grid = grid_src.dptr_; int o_n = output.size(0), o_c = output.size(1), o_h = output.size(2), o_w = output.size(3); int i_c = input.size(1), i_h = input.size(2), i_w = input.size(3); - for (index_t n = 0; n < o_n; ++n) { - for (index_t c = 0; c < o_c; ++c) { - for (index_t h = 0; h < o_h; ++h) { - for (index_t w = 0; w < o_w; ++w) { + for (index_t n = 0; n < static_cast(o_n); ++n) { + for (index_t c = 0; c < static_cast(o_c); ++c) { + for (index_t h = 0; h < static_cast(o_h); ++h) { + for (index_t w = 0; w < static_cast(o_w); ++w) { index_t out_index = n * o_c * o_h * o_w + c * o_h * o_w + h * o_w + w; index_t grid_index = n * o_h * o_w * 2 + h * o_w + w; DType y_real = (*(grid + grid_index + o_h * o_w) + 1) * (i_h - 1) / 2; @@ -71,9 +71,9 @@ inline void BilinearSamplerBackward(const Tensor &gdata, int o_n = output_grad.size(0), o_c = output_grad.size(1), o_h = output_grad.size(2), o_w = output_grad.size(3); int i_c = input_data.size(1), i_h = input_data.size(2), i_w = input_data.size(3); - for (index_t n = 0; n < o_n; ++n) { - for (index_t h = 0; h < o_h; ++h) { - for (index_t w = 0; w < o_w; ++w) { + for (index_t n = 0; n < static_cast(o_n); ++n) { + for (index_t h = 0; h < static_cast(o_h); ++h) { + for (index_t w = 0; w < static_cast(o_w); ++w) { DType top_left_y_gw = 0.0; DType top_left_x_gw = 0.0; index_t grid_src_index = n * o_h * o_w * 2 + h * o_w + w; @@ -83,7 +83,7 @@ inline void BilinearSamplerBackward(const Tensor &gdata, int top_left_x = static_cast(floor(x_real)); DType top_left_y_w = 1.0 - (y_real - top_left_y); DType top_left_x_w = 1.0 - (x_real - top_left_x); - for (index_t c = 0; c < o_c; ++c) { + for (index_t c = 0; c < static_cast(o_c); ++c) { index_t grad_index = n * o_c * o_h * o_w + c * o_h * o_w + h * o_w + w; int data_index = n * i_c * i_h * i_w + c * i_h * i_w + top_left_y * i_w + top_left_x; diff --git a/src/operator/contrib/proposal.cc b/src/operator/contrib/proposal.cc index 06a0565bf822..fe4fe98a9f4b 100644 --- a/src/operator/contrib/proposal.cc +++ b/src/operator/contrib/proposal.cc @@ -326,9 +326,9 @@ class ProposalOp : public Operator{ std::memcpy(workspace_proposals.dptr_, &anchors[0], sizeof(float) * anchors.size()); // Enumerate all shifted anchors - for (index_t i = 0; i < num_anchors; ++i) { - for (index_t j = 0; j < height; ++j) { - for (index_t k = 0; k < width; ++k) { + for (index_t i = 0; i < static_cast(num_anchors); ++i) { + for (index_t j = 0; j < static_cast(height); ++j) { + for (index_t k = 0; k < static_cast(width); ++k) { index_t index = j * (width * num_anchors) + k * (num_anchors) + i; workspace_proposals[index][0] = workspace_proposals[i][0] + k * param_.feature_stride; workspace_proposals[index][1] = workspace_proposals[i][1] + j * param_.feature_stride; diff --git a/src/operator/correlation.cc b/src/operator/correlation.cc index 5160d1f245ad..18a3e1cb06b9 100644 --- a/src/operator/correlation.cc +++ b/src/operator/correlation.cc @@ -33,22 +33,23 @@ inline void CorrelationForward(const Tensor &out, const int bchannels = data1.size(1); const int sumelems = kernel_size_ * kernel_size_ * bchannels; AddPad(data1, tmp1, pad_size_); + index_t top_channels_unsigned_ = static_cast(top_channels_); AddPad(data2, tmp2, pad_size_); for (index_t i = 0 ; i < static_cast(top_height_) ; i++) for (index_t j = 0 ; j < static_cast(top_width_); j++) for (index_t nbatch = 0 ; nbatch < bnum ; nbatch++) { int x1 = j*stride1_+max_displacement_; int y1 = i*stride1_+max_displacement_; - for (index_t top_channel = 0 ; top_channel < top_channels_ ; top_channel++) { + for (index_t top_channel = 0 ; top_channel < top_channels_unsigned_ ; top_channel++) { int s2o = (top_channel % neighborhood_grid_width_ -\ neighborhood_grid_radius_) * stride2_; int s2p = (top_channel / neighborhood_grid_width_ -\ neighborhood_grid_radius_) * stride2_; int x2 = x1 + s2o; int y2 = y1 + s2p; - for (index_t h = 0; h < kernel_size_; h++) - for (index_t w = 0; w < kernel_size_; w++) - for (index_t channel = 0; channel < bchannels; channel++) { + for (index_t h = 0; h < static_cast(kernel_size_); h++) + for (index_t w = 0; w < static_cast(kernel_size_); w++) + for (index_t channel = 0; channel < static_cast(bchannels); channel++) { if (is_multiply == true) out[nbatch][top_channel][i][j] += \ tmp1[nbatch][y1+h][x1+w][channel]*tmp2[nbatch][y2+h][x2+w][channel]; @@ -76,9 +77,9 @@ inline void CorrelationBackward(const Tensor &out_grad, int channels, int height, int width ) { const float sumelems = kernel_size_ * kernel_size_ * channels; - for (int i = 0 ; i < static_cast(top_height_) ; i++) - for (int j = 0 ; j < static_cast(top_width_); j++) - for (int nbatch = 0 ; nbatch < static_cast(num) ; nbatch++) { + for (index_t i = 0 ; i < static_cast(top_height_) ; i++) + for (index_t j = 0 ; j < static_cast(top_width_); j++) + for (index_t nbatch = 0 ; nbatch < static_cast(num) ; nbatch++) { int x1 = j*stride1_+max_displacement_; int y1 = i*stride1_+max_displacement_; for (int top_channel = 0 ; top_channel < top_channels_ ; top_channel++) { diff --git a/src/operator/deconvolution-inl.h b/src/operator/deconvolution-inl.h index b15777c69517..909a6fd5fed6 100644 --- a/src/operator/deconvolution-inl.h +++ b/src/operator/deconvolution-inl.h @@ -96,7 +96,7 @@ struct DeconvolutionParam : public dmlc::Parameter { // Use tag to control the calculation of pad bool bCal = false; if (target_shape.ndim() != 0) { - for (int i = 0; i < target_shape.ndim(); i++) { + for (index_t i = 0; i < target_shape.ndim(); i++) { if (target_shape[i] != 0) bCal = true; } } @@ -104,7 +104,7 @@ struct DeconvolutionParam : public dmlc::Parameter { if (bCal) { size_t input_ndim = input.ndim(); - for (unsigned int i = 0; i < ndim; i++) { + for (index_t i = 0; i < ndim; i++) { // input.ndim() can be larger than ndim, in case that the complete input // shape was passed and not only the ndim last ones o_pad[i] = stride[i] * (input[(input_ndim - ndim) + i] - 1) + DilatedKernelSize(i); @@ -114,7 +114,7 @@ struct DeconvolutionParam : public dmlc::Parameter { o_pad[i] = (o_pad[i] + 1) / 2; } } else { - for (unsigned int i = 0; i < ndim; i++) { + for (index_t i = 0; i < ndim; i++) { o_pad[i] = pad[i]; o_adj[i] = adj[i]; } diff --git a/src/operator/identity_attach_KL_sparse_reg-inl.h b/src/operator/identity_attach_KL_sparse_reg-inl.h index ca7eab0f399e..413bac90c0ac 100644 --- a/src/operator/identity_attach_KL_sparse_reg-inl.h +++ b/src/operator/identity_attach_KL_sparse_reg-inl.h @@ -151,7 +151,7 @@ class IdentityAttachKLSparseRegProp : public OperatorProperty { const std::vector &out_grad, const std::vector &in_data, const std::vector &out_data, - const std::vector &in_grad) const { + const std::vector &in_grad) const override { return { {out_grad[sparsereg::kOut], in_grad[sparsereg::kData]} }; } diff --git a/src/operator/nn/softmax-inl.h b/src/operator/nn/softmax-inl.h index 11eeb5d23624..749ad7374b96 100644 --- a/src/operator/nn/softmax-inl.h +++ b/src/operator/nn/softmax-inl.h @@ -43,7 +43,7 @@ inline void Softmax(Stream *s, DType *in, DType *out, index_t sa = stride[axis]; #pragma omp parallel for - for (int i = 0; i < N; ++i) { + for (int i = 0; i < static_cast(N); ++i) { index_t base = unravel_dot(i, sshape, stride); DType mmax = in[base]; @@ -90,7 +90,7 @@ inline void SoftmaxGrad(Stream *s, DType *out, DType *ograd, index_t sa = stride[axis]; #pragma omp parallel for - for (int i = 0; i < N; ++i) { + for (int i = 0; i < static_cast(N); ++i) { index_t base = unravel_dot(i, sshape, stride); DType sum = DType(0); diff --git a/src/operator/pad.cc b/src/operator/pad.cc index 5d1afca588fb..77177b5758ec 100644 --- a/src/operator/pad.cc +++ b/src/operator/pad.cc @@ -121,12 +121,18 @@ void single_image_constant(const Tensor &dst, const int pad_t = pad[4]; const int pad_l = pad[6]; int c, w, h; + // using these vars to avoid casting overhead each loop iteration + const int dst0 = dst.size(0); + const int dst1 = dst.size(1); + const int dst2 = dst.size(2); + const int src1 = src.size(1); + const int src2 = src.size(2); #pragma omp parallel for private(c, w, h) - for (c = 0; c < dst.size(0); ++c) { - for (h = 0; h < dst.size(1); ++h) { - for (w = 0; w < dst.size(2); ++w) { - if ((w < pad_l) || (h < pad_t) || (h >= (src.size(1) + pad_t)) || - (w >= (src.size(2) + pad_l))) { + for (c = 0; c < dst0; ++c) { + for (h = 0; h < dst1; ++h) { + for (w = 0; w < dst2; ++w) { + if ((w < pad_l) || (h < pad_t) || (h >= (src1 + pad_t)) || + (w >= (src2 + pad_l))) { dst[c][h][w] = constant_value; } else { dst[c][h][w] = src[c][h - pad_t][w - pad_l]; @@ -142,11 +148,15 @@ void single_image_constant_grad(const Tensor &in_grad, mxnet::TShape pad) { const int pad_t = pad[4]; const int pad_l = pad[6]; + + const int in_grad0 = in_grad.size(0); + const int in_grad1 = in_grad.size(1); + const int in_grad2 = in_grad.size(2); int c, h, w; #pragma omp parallel for private(c, w, h) - for (c = 0; c < in_grad.size(0); ++c) { - for (h = 0; h < in_grad.size(1); ++h) { - for (w = 0; w < in_grad.size(2); ++w) { + for (c = 0; c < in_grad0; ++c) { + for (h = 0; h < in_grad1; ++h) { + for (w = 0; w < in_grad2; ++w) { in_grad[c][h][w] += out_grad[c][h + pad_t][w + pad_l]; } } @@ -404,15 +414,24 @@ void single_image_constant(const Tensor &dst, const int pad_f = pad[4]; const int pad_t = pad[6]; const int pad_l = pad[8]; + + const int dst0 = dst.size(0); + const int dst1 = dst.size(1); + const int dst2 = dst.size(2); + const int dst3 = dst.size(3); + const int src1 = src.size(1); + const int src2 = src.size(2); + const int src3 = src.size(3); + int c, d, w, h; #pragma omp parallel for private(c, d, w, h) - for (c = 0; c < dst.size(0); ++c) { - for (d = 0; d < dst.size(1); ++d) { - for (h = 0; h < dst.size(2); ++h) { - for (w = 0; w < dst.size(3); ++w) { + for (c = 0; c < dst0; ++c) { + for (d = 0; d < dst1; ++d) { + for (h = 0; h < dst2; ++h) { + for (w = 0; w < dst3; ++w) { if ((w < pad_l) || (h < pad_t) || (d < pad_f) || - (d >= (src.size(1) + pad_f)) || (h >= (src.size(2) + pad_t)) || - (w >= (src.size(3) + pad_l))) { + (d >= (src1 + pad_f)) || (h >= (src2 + pad_t)) || + (w >= (src3 + pad_l))) { dst[c][d][h][w] = constant_value; } else { dst[c][d][h][w] = src[c][d - pad_f][h - pad_t][w - pad_l]; @@ -430,12 +449,16 @@ void single_image_constant_grad(const Tensor &in_grad, const int pad_f = pad[4]; const int pad_t = pad[6]; const int pad_l = pad[8]; + const int in_grad0 = in_grad.size(0); + const int in_grad1 = in_grad.size(1); + const int in_grad2 = in_grad.size(2); + const int in_grad3 = in_grad.size(3); int c, d, w, h; #pragma omp parallel for private(c, d, w, h) - for (c = 0; c < in_grad.size(0); ++c) { - for (d = 0; d < in_grad.size(1); ++d) { - for (h = 0; h < in_grad.size(2); ++h) { - for (w = 0; w < in_grad.size(3); ++w) { + for (c = 0; c < in_grad0; ++c) { + for (d = 0; d < in_grad1; ++d) { + for (h = 0; h < in_grad2; ++h) { + for (w = 0; w < in_grad3; ++w) { in_grad[c][d][h][w] += out_grad[c][d + pad_f][h + pad_t][w + pad_l]; } } diff --git a/src/operator/sequence_reverse-inl.h b/src/operator/sequence_reverse-inl.h index 44a1bd7ceaad..0ead25ceba72 100644 --- a/src/operator/sequence_reverse-inl.h +++ b/src/operator/sequence_reverse-inl.h @@ -55,7 +55,7 @@ struct ReverseKernel { : max_seq_len; const index_t padded_periods = max_seq_len - num_seq; // padded part - if (padded_periods > 0 && i < padded_periods) { + if (padded_periods > 0 && i < static_cast(padded_periods)) { const int padded_in_offset = (i + num_seq) * batch_size * other_dim + batch * other_dim; @@ -65,7 +65,7 @@ struct ReverseKernel { } } // unpadded part - if (i < num_seq) { + if (i < static_cast(num_seq)) { const int in_offset = i * batch_size * other_dim + batch * other_dim; const int out_offset = numel - (i + 1 + padded_periods) * batch_size * other_dim + diff --git a/src/operator/spatial_transformer.cc b/src/operator/spatial_transformer.cc index 45c1d8588776..409339b3c445 100644 --- a/src/operator/spatial_transformer.cc +++ b/src/operator/spatial_transformer.cc @@ -20,7 +20,7 @@ inline void BilinearSamplingForward(const Tensor &output, for (index_t n = 0; n < static_cast(o_n); ++n) { for (index_t c = 0; c < static_cast(o_c); ++c) { for (index_t h = 0; h < static_cast(o_h); ++h) { - for (index_t w = 0; w < o_w; ++w) { + for (index_t w = 0; w < static_cast(o_w); ++w) { index_t out_index = n * o_c * o_h * o_w + c * o_h * o_w + h * o_w + w; index_t grid_index = n * o_h * o_w * 2 + h * o_w + w; DType y_real = (*(grid + grid_index + o_h * o_w) + 1) * (i_h - 1) / 2; @@ -68,7 +68,7 @@ inline void BilinearSamplingBackward(const Tensor &input_grad, index_t top_left_x = std::min(i_w, std::max(0, static_cast(floor(x_real)))); DType top_left_y_w = 1.0 - (y_real - top_left_y); DType top_left_x_w = 1.0 - (x_real - top_left_x); - for (index_t c = 0; c < o_c; ++c) { + for (index_t c = 0; c < static_cast(o_c); ++c) { index_t grad_index = n * o_c * o_h * o_w + c * o_h * o_w + h * o_w + w; index_t data_index = n * i_c * i_h * i_w + c * i_h * i_w + top_left_y * i_w + top_left_x; diff --git a/src/operator/tensor/broadcast_reduce_op.h b/src/operator/tensor/broadcast_reduce_op.h index 9ed56f4c997e..1de33bc86f89 100644 --- a/src/operator/tensor/broadcast_reduce_op.h +++ b/src/operator/tensor/broadcast_reduce_op.h @@ -623,7 +623,7 @@ void PickOpForward(const nnvm::NodeAttrs& attrs, const PickParam& param = nnvm::get(attrs.parsed); const TShape& ishape = inputs[0].shape_; - int axis = CheckAxis(param.axis.value(), ishape.ndim()); + index_t axis = CheckAxis(param.axis.value(), ishape.ndim()); int leading = 1, trailing = 1, M = ishape[axis]; for (index_t i = 0; i < axis; ++i) leading *= ishape[i]; for (index_t i = axis+1; i < ishape.ndim(); ++i) trailing *= ishape[i]; diff --git a/src/operator/tensor/matrix_op-inl.h b/src/operator/tensor/matrix_op-inl.h index 72fd2773c8f8..2e1aa6661b67 100644 --- a/src/operator/tensor/matrix_op-inl.h +++ b/src/operator/tensor/matrix_op-inl.h @@ -1736,7 +1736,7 @@ void ReverseOpForward(const nnvm::NodeAttrs& attrs, CHECK_LT(*axis_iter, static_cast(ishape.ndim())); stride_[reverse_index] = ishape[*axis_iter]; trailing_[reverse_index] = 1; - for (int i2 = *axis_iter + 1; i2 < ishape.ndim(); ++i2) { + for (index_t i2 = *axis_iter + 1; i2 < ishape.ndim(); ++i2) { trailing_[reverse_index] *= ishape[i2]; } reverse_index++; diff --git a/src/operator/tensor/sort_op.h b/src/operator/tensor/sort_op.h index 42ae43cc7584..ac8a69846ea1 100644 --- a/src/operator/tensor/sort_op.h +++ b/src/operator/tensor/sort_op.h @@ -31,7 +31,7 @@ inline void SortByKey(mshadow::Tensor keys, mshadow::Tensor idx(keys.size(0)); std::vector keys_vec(keys.size(0)); std::vector values_vec(values.size(0)); - for (int i = 0; i < keys.size(0); i++) { + for (index_t i = 0; i < keys.size(0); i++) { idx[i] = i; keys_vec[i] = keys[i]; values_vec[i] = values[i]; From fada5c1df8bc31ee6f1bd7bfa3efa055f1166267 Mon Sep 17 00:00:00 2001 From: Yanbo Liang Date: Thu, 20 Jul 2017 18:51:58 +0800 Subject: [PATCH 237/834] Fix svm_mnist.py. (#7036) --- example/svm_mnist/svm_mnist.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/example/svm_mnist/svm_mnist.py b/example/svm_mnist/svm_mnist.py index 177a927a4548..ac2702e1260d 100644 --- a/example/svm_mnist/svm_mnist.py +++ b/example/svm_mnist/svm_mnist.py @@ -41,8 +41,7 @@ Y = mnist.target[p] X_show = mnist.data[p] -# This is just to normalize the input to a value inside [0,1], -# and separate train set and test set +# This is just to normalize the input and separate train set and test set X = X.astype(np.float32)/255 X_train = X[:60000] X_test = X[60000:] @@ -52,12 +51,8 @@ # Article's suggestion on batch size batch_size = 200 -train_iter = mx.io.NDArrayIter(X_train, Y_train, batch_size=batch_size) -test_iter = mx.io.NDArrayIter(X_test, Y_test, batch_size=batch_size) - -# A quick work around to prevent mxnet complaining the lack of a softmax_label -train_iter.label = mx.io._init_data(Y_train, allow_empty=True, default_name='svm_label') -test_iter.label = mx.io._init_data(Y_test, allow_empty=True, default_name='svm_label') +train_iter = mx.io.NDArrayIter(X_train, Y_train, batch_size=batch_size, label_name='svm_label') +test_iter = mx.io.NDArrayIter(X_test, Y_test, batch_size=batch_size, label_name='svm_label') # Here we instatiate and fit the model for our data # The article actually suggests using 400 epochs, From 59d59345701d80d4d5dbad6e528424b4aca9d95a Mon Sep 17 00:00:00 2001 From: Przemyslaw Tredak Date: Thu, 20 Jul 2017 09:53:32 -0700 Subject: [PATCH 238/834] Updating dmlc-core (#7114) --- dmlc-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dmlc-core b/dmlc-core index b647be2dee98..71bfbd3a9460 160000 --- a/dmlc-core +++ b/dmlc-core @@ -1 +1 @@ -Subproject commit b647be2dee985d77a12e8e41bc27382221938290 +Subproject commit 71bfbd3a946075cea66ca9e19bad86dd33c19b46 From 4210d857e94e607f3483e1cde83fb1d40b4129f5 Mon Sep 17 00:00:00 2001 From: solin319 Date: Fri, 21 Jul 2017 01:10:30 +0800 Subject: [PATCH 239/834] =?UTF-8?q?Resolve=20a=20problem=20when=20import?= =?UTF-8?q?=20mxnet=20in=20python=20:=20=E2=80=98cannot=20import=20name=20?= =?UTF-8?q?gluon=E2=80=99=20(#7119)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update setup.py * Update setup.py --- python/setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/setup.py b/python/setup.py index 8a8693038b3c..92dd0444ac2a 100644 --- a/python/setup.py +++ b/python/setup.py @@ -75,7 +75,8 @@ def config_cython(): description=open(os.path.join(CURRENT_DIR, 'README.md')).read(), packages=[ 'mxnet', 'mxnet.module', 'mxnet._ctypes', 'mxnet.rnn', - 'mxnet._cy2', 'mxnet._cy3', 'mxnet.notebook', 'mxnet.contrib' + 'mxnet._cy2', 'mxnet._cy3', 'mxnet.notebook', 'mxnet.contrib', + 'mxnet.gluon', 'mxnet.gluon.nn', 'mxnet.gluon.rnn' ], data_files=[('mxnet', [LIB_PATH[0]])], url='https://github.com/dmlc/mxnet', From 7606864fe71ef09fe286293ea655c73e11ad28b7 Mon Sep 17 00:00:00 2001 From: "Qiang Kou (KK)" Date: Thu, 20 Jul 2017 14:43:32 -0700 Subject: [PATCH 240/834] [R] add "allow.extra.params". Update related documents. (#7125) --- R-package/R/model.R | 23 ++++++++++++-- .../classifyRealImageWithPretrainedModel.Rmd | 31 +++++++++++++++++++ 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/R-package/R/model.R b/R-package/R/model.R index 998156d1a110..043d0e2433ea 100644 --- a/R-package/R/model.R +++ b/R-package/R/model.R @@ -414,6 +414,13 @@ mx.model.select.layout.predict <- function(X, model) { #' The names of the input symbols. #' @param output.names optional #' The names of the output symbols. +#' @param fixed.param +#' The parameters to be fixed during training. For these parameters, not gradients +#' will be calculated and thus no space will be allocated for the gradient. +#' @param allow.extra.params +#' Whether allow extra parameters that are not needed by symbol. +#' If this is TRUE, no error will be thrown when arg_params or aux_params +#' contain extra parameters that is not needed by the executor. #' @return model A trained mxnet model. #' #' @export @@ -428,7 +435,7 @@ function(symbol, X, y=NULL, ctx=NULL, begin.round=1, kvstore = "local", verbose = TRUE, arg.params = NULL, aux.params = NULL, input.names=NULL, output.names = NULL, - fixed.param = NULL, + fixed.param = NULL, allow.extra.params = FALSE, ...) { if (is.array(X) || is.matrix(X)) { if (array.layout == "auto") { @@ -458,6 +465,9 @@ function(symbol, X, y=NULL, ctx=NULL, begin.round=1, params <- mx.model.init.params(symbol, input.shape, output.shape, initializer, mx.cpu()) if (!is.null(arg.params)) params$arg.params <- arg.params if (!is.null(aux.params)) params$aux.params <- aux.params + if (allow.extra.params) { + params$arg.params[!names(params$arg.params) %in% arguments(symbol)] <- NULL + } if (is.null(ctx)) ctx <- mx.ctx.default() if (is.mx.context(ctx)) { ctx <- list(ctx) @@ -516,9 +526,13 @@ function(symbol, X, y=NULL, ctx=NULL, begin.round=1, #' "colmajor" means dim(X) = c(nfeatures, nexample) #' "auto" will auto detect the layout by match the feature size, #' and will report error when X is a square matrix to ask user to explicitly specify layout. -#' +#' @param allow.extra.params +#' Whether allow extra parameters that are not needed by symbol. +#' If this is TRUE, no error will be thrown when arg_params or aux_params +#' contain extra parameters that is not needed by the executor. #' @export -predict.MXFeedForwardModel <- function(model, X, ctx=NULL, array.batch.size=128, array.layout="auto") { +predict.MXFeedForwardModel <- function(model, X, ctx = NULL, array.batch.size = 128, + array.layout = "auto", allow.extra.params = FALSE) { if (is.serialized(model)) model <- mx.unserialize(model) if (is.null(ctx)) ctx <- mx.ctx.default() if (is.array(X) || is.matrix(X)) { @@ -536,6 +550,9 @@ predict.MXFeedForwardModel <- function(model, X, ctx=NULL, array.batch.size=128, arg_lst <- list(symbol = model$symbol, ctx = ctx, data = dim(dlist$data), grad.req="null") pexec <- do.call(mx.simple.bind, arg_lst) + if (allow.extra.params) { + model$arg.params[!names(model$arg.params) %in% arguments(model$symbol)] <- NULL + } mx.exec.update.arg.arrays(pexec, model$arg.params, match.name=TRUE) mx.exec.update.aux.arrays(pexec, model$aux.params, match.name=TRUE) packer <- mx.nd.arraypacker() diff --git a/R-package/vignettes/classifyRealImageWithPretrainedModel.Rmd b/R-package/vignettes/classifyRealImageWithPretrainedModel.Rmd index e7e53d4d2d9f..34847fd5705c 100644 --- a/R-package/vignettes/classifyRealImageWithPretrainedModel.Rmd +++ b/R-package/vignettes/classifyRealImageWithPretrainedModel.Rmd @@ -125,6 +125,37 @@ print(paste0("Predicted Top-class: ", synsets[[max.idx]])) Actually I do not know what does the word mean when I saw it. So I searched on the web to check it out.. and hmm it does get the right answer :) +Extract features +------------------ + +Besides the final classification results, we can also extract the internal features. +We need to get feature layer symbol out of internals first. Here we use `global_pool_output` +as an example. + +```{r} +internals = model$symbol$get.internals() +fea_symbol = internals[[match("global_pool_output", internals$outputs)]] +``` + +Next, we rebuild a new model using the feature symbol + +```{r} +model2 <- list(symbol = fea_symbol, + arg.params = model$arg.params, + aux.params = model$aux.params) + +class(model2) <- "MXFeedForwardModel" +``` + +Then we can do the `predict` using the new model to get the internal results. +You need to set `allow.extra.params = TRUE` since some parameters are not used this time. + +```{r} +global_pooling_feature <- predict(model2, X = normed, allow.extra.params = TRUE) +dim(global_pooling_feature) +``` + + Reference --------- [1] Ioffe, Sergey, and Christian Szegedy. "Batch normalization: Accelerating deep network training by reducing internal covariate shift." arXiv preprint arXiv:1502.03167 (2015). From 46679f15e521def082b0686227ddb0cfa2cd0064 Mon Sep 17 00:00:00 2001 From: Przemyslaw Tredak Date: Thu, 20 Jul 2017 15:55:08 -0700 Subject: [PATCH 241/834] Adding absolute tolerance to few tests (#7133) --- tests/python/unittest/test_operator.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py index 2a4c8068db65..58d39513a4a8 100644 --- a/tests/python/unittest/test_operator.py +++ b/tests/python/unittest/test_operator.py @@ -897,16 +897,16 @@ def test_batchnorm_training(): xrolling_std = np.random.uniform(size=channel_count) test = mx.symbol.BatchNorm(data, fix_gamma=True, axis=chaxis) - check_numeric_gradient(test, [data_tmp, gamma, beta], [xrolling_mean, xrolling_std], numeric_eps=1e-2, rtol=0.2) + check_numeric_gradient(test, [data_tmp, gamma, beta], [xrolling_mean, xrolling_std], numeric_eps=1e-2, rtol=0.2, atol=0.01) test = mx.symbol.BatchNorm(data, fix_gamma=True, use_global_stats=True, axis=chaxis) - check_numeric_gradient(test, [data_tmp, gamma, beta], [xrolling_mean, xrolling_std], numeric_eps=1e-2, rtol=0.2) + check_numeric_gradient(test, [data_tmp, gamma, beta], [xrolling_mean, xrolling_std], numeric_eps=1e-2, rtol=0.2, atol=0.01) test = mx.symbol.BatchNorm(data, fix_gamma=False, axis=chaxis) - check_numeric_gradient(test, [data_tmp, gamma, beta], [xrolling_mean, xrolling_std], numeric_eps=1e-2, rtol=0.2) + check_numeric_gradient(test, [data_tmp, gamma, beta], [xrolling_mean, xrolling_std], numeric_eps=1e-2, rtol=0.2, atol=0.01) test = mx.symbol.BatchNorm(data, fix_gamma=False, use_global_stats=True, axis=chaxis) - check_numeric_gradient(test, [data_tmp, gamma, beta], [xrolling_mean, xrolling_std], numeric_eps=1e-2, rtol=0.2) + check_numeric_gradient(test, [data_tmp, gamma, beta], [xrolling_mean, xrolling_std], numeric_eps=1e-2, rtol=0.2, atol=0.01) def test_convolution_grouping(): num_filter = 4 @@ -1973,7 +1973,7 @@ def check_instance_norm_with_shape(shape, xpu): exec1 = Y.bind(xpu, args = {'X':x, 'G':gamma, 'B':beta}) exec1.forward(is_train=False) out = exec1.outputs[0].asnumpy() - assert_almost_equal(out, np_out, rtol=1e-4) + assert_almost_equal(out, np_out, rtol=1e-4, atol=1e-4) check_numeric_gradient(Y, {'X':x.asnumpy(), 'G':gamma.asnumpy(), 'B':beta.asnumpy()}, numeric_eps=1e-2, rtol=1e-2, atol=1e-2) From 4f1fad3ac25da30d122d64c7302fa61bd362e0cb Mon Sep 17 00:00:00 2001 From: Przemyslaw Tredak Date: Thu, 20 Jul 2017 15:55:48 -0700 Subject: [PATCH 242/834] New algo for batchnorm for cuDNN 7 (#7131) * New algo for batchnorm for cuDNN 7 * Fix Windows build --- src/operator/cudnn_batch_norm-inl.h | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/operator/cudnn_batch_norm-inl.h b/src/operator/cudnn_batch_norm-inl.h index 90e76581fa86..5c4179057294 100644 --- a/src/operator/cudnn_batch_norm-inl.h +++ b/src/operator/cudnn_batch_norm-inl.h @@ -94,6 +94,11 @@ class CuDNNBatchNormOp : public Operator { Tensor y = out_data[cudnnbatchnorm::kOut].get_with_shape(shape_, s); +#if CUDNN_VERSION >= 7000 + auto mode = CUDNN_BATCHNORM_SPATIAL_PERSISTENT; +#else + auto mode = CUDNN_BATCHNORM_SPATIAL; +#endif MSHADOW_REAL_TYPE_SWITCH(dtype_param_, DTypeParam, { Tensor gamma = @@ -118,7 +123,7 @@ class CuDNNBatchNormOp : public Operator { out_data[cudnnbatchnorm::kInvVar] .get_with_shape(Shape1(shape_[1]), s); CUDNN_CALL(cudnnBatchNormalizationForwardTraining(s->dnn_handle_, - CUDNN_BATCHNORM_SPATIAL, + mode, &a, &b, io_desc_, @@ -178,6 +183,11 @@ class CuDNNBatchNormOp : public Operator { out_grad[cudnnbatchnorm::kOut].get_with_shape(shape_, s); #if CUDNN_VERSION >= 4007 +#if CUDNN_VERSION >= 7000 + auto mode = CUDNN_BATCHNORM_SPATIAL_PERSISTENT; +#else + auto mode = CUDNN_BATCHNORM_SPATIAL; +#endif MSHADOW_REAL_TYPE_SWITCH(dtype_param_, DTypeParam, { Tensor gamma = in_data[cudnnbatchnorm::kGamma].get_with_shape(Shape1(shape_[1]), s); @@ -199,7 +209,7 @@ class CuDNNBatchNormOp : public Operator { CUDNN_CALL(cudnnBatchNormalizationBackward( s->dnn_handle_, - CUDNN_BATCHNORM_SPATIAL, + mode, &a, &b, &a, From f45f091debac7256751cffed58c61ee5bfa6ae1a Mon Sep 17 00:00:00 2001 From: Przemyslaw Tredak Date: Thu, 20 Jul 2017 15:58:19 -0700 Subject: [PATCH 243/834] Volta arch support in Makefile and cooperative groups (#7130) * Start of cooperative groups * Changing shfl_xor in batch_norm.cu * Fixing shfl_xor * Add Volta arch support to Makefile --- Makefile | 2 +- src/operator/batch_norm.cu | 5 +++++ src/operator/tensor/indexing_op-inl.cuh | 6 ++++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 749c761f062f..f200b876db45 100644 --- a/Makefile +++ b/Makefile @@ -171,7 +171,7 @@ endif # be JIT-compiled by the updated driver from the included PTX. ifeq ($(USE_CUDA), 1) ifeq ($(origin CUDA_ARCH), undefined) - KNOWN_CUDA_ARCHS := 30 35 50 52 60 61 + KNOWN_CUDA_ARCHS := 30 35 50 52 60 61 70 # Run nvcc on a zero-length file to check architecture-level support. # Create args to include SASS in the fat binary for supported levels. CUDA_ARCH := $(foreach arch,$(KNOWN_CUDA_ARCHS), \ diff --git a/src/operator/batch_norm.cu b/src/operator/batch_norm.cu index 1d27427653b8..9f7370f00faa 100644 --- a/src/operator/batch_norm.cu +++ b/src/operator/batch_norm.cu @@ -117,6 +117,11 @@ struct GradOp { const DeviceTensor gradOutput; }; +#if CUDA_VERSION >= 9000 +#define FULLMASK 0xFFFFFFFF +#define __shfl_xor(...) __shfl_xor_sync(FULLMASK, __VA_ARGS__) +#endif + // Sum across all threads within a warp template static __device__ __forceinline__ T warpSum(T val) { diff --git a/src/operator/tensor/indexing_op-inl.cuh b/src/operator/tensor/indexing_op-inl.cuh index 93a970a90ba6..2f366c2c3e48 100644 --- a/src/operator/tensor/indexing_op-inl.cuh +++ b/src/operator/tensor/indexing_op-inl.cuh @@ -9,6 +9,12 @@ #include #include +#if CUDA_VERSION >= 9000 +#define FULLMASK 0xFFFFFFFF +#define __ballot(x) __ballot_sync(FULLMASK, (x)) +#define __all(x) __all_sync(FULLMASK, (x)) +#endif + namespace mxnet { namespace op { const int kWarpSize = 32; From a150a7402e8aa609b6c7b1b1548fd2ec3e5ac64c Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Thu, 20 Jul 2017 15:59:26 -0700 Subject: [PATCH 244/834] recursively include modules in setup.py (#7128) --- python/setup.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/python/setup.py b/python/setup.py index 92dd0444ac2a..d4b132014c84 100644 --- a/python/setup.py +++ b/python/setup.py @@ -12,6 +12,7 @@ from setuptools import setup from setuptools.extension import Extension kwargs = {'install_requires': ['numpy', 'requests', 'graphviz'], 'zip_safe': False} +from setuptools import find_packages with_cython = False if '--with-cython' in sys.argv: @@ -73,11 +74,7 @@ def config_cython(): setup(name='mxnet', version=__version__, description=open(os.path.join(CURRENT_DIR, 'README.md')).read(), - packages=[ - 'mxnet', 'mxnet.module', 'mxnet._ctypes', 'mxnet.rnn', - 'mxnet._cy2', 'mxnet._cy3', 'mxnet.notebook', 'mxnet.contrib', - 'mxnet.gluon', 'mxnet.gluon.nn', 'mxnet.gluon.rnn' - ], + packages=find_packages(where=CURRENT_DIR), data_files=[('mxnet', [LIB_PATH[0]])], url='https://github.com/dmlc/mxnet', ext_modules=config_cython(), From 376279b847da1076b4ab93219b97bd79d1c33208 Mon Sep 17 00:00:00 2001 From: Mu Li Date: Thu, 20 Jul 2017 19:37:26 -0700 Subject: [PATCH 245/834] Remove gluon tutorials from index (#7140) --- docs/tutorials/index.md | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/docs/tutorials/index.md b/docs/tutorials/index.md index afbcee8f2224..c4863acf73b4 100644 --- a/docs/tutorials/index.md +++ b/docs/tutorials/index.md @@ -4,19 +4,7 @@ These tutorials introduce a few fundamental concepts in deep learning and how to ## Python -### Basics - High-level interface - -```eval_rst -.. toctree:: - :maxdepth: 1 - - gluon/ndarray - gluon/autograd - gluon/gluon - gluon/hybrid -``` - -### Advanced -- Low-level interface +### Basic ```eval_rst .. toctree:: From 799ed45d3471d5a609dcaab9d0ff466e6eb6ecf4 Mon Sep 17 00:00:00 2001 From: Mu Li Date: Thu, 20 Jul 2017 20:57:55 -0700 Subject: [PATCH 246/834] Update index.md (#7143) --- docs/tutorials/index.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/tutorials/index.md b/docs/tutorials/index.md index c4863acf73b4..32d8bd8ae9d1 100644 --- a/docs/tutorials/index.md +++ b/docs/tutorials/index.md @@ -2,6 +2,8 @@ These tutorials introduce a few fundamental concepts in deep learning and how to implement them in _MXNet_. The _Basics_ section contains tutorials on manipulating arrays, building networks, loading/preprocessing data, etc. The _Training and Inference_ section talks about implementing Linear Regression, training a Handwritten digit classifier using MLP and CNN, running inferences using a pre-trained model, and lastly, efficiently training a large scale image classifier. +**Note:** We are working on a set of tutorials for the new imperative interface called Gluon. A preview version is hosted at [thestraightdope.mxnet.io](http://thestraightdope.mxnet.io). + ## Python ### Basic From 37c18232ee9c80386ddb210a5deb55b9ad8f4ce6 Mon Sep 17 00:00:00 2001 From: Eric Junyuan Xie Date: Fri, 21 Jul 2017 12:39:29 -0700 Subject: [PATCH 247/834] add save_params and load_params to Block (#7097) * add save_params and load_params to Block * fix --- python/mxnet/gluon/block.py | 70 ++++++++++++++++++++++---------- python/mxnet/gluon/parameter.py | 50 ++++++++++++++++++++--- tests/python/unittest/test_nn.py | 5 +++ 3 files changed, 97 insertions(+), 28 deletions(-) diff --git a/python/mxnet/gluon/block.py b/python/mxnet/gluon/block.py index 0916e2345fe4..0d47d2fc1e2c 100644 --- a/python/mxnet/gluon/block.py +++ b/python/mxnet/gluon/block.py @@ -137,15 +137,6 @@ def __init__(self, prefix=None, params=None): self._scope = _BlockScope(self) self._children = [] - def __setattr__(self, name, value): - """Registers parameters.""" - super(Block, self).__setattr__(name, value) - if isinstance(value, Block): - self.register_child(value) - - def _alias(self): - return self.__class__.__name__.lower() - def __repr__(self): s = '{name}(\n{modstr}\n)' modstr = '\n'.join([' ({key}): {block}'.format(key=key, @@ -154,20 +145,14 @@ def __repr__(self): return s.format(name=self.__class__.__name__, modstr=modstr) - @property - def params(self): - """Returns this `Block`'s parameter dictionary (does not include its - children's parameters).""" - return self._params + def __setattr__(self, name, value): + """Registers parameters.""" + super(Block, self).__setattr__(name, value) + if isinstance(value, Block): + self.register_child(value) - def collect_params(self): - """Returns a `ParameterDict` containing this `Block` and all of its - children's Parameters.""" - ret = ParameterDict(self._params.prefix) - ret.update(self.params) - for cld in self._children: - ret.update(cld.collect_params()) - return ret + def _alias(self): + return self.__class__.__name__.lower() @property def prefix(self): @@ -190,6 +175,47 @@ def name_scope(self): """ return self._scope + @property + def params(self): + """Returns this `Block`'s parameter dictionary (does not include its + children's parameters).""" + return self._params + + def collect_params(self): + """Returns a `ParameterDict` containing this `Block` and all of its + children's Parameters.""" + ret = ParameterDict(self._params.prefix) + ret.update(self.params) + for cld in self._children: + ret.update(cld.collect_params()) + return ret + + def save_params(self, filename): + """Save parameters to file. + + filename : str + Path to file. + """ + self.collect_params().save(filename, strip_prefix=self.prefix) + + def load_params(self, filename, ctx, allow_missing=False, + ignore_extra=False): + """Load parameters from file. + + filename : str + Path to parameter file. + ctx : Context or list of Context + Context(s) initialize loaded parameters on. + allow_missing : bool, default False + Whether to silently skip loading parameters not represents in the file. + ignore_extra : bool, default False + Whether to silently ignore parameters from the file that are not + present in this Block. + """ + self.collect_params().load(filename, ctx, allow_missing, ignore_extra, + self.prefix) + + def register_child(self, block): """Registers block as a child of self. `Block`s assigned to self as attributes will be registered automatically.""" diff --git a/python/mxnet/gluon/parameter.py b/python/mxnet/gluon/parameter.py index 1bf48f93a6b8..981b78b721e7 100644 --- a/python/mxnet/gluon/parameter.py +++ b/python/mxnet/gluon/parameter.py @@ -5,6 +5,7 @@ from collections import OrderedDict import numpy as np + from ..base import mx_real_t, MXNetError from .. import symbol, ndarray, initializer, context from ..context import Context @@ -425,24 +426,61 @@ def zero_grad(self): for i in self.values(): i.zero_grad() - def save(self, filename): + def save(self, filename, strip_prefix=''): + """Save parameters to file. + + filename : str + Path to parameter file. + strip_prefix : str, default '' + Strip prefix from parameter names before saving. + """ arg_dict = {} for param in self.values(): block = param.list_data() weight = sum(w.copyto(context.cpu()) for w in block) / len(block) - arg_dict[param.name] = weight + if not param.name.startswith(strip_prefix): + raise ValueError( + "Prefix %s is to be striped before saving, but Parameter " \ + "%s does not start with %s. If you are using Block.save_params, " \ + "This may be due to your Block shares parameters from other " \ + "Blocks or you forgot to use `with name_scope()`` during init. " \ + "Consider switching to Block.collect_params.save and " \ + "Block.collect_params.load instead."%( + strip_prefix, param.name, strip_prefix)) + arg_dict[param.name[len(strip_prefix):]] = weight ndarray.save(filename, arg_dict) - def load(self, filename, ctx, allow_missing=False, ignore_extra=False): - arg_dict = ndarray.load(filename) + def load(self, filename, ctx, allow_missing=False, + ignore_extra=False, restore_prefix=''): + """Load parameters from file. + + filename : str + Path to parameter file. + ctx : Context or list of Context + Context(s) initialize loaded parameters on. + allow_missing : bool, default False + Whether to silently skip loading parameters not represents in the file. + ignore_extra : bool, default False + Whether to silently ignore parameters from the file that are not + present in this ParameterDict. + restore_prefix : str, default '' + prepend prefix to names of stored parameters before loading. + """ + if restore_prefix: + for name in self.keys(): + assert name.startswith(restore_prefix), \ + "restore_prefix is %s but Parameters name %s does not start " \ + "with %s"%(restore_prefix, name, restore_prefix) + lprefix = len(restore_prefix) + arg_dict = {restore_prefix+k: v for k, v in ndarray.load(filename).items()} if not allow_missing: for name in self.keys(): assert name in arg_dict, \ - "Parameter %s is missing in file %s"%(name, filename) + "Parameter %s is missing in file %s"%(name[lprefix:], filename) for name in arg_dict: if name not in self._params: assert ignore_extra, \ "Parameter %s loaded from file %s is not present in ParameterDict"%( - name, filename) + name[lprefix:], filename) continue self[name]._load_init(arg_dict[name], ctx) diff --git a/tests/python/unittest/test_nn.py b/tests/python/unittest/test_nn.py index cc1b2dd48553..58839785b9f2 100644 --- a/tests/python/unittest/test_nn.py +++ b/tests/python/unittest/test_nn.py @@ -39,6 +39,11 @@ def forward(self, x): net1.collect_params().initialize() net2(mx.nd.zeros((3, 5))) + net1.save_params('net1.params') + + net3 = Net(prefix='net3_') + net3.load_params('net1.params', mx.cpu()) + def test_basic(): model = nn.Sequential() From 266cf3a60a8bd4100b24746e0836b08f85112917 Mon Sep 17 00:00:00 2001 From: Przemyslaw Tredak Date: Fri, 21 Jul 2017 14:12:33 -0700 Subject: [PATCH 248/834] Streamlined fp16 examples (#7150) Also added fp16 versions of inception-v3, inception-v4 and resnext --- example/image-classification/common/data.py | 8 +- example/image-classification/common/fit.py | 2 + .../image-classification/symbols/alexnet.py | 57 +++--- .../symbols/alexnet_fp16.py | 69 ------- .../symbols/inception-v3.py | 18 +- .../symbols/inception-v4.py | 57 +++--- .../symbols/resnet-v1-fp16.py | 185 ----------------- .../image-classification/symbols/resnet-v1.py | 32 ++- .../image-classification/symbols/resnet.py | 28 ++- .../symbols/resnet_fp16.py | 191 ------------------ .../image-classification/symbols/resnext.py | 27 ++- .../image-classification/train_imagenet.py | 1 + 12 files changed, 139 insertions(+), 536 deletions(-) delete mode 100755 example/image-classification/symbols/alexnet_fp16.py delete mode 100755 example/image-classification/symbols/resnet-v1-fp16.py delete mode 100755 example/image-classification/symbols/resnet_fp16.py diff --git a/example/image-classification/common/data.py b/example/image-classification/common/data.py index 7a609b77474e..fe27ec26b939 100755 --- a/example/image-classification/common/data.py +++ b/example/image-classification/common/data.py @@ -19,8 +19,6 @@ def add_data_args(parser): help='number of threads for data decoding') data.add_argument('--benchmark', type=int, default=0, help='if 1, then feed the network with synthetic data') - data.add_argument('--dtype', type=str, default='float32', - help='data type: float32 or float16') return data def add_data_aug_args(parser): @@ -93,13 +91,9 @@ def reset(self): def get_rec_iter(args, kv=None): image_shape = tuple([int(l) for l in args.image_shape.split(',')]) - dtype = np.float32; - if 'dtype' in args: - if args.dtype == 'float16': - dtype = np.float16 if 'benchmark' in args and args.benchmark: data_shape = (args.batch_size,) + image_shape - train = SyntheticDataIter(args.num_classes, data_shape, 50, dtype) + train = SyntheticDataIter(args.num_classes, data_shape, 500, np.float32) return (train, None) if kv: (rank, nworker) = (kv.rank, kv.num_workers) diff --git a/example/image-classification/common/fit.py b/example/image-classification/common/fit.py index 82bcde401336..69baed1ef4f6 100755 --- a/example/image-classification/common/fit.py +++ b/example/image-classification/common/fit.py @@ -84,6 +84,8 @@ def add_fit_args(parser): help='report the top-k accuracy. 0 means no report.') train.add_argument('--test-io', type=int, default=0, help='1 means test reading speed without training') + train.add_argument('--dtype', type=str, default='float32', + help='precision: float32 or float16') return train def fit(args, network, data_loader, **kwargs): diff --git a/example/image-classification/symbols/alexnet.py b/example/image-classification/symbols/alexnet.py index 4931c269352b..2534797a9eba 100755 --- a/example/image-classification/symbols/alexnet.py +++ b/example/image-classification/symbols/alexnet.py @@ -4,43 +4,48 @@ Krizhevsky, Alex, Ilya Sutskever, and Geoffrey E. Hinton. "Imagenet classification with deep convolutional neural networks." Advances in neural information processing systems. 2012. """ import mxnet as mx +import numpy as np -def get_symbol(num_classes, **kwargs): - input_data = mx.symbol.Variable(name="data") +def get_symbol(num_classes, dtype, **kwargs): + input_data = mx.sym.Variable(name="data") + if dtype == 'float16': + input_data = mx.sym.Cast(data=input_data, dtype=np.float16) # stage 1 - conv1 = mx.symbol.Convolution(name='conv1', + conv1 = mx.sym.Convolution(name='conv1', data=input_data, kernel=(11, 11), stride=(4, 4), num_filter=96) - relu1 = mx.symbol.Activation(data=conv1, act_type="relu") - lrn1 = mx.symbol.LRN(data=relu1, alpha=0.0001, beta=0.75, knorm=2, nsize=5) - pool1 = mx.symbol.Pooling( + relu1 = mx.sym.Activation(data=conv1, act_type="relu") + lrn1 = mx.sym.LRN(data=relu1, alpha=0.0001, beta=0.75, knorm=2, nsize=5) + pool1 = mx.sym.Pooling( data=lrn1, pool_type="max", kernel=(3, 3), stride=(2,2)) # stage 2 - conv2 = mx.symbol.Convolution(name='conv2', + conv2 = mx.sym.Convolution(name='conv2', data=pool1, kernel=(5, 5), pad=(2, 2), num_filter=256) - relu2 = mx.symbol.Activation(data=conv2, act_type="relu") - lrn2 = mx.symbol.LRN(data=relu2, alpha=0.0001, beta=0.75, knorm=2, nsize=5) - pool2 = mx.symbol.Pooling(data=lrn2, kernel=(3, 3), stride=(2, 2), pool_type="max") + relu2 = mx.sym.Activation(data=conv2, act_type="relu") + lrn2 = mx.sym.LRN(data=relu2, alpha=0.0001, beta=0.75, knorm=2, nsize=5) + pool2 = mx.sym.Pooling(data=lrn2, kernel=(3, 3), stride=(2, 2), pool_type="max") # stage 3 - conv3 = mx.symbol.Convolution(name='conv3', + conv3 = mx.sym.Convolution(name='conv3', data=pool2, kernel=(3, 3), pad=(1, 1), num_filter=384) - relu3 = mx.symbol.Activation(data=conv3, act_type="relu") - conv4 = mx.symbol.Convolution(name='conv4', + relu3 = mx.sym.Activation(data=conv3, act_type="relu") + conv4 = mx.sym.Convolution(name='conv4', data=relu3, kernel=(3, 3), pad=(1, 1), num_filter=384) - relu4 = mx.symbol.Activation(data=conv4, act_type="relu") - conv5 = mx.symbol.Convolution(name='conv5', + relu4 = mx.sym.Activation(data=conv4, act_type="relu") + conv5 = mx.sym.Convolution(name='conv5', data=relu4, kernel=(3, 3), pad=(1, 1), num_filter=256) - relu5 = mx.symbol.Activation(data=conv5, act_type="relu") - pool3 = mx.symbol.Pooling(data=relu5, kernel=(3, 3), stride=(2, 2), pool_type="max") + relu5 = mx.sym.Activation(data=conv5, act_type="relu") + pool3 = mx.sym.Pooling(data=relu5, kernel=(3, 3), stride=(2, 2), pool_type="max") # stage 4 - flatten = mx.symbol.Flatten(data=pool3) - fc1 = mx.symbol.FullyConnected(name='fc1', data=flatten, num_hidden=4096) - relu6 = mx.symbol.Activation(data=fc1, act_type="relu") - dropout1 = mx.symbol.Dropout(data=relu6, p=0.5) + flatten = mx.sym.Flatten(data=pool3) + fc1 = mx.sym.FullyConnected(name='fc1', data=flatten, num_hidden=4096) + relu6 = mx.sym.Activation(data=fc1, act_type="relu") + dropout1 = mx.sym.Dropout(data=relu6, p=0.5) # stage 5 - fc2 = mx.symbol.FullyConnected(name='fc2', data=dropout1, num_hidden=4096) - relu7 = mx.symbol.Activation(data=fc2, act_type="relu") - dropout2 = mx.symbol.Dropout(data=relu7, p=0.5) + fc2 = mx.sym.FullyConnected(name='fc2', data=dropout1, num_hidden=4096) + relu7 = mx.sym.Activation(data=fc2, act_type="relu") + dropout2 = mx.sym.Dropout(data=relu7, p=0.5) # stage 6 - fc3 = mx.symbol.FullyConnected(name='fc3', data=dropout2, num_hidden=num_classes) - softmax = mx.symbol.SoftmaxOutput(data=fc3, name='softmax') + fc3 = mx.sym.FullyConnected(name='fc3', data=dropout2, num_hidden=num_classes) + if dtype == 'float16': + fc3 = mx.sym.Cast(data=fc3, dtype=np.float32) + softmax = mx.sym.SoftmaxOutput(data=fc3, name='softmax') return softmax diff --git a/example/image-classification/symbols/alexnet_fp16.py b/example/image-classification/symbols/alexnet_fp16.py deleted file mode 100755 index 9e7d4dc0d822..000000000000 --- a/example/image-classification/symbols/alexnet_fp16.py +++ /dev/null @@ -1,69 +0,0 @@ -""" -Reference: - -Krizhevsky, Alex, Ilya Sutskever, and Geoffrey E. Hinton. "Imagenet classification with deep convolutional neural networks." Advances in neural information processing systems. 2012. -""" -import mxnet as mx -import numpy as np - -def get_symbol(num_classes, **kwargs): - input_data = mx.symbol.Variable(name="data") - input_data = mx.symbol.Cast(data=input_data, dtype=np.float16) - # stage 1 - weight = mx.symbol.Variable(name='conv1_weight', dtype=np.float16) - bias = mx.symbol.Variable(name='conv1_bias', dtype=np.float16) - conv1 = mx.symbol.Convolution(name='conv1', - data=input_data, weight=weight, bias=bias, kernel=(11, 11), stride=(4, 4), num_filter=96) - relu1 = mx.symbol.Activation(data=conv1, act_type="relu") - lrn1 = mx.symbol.LRN(data=relu1, alpha=0.0001, beta=0.75, knorm=2, nsize=5) - pool1 = mx.symbol.Pooling( - data=lrn1, pool_type="max", kernel=(3, 3), stride=(2,2)) - # stage 2 - weight = mx.symbol.Variable(name='conv2_weight', dtype=np.float16) - bias = mx.symbol.Variable(name='conv2_bias', dtype=np.float16) - conv2 = mx.symbol.Convolution(name='conv2', - data=pool1, weight=weight, bias=bias, kernel=(5, 5), pad=(2, 2), num_filter=256) - relu2 = mx.symbol.Activation(data=conv2, act_type="relu") - lrn2 = mx.symbol.LRN(data=relu2, alpha=0.0001, beta=0.75, knorm=2, nsize=5) - pool2 = mx.symbol.Pooling(data=lrn2, kernel=(3, 3), stride=(2, 2), pool_type="max") - # stage 3 - weight = mx.symbol.Variable(name='conv3_weight', dtype=np.float16) - bias = mx.symbol.Variable(name='conv3_bias', dtype=np.float16) - conv3 = mx.symbol.Convolution(name='conv3', - data=pool2, weight=weight, bias=bias, kernel=(3, 3), pad=(1, 1), num_filter=384) - relu3 = mx.symbol.Activation(data=conv3, act_type="relu") - weight = mx.symbol.Variable(name='conv4_weight', dtype=np.float16) - bias = mx.symbol.Variable(name='conv4_bias', dtype=np.float16) - conv4 = mx.symbol.Convolution(name='conv4', - data=relu3, weight=weight, bias=bias, kernel=(3, 3), pad=(1, 1), num_filter=384) - relu4 = mx.symbol.Activation(data=conv4, act_type="relu") - weight = mx.symbol.Variable(name='conv5_weight', dtype=np.float16) - bias = mx.symbol.Variable(name='conv5_bias', dtype=np.float16) - conv5 = mx.symbol.Convolution(name='conv5', - data=relu4, weight=weight, bias=bias, kernel=(3, 3), pad=(1, 1), num_filter=256) - relu5 = mx.symbol.Activation(data=conv5, act_type="relu") - pool3 = mx.symbol.Pooling(data=relu5, kernel=(3, 3), stride=(2, 2), pool_type="max") - # stage 4 - flatten = mx.symbol.Flatten(data=pool3) - weight = mx.symbol.Variable(name='fc1_weight', dtype=np.float16) - bias = mx.symbol.Variable(name='fc1_bias', dtype=np.float16) - fc1 = mx.symbol.FullyConnected(name='fc1', data=flatten, weight=weight, bias=bias, - num_hidden=4096) - relu6 = mx.symbol.Activation(data=fc1, act_type="relu") - dropout1 = mx.symbol.Dropout(data=relu6, p=0.5) - # stage 5 - weight = mx.symbol.Variable(name='fc2_weight', dtype=np.float16) - bias = mx.symbol.Variable(name='fc2_bias', dtype=np.float16) - fc2 = mx.symbol.FullyConnected(name='fc2', data=dropout1, weight=weight, bias=bias, - num_hidden=4096) - relu7 = mx.symbol.Activation(data=fc2, act_type="relu") - dropout2 = mx.symbol.Dropout(data=relu7, p=0.5) - # stage 6 - weight = mx.symbol.Variable(name='fc3_weight', dtype=np.float16) - bias = mx.symbol.Variable(name='fc3_bias', dtype=np.float16) - fc3 = mx.symbol.FullyConnected(name='fc3', data=dropout2, weight=weight, bias=bias, - num_hidden=num_classes) - label = mx.symbol.Variable(name='softmax_label') - label = mx.symbol.Cast(data=label, dtype=np.float16) - softmax = mx.symbol.SoftmaxOutput(data=fc3, name='softmax', label=label) - return softmax diff --git a/example/image-classification/symbols/inception-v3.py b/example/image-classification/symbols/inception-v3.py index 1c38ae6d57c9..35562d663745 100644 --- a/example/image-classification/symbols/inception-v3.py +++ b/example/image-classification/symbols/inception-v3.py @@ -6,6 +6,7 @@ Szegedy, Christian, et al. "Rethinking the Inception Architecture for Computer Vision." arXiv preprint arXiv:1512.00567 (2015). """ import mxnet as mx +import numpy as np def Conv(data, num_filter, kernel=(1, 1), stride=(1, 1), pad=(0, 0), name=None, suffix=''): conv = mx.sym.Convolution(data=data, num_filter=num_filter, kernel=kernel, stride=stride, pad=pad, no_bias=True, name='%s%s_conv2d' %(name, suffix)) @@ -41,7 +42,7 @@ def Inception7B(data, tower_d3x3 = Conv(data, num_d3x3_red, name=('%s_tower' % name), suffix='_conv') tower_d3x3 = Conv(tower_d3x3, num_d3x3_1, kernel=(3, 3), pad=(1, 1), stride=(1, 1), name=('%s_tower' % name), suffix='_conv_1') tower_d3x3 = Conv(tower_d3x3, num_d3x3_2, kernel=(3, 3), pad=(0, 0), stride=(2, 2), name=('%s_tower' % name), suffix='_conv_2') - pooling = mx.symbol.Pooling(data=data, kernel=(3, 3), stride=(2, 2), pad=(0,0), pool_type="max", name=('max_pool_%s_pool' % name)) + pooling = mx.sym.Pooling(data=data, kernel=(3, 3), stride=(2, 2), pad=(0,0), pool_type="max", name=('max_pool_%s_pool' % name)) concat = mx.sym.Concat(*[tower_3x3, tower_d3x3, pooling], name='ch_concat_%s_chconcat' % name) return concat @@ -104,8 +105,13 @@ def Inception7E(data, # In[49]: -def get_symbol(num_classes=1000, **kwargs): - data = mx.symbol.Variable(name="data") +def get_symbol(num_classes=1000, dtype='float32', **kwargs): + data = mx.sym.Variable(name="data") + if dtype == 'float32': + data = mx.sym.identity(data=data, name='id') + else: + if dtype == 'float16': + data = mx.sym.Cast(data=data, dtype=np.float16) # stage 1 conv = Conv(data, 32, kernel=(3, 3), stride=(2, 2), name="conv") conv_1 = Conv(conv, 32, kernel=(3, 3), name="conv_1") @@ -163,6 +169,8 @@ def get_symbol(num_classes=1000, **kwargs): # pool pool = mx.sym.Pooling(data=in5b, kernel=(8, 8), stride=(1, 1), pool_type="avg", name="global_pool") flatten = mx.sym.Flatten(data=pool, name="flatten") - fc1 = mx.symbol.FullyConnected(data=flatten, num_hidden=num_classes, name='fc1') - softmax = mx.symbol.SoftmaxOutput(data=fc1, name='softmax') + fc1 = mx.sym.FullyConnected(data=flatten, num_hidden=num_classes, name='fc1') + if dtype == 'float16': + fc1 = mx.sym.Cast(data=fc1, dtype=np.float32) + softmax = mx.sym.SoftmaxOutput(data=fc1, name='softmax') return softmax diff --git a/example/image-classification/symbols/inception-v4.py b/example/image-classification/symbols/inception-v4.py index be81e30ccd31..eead5f7c7ea0 100644 --- a/example/image-classification/symbols/inception-v4.py +++ b/example/image-classification/symbols/inception-v4.py @@ -12,13 +12,13 @@ Christian Szegedy, Sergey Ioffe, Vincent Vanhoucke arXiv.1602.07261 ''' -import find_mxnet import mxnet as mx +import numpy as np def Conv(data, num_filter, kernel=(1, 1), stride=(1, 1), pad=(0, 0), name=None, suffix=''): - conv = mx.symbol.Convolution(data=data, num_filter=num_filter, kernel=kernel, stride=stride, pad=pad, no_bias=True, name='%s%s_conv2d' %(name, suffix)) - bn = mx.symbol.BatchNorm(data=conv, name='%s%s_batchnorm' %(name, suffix), fix_gamma=True) - act = mx.symbol.Activation(data=bn, act_type='relu', name='%s%s_relu' %(name, suffix)) + conv = mx.sym.Convolution(data=data, num_filter=num_filter, kernel=kernel, stride=stride, pad=pad, no_bias=True, name='%s%s_conv2d' %(name, suffix)) + bn = mx.sym.BatchNorm(data=conv, name='%s%s_batchnorm' %(name, suffix), fix_gamma=True) + act = mx.sym.Activation(data=bn, act_type='relu', name='%s%s_relu' %(name, suffix)) return act @@ -28,9 +28,9 @@ def Inception_stem(data, name= None): c = Conv(c, 32, kernel=(3, 3), name='%s_conv2_3*3' %name) c = Conv(c, 64, kernel=(3, 3), pad=(1, 1), name='%s_conv3_3*3' %name) - p1 = mx.symbol.Pooling(c, kernel=(3, 3), stride=(2, 2), pool_type='max', name='%s_maxpool_1' %name) + p1 = mx.sym.Pooling(c, kernel=(3, 3), stride=(2, 2), pool_type='max', name='%s_maxpool_1' %name) c2 = Conv(c, 96, kernel=(3, 3), stride=(2, 2), name='%s_conv4_3*3' %name) - concat = mx.symbol.Concat(*[p1, c2], name='%s_concat_1' %name) + concat = mx.sym.Concat(*[p1, c2], name='%s_concat_1' %name) c1 = Conv(concat, 64, kernel=(1, 1), pad=(0, 0), name='%s_conv5_1*1' %name) c1 = Conv(c1, 96, kernel=(3, 3), name='%s_conv6_3*3' %name) @@ -40,18 +40,18 @@ def Inception_stem(data, name= None): c2 = Conv(c2, 64, kernel=(1, 7), pad=(0, 3), name='%s_conv9_1*7' %name) c2 = Conv(c2, 96, kernel=(3, 3), pad=(0, 0), name='%s_conv10_3*3' %name) - concat = mx.symbol.Concat(*[c1, c2], name='%s_concat_2' %name) + concat = mx.sym.Concat(*[c1, c2], name='%s_concat_2' %name) c1 = Conv(concat, 192, kernel=(3, 3), stride=(2, 2), name='%s_conv11_3*3' %name) - p1 = mx.symbol.Pooling(concat, kernel=(3, 3), stride=(2, 2), pool_type='max', name='%s_maxpool_2' %name) + p1 = mx.sym.Pooling(concat, kernel=(3, 3), stride=(2, 2), pool_type='max', name='%s_maxpool_2' %name) - concat = mx.symbol.Concat(*[c1, p1], name='%s_concat_3' %name) + concat = mx.sym.Concat(*[c1, p1], name='%s_concat_3' %name) return concat def InceptionA(input, name=None): - p1 = mx.symbol.Pooling(input, kernel=(3, 3), pad=(1, 1), pool_type='avg', name='%s_avgpool_1' %name) + p1 = mx.sym.Pooling(input, kernel=(3, 3), pad=(1, 1), pool_type='avg', name='%s_avgpool_1' %name) c1 = Conv(p1, 96, kernel=(1, 1), pad=(0, 0), name='%s_conv1_1*1' %name) c2 = Conv(input, 96, kernel=(1, 1), pad=(0, 0), name='%s_conv2_1*1' %name) @@ -63,13 +63,13 @@ def InceptionA(input, name=None): c4 = Conv(c4, 96, kernel=(3, 3), pad=(1, 1), name='%s_conv6_3*3' % name) c4 = Conv(c4, 96, kernel=(3, 3), pad=(1, 1), name='%s_conv7_3*3' %name) - concat = mx.symbol.Concat(*[c1, c2, c3, c4], name='%s_concat_1' %name) + concat = mx.sym.Concat(*[c1, c2, c3, c4], name='%s_concat_1' %name) return concat def ReductionA(input, name=None): - p1 = mx.symbol.Pooling(input, kernel=(3, 3), stride=(2, 2), pool_type='max', name='%s_maxpool_1' %name) + p1 = mx.sym.Pooling(input, kernel=(3, 3), stride=(2, 2), pool_type='max', name='%s_maxpool_1' %name) c2 = Conv(input, 384, kernel=(3, 3), stride=(2, 2), name='%s_conv1_3*3' %name) @@ -77,12 +77,12 @@ def ReductionA(input, name=None): c3 = Conv(c3, 224, kernel=(3, 3), pad=(1, 1), name='%s_conv3_3*3' %name) c3 = Conv(c3, 256, kernel=(3, 3), stride=(2, 2), pad=(0, 0), name='%s_conv4_3*3' %name) - concat = mx.symbol.Concat(*[p1, c2, c3], name='%s_concat_1' %name) + concat = mx.sym.Concat(*[p1, c2, c3], name='%s_concat_1' %name) return concat def InceptionB(input, name=None): - p1 = mx.symbol.Pooling(input, kernel=(3, 3), pad=(1, 1), pool_type='avg', name='%s_avgpool_1' %name) + p1 = mx.sym.Pooling(input, kernel=(3, 3), pad=(1, 1), pool_type='avg', name='%s_avgpool_1' %name) c1 = Conv(p1, 128, kernel=(1, 1), pad=(0, 0), name='%s_conv1_1*1' %name) c2 = Conv(input, 384, kernel=(1, 1), pad=(0, 0), name='%s_conv2_1*1' %name) @@ -103,7 +103,7 @@ def InceptionB(input, name=None): return concat def ReductionB(input,name=None): - p1 = mx.symbol.Pooling(input, kernel=(3, 3), stride=(2, 2), pool_type='max', name='%s_maxpool_1' %name) + p1 = mx.sym.Pooling(input, kernel=(3, 3), stride=(2, 2), pool_type='max', name='%s_maxpool_1' %name) c2 = Conv(input, 192, kernel=(1, 1), pad=(0, 0), name='%s_conv1_1*1' %name) c2 = Conv(c2, 192, kernel=(3, 3), stride=(2, 2), name='%s_conv2_3*3' %name) @@ -113,13 +113,13 @@ def ReductionB(input,name=None): c3 = Conv(c3, 320, kernel=(7, 1), pad=(3, 0), name='%s_conv5_7*1' %name) c3 = Conv(c3, 320, kernel=(3, 3), stride=(2, 2), name='%s_conv6_3*3' %name) - concat = mx.symbol.Concat(*[p1, c2, c3], name='%s_concat_1' %name) + concat = mx.sym.Concat(*[p1, c2, c3], name='%s_concat_1' %name) return concat def InceptionC(input, name=None): - p1 = mx.symbol.Pooling(input, kernel=(3, 3), pad=(1, 1), pool_type='avg', name='%s_avgpool_1' %name) + p1 = mx.sym.Pooling(input, kernel=(3, 3), pad=(1, 1), pool_type='avg', name='%s_avgpool_1' %name) c1 = Conv(p1, 256, kernel=(1, 1), pad=(0, 0), name='%s_conv1_1*1' %name) c2 = Conv(input, 256, kernel=(1, 1), pad=(0, 0), name='%s_conv2_1*1' %name) @@ -134,13 +134,18 @@ def InceptionC(input, name=None): c4_1 = Conv(c4, 256, kernel=(3, 1), pad=(1, 0), name='%s_conv9_1*3' %name) c4_2 = Conv(c4, 256, kernel=(1, 3), pad=(0, 1), name='%s_conv10_3*1' %name) - concat = mx.symbol.Concat(*[c1, c2, c3_1, c3_2, c4_1, c4_2], name='%s_concat' %name) + concat = mx.sym.Concat(*[c1, c2, c3_1, c3_2, c4_1, c4_2], name='%s_concat' %name) return concat -def get_symbol(num_classes=1000, **kwargs): - data = mx.symbol.Variable(name="data") +def get_symbol(num_classes=1000, dtype='float32', **kwargs): + data = mx.sym.Variable(name="data") + if dtype == 'float32': + data = mx.sym.identity(data=data, name='id') + else: + if dtype == 'float16': + data = mx.sym.Cast(data=data, dtype=np.float16) x = Inception_stem(data, name='in_stem') #4 * InceptionA @@ -179,13 +184,15 @@ def get_symbol(num_classes=1000, **kwargs): x = InceptionC(x, name='in%dC' %(i+1)) #Average Pooling - x = mx.symbol.Pooling(x, kernel=(8, 8), pad=(1, 1), pool_type='avg', name='global_avgpool') + x = mx.sym.Pooling(x, kernel=(8, 8), pad=(1, 1), pool_type='avg', name='global_avgpool') #Dropout - x = mx.symbol.Dropout(x, p=0.2) + x = mx.sym.Dropout(x, p=0.2) - flatten = mx.symbol.Flatten(x, name='flatten') - fc1 = mx.symbol.FullyConnected(flatten, num_hidden=num_classes, name='fc1') - softmax = mx.symbol.SoftmaxOutput(fc1, name='softmax') + flatten = mx.sym.Flatten(x, name='flatten') + fc1 = mx.sym.FullyConnected(flatten, num_hidden=num_classes, name='fc1') + if dtype == 'float16': + fc1 = mx.sym.Cast(data=fc1, dtype=np.float32) + softmax = mx.sym.SoftmaxOutput(fc1, name='softmax') return softmax diff --git a/example/image-classification/symbols/resnet-v1-fp16.py b/example/image-classification/symbols/resnet-v1-fp16.py deleted file mode 100755 index 1f0e2fe28023..000000000000 --- a/example/image-classification/symbols/resnet-v1-fp16.py +++ /dev/null @@ -1,185 +0,0 @@ -''' -Adapted from https://github.com/tornadomeet/ResNet/blob/master/symbol_resnet.py -(Original author Wei Wu) by Antti-Pekka Hynninen - -Implementing the original resnet ILSVRC 2015 winning network from: - -Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun. "Deep Residual Learning for Image Recognition" -''' -import mxnet as mx -import numpy as np - -def residual_unit(data, num_filter, stride, dim_match, name, bottle_neck=True, bn_mom=0.9, workspace=256, memonger=False): - """Return ResNet Unit symbol for building ResNet - Parameters - ---------- - data : str - Input data - num_filter : int - Number of output channels - bnf : int - Bottle neck channels factor with regard to num_filter - stride : tuple - Stride used in convolution - dim_match : Boolean - True means channel number between input and output is the same, otherwise means differ - name : str - Base name of the operators - workspace : int - Workspace used in convolution operator - """ - if bottle_neck: - weight = mx.symbol.Variable(name=name + '_conv1_weight', dtype=np.float16) - conv1 = mx.sym.Convolution(data=data, weight=weight, num_filter=int(num_filter*0.25), kernel=(1,1), stride=stride, pad=(0,0), - no_bias=True, workspace=workspace, name=name + '_conv1') - bn1 = mx.sym.BatchNorm(data=conv1, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn1') - act1 = mx.sym.Activation(data=bn1, act_type='relu', name=name + '_relu1') - weight = mx.symbol.Variable(name=name + '_conv2_weight', dtype=np.float16) - conv2 = mx.sym.Convolution(data=act1, weight=weight, num_filter=int(num_filter*0.25), kernel=(3,3), stride=(1,1), pad=(1,1), - no_bias=True, workspace=workspace, name=name + '_conv2') - bn2 = mx.sym.BatchNorm(data=conv2, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn2') - act2 = mx.sym.Activation(data=bn2, act_type='relu', name=name + '_relu2') - weight = mx.symbol.Variable(name=name + '_conv3_weight', dtype=np.float16) - conv3 = mx.sym.Convolution(data=act2, weight=weight, num_filter=num_filter, kernel=(1,1), stride=(1,1), pad=(0,0), no_bias=True, - workspace=workspace, name=name + '_conv3') - bn3 = mx.sym.BatchNorm(data=conv3, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn3') - - if dim_match: - shortcut = data - else: - weight = mx.symbol.Variable(name=name + '_conv1sc_weight', dtype=np.float16) - conv1sc = mx.sym.Convolution(data=data, weight=weight, num_filter=num_filter, kernel=(1,1), stride=stride, no_bias=True, - workspace=workspace, name=name+'_conv1sc') - shortcut = mx.sym.BatchNorm(data=conv1sc, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_sc') - if memonger: - shortcut._set_attr(mirror_stage='True') - return mx.sym.Activation(data=bn3 + shortcut, act_type='relu', name=name + '_relu3') - else: - weight = mx.symbol.Variable(name=name + '_conv1_weight', dtype=np.float16) - conv1 = mx.sym.Convolution(data=data, weight=weight, num_filter=num_filter, kernel=(3,3), stride=stride, pad=(1,1), - no_bias=True, workspace=workspace, name=name + '_conv1') - bn1 = mx.sym.BatchNorm(data=conv1, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_bn1') - act1 = mx.sym.Activation(data=bn1, act_type='relu', name=name + '_relu1') - weight = mx.symbol.Variable(name=name + '_conv2_weight', dtype=np.float16) - conv2 = mx.sym.Convolution(data=act1, weight=weight, num_filter=num_filter, kernel=(3,3), stride=(1,1), pad=(1,1), - no_bias=True, workspace=workspace, name=name + '_conv2') - bn2 = mx.sym.BatchNorm(data=conv2, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_bn2') - - if dim_match: - shortcut = data - else: - weight = mx.symbol.Variable(name=name + '_conv1sc_weight', dtype=np.float16) - conv1sc = mx.sym.Convolution(data=data, weight=weight, num_filter=num_filter, kernel=(1,1), stride=stride, no_bias=True, - workspace=workspace, name=name+'_conv1sc') - shortcut = mx.sym.BatchNorm(data=conv1sc, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_sc') - if memonger: - shortcut._set_attr(mirror_stage='True') - return mx.sym.Activation(data=bn2 + shortcut, act_type='relu', name=name + '_relu3') - -def resnet(units, num_stages, filter_list, num_classes, image_shape, bottle_neck=True, bn_mom=0.9, workspace=256, memonger=False): - """Return ResNet symbol of - Parameters - ---------- - units : list - Number of units in each stage - num_stages : int - Number of stage - filter_list : list - Channel size of each stage - num_classes : int - Ouput size of symbol - dataset : str - Dataset type, only cifar10 and imagenet supports - workspace : int - Workspace used in convolution operator - """ - num_unit = len(units) - assert(num_unit == num_stages) - data = mx.sym.Variable(name='data') - data = mx.symbol.Cast(data=data, dtype=np.float16) - (nchannel, height, width) = image_shape - weight = mx.symbol.Variable(name='conv0_weight', dtype=np.float16) - if height <= 32: # such as cifar10 - body = mx.sym.Convolution(data=data, weight=weight, num_filter=filter_list[0], kernel=(3, 3), stride=(1,1), pad=(1, 1), - no_bias=True, name="conv0", workspace=workspace) - # Is this BatchNorm supposed to be here? - body = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn0') - else: # often expected to be 224 such as imagenet - body = mx.sym.Convolution(data=data, weight=weight, num_filter=filter_list[0], kernel=(7, 7), stride=(2,2), pad=(3, 3), - no_bias=True, name="conv0", workspace=workspace) - body = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn0') - body = mx.sym.Activation(data=body, act_type='relu', name='relu0') - body = mx.symbol.Pooling(data=body, kernel=(3, 3), stride=(2,2), pad=(1,1), pool_type='max') - - for i in range(num_stages): - body = residual_unit(body, filter_list[i+1], (1 if i==0 else 2, 1 if i==0 else 2), False, - name='stage%d_unit%d' % (i + 1, 1), bottle_neck=bottle_neck, workspace=workspace, - memonger=memonger) - for j in range(units[i]-1): - body = residual_unit(body, filter_list[i+1], (1,1), True, name='stage%d_unit%d' % (i + 1, j + 2), - bottle_neck=bottle_neck, workspace=workspace, memonger=memonger) - # bn1 = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn1') - # relu1 = mx.sym.Activation(data=bn1, act_type='relu', name='relu1') - # Although kernel is not used here when global_pool=True, we should put one - pool1 = mx.symbol.Pooling(data=body, global_pool=True, kernel=(7, 7), pool_type='avg', name='pool1') - flat = mx.symbol.Flatten(data=pool1) - weight = mx.symbol.Variable(name='fc1_weight', dtype=np.float16) - bias = mx.symbol.Variable(name='fc1_bias', dtype=np.float16) - fc1 = mx.symbol.FullyConnected(data=flat, weight=weight, bias=bias, num_hidden=num_classes, name='fc1') - fc1 = mx.symbol.Cast(data=fc1, dtype=np.float32) - return mx.symbol.SoftmaxOutput(data=fc1, name='softmax') - -def get_symbol(num_classes, num_layers, image_shape, conv_workspace=256, **kwargs): - """ - Adapted from https://github.com/tornadomeet/ResNet/blob/master/symbol_resnet.py - (Original author Wei Wu) by Antti-Pekka Hynninen - Implementing the original resnet ILSVRC 2015 winning network from: - Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun. "Deep Residual Learning for Image Recognition" - """ - image_shape = [int(l) for l in image_shape.split(',')] - (nchannel, height, width) = image_shape - if height <= 28: - num_stages = 3 - if (num_layers-2) % 9 == 0 and num_layers >= 164: - per_unit = [(num_layers-2)//9] - filter_list = [16, 64, 128, 256] - bottle_neck = True - elif (num_layers-2) % 6 == 0 and num_layers < 164: - per_unit = [(num_layers-2)//6] - filter_list = [16, 16, 32, 64] - bottle_neck = False - else: - raise ValueError("no experiments done on num_layers {}, you can do it yourself".format(num_layers)) - units = per_unit * num_stages - else: - if num_layers >= 50: - filter_list = [64, 256, 512, 1024, 2048] - bottle_neck = True - else: - filter_list = [64, 64, 128, 256, 512] - bottle_neck = False - num_stages = 4 - if num_layers == 18: - units = [2, 2, 2, 2] - elif num_layers == 34: - units = [3, 4, 6, 3] - elif num_layers == 50: - units = [3, 4, 6, 3] - elif num_layers == 101: - units = [3, 4, 23, 3] - elif num_layers == 152: - units = [3, 8, 36, 3] - elif num_layers == 200: - units = [3, 24, 36, 3] - elif num_layers == 269: - units = [3, 30, 48, 8] - else: - raise ValueError("no experiments done on num_layers {}, you can do it yourself".format(num_layers)) - - return resnet(units = units, - num_stages = num_stages, - filter_list = filter_list, - num_classes = num_classes, - image_shape = image_shape, - bottle_neck = bottle_neck, - workspace = conv_workspace) diff --git a/example/image-classification/symbols/resnet-v1.py b/example/image-classification/symbols/resnet-v1.py index 2b898e5b8564..0d7bee1d16cd 100755 --- a/example/image-classification/symbols/resnet-v1.py +++ b/example/image-classification/symbols/resnet-v1.py @@ -7,6 +7,7 @@ Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun. "Deep Residual Learning for Image Recognition" ''' import mxnet as mx +import numpy as np def residual_unit(data, num_filter, stride, dim_match, name, bottle_neck=True, bn_mom=0.9, workspace=256, memonger=False): """Return ResNet Unit symbol for building ResNet @@ -68,7 +69,7 @@ def residual_unit(data, num_filter, stride, dim_match, name, bottle_neck=True, b shortcut._set_attr(mirror_stage='True') return mx.sym.Activation(data=bn2 + shortcut, act_type='relu', name=name + '_relu3') -def resnet(units, num_stages, filter_list, num_classes, image_shape, bottle_neck=True, bn_mom=0.9, workspace=256, memonger=False): +def resnet(units, num_stages, filter_list, num_classes, image_shape, bottle_neck=True, bn_mom=0.9, workspace=256, dtype='float32', memonger=False): """Return ResNet symbol of Parameters ---------- @@ -84,11 +85,17 @@ def resnet(units, num_stages, filter_list, num_classes, image_shape, bottle_neck Dataset type, only cifar10 and imagenet supports workspace : int Workspace used in convolution operator + dtype : str + Precision (float32 or float16) """ num_unit = len(units) assert(num_unit == num_stages) data = mx.sym.Variable(name='data') - data = mx.sym.identity(data=data, name='id') + if dtype == 'float32': + data = mx.sym.identity(data=data, name='id') + else: + if dtype == 'float16': + data = mx.sym.Cast(data=data, dtype=np.float16) (nchannel, height, width) = image_shape if height <= 32: # such as cifar10 body = mx.sym.Convolution(data=data, num_filter=filter_list[0], kernel=(3, 3), stride=(1,1), pad=(1, 1), @@ -100,7 +107,7 @@ def resnet(units, num_stages, filter_list, num_classes, image_shape, bottle_neck no_bias=True, name="conv0", workspace=workspace) body = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn0') body = mx.sym.Activation(data=body, act_type='relu', name='relu0') - body = mx.symbol.Pooling(data=body, kernel=(3, 3), stride=(2,2), pad=(1,1), pool_type='max') + body = mx.sym.Pooling(data=body, kernel=(3, 3), stride=(2,2), pad=(1,1), pool_type='max') for i in range(num_stages): body = residual_unit(body, filter_list[i+1], (1 if i==0 else 2, 1 if i==0 else 2), False, @@ -112,12 +119,14 @@ def resnet(units, num_stages, filter_list, num_classes, image_shape, bottle_neck # bn1 = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn1') # relu1 = mx.sym.Activation(data=bn1, act_type='relu', name='relu1') # Although kernel is not used here when global_pool=True, we should put one - pool1 = mx.symbol.Pooling(data=body, global_pool=True, kernel=(7, 7), pool_type='avg', name='pool1') - flat = mx.symbol.Flatten(data=pool1) - fc1 = mx.symbol.FullyConnected(data=flat, num_hidden=num_classes, name='fc1') - return mx.symbol.SoftmaxOutput(data=fc1, name='softmax') + pool1 = mx.sym.Pooling(data=body, global_pool=True, kernel=(7, 7), pool_type='avg', name='pool1') + flat = mx.sym.Flatten(data=pool1) + fc1 = mx.sym.FullyConnected(data=flat, num_hidden=num_classes, name='fc1') + if dtype == 'float16': + fc1 = mx.sym.Cast(data=fc1, dtype=np.float32) + return mx.sym.SoftmaxOutput(data=fc1, name='softmax') -def get_symbol(num_classes, num_layers, image_shape, conv_workspace=256, **kwargs): +def get_symbol(num_classes, num_layers, image_shape, conv_workspace=256, dtype='float32', **kwargs): """ Adapted from https://github.com/tornadomeet/ResNet/blob/master/symbol_resnet.py (Original author Wei Wu) by Antti-Pekka Hynninen @@ -137,7 +146,7 @@ def get_symbol(num_classes, num_layers, image_shape, conv_workspace=256, **kwarg filter_list = [16, 16, 32, 64] bottle_neck = False else: - raise ValueError("no experiments done on num_layers {}, you can do it youself".format(num_layers)) + raise ValueError("no experiments done on num_layers {}, you can do it yourself".format(num_layers)) units = per_unit * num_stages else: if num_layers >= 50: @@ -162,7 +171,7 @@ def get_symbol(num_classes, num_layers, image_shape, conv_workspace=256, **kwarg elif num_layers == 269: units = [3, 30, 48, 8] else: - raise ValueError("no experiments done on num_layers {}, you can do it youself".format(num_layers)) + raise ValueError("no experiments done on num_layers {}, you can do it yourself".format(num_layers)) return resnet(units = units, num_stages = num_stages, @@ -170,4 +179,5 @@ def get_symbol(num_classes, num_layers, image_shape, conv_workspace=256, **kwarg num_classes = num_classes, image_shape = image_shape, bottle_neck = bottle_neck, - workspace = conv_workspace) + workspace = conv_workspace, + dtype = dtype) diff --git a/example/image-classification/symbols/resnet.py b/example/image-classification/symbols/resnet.py index a0c7002dcad9..41cbc82bcb45 100644 --- a/example/image-classification/symbols/resnet.py +++ b/example/image-classification/symbols/resnet.py @@ -7,6 +7,7 @@ Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun. "Identity Mappings in Deep Residual Networks" ''' import mxnet as mx +import numpy as np def residual_unit(data, num_filter, stride, dim_match, name, bottle_neck=True, bn_mom=0.9, workspace=256, memonger=False): """Return ResNet Unit symbol for building ResNet @@ -67,7 +68,7 @@ def residual_unit(data, num_filter, stride, dim_match, name, bottle_neck=True, b shortcut._set_attr(mirror_stage='True') return conv2 + shortcut -def resnet(units, num_stages, filter_list, num_classes, image_shape, bottle_neck=True, bn_mom=0.9, workspace=256, memonger=False): +def resnet(units, num_stages, filter_list, num_classes, image_shape, bottle_neck=True, bn_mom=0.9, workspace=256, dtype='float32', memonger=False): """Return ResNet symbol of Parameters ---------- @@ -83,11 +84,17 @@ def resnet(units, num_stages, filter_list, num_classes, image_shape, bottle_neck Dataset type, only cifar10 and imagenet supports workspace : int Workspace used in convolution operator + dtype : str + Precision (float32 or float16) """ num_unit = len(units) assert(num_unit == num_stages) data = mx.sym.Variable(name='data') - data = mx.sym.identity(data=data, name='id') + if dtype == 'float32': + data = mx.sym.identity(data=data, name='id') + else: + if dtype == 'float16': + data = mx.sym.Cast(data=data, dtype=np.float16) data = mx.sym.BatchNorm(data=data, fix_gamma=True, eps=2e-5, momentum=bn_mom, name='bn_data') (nchannel, height, width) = image_shape if height <= 32: # such as cifar10 @@ -98,7 +105,7 @@ def resnet(units, num_stages, filter_list, num_classes, image_shape, bottle_neck no_bias=True, name="conv0", workspace=workspace) body = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn0') body = mx.sym.Activation(data=body, act_type='relu', name='relu0') - body = mx.symbol.Pooling(data=body, kernel=(3, 3), stride=(2,2), pad=(1,1), pool_type='max') + body = mx.sym.Pooling(data=body, kernel=(3, 3), stride=(2,2), pad=(1,1), pool_type='max') for i in range(num_stages): body = residual_unit(body, filter_list[i+1], (1 if i==0 else 2, 1 if i==0 else 2), False, @@ -110,12 +117,14 @@ def resnet(units, num_stages, filter_list, num_classes, image_shape, bottle_neck bn1 = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn1') relu1 = mx.sym.Activation(data=bn1, act_type='relu', name='relu1') # Although kernel is not used here when global_pool=True, we should put one - pool1 = mx.symbol.Pooling(data=relu1, global_pool=True, kernel=(7, 7), pool_type='avg', name='pool1') - flat = mx.symbol.Flatten(data=pool1) - fc1 = mx.symbol.FullyConnected(data=flat, num_hidden=num_classes, name='fc1') - return mx.symbol.SoftmaxOutput(data=fc1, name='softmax') + pool1 = mx.sym.Pooling(data=relu1, global_pool=True, kernel=(7, 7), pool_type='avg', name='pool1') + flat = mx.sym.Flatten(data=pool1) + fc1 = mx.sym.FullyConnected(data=flat, num_hidden=num_classes, name='fc1') + if dtype == 'float16': + fc1 = mx.sym.Cast(data=fc1, dtype=np.float32) + return mx.sym.SoftmaxOutput(data=fc1, name='softmax') -def get_symbol(num_classes, num_layers, image_shape, conv_workspace=256, **kwargs): +def get_symbol(num_classes, num_layers, image_shape, conv_workspace=256, dtype='float32', **kwargs): """ Adapted from https://github.com/tornadomeet/ResNet/blob/master/train_resnet.py Original author Wei Wu @@ -166,4 +175,5 @@ def get_symbol(num_classes, num_layers, image_shape, conv_workspace=256, **kwarg num_classes = num_classes, image_shape = image_shape, bottle_neck = bottle_neck, - workspace = conv_workspace) + workspace = conv_workspace, + dtype = dtype) diff --git a/example/image-classification/symbols/resnet_fp16.py b/example/image-classification/symbols/resnet_fp16.py deleted file mode 100755 index 22d6d39dc36f..000000000000 --- a/example/image-classification/symbols/resnet_fp16.py +++ /dev/null @@ -1,191 +0,0 @@ -''' -Adapted from https://github.com/tornadomeet/ResNet/blob/master/symbol_resnet.py -Original author Wei Wu - -Implemented the following paper: - -Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun. "Identity Mappings in Deep Residual Networks" -''' -import mxnet as mx -import numpy as np - -def residual_unit(data, num_filter, stride, dim_match, name, bottle_neck=True, bn_mom=0.9, workspace=256, memonger=False): - """Return ResNet Unit symbol for building ResNet - Parameters - ---------- - data : str - Input data - num_filter : int - Number of output channels - bnf : int - Bottle neck channels factor with regard to num_filter - stride : tuple - Stride used in convolution - dim_match : Boolean - True means channel number between input and output is the same, otherwise means differ - name : str - Base name of the operators - workspace : int - Workspace used in convolution operator - """ - if bottle_neck: - # the same as https://github.com/facebook/fb.resnet.torch#notes, a bit difference with origin paper - bn1 = mx.sym.BatchNorm(data=data, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn1') - act1 = mx.sym.Activation(data=bn1, act_type='relu', name=name + '_relu1') - weight = mx.symbol.Variable(name=name + '_conv1_weight', dtype=np.float32) - weight = mx.symbol.Cast(data=weight, dtype=np.float16) - conv1 = mx.sym.Convolution(data=act1, weight=weight, num_filter=int(num_filter*0.25), kernel=(1,1), stride=(1,1), pad=(0,0), - no_bias=True, workspace=workspace, name=name + '_conv1') - bn2 = mx.sym.BatchNorm(data=conv1, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn2') - act2 = mx.sym.Activation(data=bn2, act_type='relu', name=name + '_relu2') - weight = mx.symbol.Variable(name=name + '_conv2_weight', dtype=np.float32) - weight = mx.symbol.Cast(data=weight, dtype=np.float16) - conv2 = mx.sym.Convolution(data=act2, weight=weight, num_filter=int(num_filter*0.25), kernel=(3,3), stride=stride, pad=(1,1), - no_bias=True, workspace=workspace, name=name + '_conv2') - bn3 = mx.sym.BatchNorm(data=conv2, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn3') - act3 = mx.sym.Activation(data=bn3, act_type='relu', name=name + '_relu3') - weight = mx.symbol.Variable(name=name + '_conv3_weight', dtype=np.float32) - weight = mx.symbol.Cast(data=weight, dtype=np.float16) - conv3 = mx.sym.Convolution(data=act3, weight=weight, num_filter=num_filter, kernel=(1,1), stride=(1,1), pad=(0,0), no_bias=True, - workspace=workspace, name=name + '_conv3') - if dim_match: - shortcut = data - else: - weight = mx.symbol.Variable(name=name + '_sc_weight', dtype=np.float32) - weight = mx.symbol.Cast(data=weight, dtype=np.float16) - shortcut = mx.sym.Convolution(data=act1, weight=weight, num_filter=num_filter, kernel=(1,1), stride=stride, no_bias=True, - workspace=workspace, name=name+'_sc') - if memonger: - shortcut._set_attr(mirror_stage='True') - return conv3 + shortcut - else: - bn1 = mx.sym.BatchNorm(data=data, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_bn1') - act1 = mx.sym.Activation(data=bn1, act_type='relu', name=name + '_relu1') - weight = mx.symbol.Variable(name=name + '_conv1_weight', dtype=np.float32) - weight = mx.symbol.Cast(data=weight, dtype=np.float16) - conv1 = mx.sym.Convolution(data=act1, weight=weight, num_filter=num_filter, kernel=(3,3), stride=stride, pad=(1,1), - no_bias=True, workspace=workspace, name=name + '_conv1') - bn2 = mx.sym.BatchNorm(data=conv1, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_bn2') - act2 = mx.sym.Activation(data=bn2, act_type='relu', name=name + '_relu2') - weight = mx.symbol.Variable(name=name + '_conv2_weight', dtype=np.float32) - weight = mx.symbol.Cast(data=weight, dtype=np.float16) - conv2 = mx.sym.Convolution(data=act2, weight=weight, num_filter=num_filter, kernel=(3,3), stride=(1,1), pad=(1,1), - no_bias=True, workspace=workspace, name=name + '_conv2') - if dim_match: - shortcut = data - else: - weight = mx.symbol.Variable(name=name + '_sc_weight', dtype=np.float32) - weight = mx.symbol.Cast(data=weight, dtype=np.float16) - shortcut = mx.sym.Convolution(data=act1, weight=weight, num_filter=num_filter, kernel=(1,1), stride=stride, no_bias=True, - workspace=workspace, name=name+'_sc') - if memonger: - shortcut._set_attr(mirror_stage='True') - return conv2 + shortcut - -def resnet(units, num_stages, filter_list, num_classes, image_shape, bottle_neck=True, bn_mom=0.9, workspace=256, memonger=False): - """Return ResNet symbol of - Parameters - ---------- - units : list - Number of units in each stage - num_stages : int - Number of stage - filter_list : list - Channel size of each stage - num_classes : int - Ouput size of symbol - dataset : str - Dataset type, only cifar10 and imagenet supports - workspace : int - Workspace used in convolution operator - """ - num_unit = len(units) - assert(num_unit == num_stages) - data = mx.sym.Variable(name='data') - data = mx.symbol.Cast(data=data, dtype=np.float16) - data = mx.sym.BatchNorm(data=data, fix_gamma=True, eps=2e-5, momentum=bn_mom, name='bn_data') - (nchannel, height, width) = image_shape - weight = mx.symbol.Variable(name='conv0_weight', dtype=np.float32) - weight = mx.symbol.Cast(data=weight, dtype=np.float16) - if height <= 32: # such as cifar10 - body = mx.sym.Convolution(data=data, weight=weight, num_filter=filter_list[0], kernel=(3, 3), stride=(1,1), pad=(1, 1), - no_bias=True, name="conv0", workspace=workspace) - else: # often expected to be 224 such as imagenet - body = mx.sym.Convolution(data=data, weight=weight, num_filter=filter_list[0], kernel=(7, 7), stride=(2,2), pad=(3, 3), - no_bias=True, name="conv0", workspace=workspace) - body = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn0') - body = mx.sym.Activation(data=body, act_type='relu', name='relu0') - body = mx.symbol.Pooling(data=body, kernel=(3, 3), stride=(2,2), pad=(1,1), pool_type='max') - - for i in range(num_stages): - body = residual_unit(body, filter_list[i+1], (1 if i==0 else 2, 1 if i==0 else 2), False, - name='stage%d_unit%d' % (i + 1, 1), bottle_neck=bottle_neck, workspace=workspace, - memonger=memonger) - for j in range(units[i]-1): - body = residual_unit(body, filter_list[i+1], (1,1), True, name='stage%d_unit%d' % (i + 1, j + 2), - bottle_neck=bottle_neck, workspace=workspace, memonger=memonger) - bn1 = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn1') - relu1 = mx.sym.Activation(data=bn1, act_type='relu', name='relu1') - # Although kernel is not used here when global_pool=True, we should put one - pool1 = mx.symbol.Pooling(data=relu1, global_pool=True, kernel=(7, 7), pool_type='avg', name='pool1') - flat = mx.symbol.Flatten(data=pool1) - weight = mx.symbol.Variable(name='fc1_weight', dtype=np.float32) - bias = mx.symbol.Variable(name='fc1_bias', dtype=np.float32) - weight = mx.symbol.Cast(data=weight, dtype=np.float16) - bias = mx.symbol.Cast(data=bias, dtype=np.float16) - fc1 = mx.symbol.FullyConnected(data=flat, weight=weight, bias=bias, num_hidden=num_classes, name='fc1') - fc1 = mx.symbol.Cast(data=fc1, dtype=np.float32) - return mx.symbol.SoftmaxOutput(data=fc1, name='softmax') - -def get_symbol(num_classes, num_layers, image_shape, conv_workspace=256, **kwargs): - """ - Adapted from https://github.com/tornadomeet/ResNet/blob/master/train_resnet.py - Original author Wei Wu - """ - image_shape = [int(l) for l in image_shape.split(',')] - (nchannel, height, width) = image_shape - if height <= 28: - num_stages = 3 - if (num_layers-2) % 9 == 0 and num_layers >= 164: - per_unit = [(num_layers-2)//9] - filter_list = [16, 64, 128, 256] - bottle_neck = True - elif (num_layers-2) % 6 == 0 and num_layers < 164: - per_unit = [(num_layers-2)//6] - filter_list = [16, 16, 32, 64] - bottle_neck = False - else: - raise ValueError("no experiments done on num_layers {}, you can do it yourself".format(num_layers)) - units = per_unit * num_stages - else: - if num_layers >= 50: - filter_list = [64, 256, 512, 1024, 2048] - bottle_neck = True - else: - filter_list = [64, 64, 128, 256, 512] - bottle_neck = False - num_stages = 4 - if num_layers == 18: - units = [2, 2, 2, 2] - elif num_layers == 34: - units = [3, 4, 6, 3] - elif num_layers == 50: - units = [3, 4, 6, 3] - elif num_layers == 101: - units = [3, 4, 23, 3] - elif num_layers == 152: - units = [3, 8, 36, 3] - elif num_layers == 200: - units = [3, 24, 36, 3] - elif num_layers == 269: - units = [3, 30, 48, 8] - else: - raise ValueError("no experiments done on num_layers {}, you can do it yourself".format(num_layers)) - - return resnet(units = units, - num_stages = num_stages, - filter_list = filter_list, - num_classes = num_classes, - image_shape = image_shape, - bottle_neck = bottle_neck, - workspace = conv_workspace) diff --git a/example/image-classification/symbols/resnext.py b/example/image-classification/symbols/resnext.py index b6e7d7101fc3..bd5b65621722 100644 --- a/example/image-classification/symbols/resnext.py +++ b/example/image-classification/symbols/resnext.py @@ -6,6 +6,7 @@ Saining Xie, Ross Girshick, Piotr Dollar, Zhuowen Tu, Kaiming He. "Aggregated Residual Transformations for Deep Neural Network" ''' import mxnet as mx +import numpy as np def residual_unit(data, num_filter, stride, dim_match, name, bottle_neck=True, num_group=32, bn_mom=0.9, workspace=256, memonger=False): """Return ResNet Unit symbol for building ResNet @@ -80,7 +81,7 @@ def residual_unit(data, num_filter, stride, dim_match, name, bottle_neck=True, n eltwise = bn2 + shortcut return mx.sym.Activation(data=eltwise, act_type='relu', name=name + '_relu') -def resnext(units, num_stages, filter_list, num_classes, num_group, image_shape, bottle_neck=True, bn_mom=0.9, workspace=256, memonger=False): +def resnext(units, num_stages, filter_list, num_classes, num_group, image_shape, bottle_neck=True, bn_mom=0.9, workspace=256, dtype='float32', memonger=False): """Return ResNeXt symbol of Parameters ---------- @@ -98,10 +99,17 @@ def resnext(units, num_stages, filter_list, num_classes, num_group, image_shape, Dataset type, only cifar10 and imagenet supports workspace : int Workspace used in convolution operator + dtype : str + Precision (float32 or float16) """ num_unit = len(units) assert(num_unit == num_stages) data = mx.sym.Variable(name='data') + if dtype == 'float32': + data = mx.sym.identity(data=data, name='id') + else: + if dtype == 'float16': + data = mx.sym.Cast(data=data, dtype=np.float16) data = mx.sym.BatchNorm(data=data, fix_gamma=True, eps=2e-5, momentum=bn_mom, name='bn_data') (nchannel, height, width) = image_shape if height <= 32: # such as cifar10 @@ -112,7 +120,7 @@ def resnext(units, num_stages, filter_list, num_classes, num_group, image_shape, no_bias=True, name="conv0", workspace=workspace) body = mx.sym.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn0') body = mx.sym.Activation(data=body, act_type='relu', name='relu0') - body = mx.symbol.Pooling(data=body, kernel=(3, 3), stride=(2,2), pad=(1,1), pool_type='max') + body = mx.sym.Pooling(data=body, kernel=(3, 3), stride=(2,2), pad=(1,1), pool_type='max') for i in range(num_stages): body = residual_unit(body, filter_list[i+1], (1 if i==0 else 2, 1 if i==0 else 2), False, @@ -122,12 +130,14 @@ def resnext(units, num_stages, filter_list, num_classes, num_group, image_shape, body = residual_unit(body, filter_list[i+1], (1,1), True, name='stage%d_unit%d' % (i + 1, j + 2), bottle_neck=bottle_neck, num_group=num_group, bn_mom=bn_mom, workspace=workspace, memonger=memonger) - pool1 = mx.symbol.Pooling(data=body, global_pool=True, kernel=(7, 7), pool_type='avg', name='pool1') - flat = mx.symbol.Flatten(data=pool1) - fc1 = mx.symbol.FullyConnected(data=flat, num_hidden=num_classes, name='fc1') - return mx.symbol.SoftmaxOutput(data=fc1, name='softmax') + pool1 = mx.sym.Pooling(data=body, global_pool=True, kernel=(7, 7), pool_type='avg', name='pool1') + flat = mx.sym.Flatten(data=pool1) + fc1 = mx.sym.FullyConnected(data=flat, num_hidden=num_classes, name='fc1') + if dtype == 'float16': + fc1 = mx.sym.Cast(data=fc1, dtype=np.float32) + return mx.sym.SoftmaxOutput(data=fc1, name='softmax') -def get_symbol(num_classes, num_layers, image_shape, num_group=32, conv_workspace=256, **kwargs): +def get_symbol(num_classes, num_layers, image_shape, num_group=32, conv_workspace=256, dtype='float32', **kwargs): """ Adapted from https://github.com/tornadomeet/ResNet/blob/master/train_resnet.py Original author Wei Wu @@ -179,4 +189,5 @@ def get_symbol(num_classes, num_layers, image_shape, num_group=32, conv_workspac num_group = num_group, image_shape = image_shape, bottle_neck = bottle_neck, - workspace = conv_workspace) + workspace = conv_workspace, + dtype = dtype) diff --git a/example/image-classification/train_imagenet.py b/example/image-classification/train_imagenet.py index 710398b3cb57..760ea6b22391 100644 --- a/example/image-classification/train_imagenet.py +++ b/example/image-classification/train_imagenet.py @@ -28,6 +28,7 @@ # train num_epochs = 80, lr_step_epochs = '30,60', + dtype = 'float32' ) args = parser.parse_args() From f3a4b7d75d0988a46cc8b390e8aa43074ef6774b Mon Sep 17 00:00:00 2001 From: Aston <22279212+astonzhang@users.noreply.github.com> Date: Fri, 21 Jul 2017 14:13:19 -0700 Subject: [PATCH 249/834] Fix a typo (#7151) --- python/mxnet/gluon/block.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/mxnet/gluon/block.py b/python/mxnet/gluon/block.py index 0d47d2fc1e2c..e8ec12be570b 100644 --- a/python/mxnet/gluon/block.py +++ b/python/mxnet/gluon/block.py @@ -122,7 +122,7 @@ def forward(self, x): Parameters ---------- prefix : str - Prefix acts like a name space. It will be prepended to the name of all + Prefix acts like a name space. It will be prepended to the names of all Parameters and child `Block`s in this `Block`'s `name_scope`. Prefix should be unique within one model to prevent name collisions. params : ParameterDict or None From 4b6f2171742186be0b017cbaea9c19fe2811642d Mon Sep 17 00:00:00 2001 From: Jencir Lee Date: Fri, 21 Jul 2017 22:18:26 +0100 Subject: [PATCH 250/834] Khatri-Rao product (#6567) * add Khatri-Rao product API * upgrade to std=c++11 * fix include path for macOS * fix setting result variable to all zero * add more comment about the memory layout of input matrices * rename krprod to row_wise_kronecker * add Khatri-Rao product * add inverse Khatri-Rao product * enforce non-empty input matrices * rename functions * small fix --- include/mxnet/c_lapack_api.h | 103 ++++++- src/operator/contrib/krprod.h | 234 ++++++++++++++++ tests/cpp/operator/krprod_test.cc | 429 ++++++++++++++++++++++++++++++ tests/cpp/unittest.mk | 20 +- 4 files changed, 775 insertions(+), 11 deletions(-) create mode 100644 src/operator/contrib/krprod.h create mode 100644 tests/cpp/operator/krprod_test.cc diff --git a/include/mxnet/c_lapack_api.h b/include/mxnet/c_lapack_api.h index 28f34ee20d54..440d284a8636 100644 --- a/include/mxnet/c_lapack_api.h +++ b/include/mxnet/c_lapack_api.h @@ -1,7 +1,7 @@ /*! * Copyright (c) 2017 by Contributors * \file c_lapack_api.h - * \brief Unified interface for LAPACK calls from within mxnet. + * \brief Unified interface for LAPACK calls from within mxnet. * Purpose is to hide the platform specific differences. */ #ifndef MXNET_C_LAPACK_API_H_ @@ -31,6 +31,9 @@ // - Must support compilation without lapack-package but issue runtime error in this case. #include +#include "mshadow/tensor.h" + +using namespace mshadow; extern "C" { // Fortran signatures @@ -41,6 +44,12 @@ extern "C" { MXNET_LAPACK_FSIGNATURE1(dpotrf, double) MXNET_LAPACK_FSIGNATURE1(spotri, float) MXNET_LAPACK_FSIGNATURE1(dpotri, double) + + void dposv_(char *uplo, int *n, int *nrhs, + double *a, int *lda, double *b, int *ldb, int *info); + + void sposv_(char *uplo, int *n, int *nrhs, + float *a, int *lda, float *b, int *ldb, int *info); } #define MXNET_LAPACK_ROW_MAJOR 101 @@ -54,6 +63,40 @@ extern "C" { inline char loup(char uplo, bool invert) { return invert ? (uplo == 'U' ? 'L' : 'U') : uplo; } + +/*! + * \brief Transpose matrix data in memory + * + * Equivalently we can see it as flipping the layout of the matrix + * between row-major and column-major. + * + * \param m number of rows of input matrix a + * \param n number of columns of input matrix a + * \param b output matrix + * \param ldb leading dimension of b + * \param a input matrix + * \param lda leading dimension of a + */ +template +inline void flip(int m, int n, DType *b, int ldb, DType *a, int lda); + +template <> +inline void flip(int m, int n, + float *b, int ldb, float *a, int lda) { + for (int i = 0; i < m; ++i) + for (int j = 0; j < n; ++j) + b[j * ldb + i] = a[i * lda + j]; +} + +template <> +inline void flip(int m, int n, + double *b, int ldb, double *a, int lda) { + for (int i = 0; i < m; ++i) + for (int j = 0; j < n; ++j) + b[j * ldb + i] = a[i * lda + j]; +} + + #if MXNET_USE_LAPACK #define MXNET_LAPACK_CWRAPPER1(func, dtype) \ @@ -70,6 +113,38 @@ inline char loup(char uplo, bool invert) { return invert ? (uplo == 'U' ? 'L' : MXNET_LAPACK_CWRAPPER1(spotri, float) MXNET_LAPACK_CWRAPPER1(dpotri, double) + inline int mxnet_lapack_sposv(int matrix_layout, char uplo, int n, int nrhs, + float *a, int lda, float *b, int ldb) { + int info; + if (matrix_layout == MXNET_LAPACK_ROW_MAJOR) { + // Transpose b to b_t of shape (nrhs, n) + float *b_t = new float[nrhs * n]; + flip(n, nrhs, b_t, n, b, ldb); + sposv_(&uplo, &n, &nrhs, a, &lda, b_t, &n, &info); + flip(nrhs, n, b, ldb, b_t, n); + delete [] b_t; + return info; + } + sposv_(&uplo, &n, &nrhs, a, &lda, b, &ldb, &info); + return info; + } + + inline int mxnet_lapack_dposv(int matrix_layout, char uplo, int n, int nrhs, + double *a, int lda, double *b, int ldb) { + int info; + if (matrix_layout == MXNET_LAPACK_ROW_MAJOR) { + // Transpose b to b_t of shape (nrhs, n) + double *b_t = new double[nrhs * n]; + flip(n, nrhs, b_t, n, b, ldb); + dposv_(&uplo, &n, &nrhs, a, &lda, b_t, &n, &info); + flip(nrhs, n, b, ldb, b_t, n); + delete [] b_t; + return info; + } + dposv_(&uplo, &n, &nrhs, a, &lda, b, &ldb, &info); + return info; + } + #else // use pragma message instead of warning @@ -83,11 +158,37 @@ inline char loup(char uplo, bool invert) { return invert ? (uplo == 'U' ? 'L' : LOG(FATAL) << "MXNet build without lapack. Function " << #func << " is not available."; \ return 1; \ } + + #define MXNET_LAPACK_UNAVAILABLE(func) \ + inline int mxnet_lapack_##func(...) { \ + LOG(FATAL) << "MXNet build without lapack. Function " << #func << " is not available."; \ + return 1; \ + } + MXNET_LAPACK_CWRAPPER1(spotrf, float) MXNET_LAPACK_CWRAPPER1(dpotrf, double) MXNET_LAPACK_CWRAPPER1(spotri, float) MXNET_LAPACK_CWRAPPER1(dpotri, double) + MXNET_LAPACK_UNAVAILABLE(sposv) + MXNET_LAPACK_UNAVAILABLE(dposv) + #endif +template +inline int MXNET_LAPACK_posv(int matrix_layout, char uplo, int n, int nrhs, + DType *a, int lda, DType *b, int ldb); + +template <> +inline int MXNET_LAPACK_posv(int matrix_layout, char uplo, int n, + int nrhs, float *a, int lda, float *b, int ldb) { + return mxnet_lapack_sposv(matrix_layout, uplo, n, nrhs, a, lda, b, ldb); +} + +template <> +inline int MXNET_LAPACK_posv(int matrix_layout, char uplo, int n, + int nrhs, double *a, int lda, double *b, int ldb) { + return mxnet_lapack_dposv(matrix_layout, uplo, n, nrhs, a, lda, b, ldb); +} + #endif // MXNET_C_LAPACK_API_H_ diff --git a/src/operator/contrib/krprod.h b/src/operator/contrib/krprod.h new file mode 100644 index 000000000000..a713f1e093a7 --- /dev/null +++ b/src/operator/contrib/krprod.h @@ -0,0 +1,234 @@ +/*! + * Copyright (c) 2017 by Contributors + * \file krprod.h + * \brief Core function for Khatri-Rao product + * \author Jencir Lee + */ +#ifndef MXNET_OPERATOR_CONTRIB_KRPROD_H_ +#define MXNET_OPERATOR_CONTRIB_KRPROD_H_ +#include +#include "mshadow/tensor.h" +#include "mxnet/c_lapack_api.h" + +namespace mxnet { +namespace op { + +using namespace mshadow; +using namespace mshadow::expr; + +/*! + * \brief Computes row-wise Kronecker product + * + * Given input matrices, this function computes the Kronecker product + * row-wise. E.g. if the input matrices are of shape (3, 2), (3, 4), + * (3, 5), the result matrix will be of shape (3, 2 * 4 * 5), which is + * (3, 40). + * + * \param out result matrix + * \param ts_arr vector of input matrices + */ +template +inline void row_wise_kronecker + (Tensor out, + const std::vector > &ts_arr) { + CHECK_GE(ts_arr.size(), 1) << "The input matrices must be non-empty."; + + // Check all input and output matrices have the same number of rows + // and the output matrix has the right number of columns + int nrows = static_cast(out.size(0)); + int ncols = 1; + for (auto & ts : ts_arr) { + CHECK_EQ(nrows, static_cast(ts.size(0))) + << "All input and output matrices must have the same number of rows."; + ncols *= ts.size(1); + } + CHECK_EQ(ncols, static_cast(out.size(1))); + + // Create an intermediate space of the same shape as out + // + // Suppose storage stores the result at step i-1, we'd + // compute and store the result into out for step i; + // we then proceed to compute and store the result in storage + // for step i+1 and so on and so forth, by alternating using + // storage and out to store the given variable and the result variable + Tensor storage(out.shape_); + AllocSpace(&storage); + + // Pointers to the given variable and result variable + // We exchange what given and result point to at every step + Tensor *given = &storage, + *result = &out, *tmp; + + // Compute each intermediate row-wise Kronecker product + storage = 1; + ncols = 1; + for (auto & ts : ts_arr) { + expr::BLASEngine::SetStream + (result->stream_); + + // Compute the current row-wise Kronecker product + *result = 0; + for (int i = 0; i < nrows; ++i) { + // BLAS signature + // + // dger( + // m : ts.size(1), length of each row of current matrix + // n : ncols, length of each row of previous result + // alpha : 1, scaling to the outer product of x and y + // x : ts[i].dptr_, current row of current matrix + // incx : 1, as each element in the row is contiguous + // y : (*given)[i].dptr_, current row of the given variable + // incy : 1, as each element in the row is contiguous + // a : (*result)[i].dptr_, current row of the result variable + // lda : ts.size(1), as the outer product is stored as one row + // which occupies contiguous memory, and as BLASEngine::ger() + // assumes column-major matrix, lda has to be precisely + // the length of x, i.e. ts[i].size(1) + // ) + expr::BLASEngine::ger + (result->stream_, + ts.size(1), ncols, 1, + ts[i].dptr_, 1, + (*given)[i].dptr_, 1, + (*result)[i].dptr_, ts.size(1)); + } + ncols *= ts.size(1); + + tmp = given; + given = result; + result = tmp; + } + + // If the final result is stored in storage, + // copy its value to out + if (given != &out) + Copy(out, storage); + + FreeSpace(&storage); +} + +/*! + * \brief Khatri-Rao product + * + * \param out result matrix + * \param ts_arr vector of input matrices + */ +template +inline void khatri_rao + (Tensor out, + const std::vector > &ts_arr) { + CHECK_GE(ts_arr.size(), 1) << "The input matrices must be non-empty."; + + // Check all input and output matrices have the same number + // of columns and the output matrix has the right number of rows + int ncols = static_cast(out.size(1)); + int nrows = 1; + for (auto & ts : ts_arr) { + CHECK_EQ(ncols, static_cast(ts.size(1))) + << "All input and output matrices must have the same number of columns."; + nrows *= ts.size(0); + } + CHECK_EQ(nrows, static_cast(out.size(0))); + + // Change the layout of matrices to column-major + Tensor out_t(Shape2(out.size(1), out.size(0))); + AllocSpace(&out_t); + flip(out.size(0), out.size(1), out_t.dptr_, out_t.stride_, + out.dptr_, out.stride_); + + std::vector > ts_t_arr; + for (int i = 0; i < static_cast(ts_arr.size()); ++i) { + ts_t_arr.emplace_back(Shape2(ts_arr[i].size(1), ts_arr[i].size(0))); + AllocSpace(&ts_t_arr[i]); + flip(ts_arr[i].size(0), ts_arr[i].size(1), ts_t_arr[i].dptr_, + ts_t_arr[i].stride_, ts_arr[i].dptr_, ts_arr[i].stride_); + } + + // Perform row-wise Kronecker product + row_wise_kronecker(out_t, ts_t_arr); + + // Change the layout of result matrix back to row-major + flip(out.size(1), out.size(0), out.dptr_, out.stride_, + out_t.dptr_, out_t.stride_); + + FreeSpace(&out_t); + for (auto &ts_t : ts_t_arr) + FreeSpace(&ts_t); +} + +/*! + * \brief Moore-Penrose pseudoinverse of the Khatri-Rao product + * + * Given input matrices A_1, ..., A_n, of shape (l_1, k), ..., (l_n, k) respectively, the pseudoinverse of the Khatri-Rao product is + * + * pinv(A_1 khatri-rao A_2 khatri-rao ... khatri-rao A_n) = + * ((A_1^T A_1) hadamard-dot ... hadamard-dot (A_n^T A_n)) + * (A_1 khatri-rao ... khatri-rao A_n)^T + * + * As the first term of the r.h.s is a square matrix, the result is always of the same shape as the transpose of the Khatri-Rao product of the input matrices. The input argument ts_arr could contain the original input matrices, or transposed ones. + * + * \param out result matrix + * \param ts_arr vector of input matrices + * \param input_transposed if every input matrices is transposed + */ +template +inline void inv_khatri_rao + (Tensor out, + const std::vector > &ts_arr, + bool input_transposed = false) { + CHECK_GE(ts_arr.size(), 1) << "Input tensor array must be non-empty"; + + // Initialise the Hadamard product to eye(k) + // where k is the number of "factors" + int k = out.size(0); + Tensor hadamard_prod(Shape2(k, k)); + AllocSpace(&hadamard_prod); + hadamard_prod = 1; + + // Note that out is of the same shape as the transpose of + // the Khatri-Rao product + // + // When input is transposed, we could first put the transpose of + // the Khatri-Rao product in out, then call the linear solver, which + // will update the out's content to the final result; + // + // If the input is not transposed, we need to create an intermediate + // tensor to store the Khatri-Rao product, call the linear solver with + // MXNET_LAPACK_COL_MAJOR as the matrix layout, and transpose + // the final result into out + + int info; + if (input_transposed) { + row_wise_kronecker(out, ts_arr); + for (auto &ts : ts_arr) + hadamard_prod *= implicit_dot(ts, ts.T()); + + info = MXNET_LAPACK_posv(MXNET_LAPACK_ROW_MAJOR, 'U', + k, out.size(1), hadamard_prod.dptr_, hadamard_prod.stride_, + out.dptr_, out.stride_); + } else { + Tensor kr(Shape2(out.size(1), out.size(0))); + AllocSpace(&kr); + khatri_rao(kr, ts_arr); + + for (auto &ts : ts_arr) + hadamard_prod *= implicit_dot(ts.T(), ts); + + info = MXNET_LAPACK_posv(MXNET_LAPACK_COL_MAJOR, 'U', + k, out.size(1), hadamard_prod.dptr_, hadamard_prod.stride_, + kr.dptr_, kr.stride_); + + flip(out.size(1), out.size(0), out.dptr_, out.stride_, + kr.dptr_, kr.stride_); + FreeSpace(&kr); + } + + FreeSpace(&hadamard_prod); + if (info != 0) + LOG(FATAL) << "The linear solver in inv_prod() returns " << info; +} + +} // namespace op +} // namespace mxnet + +#endif // MXNET_OPERATOR_CONTRIB_KRPROD_H_ diff --git a/tests/cpp/operator/krprod_test.cc b/tests/cpp/operator/krprod_test.cc new file mode 100644 index 000000000000..6e10221f7ab3 --- /dev/null +++ b/tests/cpp/operator/krprod_test.cc @@ -0,0 +1,429 @@ +/*! + * Copyright (c) 2017 by Contributors + * \file krprod_test.cc + * \brief Test Khatri-Rao product + * \author Jencir Lee + */ +#include +#include +#include +#include "gtest/gtest.h" +#include "operator/contrib/krprod.h" + +namespace mxnet { +namespace op { + +using namespace mshadow; +using namespace mshadow::expr; +using DType = double; + +#define EXPECT_DOUBLE_EQ_MATRIX(expected, actual) \ +{ \ + for (int i = 0; i < static_cast(actual.size(0)); ++i) \ + for (int j = 0; j < static_cast(actual.size(1)); ++j) \ + EXPECT_LE(std::abs(actual[i][j] - expected[i][j]), 1e-10); \ +} \ + +TEST(row_wise_kronecker, OneInputMatrix) { + // Input matrices of shape (2, 4) which is also the expected result + DType mat[8] {1, 2, 3, 4, 5, 6, 7, 8}; + + // Make input tensors + std::vector > ts_arr; + ts_arr.emplace_back(mat, Shape2(2, 4), 4, nullptr); + + // Compute Khatri-Rao product + Tensor result(Shape2(2, 4)); + AllocSpace(&result); + row_wise_kronecker(result, ts_arr); + + // Check against expected result + EXPECT_DOUBLE_EQ_MATRIX(ts_arr[0], result); + + FreeSpace(&result); +} + +TEST(row_wise_kronecker, TwoInputMatrices) { + // Input matrices of shape (2, 3) and (2, 4) + DType mat1[6] {1, 2, 3, 4, 5, 6}; + DType mat2[8] {1, 2, 3, 4, 5, 6, 7, 8}; + + // Expect result of shape (2, 12) + DType expected[24] {1, 2, 3, 4, 2, 4, 6, 8, 3, 6, 9, 12, + 20, 24, 28, 32, 25, 30, 35, 40, 30, 36, 42, 48}; + + // Make input tensors + std::vector > ts_arr; + ts_arr.emplace_back(mat1, Shape2(2, 3), 3, nullptr); + ts_arr.emplace_back(mat2, Shape2(2, 4), 4, nullptr); + + // Compute Khatri-Rao product + Tensor result(Shape2(2, 12)); + AllocSpace(&result); + row_wise_kronecker(result, ts_arr); + + // Check against expected result + Tensor ts_expected(expected, Shape2(2, 12), 12, nullptr); + EXPECT_DOUBLE_EQ_MATRIX(ts_expected, result); + + FreeSpace(&result); +} + +TEST(row_wise_kronecker, TwoInputMatrices2) { + // Input matrices of shape (2, 3) and (2, 1) + DType mat1[6] {1, 2, 3, 4, 5, 6}; + DType mat2[2] {1, 2}; + + // Expect result of shape (2, 3) + DType expected[6] {1, 2, 3, 8, 10, 12}; + + // Make input tensors + std::vector > ts_arr; + ts_arr.emplace_back(mat1, Shape2(2, 3), 3, nullptr); + ts_arr.emplace_back(mat2, Shape2(2, 1), 1, nullptr); + + // Compute Khatri-Rao product + Tensor result(Shape2(2, 3)); + AllocSpace(&result); + row_wise_kronecker(result, ts_arr); + + // Check against expected result + Tensor ts_expected(expected, Shape2(2, 3), 3, nullptr); + EXPECT_DOUBLE_EQ_MATRIX(ts_expected, result); + + FreeSpace(&result); +} + +TEST(row_wise_kronecker, ThreeInputMatrices) { + std::default_random_engine generator; + std::uniform_int_distribution distribution(1, 6); + + Tensor in1(Shape2(3, 4)), in2(Shape2(3, 2)), + in3(Shape2(3, 3)), kr12(Shape2(3, 8)), kr13(Shape2(3, 24)), + result(Shape2(3, 24)); + AllocSpace(&in1); + AllocSpace(&in2); + AllocSpace(&in3); + AllocSpace(&kr12); + AllocSpace(&kr13); + AllocSpace(&result); + + std::vector > ts_arr {in1, in2, in3}; + for (auto & in : ts_arr) { + for (int i = 0; i < static_cast(in.size(0)); ++i) + for (int j = 0; j < static_cast(in.size(1)); ++j) + in[i][j] = distribution(generator); + } + + row_wise_kronecker(kr12, {in1, in2}); + row_wise_kronecker(kr13, {kr12, in3}); + row_wise_kronecker(result, ts_arr); + EXPECT_DOUBLE_EQ_MATRIX(kr13, result); + + for (auto & in : ts_arr) + FreeSpace(&in); + FreeSpace(&kr12); + FreeSpace(&kr13); + FreeSpace(&result); +} + +TEST(row_wise_kronecker, ThreeInputMatrices2) { + std::default_random_engine generator; + std::uniform_int_distribution distribution(1, 6); + + Tensor in1(Shape2(3, 4)), in2(Shape2(3, 1)), + in3(Shape2(3, 3)), kr12(Shape2(3, 4)), kr13(Shape2(3, 12)), + result(Shape2(3, 12)); + AllocSpace(&in1); + AllocSpace(&in2); + AllocSpace(&in3); + AllocSpace(&kr12); + AllocSpace(&kr13); + AllocSpace(&result); + + std::vector > ts_arr {in1, in2, in3}; + for (auto & in : ts_arr) { + for (int i = 0; i < static_cast(in.size(0)); ++i) + for (int j = 0; j < static_cast(in.size(1)); ++j) + in[i][j] = distribution(generator); + } + + row_wise_kronecker(kr12, {in1, in2}); + row_wise_kronecker(kr13, {kr12, in3}); + row_wise_kronecker(result, ts_arr); + EXPECT_DOUBLE_EQ_MATRIX(kr13, result); + + for (auto & in : ts_arr) + FreeSpace(&in); + FreeSpace(&kr12); + FreeSpace(&kr13); + FreeSpace(&result); +} + +TEST(row_wise_kronecker, ThreeInputMatrices3) { + std::default_random_engine generator; + std::uniform_int_distribution distribution(1, 6); + + Tensor in1(Shape2(3, 1)), in2(Shape2(3, 4)), + in3(Shape2(3, 3)), kr12(Shape2(3, 4)), kr13(Shape2(3, 12)), + result(Shape2(3, 12)); + AllocSpace(&in1); + AllocSpace(&in2); + AllocSpace(&in3); + AllocSpace(&kr12); + AllocSpace(&kr13); + AllocSpace(&result); + + std::vector > ts_arr {in1, in2, in3}; + for (auto & in : ts_arr) { + for (int i = 0; i < static_cast(in.size(0)); ++i) + for (int j = 0; j < static_cast(in.size(1)); ++j) + in[i][j] = distribution(generator); + } + + row_wise_kronecker(kr12, {in1, in2}); + row_wise_kronecker(kr13, {kr12, in3}); + row_wise_kronecker(result, ts_arr); + EXPECT_DOUBLE_EQ_MATRIX(kr13, result); + + for (auto & in : ts_arr) + FreeSpace(&in); + FreeSpace(&kr12); + FreeSpace(&kr13); + FreeSpace(&result); +} + +TEST(row_wise_kronecker, FourInputMatrices) { + std::default_random_engine generator; + std::uniform_int_distribution distribution(1, 6); + + Tensor in1(Shape2(3, 47)), in2(Shape2(3, 1)), + in3(Shape2(3, 5)), in4(Shape2(3, 2173)), kr12(Shape2(3, 47)), + kr13(Shape2(3, 47 * 5)), kr14(Shape2(3, 47 * 5 * 2173)), + result(Shape2(3, 47 * 5 * 2173)); + AllocSpace(&in1); + AllocSpace(&in2); + AllocSpace(&in3); + AllocSpace(&in4); + AllocSpace(&kr12); + AllocSpace(&kr13); + AllocSpace(&kr14); + AllocSpace(&result); + + std::vector > ts_arr {in1, in2, in3, in4}; + for (auto & in : ts_arr) { + for (int i = 0; i < static_cast(in.size(0)); ++i) + for (int j = 0; j < static_cast(in.size(1)); ++j) + in[i][j] = distribution(generator); + } + + row_wise_kronecker(kr12, {in1, in2}); + row_wise_kronecker(kr13, {kr12, in3}); + row_wise_kronecker(kr14, {kr13, in4}); + row_wise_kronecker(result, ts_arr); + EXPECT_DOUBLE_EQ_MATRIX(kr14, result); + + for (auto & in : ts_arr) + FreeSpace(&in); + FreeSpace(&kr12); + FreeSpace(&kr13); + FreeSpace(&kr14); + FreeSpace(&result); +} + +TEST(khatri_rao, OneInputMatrix) { + // Input matrices of shape (2, 4) which is also the expected result + DType mat[8] {1, 2, 3, 4, 5, 6, 7, 8}; + + // Make input tensors + std::vector > ts_arr; + ts_arr.emplace_back(mat, Shape2(2, 4), 4, nullptr); + + // Compute Khatri-Rao product + Tensor result(Shape2(2, 4)); + AllocSpace(&result); + khatri_rao(result, ts_arr); + + // Check against expected result + EXPECT_DOUBLE_EQ_MATRIX(ts_arr[0], result); + + FreeSpace(&result); +} + +TEST(khatri_rao, TwoInputMatrices) { + // Input matrices of shape (3, 2) and (4, 2) + DType mat1[6] {1, 4, 2, 5, 3, 6}; + DType mat2[8] {1, 5, 2, 6, 3, 7, 4, 8}; + + // Expect result of shape (12, 2) + DType expected[24] {1, 20, 2, 24, 3, 28, 4, 32, 2, 25, 4, 30, + 6, 35, 8, 40, 3, 30, 6, 36, 9, 42, 12, 48}; + + // Make input tensors + std::vector > ts_arr; + ts_arr.emplace_back(mat1, Shape2(3, 2), 2, nullptr); + ts_arr.emplace_back(mat2, Shape2(4, 2), 2, nullptr); + + // Compute Khatri-Rao product + Tensor result(Shape2(12, 2)); + AllocSpace(&result); + khatri_rao(result, ts_arr); + + // Check against expected result + Tensor ts_expected(expected, Shape2(12, 2), 2, nullptr); + EXPECT_DOUBLE_EQ_MATRIX(ts_expected, result); + + FreeSpace(&result); +} + +TEST(khatri_rao, ThreeInputMatrices) { + std::default_random_engine generator; + std::uniform_int_distribution distribution(1, 6); + + Tensor in1(Shape2(4, 3)), in2(Shape2(2, 3)), + in3(Shape2(3, 3)), kr12(Shape2(8, 3)), kr13(Shape2(24, 3)), + result(Shape2(24, 3)); + AllocSpace(&in1); + AllocSpace(&in2); + AllocSpace(&in3); + AllocSpace(&kr12); + AllocSpace(&kr13); + AllocSpace(&result); + + std::vector > ts_arr {in1, in2, in3}; + for (auto & in : ts_arr) { + for (int i = 0; i < static_cast(in.size(0)); ++i) + for (int j = 0; j < static_cast(in.size(1)); ++j) + in[i][j] = distribution(generator); + } + + khatri_rao(kr12, {in1, in2}); + khatri_rao(kr13, {kr12, in3}); + khatri_rao(result, ts_arr); + EXPECT_DOUBLE_EQ_MATRIX(kr13, result); + + for (auto & in : ts_arr) + FreeSpace(&in); + FreeSpace(&kr12); + FreeSpace(&kr13); + FreeSpace(&result); +} + +TEST(inv_khatri_rao, OneInputMatrixTransposed) { + DType mat[8] {1, 2, 3, 4, 5, 6, 7, 8}; + + // Make input tensors + std::vector > ts_arr; + ts_arr.emplace_back(mat, Shape2(2, 4), 4, nullptr); + + // Compute inverse Khatri-Rao product + Tensor inv_kr(Shape2(2, 4)); + AllocSpace(&inv_kr); + inv_khatri_rao(inv_kr, ts_arr, true); + + // Check against expected result + Tensor actual_dot(Shape2(2, 4)); + AllocSpace(&actual_dot); + actual_dot = implicit_dot(implicit_dot(inv_kr, ts_arr[0].T()), inv_kr); + EXPECT_DOUBLE_EQ_MATRIX(inv_kr, actual_dot); + + FreeSpace(&inv_kr); + FreeSpace(&actual_dot); +} + +TEST(inv_khatri_rao, TwoInputMatrices) { + // Input matrices of shape (3, 2) and (4, 2) + DType mat1[6] {1, 4, 2, 5, 3, 6}; + DType mat2[8] {1, 5, 2, 6, 3, 7, 4, 8}; + + // Make input tensors + std::vector > ts_arr; + ts_arr.emplace_back(mat1, Shape2(3, 2), 2, nullptr); + ts_arr.emplace_back(mat2, Shape2(4, 2), 2, nullptr); + + // Compute inverse Khatri-Rao product + Tensor inv_kr(Shape2(2, 12)), kr(Shape2(12, 2)); + AllocSpace(&inv_kr); + AllocSpace(&kr); + inv_khatri_rao(inv_kr, ts_arr, false); + khatri_rao(kr, ts_arr); + + // Check against expected result + Tensor actual_dot(Shape2(2, 12)); + AllocSpace(&actual_dot); + actual_dot = implicit_dot(implicit_dot(inv_kr, kr), inv_kr); + EXPECT_DOUBLE_EQ_MATRIX(inv_kr, actual_dot); + + FreeSpace(&inv_kr); + FreeSpace(&kr); + FreeSpace(&actual_dot); +} + +TEST(inv_khatri_rao, TwoInputMatricesTransposed) { + // Transposed input matrices of shape (2, 3) and (2, 4) + DType mat1[6] {1, 2, 3, 4, 5, 6}; + DType mat2[8] {1, 2, 3, 4, 5, 6, 7, 8}; + + // Make input tensors + std::vector > ts_arr; + ts_arr.emplace_back(mat1, Shape2(2, 3), 3, nullptr); + ts_arr.emplace_back(mat2, Shape2(2, 4), 4, nullptr); + + // Compute invser Khatri-Rao product + Tensor inv_kr(Shape2(2, 12)), kr_t(Shape2(2, 12)); + AllocSpace(&inv_kr); + AllocSpace(&kr_t); + inv_khatri_rao(inv_kr, ts_arr, true); + row_wise_kronecker(kr_t, ts_arr); + + // Check against expected result + Tensor actual_dot(Shape2(2, 12)); + AllocSpace(&actual_dot); + actual_dot = implicit_dot(implicit_dot(inv_kr, kr_t.T()), inv_kr); + EXPECT_DOUBLE_EQ_MATRIX(inv_kr, actual_dot); + + FreeSpace(&inv_kr); + FreeSpace(&kr_t); + FreeSpace(&actual_dot); +} + +TEST(inv_khatri_rao, ThreeInputMatricesTranposed) { + // Randomly initialise the transposed input matrices + std::default_random_engine generator; + std::uniform_int_distribution distribution(1, 6); + + Tensor in1(Shape2(3, 4)), in2(Shape2(3, 2)), + in3(Shape2(3, 3)); + AllocSpace(&in1); + AllocSpace(&in2); + AllocSpace(&in3); + + std::vector > ts_arr {in1, in2, in3}; + for (auto & in : ts_arr) { + for (int i = 0; i < static_cast(in.size(0)); ++i) + for (int j = 0; j < static_cast(in.size(1)); ++j) + in[i][j] = distribution(generator); + } + + // Compute inv_kr & kr + Tensor inv_kr(Shape2(3, 24)), kr_t(Shape2(3, 24)); + AllocSpace(&inv_kr); + AllocSpace(&kr_t); + + inv_khatri_rao(inv_kr, ts_arr, true); + row_wise_kronecker(kr_t, ts_arr); + + // Check dot result + Tensor actual_dot(Shape2(3, 24)); + AllocSpace(&actual_dot); + actual_dot = implicit_dot(implicit_dot(inv_kr, kr_t.T()), inv_kr); + EXPECT_DOUBLE_EQ_MATRIX(inv_kr, actual_dot); + + for (auto & in : ts_arr) + FreeSpace(&in); + FreeSpace(&inv_kr); + FreeSpace(&kr_t); + FreeSpace(&actual_dot); +} +} // namespace op +} // namespace mxnet diff --git a/tests/cpp/unittest.mk b/tests/cpp/unittest.mk index 808b655e9dba..11ea6d141a53 100644 --- a/tests/cpp/unittest.mk +++ b/tests/cpp/unittest.mk @@ -17,26 +17,26 @@ endif build/tests/cpp/%.o : tests/cpp/%.cc @mkdir -p $(@D) - $(CXX) -std=c++0x $(TEST_CFLAGS) -MM -MT tests/cpp/$* $< > build/tests/cpp/$*.d - $(CXX) -c -std=c++0x $(TEST_CFLAGS) -I$(GTEST_INC) -o build/tests/cpp/$*.o $(filter %.cc %.a, $^) + $(CXX) -std=c++11 $(TEST_CFLAGS) -I$(GTEST_INC) -MM -MT tests/cpp/$* $< > build/tests/cpp/$*.d + $(CXX) -c -std=c++11 $(TEST_CFLAGS) -I$(GTEST_INC) -o build/tests/cpp/$*.o $(filter %.cc %.a, $^) build/tests/cpp/operator/%.o : tests/cpp/operator/%.cc @mkdir -p $(@D) - $(CXX) -std=c++0x $(TEST_CFLAGS) -MM -MT tests/cpp/operator/$* $< > build/tests/cpp/operator/$*.d - $(CXX) -c -std=c++0x $(TEST_CFLAGS) -I$(GTEST_INC) -o build/tests/cpp/operator/$*.o $(filter %.cc %.a, $^) + $(CXX) -std=c++11 $(TEST_CFLAGS) -I$(GTEST_INC) -MM -MT tests/cpp/operator/$* $< > build/tests/cpp/operator/$*.d + $(CXX) -c -std=c++11 $(TEST_CFLAGS) -I$(GTEST_INC) -o build/tests/cpp/operator/$*.o $(filter %.cc %.a, $^) build/tests/cpp/storage/%.o : tests/cpp/storage/%.cc @mkdir -p $(@D) - $(CXX) -std=c++0x $(TEST_CFLAGS) -MM -MT tests/cpp/storage/$* $< > build/tests/cpp/storage/$*.d - $(CXX) -c -std=c++0x $(TEST_CFLAGS) -I$(GTEST_INC) -o build/tests/cpp/storage/$*.o $(filter %.cc %.a, $^) + $(CXX) -std=c++11 $(TEST_CFLAGS) -I$(GTEST_INC) -MM -MT tests/cpp/storage/$* $< > build/tests/cpp/storage/$*.d + $(CXX) -c -std=c++11 $(TEST_CFLAGS) -I$(GTEST_INC) -o build/tests/cpp/storage/$*.o $(filter %.cc %.a, $^) build/tests/cpp/engine/%.o : tests/cpp/engine/%.cc @mkdir -p $(@D) - $(CXX) -std=c++0x $(TEST_CFLAGS) -MM -MT tests/cpp/engine/$* $< > build/tests/cpp/engine/$*.d - $(CXX) -c -std=c++0x $(TEST_CFLAGS) -I$(GTEST_INC) -o build/tests/cpp/engine/$*.o $(filter %.cc %.a, $^) + $(CXX) -std=c++11 $(TEST_CFLAGS) -I$(GTEST_INC) -MM -MT tests/cpp/engine/$* $< > build/tests/cpp/engine/$*.d + $(CXX) -c -std=c++11 $(TEST_CFLAGS) -I$(GTEST_INC) -o build/tests/cpp/engine/$*.o $(filter %.cc %.a, $^) $(TEST): $(TEST_OBJ) lib/libmxnet.so - $(CXX) -std=c++0x $(TEST_CFLAGS) -I$(GTEST_INC) -o $@ $^ $(TEST_LDFLAGS) -L$(GTEST_LIB) -lgtest + $(CXX) -std=c++11 $(TEST_CFLAGS) -I$(GTEST_INC) -o $@ $^ $(TEST_LDFLAGS) -L$(GTEST_LIB) -lgtest runtest: $(TEST) LD_LIBRARY_PATH=$(shell pwd)/lib:$(LD_LIBRARY_PATH) $(TEST) @@ -47,4 +47,4 @@ testclean: -include build/tests/cpp/*.d -include build/tests/cpp/operator/*.d -include build/tests/cpp/storage/*.d --include build/tests/cpp/engine/*.d \ No newline at end of file +-include build/tests/cpp/engine/*.d From a242e9f4b77bf705ae679de05642fbe88f870a9f Mon Sep 17 00:00:00 2001 From: "Joshua Z. Zhang" Date: Fri, 21 Jul 2017 21:37:30 -0700 Subject: [PATCH 251/834] Mx image2 (#6961) * add detection to mx.image fix bug, extend makeBorder value option squeezed multiple commits fix eval_metric count, fix crop border convert to class call change overlap constraint * add doc and init unittest * improve doc fix lint * update doc * fix lint * fix id for cropping * remove clip operations * add test for image * fix tests * add thread instruction to logging * fix lint * fix true divide --- docs/api/python/image.md | 206 +++++ docs/api/python/index.md | 1 + docs/api/python/io.md | 30 +- example/ssd/evaluate/eval_metric.py | 17 +- python/mxnet/image/__init__.py | 10 + python/mxnet/image/detection.py | 923 +++++++++++++++++++++ python/mxnet/{ => image}/image.py | 704 ++++++++++++---- python/mxnet/test_utils.py | 7 +- src/io/image_io.cc | 17 +- tests/python/unittest/test_image.py | 167 ++++ tools/caffe_converter/caffe_parser.py | 2 +- tools/caffe_converter/caffe_proto_utils.py | 2 +- 12 files changed, 1903 insertions(+), 183 deletions(-) create mode 100644 docs/api/python/image.md create mode 100644 python/mxnet/image/__init__.py create mode 100644 python/mxnet/image/detection.py rename python/mxnet/{ => image}/image.py (51%) create mode 100644 tests/python/unittest/test_image.py diff --git a/docs/api/python/image.md b/docs/api/python/image.md new file mode 100644 index 000000000000..23b5ee3d1415 --- /dev/null +++ b/docs/api/python/image.md @@ -0,0 +1,206 @@ +# Image API + +## Overview +This document summarizes supporting functions and iterators to read and process +images provided in +```eval_rst +.. autosummary:: + :nosignatures: + + mxnet.image +``` + +## Image processing functions +```eval_rst +.. currentmodule:: mxnet +.. autosummary:: + :nosignatures: + + image.imdecode + image.scale_down + image.resize_short + image.fixed_crop + image.random_crop + image.center_crop + image.color_normalize + image.random_size_crop +``` + +## Image iterators +Iterators support loading image from binary `Record IO` and raw image files. +```eval_rst +.. autosummary:: + :nosignatures: + + image.ImageIter +``` +```python +>>> data_iter = mx.image.ImageIter(batch_size=4, data_shape=(3, 224, 224), label_width=1, + path_imglist='data/custom.lst') +>>> data_iter.reset() +>>> for data in data_iter: +... d = data.data[0] +... print(d.shape) +>>> # we can apply lots of augmentations as well +>>> data_iter = mx.image.ImageIter(4, (3, 224, 224), path_imglist='data/custom.lst', + rand_crop=resize=True, rand_mirror=True, mean=True, + brightness=0.1, contrast=0.1, saturation=0.1, hue=0.1, + pca_noise=0.1, rand_gray=0.05) +>>> data = data_iter.next() +>>> # specify augmenters manually is also supported +>>> data_iter = mx.image.ImageIter(32, (3, 224, 224), path_rec='data/caltech.rec', + path_imgidx='data/caltech.idx', shuffle=True, + aug_list=[mx.image.HorizontalFlipAug(0.5), + mx.image.ColorJitterAug(0.1, 0.1, 0.1)]) +``` + +We use helper function to initialize augmenters +```eval_rst + .. currentmodule:: mxnet +.. autosummary:: + :nosignatures: + + image.CreateAugmenter +``` + +A list of supporting augmenters +```eval_rst +.. autosummary:: + :nosignatures: + + image.Augmenter + image.ResizeAug + image.ForceResizeAug + image.RandomCropAug + image.RandomSizedCropAug + image.CenterCropAug + image.RandomOrderAug + image.BrightnessJitterAug + image.ContrastJitterAug + image.SaturationJitterAug + image.HueJitterAug + image.ColorJitterAug + image.LightingAug + image.ColorNormalizeAug + image.RandomGrayAug + image.HorizontalFlipAug + image.CastAug +``` + +Similar to `ImageIter`, `ImageDetIter` is designed for `Object Detection` tasks. +```eval_rst +.. autosummary:: + :nosignatures: + + image.ImageDetIter +``` + +```python +>>> data_iter = mx.image.ImageDetIter(batch_size=4, data_shape=(3, 224, 224), + path_imglist='data/train.lst') +>>> data_iter.reset() +>>> for data in data_iter: +... d = data.data[0] +... l = data.label[0] +... print(d.shape) +... print(l.shape) +``` + +Unlike object classification with fixed label_width, object count may vary from +image to image. Thus we have special format for object detection labels. +Usually the `lst` file generated by `tools/im2rec.py` is a list of +``` +index_0 label_0 image_path_0 +index_1 label_1 image_path_1 +``` +Where `label_N` is a number a of fixed-width vector. +The format of label used in object detection is a variable length vector +``` +A B [header] [(object0), (object1), ... (objectN)] +``` +Where A is the width of header, B is the width of each object. +Header is optional and used for inserting helper information such as (width, height). +Each object is usually 5 or 6 numbers describing the object properties, for example: +[id, xmin, ymin, xmax, ymax, difficulty] +Putting all together, we have a `lst` file for object detection: +``` +0 2 5 640 480 1 0.1 0.2 0.8 0.9 2 0.5 0.3 0.6 0.8 data/xxx.jpg +1 2 5 480 640 3 0.05 0.16 0.75 0.9 data/xxx.jpg +2 2 5 500 600 2 0.6 0.1 0.7 0.5 0 0.1 0.3 0.2 0.4 3 0.25 0.25 0.3 0.3 data/xxx.jpg +... +``` + +A helper function to initialize Augmenters for `Object detection` task +```eval_rst +.. autosummary:: + :nosignatures: + + image.CreateDetAugmenter + ``` + +Since `Detection` task is sensitive to object localization, any modification +to image that introduced localization shift will require correction to label, +and a list of augmenters specific for `Object detection` is provided +```eval_rst +.. autosummary:: + :nosignatures: + + image.DetBorrowAug + image.DetRandomSelectAug + image.DetHorizontalFlipAug + image.DetRandomCropAug + image.DetRandomPadAug +``` + +## API Reference + + + +```eval_rst +.. automodule:: mxnet.image +.. autoclass:: mxnet.image.ImageIter + :members: + +.. automethod:: mxnet.image.imdecode +.. automethod:: mxnet.image.scale_down +.. automethod:: mxnet.image.resize_short +.. automethod:: mxnet.image.fixed_crop +.. automethod:: mxnet.image.random_crop +.. automethod:: mxnet.image.center_crop +.. automethod:: mxnet.image.color_normalize +.. automethod:: mxnet.image.random_size_crop + +.. autoclass:: mxnet.image.Augmenter + :members: +.. autoclass:: mxnet.image.ResizeAug +.. autoclass:: mxnet.image.ForceResizeAug +.. autoclass:: mxnet.image.RandomCropAug +.. autoclass:: mxnet.image.RandomSizedCropAug +.. autoclass:: mxnet.image.CenterCropAug +.. autoclass:: mxnet.image.RandomOrderAug +.. autoclass:: mxnet.image.BrightnessJitterAug +.. autoclass:: mxnet.image.ContrastJitterAug +.. autoclass:: mxnet.image.SaturationJitterAug +.. autoclass:: mxnet.image.HueJitterAug +.. autoclass:: mxnet.image.ColorJitterAug +.. autoclass:: mxnet.image.LightingAug +.. autoclass:: mxnet.image.ColorNormalizeAug +.. autoclass:: mxnet.image.RandomGrayAug +.. autoclass:: mxnet.image.HorizontalFlipAug +.. autoclass:: mxnet.image.CastAug + +.. automethod:: mxnet.image.CreateAugmenter + +.. autoclass:: mxnet.image.ImageDetIter + :members: +.. autoclass:: mxnet.image.DetAugmenter + :members: +.. autoclass:: mxnet.image.DetBorrowAug +.. autoclass:: mxnet.image.DetRandomSelectAug +.. autoclass:: mxnet.image.DetHorizontalFlipAug +.. autoclass:: mxnet.image.DetRandomCropAug +.. autoclass:: mxnet.image.DetRandomPadAug + +.. automethod:: mxnet.image.CreateDetAugmenter +``` + diff --git a/docs/api/python/index.md b/docs/api/python/index.md index 6035e914b5b7..4014a886a0d4 100644 --- a/docs/api/python/index.md +++ b/docs/api/python/index.md @@ -32,6 +32,7 @@ imported by running: rnn kvstore io + image optimization callback metric diff --git a/docs/api/python/io.md b/docs/api/python/io.md index 9cbffc91aa63..15f8aa3ce354 100644 --- a/docs/api/python/io.md +++ b/docs/api/python/io.md @@ -62,6 +62,7 @@ A detailed tutorial is available at recordio.MXRecordIO recordio.MXIndexedRecordIO image.ImageIter + image.ImageDetIter ``` ## Helper classes and functions @@ -81,33 +82,6 @@ Data structures and other iterators provided in the ``mxnet.io`` packages. io.MXDataIter ``` -A list of image modification functions provided by ``mxnet.image``. - -```eval_rst -.. autosummary:: - :nosignatures: - - image.imdecode - image.scale_down - image.resize_short - image.fixed_crop - image.random_crop - image.center_crop - image.color_normalize - image.random_size_crop - image.ResizeAug - image.RandomCropAug - image.RandomSizedCropAug - image.CenterCropAug - image.RandomOrderAug - image.ColorJitterAug - image.LightingAug - image.ColorNormalizeAug - image.HorizontalFlipAug - image.CastAug - image.CreateAugmenter -``` - Functions to read and write RecordIO files. ```eval_rst @@ -179,8 +153,6 @@ The backend engine will recognize the index of `N` in the `layout` as the axis f ```eval_rst .. automodule:: mxnet.io :members: -.. automodule:: mxnet.image - :members: .. automodule:: mxnet.recordio :members: ``` diff --git a/example/ssd/evaluate/eval_metric.py b/example/ssd/evaluate/eval_metric.py index 68d0fb2da443..bb03e0133ece 100644 --- a/example/ssd/evaluate/eval_metric.py +++ b/example/ssd/evaluate/eval_metric.py @@ -110,6 +110,8 @@ def iou(x, ys): for i in range(labels[0].shape[0]): # get as numpy arrays label = labels[0][i].asnumpy() + if np.sum(label[:, 0] >= 0) < 1: + continue pred = preds[self.pred_idx][i].asnumpy() # calculate for each class while (pred.shape[0] > 0): @@ -124,7 +126,9 @@ def iou(x, ys): dets[dets[:,1].argsort()[::-1]] records = np.hstack((dets[:, 1][:, np.newaxis], np.zeros((dets.shape[0], 1)))) # ground-truths - gts = label[np.where(label[:, 0].astype(int) == cid)[0], :] + label_indices = np.where(label[:, 0].astype(int) == cid)[0] + gts = label[label_indices, :] + label = np.delete(label, label_indices, axis=0) if gts.size > 0: found = [False] * gts.shape[0] for j in range(dets.shape[0]): @@ -163,6 +167,16 @@ def iou(x, ys): if records.size > 0: self._insert(cid, records, gt_count) + # add missing class if not present in prediction + while (label.shape[0] > 0): + cid = int(label[0, 0]) + label_indices = np.where(label[:, 0].astype(int) == cid)[0] + label = np.delete(label, label_indices, axis=0) + if cid < 0: + continue + gt_count = label_indices.size + self._insert(cid, np.array([[0, 0]]), gt_count) + def _update(self): """ update num_inst and sum_metric """ aps = [] @@ -182,6 +196,7 @@ def _update(self): def _recall_prec(self, record, count): """ get recall and precision from internal records """ + record = np.delete(record, np.where(record[:, 1].astype(int) == 0)[0], axis=0) sorted_records = record[record[:,0].argsort()[::-1]] tp = np.cumsum(sorted_records[:, 1].astype(int) == 1) fp = np.cumsum(sorted_records[:, 1].astype(int) == 2) diff --git a/python/mxnet/image/__init__.py b/python/mxnet/image/__init__.py new file mode 100644 index 000000000000..d484006ad7ce --- /dev/null +++ b/python/mxnet/image/__init__.py @@ -0,0 +1,10 @@ +# coding: utf-8 +# pylint: disable=wildcard-import +"""Image Iterators and image augmentation functions""" + +from . import image +from .image import * + +from . import detection +from . import detection as det +from .detection import * diff --git a/python/mxnet/image/detection.py b/python/mxnet/image/detection.py new file mode 100644 index 000000000000..d5e5c1e7e691 --- /dev/null +++ b/python/mxnet/image/detection.py @@ -0,0 +1,923 @@ +# pylint: disable=unused-import +"""Read images and perform augmentations for object detection.""" + +from __future__ import absolute_import, print_function + +import random +import logging +import json +import numpy as np + +from ..base import numeric_types +from .. import ndarray as nd +from .._ndarray_internal import _cvcopyMakeBorder as copyMakeBorder +from .. import io +from .image import RandomOrderAug, ColorJitterAug, LightingAug, ColorNormalizeAug +from .image import ResizeAug, ForceResizeAug, CastAug, HueJitterAug, RandomGrayAug +from .image import fixed_crop, ImageIter, Augmenter + + +class DetAugmenter(object): + """Detection base augmenter""" + def __init__(self, **kwargs): + self._kwargs = kwargs + for k, v in self._kwargs.items(): + if isinstance(v, nd.NDArray): + v = v.asnumpy() + if isinstance(v, np.ndarray): + v = v.tolist() + self._kwargs[k] = v + + def dumps(self): + """Saves the Augmenter to string + + Returns + ------- + str + JSON formatted string that describes the Augmenter. + """ + return json.dumps([self.__class__.__name__.lower(), self._kwargs]) + + def __call__(self, src, label): + """Abstract implementation body""" + raise NotImplementedError("Must override implementation.") + + +class DetBorrowAug(DetAugmenter): + """Borrow standard augmenter from image classification. + Which is good once you know label won't be affected after this augmenter. + + Parameters + ---------- + augmenter : mx.image.Augmenter + The borrowed standard augmenter which has no effect on label + """ + def __init__(self, augmenter): + if not isinstance(augmenter, Augmenter): + raise TypeError('Borrowing from invalid Augmenter') + super(DetBorrowAug, self).__init__(augmenter=augmenter.dumps()) + self.augmenter = augmenter + + def dumps(self): + """Override the default one to avoid duplicate dump.""" + return [self.__class__.__name__.lower(), self.augmenter.dumps()] + + def __call__(self, src, label): + """Augmenter implementation body""" + src = self.augmenter(src)[0] + return (src, label) + + +class DetRandomSelectAug(DetAugmenter): + """Randomly select one augmenter to apply, with chance to skip all. + + Parameters + ---------- + aug_list : list of DetAugmenter + The random selection will be applied to one of the augmenters + skip_prob : float + The probability to skip all augmenters and return input directly + """ + def __init__(self, aug_list, skip_prob=0): + super(DetRandomSelectAug, self).__init__(skip_prob=skip_prob) + if not isinstance(aug_list, (list, tuple)): + aug_list = [aug_list] + for aug in aug_list: + if not isinstance(aug, DetAugmenter): + raise ValueError('Allow DetAugmenter in list only') + if not aug_list: + skip_prob = 1 # disabled + + self.aug_list = aug_list + self.skip_prob = skip_prob + + def dumps(self): + """Override default.""" + return [self.__class__.__name__.lower(), [x.dumps() for x in self.aug_list]] + + def __call__(self, src, label): + """Augmenter implementation body""" + if random.random() < self.skip_prob: + return (src, label) + else: + random.shuffle(self.aug_list) + return self.aug_list[0](src, label) + + +class DetHorizontalFlipAug(DetAugmenter): + """Random horizontal flipping. + + Parameters + ---------- + p : float + chance [0, 1] to flip + """ + def __init__(self, p): + super(DetHorizontalFlipAug, self).__init__(p=p) + self.p = p + + def __call__(self, src, label): + """Augmenter implementation""" + if random.random() < self.p: + src = nd.flip(src, axis=1) + self._flip_label(label) + return (src, label) + + def _flip_label(self, label): + """Helper function to flip label.""" + tmp = 1.0 - label[:, 1] + label[:, 1] = 1.0 - label[:, 3] + label[:, 3] = tmp + + +class DetRandomCropAug(DetAugmenter): + """Random cropping with constraints + + Parameters + ---------- + min_object_covered : float, default=0.1 + The cropped area of the image must contain at least this fraction of + any bounding box supplied. The value of this parameter should be non-negative. + In the case of 0, the cropped area does not need to overlap any of the + bounding boxes supplied. + min_eject_coverage : float, default=0.3 + The minimum coverage of cropped sample w.r.t its original size. With this + constraint, objects that have marginal area after crop will be discarded. + aspect_ratio_range : tuple of floats, default=(0.75, 1.33) + The cropped area of the image must have an aspect ratio = width / height + within this range. + area_range : tuple of floats, default=(0.05, 1.0) + The cropped area of the image must contain a fraction of the supplied + image within in this range. + max_attempts : int, default=50 + Number of attempts at generating a cropped/padded region of the image of the + specified constraints. After max_attempts failures, return the original image. + """ + def __init__(self, min_object_covered=0.1, aspect_ratio_range=(0.75, 1.33), + area_range=(0.05, 1.0), min_eject_coverage=0.3, max_attempts=50): + if not isinstance(aspect_ratio_range, (tuple, list)): + assert isinstance(aspect_ratio_range, numeric_types) + logging.info('Using fixed aspect ratio: %s in DetRandomCropAug', + str(aspect_ratio_range)) + aspect_ratio_range = (aspect_ratio_range, aspect_ratio_range) + if not isinstance(area_range, (tuple, list)): + assert isinstance(area_range, numeric_types) + logging.info('Using fixed area range: %s in DetRandomCropAug', area_range) + area_range = (area_range, area_range) + super(DetRandomCropAug, self).__init__(min_object_covered=min_object_covered, + aspect_ratio_range=aspect_ratio_range, + area_range=area_range, + min_eject_coverage=min_eject_coverage, + max_attempts=max_attempts) + self.min_object_covered = min_object_covered + self.min_eject_coverage = min_eject_coverage + self.max_attempts = max_attempts + self.aspect_ratio_range = aspect_ratio_range + self.area_range = area_range + self.enabled = False + if (area_range[1] <= 0 or area_range[0] > area_range[1]): + logging.warn('Skip DetRandomCropAug due to invalid area_range: %s', area_range) + elif (aspect_ratio_range[0] > aspect_ratio_range[1] or aspect_ratio_range[0] <= 0): + logging.warn('Skip DetRandomCropAug due to invalid aspect_ratio_range: %s', + aspect_ratio_range) + else: + self.enabled = True + + def __call__(self, src, label): + """Augmenter implementation body""" + crop = self._random_crop_proposal(label, src.shape[0], src.shape[1]) + if crop: + x, y, w, h, label = crop + src = fixed_crop(src, x, y, w, h, None) + return (src, label) + + def _calculate_areas(self, label): + """Calculate areas for multiple labels""" + heights = np.maximum(0, label[:, 3] - label[:, 1]) + widths = np.maximum(0, label[:, 2] - label[:, 0]) + return heights * widths + + + def _intersect(self, label, xmin, ymin, xmax, ymax): + """Calculate intersect areas, normalized.""" + left = np.maximum(label[:, 0], xmin) + right = np.minimum(label[:, 2], xmax) + top = np.maximum(label[:, 1], ymin) + bot = np.minimum(label[:, 3], ymax) + invalid = np.where(np.logical_or(left >= right, top >= bot))[0] + out = label.copy() + out[:, 0] = left + out[:, 1] = top + out[:, 2] = right + out[:, 3] = bot + out[invalid, :] = 0 + return out + + def _check_satisfy_constraints(self, label, xmin, ymin, xmax, ymax, width, height): + """Check if constrains are satisfied""" + if (xmax - xmin) * (ymax - ymin) < 2: + return False # only 1 pixel + x1 = float(xmin) / width + y1 = float(ymin) / height + x2 = float(xmax) / width + y2 = float(ymax) / height + object_areas = self._calculate_areas(label[:, 1:]) + valid_objects = np.where(object_areas * width * height > 2)[0] + if valid_objects.size < 1: + return False + intersects = self._intersect(label[valid_objects, 1:], x1, y1, x2, y2) + coverages = self._calculate_areas(intersects) / object_areas + coverages = coverages[np.where(coverages > 0)[0]] + if coverages.size > 0 and np.amin(coverages) > self.min_object_covered: + return True + + def _update_labels(self, label, crop_box, height, width): + """Convert labels according to crop box""" + xmin = float(crop_box[0]) / width + ymin = float(crop_box[1]) / height + w = float(crop_box[2]) / width + h = float(crop_box[3]) / height + out = label.copy() + out[:, (1, 3)] -= xmin + out[:, (2, 4)] -= ymin + out[:, (1, 3)] /= w + out[:, (2, 4)] /= h + out[:, 1:5] = np.maximum(0, out[:, 1:5]) + out[:, 1:5] = np.minimum(1, out[:, 1:5]) + coverage = self._calculate_areas(out[:, 1:]) * w * h / self._calculate_areas(label[:, 1:]) + valid = np.logical_and(out[:, 3] > out[:, 1], out[:, 4] > out[:, 2]) + valid = np.logical_and(valid, coverage > self.min_eject_coverage) + valid = np.where(valid)[0] + if valid.size < 1: + return None + out = out[valid, :] + return out + + def _random_crop_proposal(self, label, height, width): + """Propose cropping areas""" + from math import sqrt + + if not self.enabled or height <= 0 or width <= 0: + return None + min_area = self.area_range[0] * height * width + max_area = self.area_range[1] * height * width + for _ in range(self.max_attempts): + ratio = random.uniform(*self.aspect_ratio_range) + if ratio <= 0: + continue + h = int(round(sqrt(min_area / ratio))) + max_h = int(round(sqrt(max_area / ratio))) + if round(max_h * ratio) > width: + # find smallest max_h satifying round(max_h * ratio) <= width + max_h = int((width + 0.4999999) / ratio) + if max_h > height: + max_h = height + if h > max_h: + h = max_h + if h < max_h: + # generate random h in range [h, max_h] + h = random.randint(h, max_h) + w = int(round(h * ratio)) + assert w <= width + + # trying to fix rounding problems + area = w * h + if area < min_area: + h += 1 + w = int(round(h * ratio)) + area = w * h + if area > max_area: + h -= 1 + w = int(round(h * ratio)) + area = w * h + if (area < min_area or area > max_area or w > width or h > height \ + or w <= 0 or h <= 0): + continue + + y = random.randint(0, max(0, height - h)) + x = random.randint(0, max(0, width - w)) + if self._check_satisfy_constraints(label, x, y, x + w, y + h, width, height): + new_label = self._update_labels(label, (x, y, w, h), height, width) + if new_label is not None: + return (x, y, w, h, new_label) + return None + + +class DetRandomPadAug(DetAugmenter): + """Random padding augmenter. + + Parameters + ---------- + aspect_ratio_range : tuple of floats, default=(0.75, 1.33) + The padded area of the image must have an aspect ratio = width / height + within this range. + area_range : tuple of floats, default=(1.0, 3.0) + The padded area of the image must be larger than the original area + max_attempts : int, default=50 + Number of attempts at generating a padded region of the image of the + specified constraints. After max_attempts failures, return the original image. + pad_val: float or tuple of float, default=(128, 128, 128) + pixel value to be filled when padding is enabled. + """ + def __init__(self, aspect_ratio_range=(0.75, 1.33), area_range=(1.0, 3.0), + max_attempts=50, pad_val=(128, 128, 128)): + if not isinstance(pad_val, (list, tuple)): + assert isinstance(pad_val, numeric_types) + pad_val = (pad_val) + if not isinstance(aspect_ratio_range, (list, tuple)): + assert isinstance(aspect_ratio_range, numeric_types) + logging.info('Using fixed aspect ratio: %s in DetRandomPadAug', + str(aspect_ratio_range)) + aspect_ratio_range = (aspect_ratio_range, aspect_ratio_range) + if not isinstance(area_range, (tuple, list)): + assert isinstance(area_range, numeric_types) + logging.info('Using fixed area range: %s in DetRandomPadAug', area_range) + area_range = (area_range, area_range) + super(DetRandomPadAug, self).__init__(aspect_ratio_range=aspect_ratio_range, + area_range=area_range, max_attempts=max_attempts, + pad_val=pad_val) + self.pad_val = pad_val + self.aspect_ratio_range = aspect_ratio_range + self.area_range = area_range + self.max_attempts = max_attempts + self.enabled = False + if (area_range[1] <= 1.0 or area_range[0] > area_range[1]): + logging.warn('Skip DetRandomPadAug due to invalid parameters: %s', area_range) + elif (aspect_ratio_range[0] <= 0 or aspect_ratio_range[0] > aspect_ratio_range[1]): + logging.warn('Skip DetRandomPadAug due to invalid aspect_ratio_range: %s', + aspect_ratio_range) + else: + self.enabled = True + + def __call__(self, src, label): + """Augmenter body""" + height, width, _ = src.shape + pad = self._random_pad_proposal(label, height, width) + if pad: + x, y, w, h, label = pad + src = copyMakeBorder(src, y, h-y-height, x, w-x-width, 16, values=self.pad_val) + return (src, label) + + def _update_labels(self, label, pad_box, height, width): + """Update label according to padding region""" + out = label.copy() + out[:, (1, 3)] = (out[:, (1, 3)] * width + pad_box[0]) / pad_box[2] + out[:, (2, 4)] = (out[:, (2, 4)] * height + pad_box[1]) / pad_box[3] + return out + + def _random_pad_proposal(self, label, height, width): + """Generate random padding region""" + from math import sqrt + if not self.enabled or height <= 0 or width <= 0: + return None + min_area = self.area_range[0] * height * width + max_area = self.area_range[1] * height * width + for _ in range(self.max_attempts): + ratio = random.uniform(*self.aspect_ratio_range) + if ratio <= 0: + continue + h = int(round(sqrt(min_area / ratio))) + max_h = int(round(sqrt(max_area / ratio))) + if round(h * ratio) < width: + h = int((width + 0.499999) / ratio) + if h < height: + h = height + if h > max_h: + h = max_h + if h < max_h: + h = random.randint(h, max_h) + w = int(round(h * ratio)) + if (h - height) < 2 or (w - width) < 2: + continue # marginal padding is not helpful + + y = random.randint(0, max(0, h - height)) + x = random.randint(0, max(0, w - width)) + new_label = self._update_labels(label, (x, y, w, h), height, width) + return (x, y, w, h, new_label) + return None + + +def CreateMultiRandCropAugmenter(min_object_covered=0.1, aspect_ratio_range=(0.75, 1.33), + area_range=(0.05, 1.0), min_eject_coverage=0.3, + max_attempts=50, skip_prob=0): + """Helper function to create multiple random crop augmenters. + + Parameters + ---------- + min_object_covered : float or list of float, default=0.1 + The cropped area of the image must contain at least this fraction of + any bounding box supplied. The value of this parameter should be non-negative. + In the case of 0, the cropped area does not need to overlap any of the + bounding boxes supplied. + min_eject_coverage : float or list of float, default=0.3 + The minimum coverage of cropped sample w.r.t its original size. With this + constraint, objects that have marginal area after crop will be discarded. + aspect_ratio_range : tuple of floats or list of tuple of floats, default=(0.75, 1.33) + The cropped area of the image must have an aspect ratio = width / height + within this range. + area_range : tuple of floats or list of tuple of floats, default=(0.05, 1.0) + The cropped area of the image must contain a fraction of the supplied + image within in this range. + max_attempts : int or list of int, default=50 + Number of attempts at generating a cropped/padded region of the image of the + specified constraints. After max_attempts failures, return the original image. + + Examples + -------- + >>> # An example of creating multiple random crop augmenters + >>> min_object_covered = [0.1, 0.3, 0.5, 0.7, 0.9] # use 5 augmenters + >>> aspect_ratio_range = (0.75, 1.33) # use same range for all augmenters + >>> area_range = [(0.1, 1.0), (0.2, 1.0), (0.2, 1.0), (0.3, 0.9), (0.5, 1.0)] + >>> min_eject_coverage = 0.3 + >>> max_attempts = 50 + >>> aug = mx.image.det.CreateMultiRandCropAugmenter(min_object_covered=min_object_covered, + aspect_ratio_range=aspect_ratio_range, area_range=area_range, + min_eject_coverage=min_eject_coverage, max_attempts=max_attempts, + skip_prob=0) + >>> aug.dumps() # show some details + + """ + def align_parameters(params): + """Align parameters as pairs""" + out_params = [] + num = 1 + for p in params: + if not isinstance(p, list): + p = [p] + out_params.append(p) + num = max(num, len(p)) + # align for each param + for k, p in enumerate(out_params): + if len(p) != num: + assert len(p) == 1 + out_params[k] = p * num + return out_params + + aligned_params = align_parameters([min_object_covered, aspect_ratio_range, area_range, + min_eject_coverage, max_attempts]) + augs = [] + for moc, arr, ar, mec, ma in zip(*aligned_params): + augs.append(DetRandomCropAug(min_object_covered=moc, aspect_ratio_range=arr, + area_range=ar, min_eject_coverage=mec, max_attempts=ma)) + return DetRandomSelectAug(augs, skip_prob=skip_prob) + + +def CreateDetAugmenter(data_shape, resize=0, rand_crop=0, rand_pad=0, rand_gray=0, + rand_mirror=False, mean=None, std=None, brightness=0, contrast=0, + saturation=0, pca_noise=0, hue=0, inter_method=2, min_object_covered=0.1, + aspect_ratio_range=(0.75, 1.33), area_range=(0.05, 3.0), + min_eject_coverage=0.3, max_attempts=50, pad_val=(128, 128, 128)): + """Create augmenters for detection. + + Parameters + ---------- + data_shape : tuple of int + shape for output data + resize : int + resize shorter edge if larger than 0 at the begining + rand_crop : float + [0, 1], probability to apply random cropping + rand_pad : float + [0, 1], probability to apply random padding + rand_gray : float + [0, 1], probability to convert to grayscale for all channels + rand_mirror : bool + whether apply horizontal flip to image with probability 0.5 + mean : np.ndarray or None + mean pixel values for [r, g, b] + std : np.ndarray or None + standard deviations for [r, g, b] + brightness : float + brightness jittering range (percent) + contrast : float + contrast jittering range + saturation : float + saturation jittering range + hue : float + hue jittering range + pca_noise : float + pca noise level + inter_method : int, default=2(Area-based) + interpolation method for all resizing operations + + Possible values: + 0: Nearest Neighbors Interpolation. + 1: Bilinear interpolation. + 2: Area-based (resampling using pixel area relation). It may be a + preferred method for image decimation, as it gives moire-free + results. But when the image is zoomed, it is similar to the Nearest + Neighbors method. (used by default). + 3: Bicubic interpolation over 4x4 pixel neighborhood. + 4: Lanczos interpolation over 8x8 pixel neighborhood. + 9: Cubic for enlarge, area for shrink, bilinear for others + 10: Random select from interpolation method metioned above. + Note: + When shrinking an image, it will generally look best with AREA-based + interpolation, whereas, when enlarging an image, it will generally look best + with Bicubic (slow) or Bilinear (faster but still looks OK). + min_object_covered : float + The cropped area of the image must contain at least this fraction of + any bounding box supplied. The value of this parameter should be non-negative. + In the case of 0, the cropped area does not need to overlap any of the + bounding boxes supplied. + min_eject_coverage : float + The minimum coverage of cropped sample w.r.t its original size. With this + constraint, objects that have marginal area after crop will be discarded. + aspect_ratio_range : tuple of floats + The cropped area of the image must have an aspect ratio = width / height + within this range. + area_range : tuple of floats + The cropped area of the image must contain a fraction of the supplied + image within in this range. + max_attempts : int + Number of attempts at generating a cropped/padded region of the image of the + specified constraints. After max_attempts failures, return the original image. + pad_val: float + pixel value to be filled when padding is enabled. pad_val will automatically + be subtracted by mean and divided by std if applicable. + + Examples + -------- + >>> # An example of creating multiple augmenters + >>> augs = mx.image.CreateDetAugmenter(data_shape=(3, 300, 300), rand_crop=0.5, + ... rand_pad=0.5, rand_mirror=True, mean=True, brightness=0.125, contrast=0.125, + ... saturation=0.125, pca_noise=0.05, inter_method=10, min_object_covered=[0.3, 0.5, 0.9], + ... area_range=(0.3, 3.0)) + >>> # dump the details + >>> for aug in augs: + ... aug.dumps() + """ + auglist = [] + + if resize > 0: + auglist.append(DetBorrowAug(ResizeAug(resize, inter_method))) + + if rand_crop > 0: + crop_augs = CreateMultiRandCropAugmenter(min_object_covered, aspect_ratio_range, + area_range, min_eject_coverage, + max_attempts, skip_prob=(1 - rand_crop)) + auglist.append(crop_augs) + + if rand_mirror > 0: + auglist.append(DetHorizontalFlipAug(0.5)) + + # apply random padding as late as possible to save computation + if rand_pad > 0: + pad_aug = DetRandomPadAug(aspect_ratio_range, + (1.0, area_range[1]), max_attempts, pad_val) + auglist.append(DetRandomSelectAug([pad_aug], 1 - rand_pad)) + + # force resize + auglist.append(DetBorrowAug(ForceResizeAug((data_shape[2], data_shape[1]), inter_method))) + + auglist.append(DetBorrowAug(CastAug())) + + if brightness or contrast or saturation: + auglist.append(DetBorrowAug(ColorJitterAug(brightness, contrast, saturation))) + + if hue: + auglist.append(DetBorrowAug(HueJitterAug(hue))) + + if pca_noise > 0: + eigval = np.array([55.46, 4.794, 1.148]) + eigvec = np.array([[-0.5675, 0.7192, 0.4009], + [-0.5808, -0.0045, -0.8140], + [-0.5836, -0.6948, 0.4203]]) + auglist.append(DetBorrowAug(LightingAug(pca_noise, eigval, eigvec))) + + if rand_gray > 0: + auglist.append(DetBorrowAug(RandomGrayAug(rand_gray))) + + if mean is True: + mean = np.array([123.68, 116.28, 103.53]) + elif mean is not None: + assert isinstance(mean, np.ndarray) and mean.shape[0] in [1, 3] + + if std is True: + std = np.array([58.395, 57.12, 57.375]) + elif std is not None: + assert isinstance(std, np.ndarray) and std.shape[0] in [1, 3] + + if mean is not None or std is not None: + auglist.append(DetBorrowAug(ColorNormalizeAug(mean, std))) + + return auglist + + +class ImageDetIter(ImageIter): + """Image iterator with a large number of augmentation choices for detection. + + Parameters + ---------- + aug_list : list or None + augmenter list for generating distorted images + batch_size : int + Number of examples per batch. + data_shape : tuple + Data shape in (channels, height, width) format. + For now, only RGB image with 3 channels is supported. + path_imgrec : str + Path to image record file (.rec). + Created with tools/im2rec.py or bin/im2rec. + path_imglist : str + Path to image list (.lst). + Created with tools/im2rec.py or with custom script. + Format: Tab separated record of index, one or more labels and relative_path_from_root. + imglist: list + A list of images with the label(s). + Each item is a list [imagelabel: float or list of float, imgpath]. + path_root : str + Root folder of image files. + path_imgidx : str + Path to image index file. Needed for partition and shuffling when using .rec source. + shuffle : bool + Whether to shuffle all images at the start of each iteration or not. + Can be slow for HDD. + part_index : int + Partition index. + num_parts : int + Total number of partitions. + data_name : str + Data name for provided symbols. + label_name : str + name for detection labels + kwargs : ... + More arguments for creating augmenter. See mx.image.CreateDetAugmenter. + """ + def __init__(self, batch_size, data_shape, + path_imgrec=None, path_imglist=None, path_root=None, path_imgidx=None, + shuffle=False, part_index=0, num_parts=1, aug_list=None, imglist=None, + data_name='data', label_name='label', **kwargs): + super(ImageDetIter, self).__init__(batch_size=batch_size, data_shape=data_shape, + path_imgrec=path_imgrec, path_imglist=path_imglist, + path_root=path_root, path_imgidx=path_imgidx, + shuffle=shuffle, part_index=part_index, + num_parts=num_parts, aug_list=[], imglist=imglist, + data_name=data_name, label_name=label_name) + + if aug_list is None: + self.auglist = CreateDetAugmenter(data_shape, **kwargs) + else: + self.auglist = aug_list + + # went through all labels to get the proper label shape + label_shape = self._estimate_label_shape() + self.provide_label = [(label_name, (self.batch_size, label_shape[0], label_shape[1]))] + self.label_shape = label_shape + + def _check_valid_label(self, label): + """Validate label and its shape.""" + if len(label.shape) != 2 or label.shape[1] < 5: + msg = "Label with shape (1+, 5+) required, %s received." % str(label) + raise RuntimeError(msg) + valid_label = np.where(np.logical_and(label[:, 0] >= 0, label[:, 3] > label[:, 1], + label[:, 4] > label[:, 2]))[0] + if valid_label.size < 1: + raise RuntimeError('Invalid label occurs.') + + def _estimate_label_shape(self): + """Helper function to estimate label shape""" + max_count = 0 + self.reset() + try: + while True: + label, _ = self.next_sample() + label = self._parse_label(label) + max_count = max(max_count, label.shape[0]) + except StopIteration: + pass + self.reset() + return (max_count, label.shape[1]) + + def _parse_label(self, label): + """Helper function to parse object detection label. + + Format for raw label: + n \t k \t ... \t [id \t xmin\t ymin \t xmax \t ymax \t ...] \t [repeat] + where n is the width of header, 2 or larger + k is the width of each object annotation, can be arbitrary, at least 5 + """ + if isinstance(label, nd.NDArray): + label = label.asnumpy() + raw = label.ravel() + if raw.size < 7: + raise RuntimeError("Label shape is invalid: " + str(raw.shape)) + header_width = int(raw[0]) + obj_width = int(raw[1]) + if (raw.size - header_width) % obj_width != 0: + msg = "Label shape %s inconsistent with annotation width %d." \ + %(str(raw.shape, obj_width)) + raise RuntimeError(msg) + out = np.reshape(raw[header_width:], (-1, obj_width)) + # remove bad ground-truths + valid = np.where(np.logical_and(out[:, 3] > out[:, 1], out[:, 4] > out[:, 2]))[0] + if valid.size < 1: + raise RuntimeError('Encounter sample with no valid label.') + return out[valid, :] + + def reshape(self, data_shape=None, label_shape=None): + """Reshape iterator for data_shape or label_shape. + + Parameters + ---------- + data_shape : tuple or None + reshape the data_shape to the new shape if not None + label_shape : tuple or None + reshape label shape to new shape if not None + """ + if data_shape is not None: + self.check_data_shape(data_shape) + self.provide_data = [(self.provide_data[0][0], (self.batch_size,) + data_shape)] + if label_shape is not None: + self.check_label_shape(label_shape) + self.provide_label = [(self.provide_label[0][0], (self.batch_size,) + label_shape)] + + def next(self): + """Override the function for returning next batch.""" + batch_size = self.batch_size + c, h, w = self.data_shape + batch_data = nd.zeros((batch_size, c, h, w)) + batch_label = nd.empty(self.provide_label[0][1]) + batch_label[:] = -1 + i = 0 + try: + while i < batch_size: + label, s = self.next_sample() + data = self.imdecode(s) + try: + self.check_valid_image([data]) + label = self._parse_label(label) + data, label = self.augmentation_transform(data, label) + self._check_valid_label(label) + except RuntimeError as e: + logging.debug('Invalid image, skipping: %s', str(e)) + continue + for datum in [data]: + assert i < batch_size, 'Batch size must be multiples of augmenter output length' + batch_data[i][:] = self.postprocess_data(datum) + num_object = label.shape[0] + batch_label[i][0:num_object][:] = nd.array(label) + batch_label[i][num_object:][:] = -1 + i += 1 + except StopIteration: + if not i: + raise StopIteration + + return io.DataBatch([batch_data], [batch_label], batch_size - i) + + def augmentation_transform(self, data, label): # pylint: disable=arguments-differ + """Override Transforms input data with specified augmentations.""" + for aug in self.auglist: + data, label = aug(data, label) + return (data, label) + + def check_label_shape(self, label_shape): + """Checks if the new label shape is valid""" + if not len(label_shape) == 2: + raise ValueError('label_shape should have length 2') + if label_shape[0] < self.label_shape[0]: + msg = 'Attempts to reduce label count from %d to %d, not allowed.' \ + % (self.label_shape[0], label_shape[0]) + raise ValueError(msg) + if label_shape[1] != self.provide_label[0][1][2]: + msg = 'label_shape object width inconsistent: %d vs %d.' \ + % (self.provide_label[0][1][2], label_shape[1]) + raise ValueError(msg) + + def draw_next(self, color=None, thickness=2, mean=None, std=None, clip=True, + waitKey=None, window_name='draw_next'): + """Display next image with bounding boxes drawn. + + Parameters + ---------- + color : tuple + Bounding box color in RGB, use None for random color + thickness : int + Bounding box border thickness + mean : True or numpy.ndarray + Compensate for the mean to have better visual effect + std : True or numpy.ndarray + Revert standard deviations + clip : bool + If true, clip to [0, 255] for better visual effect + waitKey : None or int + Hold the window for waitKey milliseconds if set, skip ploting if None + window_name : str + Plot window name if waitKey is set. + + Returns + ------- + numpy.ndarray + + Examples + -------- + >>> # use draw_next to get images with bounding boxes drawn + >>> iterator = mx.image.ImageDetIter(1, (3, 600, 600), path_imgrec='train.rec') + >>> for image in iterator.draw_next(waitKey=None): + ... # display image + >>> # or let draw_next display using cv2 module + >>> for image in iterator.draw_next(waitKey=0, window_name='disp'): + ... pass + """ + try: + import cv2 + except ImportError as e: + logging.warn('Unable to import cv2, skip drawing: %s', str(e)) + raise StopIteration + count = 0 + try: + while True: + label, s = self.next_sample() + data = self.imdecode(s) + try: + self.check_valid_image([data]) + label = self._parse_label(label) + except RuntimeError as e: + logging.debug('Invalid image, skipping: %s', str(e)) + continue + count += 1 + data, label = self.augmentation_transform(data, label) + image = data.asnumpy() + + # revert color_normalize + if std is True: + std = np.array([58.395, 57.12, 57.375]) + elif std is not None: + assert isinstance(std, np.ndarray) and std.shape[0] in [1, 3] + if std is not None: + image *= std + + if mean is True: + mean = np.array([123.68, 116.28, 103.53]) + elif mean is not None: + assert isinstance(mean, np.ndarray) and mean.shape[0] in [1, 3] + if mean is not None: + image += mean + + # swap RGB + image[:, :, (0, 1, 2)] = image[:, :, (2, 1, 0)] + if clip: + image = np.maximum(0, np.minimum(255, image)) + if color: + color = color[::-1] + image = image.astype(np.uint8) + height, width, _ = image.shape + for i in range(label.shape[0]): + x1 = int(label[i, 1] * width) + if x1 < 0: + continue + y1 = int(label[i, 2] * height) + x2 = int(label[i, 3] * width) + y2 = int(label[i, 4] * height) + bc = np.random.rand(3) * 255 if not color else color + cv2.rectangle(image, (x1, y1), (x2, y2), bc, thickness) + if waitKey is not None: + cv2.imshow(window_name, image) + cv2.waitKey(waitKey) + yield image + except StopIteration: + if not count: + raise StopIteration + + def sync_label_shape(self, it, verbose=False): + """Synchronize label shape with the input iterator. This is useful when + train/validation iterators have different label padding. + + Parameters + ---------- + it : ImageDetIter + The other iterator to synchronize + verbose : bool + Print verbose log if true + + Returns + ------- + ImageDetIter + The synchronized other iterator, the internal label shape is updated as well. + + Examples + -------- + >>> train_iter = mx.image.ImageDetIter(32, (3, 300, 300), path_imgrec='train.rec') + >>> val_iter = mx.image.ImageDetIter(32, (3, 300, 300), path.imgrec='val.rec') + >>> train_iter.label_shape + (30, 6) + >>> val_iter.label_shape + (25, 6) + >>> val_iter = train_iter.sync_label_shape(val_iter, verbose=False) + >>> train_iter.label_shape + (30, 6) + >>> val_iter.label_shape + (30, 6) + """ + assert isinstance(it, ImageDetIter), 'Synchronize with invalid iterator.' + train_label_shape = self.label_shape + val_label_shape = it.label_shape + assert train_label_shape[1] == val_label_shape[1], "object width mismatch." + max_count = max(train_label_shape[0], val_label_shape[0]) + if max_count > train_label_shape[0]: + self.reshape(None, (max_count, train_label_shape[1])) + if max_count > val_label_shape[0]: + it.reshape(None, (max_count, val_label_shape[1])) + if verbose and max_count > min(train_label_shape[0], val_label_shape[0]): + logging.info('Resized label_shape to (%d, %d).', max_count, train_label_shape[1]) + return it diff --git a/python/mxnet/image.py b/python/mxnet/image/image.py similarity index 51% rename from python/mxnet/image.py rename to python/mxnet/image/image.py index 890de7d0ffb8..5bf2afd09204 100644 --- a/python/mxnet/image.py +++ b/python/mxnet/image/image.py @@ -7,6 +7,7 @@ import os import random import logging +import json import numpy as np try: @@ -14,13 +15,13 @@ except ImportError: cv2 = None -from .base import numeric_types -from . import ndarray as nd -from . import _ndarray_internal as _internal -from ._ndarray_internal import _cvimresize as imresize -from ._ndarray_internal import _cvcopyMakeBorder as copyMakeBorder -from . import io -from . import recordio +from ..base import numeric_types +from .. import ndarray as nd +from .. import _ndarray_internal as _internal +from .._ndarray_internal import _cvimresize as imresize +from .._ndarray_internal import _cvcopyMakeBorder as copyMakeBorder +from .. import io +from .. import recordio def imdecode(buf, **kwargs): @@ -112,6 +113,61 @@ def scale_down(src_size, size): return int(w), int(h) +def _get_interp_method(interp, sizes=None): + """Get the interpolation method for resize functions. + The major purpose of this function is to wrap a random interp method selection + and a auto-estimation method. + + Parameters + ---------- + interp : int + interpolation method for all resizing operations + + Possible values: + 0: Nearest Neighbors Interpolation. + 1: Bilinear interpolation. + 2: Area-based (resampling using pixel area relation). It may be a + preferred method for image decimation, as it gives moire-free + results. But when the image is zoomed, it is similar to the Nearest + Neighbors method. (used by default). + 3: Bicubic interpolation over 4x4 pixel neighborhood. + 4: Lanczos interpolation over 8x8 pixel neighborhood. + 9: Cubic for enlarge, area for shrink, bilinear for others + 10: Random select from interpolation method metioned above. + Note: + When shrinking an image, it will generally look best with AREA-based + interpolation, whereas, when enlarging an image, it will generally look best + with Bicubic (slow) or Bilinear (faster but still looks OK). + More details can be found in the documentation of OpenCV, please refer to + http://docs.opencv.org/master/da/d54/group__imgproc__transform.html. + sizes : tuple of int + (old_height, old_width, new_height, new_width), if None provided, auto(9) + will return Area(2) anyway. + + Returns + ------- + int + interp method from 0 to 4 + """ + if interp == 9: + if sizes: + assert len(sizes) == 4 + oh, ow, nh, nw = sizes + if nh > oh and nw > ow: + return 2 + elif nh < oh and nw < ow: + return 3 + else: + return 1 + else: + return 2 + if interp == 10: + return random.randint(0, 4) + if interp not in (0, 1, 2, 3, 4): + raise ValueError('Unknown interp method %d' % interp) + return interp + + def resize_short(src, size, interp=2): """Resizes shorter edge to size. @@ -130,7 +186,21 @@ def resize_short(src, size, interp=2): The length to be set for the shorter edge. interp : int, optional, default=2 Interpolation method used for resizing the image. - Default method is bicubic interpolation. + Possible values: + 0: Nearest Neighbors Interpolation. + 1: Bilinear interpolation. + 2: Area-based (resampling using pixel area relation). It may be a + preferred method for image decimation, as it gives moire-free + results. But when the image is zoomed, it is similar to the Nearest + Neighbors method. (used by default). + 3: Bicubic interpolation over 4x4 pixel neighborhood. + 4: Lanczos interpolation over 8x8 pixel neighborhood. + 9: Cubic for enlarge, area for shrink, bilinear for others + 10: Random select from interpolation method metioned above. + Note: + When shrinking an image, it will generally look best with AREA-based + interpolation, whereas, when enlarging an image, it will generally look best + with Bicubic (slow) or Bilinear (faster but still looks OK). More details can be found in the documentation of OpenCV, please refer to http://docs.opencv.org/master/da/d54/group__imgproc__transform.html. @@ -154,17 +224,41 @@ def resize_short(src, size, interp=2): """ h, w, _ = src.shape if h > w: - new_h, new_w = size * h / w, size + new_h, new_w = size * h // w, size else: - new_h, new_w = size, size * w / h - return imresize(src, new_w, new_h, interp=interp) + new_h, new_w = size, size * w // h + return imresize(src, new_w, new_h, interp=_get_interp_method(interp, (h, w, new_h, new_w))) def fixed_crop(src, x0, y0, w, h, size=None, interp=2): - """Crop src at fixed location, and (optionally) resize it to size.""" + """Crop src at fixed location, and (optionally) resize it to size. + + Parameters + ---------- + src : NDArray + Input image + x0 : int + Left boundary of the cropping area + y0 : int + Top boundary of the cropping area + w : int + Width of the cropping area + h : int + Height of the cropping area + size : tuple of (w, h) + Optional, resize to new size after cropping + interp : int, optional, default=2 + Interpolation method. See resize_short for details. + + Returns + ------- + NDArray + An `NDArray` containing the cropped image. + """ out = nd.crop(src, begin=(y0, x0, 0), end=(y0 + h, x0 + w, int(src.shape[2]))) if size is not None and (w, h) != size: - out = imresize(out, *size, interp=interp) + sizes = (h, w, size[1], size[0]) + out = imresize(out, *size, interp=_get_interp_method(interp, sizes)) return out @@ -177,9 +271,8 @@ def random_crop(src, size, interp=2): src: Source image `NDArray` size: Size of the crop formatted as (width, height). If the `size` is larger than the image, then the source image is upsampled to `size` and returned. - interp: Interpolation method to be used in case the size is larger (default: bicubic). - Uses OpenCV convention for the parameters. Nearest - 0, Bilinear - 1, Bicubic - 2, - Area - 3. See OpenCV imresize function for more details. + interp: int, optional, default=2 + Interpolation method. See resize_short for details. Returns ------- NDArray @@ -221,27 +314,8 @@ def center_crop(src, size, interp=2): Binary source image data. size : list or tuple of int The desired output image size. - interp : interpolation, optional, default=Area-based - The type of interpolation that is done to the image. - - Possible values: - - 0: Nearest Neighbors Interpolation. - - 1: Bilinear interpolation. - - 2: Area-based (resampling using pixel area relation). It may be a - preferred method for image decimation, as it gives moire-free - results. But when the image is zoomed, it is similar to the Nearest - Neighbors method. (used by default). - - 3: Bicubic interpolation over 4x4 pixel neighborhood. - - 4: Lanczos interpolation over 8x8 pixel neighborhood. - - When shrinking an image, it will generally look best with AREA-based - interpolation, whereas, when enlarging an image, it will generally look best - with Bicubic (slow) or Bilinear (faster but still looks OK). + interp : int, optional, default=2 + Interpolation method. See resize_short for details. Returns ------- @@ -277,15 +351,53 @@ def center_crop(src, size, interp=2): def color_normalize(src, mean, std=None): - """Normalize src with mean and std.""" - src -= mean + """Normalize src with mean and std. + + Parameters + ---------- + src : NDArray + Input image + mean : NDArray + RGB mean to be subtracted + std : NDArray + RGB standard deviation to be divided + + Returns + ------- + NDArray + An `NDArray` containing the normalized image. + """ + if mean is not None: + src -= mean if std is not None: src /= std return src def random_size_crop(src, size, min_area, ratio, interp=2): - """Randomly crop src with size. Randomize area and aspect ratio.""" + """Randomly crop src with size. Randomize area and aspect ratio. + + Parameters + ---------- + src : NDArray + Input image + size : tuple of (int, int) + Size of the crop formatted as (width, height). + min_area : int + Minimum area to be maintained after cropping + ratio : tuple of (float, float) + Aspect ratio range as (min_aspect_ratio, max_aspect_ratio) + interp: int, optional, default=2 + Interpolation method. See resize_short for details. + Returns + ------- + NDArray + An `NDArray` containing the cropped image. + Tuple + A tuple (x, y, width, height) where (x, y) is top-left position of the crop in the + original image and (width, height) are the dimensions of the cropped image. + + """ h, w, _ = src.shape new_ratio = random.uniform(*ratio) if new_ratio * h > w: @@ -308,152 +420,446 @@ def random_size_crop(src, size, min_area, ratio, interp=2): return out, (x0, y0, new_w, new_h) -def ResizeAug(size, interp=2): - """Make resize shorter edge to size augmenter.""" +class Augmenter(object): + """Image Augmenter base class""" + def __init__(self, **kwargs): + self._kwargs = kwargs + for k, v in self._kwargs.items(): + if isinstance(v, nd.NDArray): + v = v.asnumpy() + if isinstance(v, np.ndarray): + v = v.tolist() + self._kwargs[k] = v + + def dumps(self): + """Saves the Augmenter to string + + Returns + ------- + str + JSON formatted string that describes the Augmenter. + """ + return json.dumps([self.__class__.__name__.lower(), self._kwargs]) + + def __call__(self, src): + """Abstract implementation body""" + raise NotImplementedError("Must override implementation.") + - def aug(src): +class ResizeAug(Augmenter): + """Make resize shorter edge to size augmenter. + + Parameters + ---------- + size : int + The length to be set for the shorter edge. + interp : int, optional, default=2 + Interpolation method. See resize_short for details. + """ + def __init__(self, size, interp=2): + super(ResizeAug, self).__init__(size=size, interp=interp) + self.size = size + self.interp = interp + + def __call__(self, src): """Augmenter body""" - return [resize_short(src, size, interp)] + return [resize_short(src, self.size, self.interp)] - return aug +class ForceResizeAug(Augmenter): + """Force resize to size regardless of aspect ratio -def RandomCropAug(size, interp=2): - """Make random crop augmenter""" + Parameters + ---------- + size : tuple of (int, int) + The desired size as in (width, height) + interp : int, optional, default=2 + Interpolation method. See resize_short for details. + """ + def __init__(self, size, interp=2): + super(ForceResizeAug, self).__init__(size=size, interp=interp) + self.size = size + self.interp = interp - def aug(src): + def __call__(self, src): """Augmenter body""" - return [random_crop(src, size, interp)[0]] + sizes = (src.shape[0], src.shape[1], self.size[1], self.size[0]) + return [imresize(src, *self.size, interp=_get_interp_method(self.interp, sizes))] + - return aug +class RandomCropAug(Augmenter): + """Make random crop augmenter + Parameters + ---------- + size : int + The length to be set for the shorter edge. + interp : int, optional, default=2 + Interpolation method. See resize_short for details. + """ + def __init__(self, size, interp=2): + super(RandomCropAug, self).__init__(size=size, interp=interp) + self.size = size + self.interp = interp -def RandomSizedCropAug(size, min_area, ratio, interp=2): - """Make random crop with random resizing and random aspect ratio jitter augmenter.""" + def __call__(self, src): + """Augmenter body""" + return [random_crop(src, self.size, self.interp)[0]] - def aug(src): + +class RandomSizedCropAug(Augmenter): + """Make random crop with random resizing and random aspect ratio jitter augmenter. + + Parameters + ---------- + size : tuple of (int, int) + Size of the crop formatted as (width, height). + min_area : int + Minimum area to be maintained after cropping + ratio : tuple of (float, float) + Aspect ratio range as (min_aspect_ratio, max_aspect_ratio) + interp: int, optional, default=2 + Interpolation method. See resize_short for details. + """ + def __init__(self, size, min_area, ratio, interp=2): + super(RandomSizedCropAug, self).__init__(size=size, min_area=min_area, + ratio=ratio, interp=interp) + self.size = size + self.min_area = min_area + self.ratio = ratio + self.interp = interp + + def __call__(self, src): """Augmenter body""" - return [random_size_crop(src, size, min_area, ratio, interp)[0]] + return [random_size_crop(src, self.size, self.min_area, self.ratio, self.interp)[0]] - return aug +class CenterCropAug(Augmenter): + """Make center crop augmenter. -def CenterCropAug(size, interp=2): - """Make center crop augmenter.""" + Parameters + ---------- + size : list or tuple of int + The desired output image size. + interp : int, optional, default=2 + Interpolation method. See resize_short for details. + """ + def __init__(self, size, interp=2): + super(CenterCropAug, self).__init__(size=size, interp=interp) + self.size = size + self.interp = interp - def aug(src): + def __call__(self, src): """Augmenter body""" - return [center_crop(src, size, interp)[0]] + return [center_crop(src, self.size, self.interp)[0]] - return aug +class RandomOrderAug(Augmenter): + """Apply list of augmenters in random order -def RandomOrderAug(ts): - """Apply list of augmenters in random order""" + Parameters + ---------- + ts : list of augmenters + A series of augmenters to be applied in random order + """ + def __init__(self, ts): + super(RandomOrderAug, self).__init__() + self.ts = ts + + def dumps(self): + """Override the default to avoid duplicate dump.""" + return [self.__class__.__name__.lower(), [x.dumps() for x in self.ts]] - def aug(src): + def __call__(self, src): """Augmenter body""" src = [src] - random.shuffle(ts) - for t in ts: + random.shuffle(self.ts) + for t in self.ts: src = [j for i in src for j in t(i)] return src - return aug - - -def ColorJitterAug(brightness, contrast, saturation): - """Apply random brightness, contrast and saturation jitter in random order.""" - ts = [] - coef = nd.array([[[0.299, 0.587, 0.114]]]) - if brightness > 0: - def baug(src): - """Augmenter body""" - alpha = 1.0 + random.uniform(-brightness, brightness) - src *= alpha - return [src] - - ts.append(baug) - - if contrast > 0: - def caug(src): - """Augmenter body""" - alpha = 1.0 + random.uniform(-contrast, contrast) - gray = src * coef - gray = (3.0 * (1.0 - alpha) / gray.size) * nd.sum(gray) - src *= alpha - src += gray - return [src] - - ts.append(caug) - - if saturation > 0: - def saug(src): - """Augmenter body""" - alpha = 1.0 + random.uniform(-saturation, saturation) - gray = src * coef - gray = nd.sum(gray, axis=2, keepdims=True) - gray *= (1.0 - alpha) - src *= alpha - src += gray - return [src] - - ts.append(saug) - return RandomOrderAug(ts) - - -def LightingAug(alphastd, eigval, eigvec): - """Add PCA based noise.""" - - def aug(src): + +class BrightnessJitterAug(Augmenter): + """Random brightness jitter augmentation. + + Parameters + ---------- + brightness : float + The brightness jitter ratio range, [0, 1] + """ + def __init__(self, brightness): + super(BrightnessJitterAug, self).__init__(brightness=brightness) + self.brightness = brightness + + def __call__(self, src): + """Augmenter body""" + alpha = 1.0 + random.uniform(-self.brightness, self.brightness) + src *= alpha + return [src] + + +class ContrastJitterAug(Augmenter): + """Random contrast jitter augmentation. + + Parameters + ---------- + contrast : float + The contrast jitter ratio range, [0, 1] + """ + def __init__(self, contrast): + super(ContrastJitterAug, self).__init__(contrast=contrast) + self.contrast = contrast + self.coef = nd.array([[[0.299, 0.587, 0.114]]]) + + def __call__(self, src): + """Augmenter body""" + alpha = 1.0 + random.uniform(-self.contrast, self.contrast) + gray = src * self.coef + gray = (3.0 * (1.0 - alpha) / gray.size) * nd.sum(gray) + src *= alpha + src += gray + return [src] + + +class SaturationJitterAug(Augmenter): + """Random saturation jitter augmentation. + + Parameters + ---------- + saturation : float + The saturation jitter ratio range, [0, 1] + """ + def __init__(self, saturation): + super(SaturationJitterAug, self).__init__(saturation=saturation) + self.saturation = saturation + self.coef = nd.array([[[0.299, 0.587, 0.114]]]) + + def __call__(self, src): + """Augmenter body""" + alpha = 1.0 + random.uniform(-self.saturation, self.saturation) + gray = src * self.coef + gray = nd.sum(gray, axis=2, keepdims=True) + gray *= (1.0 - alpha) + src *= alpha + src += gray + return [src] + + +class HueJitterAug(Augmenter): + """Random hue jitter augmentation. + + Parameters + ---------- + hue : float + The hue jitter ratio range, [0, 1] + """ + def __init__(self, hue): + super(HueJitterAug, self).__init__(hue=hue) + self.hue = hue + self.tyiq = np.array([[0.299, 0.587, 0.114], + [0.596, -0.274, -0.321], + [0.211, -0.523, 0.311]]) + self.ityiq = np.array([[1.0, 0.956, 0.621], + [1.0, -0.272, -0.647], + [1.0, -1.107, 1.705]]) + + def __call__(self, src): + """Augmenter body. + Using approximate linear transfomation described in: + https://beesbuzz.biz/code/hsv_color_transforms.php + """ + alpha = random.uniform(-self.hue, self.hue) + vsu = np.cos(alpha * np.pi) + vsw = np.sin(alpha * np.pi) + bt = np.array([[1.0, 0.0, 0.0], + [0.0, vsu, -vsw], + [0.0, vsw, vsu]]) + t = np.dot(np.dot(self.tyiq, bt), self.ityiq).T + src = nd.dot(src, nd.array(t)) + return [src] + + +class ColorJitterAug(RandomOrderAug): + """Apply random brightness, contrast and saturation jitter in random order. + + Parameters + ---------- + brightness : float + The brightness jitter ratio range, [0, 1] + contrast : float + The contrast jitter ratio range, [0, 1] + saturation : float + The saturation jitter ratio range, [0, 1] + """ + def __init__(self, brightness, contrast, saturation): + ts = [] + if brightness > 0: + ts.append(BrightnessJitterAug(brightness)) + if contrast > 0: + ts.append(ContrastJitterAug(contrast)) + if saturation > 0: + ts.append(SaturationJitterAug(saturation)) + super(ColorJitterAug, self).__init__(ts) + + +class LightingAug(Augmenter): + """Add PCA based noise. + + Parameters + ---------- + alphastd : float + Noise level + eigval : 3x1 np.array + Eigen values + eigvec : 3x3 np.array + Eigen vectors + """ + def __init__(self, alphastd, eigval, eigvec): + super(LightingAug, self).__init__(alphastd=alphastd, eigval=eigval, eigvec=eigvec) + self.alphastd = alphastd + self.eigval = eigval + self.eigvec = eigvec + + def __call__(self, src): """Augmenter body""" - alpha = np.random.normal(0, alphastd, size=(3,)) - rgb = np.dot(eigvec * alpha, eigval) + alpha = np.random.normal(0, self.alphastd, size=(3,)) + rgb = np.dot(self.eigvec * alpha, self.eigval) src += nd.array(rgb) return [src] - return aug +class ColorNormalizeAug(Augmenter): + """Mean and std normalization. -def ColorNormalizeAug(mean, std): - """Mean and std normalization.""" - mean = nd.array(mean) - std = nd.array(std) + Parameters + ---------- + mean : NDArray + RGB mean to be subtracted + std : NDArray + RGB standard deviation to be divided + """ + def __init__(self, mean, std): + super(ColorNormalizeAug, self).__init__(mean=mean, std=std) + self.mean = nd.array(mean) if mean is not None else None + self.std = nd.array(std) if std is not None else None - def aug(src): + def __call__(self, src): + """Augmenter body""" + return [color_normalize(src, self.mean, self.std)] + + +class RandomGrayAug(Augmenter): + """Randomly convert to gray image. + + Parameters + ---------- + p : float + Probability to convert to grayscale + """ + def __init__(self, p): + super(RandomGrayAug, self).__init__(p=p) + self.p = p + self.mat = nd.array([[0.21, 0.21, 0.21], + [0.72, 0.72, 0.72], + [0.07, 0.07, 0.07]]) + + def __call__(self, src): """Augmenter body""" - return [color_normalize(src, mean, std)] + if random.random() < self.p: + src = nd.dot(src, self.mat) + return [src] - return aug +class HorizontalFlipAug(Augmenter): + """Random horizontal flip. -def HorizontalFlipAug(p): - """Random horizontal flipping.""" + Parameters + ---------- + p : float + Probability to flip image horizontally + """ + def __init__(self, p): + super(HorizontalFlipAug, self).__init__(p=p) + self.p = p - def aug(src): + def __call__(self, src): """Augmenter body""" - if random.random() < p: + if random.random() < self.p: src = nd.flip(src, axis=1) return [src] - return aug - -def CastAug(): +class CastAug(Augmenter): """Cast to float32""" + def __init__(self): + super(CastAug, self).__init__(type='float32') - def aug(src): + def __call__(self, src): """Augmenter body""" src = src.astype(np.float32) return [src] - return aug - def CreateAugmenter(data_shape, resize=0, rand_crop=False, rand_resize=False, rand_mirror=False, - mean=None, std=None, brightness=0, contrast=0, saturation=0, - pca_noise=0, inter_method=2): - """Creates an augmenter list.""" + mean=None, std=None, brightness=0, contrast=0, saturation=0, hue=0, + pca_noise=0, rand_gray=0, inter_method=2): + """Creates an augmenter list. + + Parameters + ---------- + data_shape : tuple of int + shape for output data + resize : int + resize shorter edge if larger than 0 at the begining + rand_resize : float + [0, 1], probability to apply random resizing + rand_gray : float + [0, 1], probability to convert to grayscale for all channels + rand_mirror : bool + whether apply horizontal flip to image with probability 0.5 + mean : np.ndarray or None + mean pixel values for [r, g, b] + std : np.ndarray or None + standard deviations for [r, g, b] + brightness : float + brightness jittering range (percent) + contrast : float + contrast jittering range + saturation : float + saturation jittering range + hue : float + hue jittering range + pca_noise : float + pca noise level + inter_method : int, default=2(Area-based) + interpolation method for all resizing operations + + Possible values: + 0: Nearest Neighbors Interpolation. + 1: Bilinear interpolation. + 2: Area-based (resampling using pixel area relation). It may be a + preferred method for image decimation, as it gives moire-free + results. But when the image is zoomed, it is similar to the Nearest + Neighbors method. (used by default). + 3: Bicubic interpolation over 4x4 pixel neighborhood. + 4: Lanczos interpolation over 8x8 pixel neighborhood. + 9: Cubic for enlarge, area for shrink, bilinear for others + 10: Random select from interpolation method metioned above. + Note: + When shrinking an image, it will generally look best with AREA-based + interpolation, whereas, when enlarging an image, it will generally look best + with Bicubic (slow) or Bilinear (faster but still looks OK). + + Examples + -------- + >>> # An example of creating multiple augmenters + >>> augs = mx.image.CreateAugmenter(data_shape=(3, 300, 300), rand_mirror=True, + ... mean=True, brightness=0.125, contrast=0.125, rand_gray=0.05, + ... saturation=0.125, pca_noise=0.05, inter_method=10) + >>> # dump the details + >>> for aug in augs: + ... aug.dumps() + """ auglist = [] if resize > 0: @@ -476,6 +882,9 @@ def CreateAugmenter(data_shape, resize=0, rand_crop=False, rand_resize=False, ra if brightness or contrast or saturation: auglist.append(ColorJitterAug(brightness, contrast, saturation)) + if hue: + auglist.append(HueJitterAug(hue)) + if pca_noise > 0: eigval = np.array([55.46, 4.794, 1.148]) eigvec = np.array([[-0.5675, 0.7192, 0.4009], @@ -483,6 +892,9 @@ def CreateAugmenter(data_shape, resize=0, rand_crop=False, rand_resize=False, ra [-0.5836, -0.6948, 0.4203]]) auglist.append(LightingAug(pca_noise, eigval, eigvec)) + if rand_gray > 0: + auglist.append(RandomGrayAug(rand_gray)) + if mean is True: mean = np.array([123.68, 116.28, 103.53]) elif mean is not None: @@ -493,7 +905,7 @@ def CreateAugmenter(data_shape, resize=0, rand_crop=False, rand_resize=False, ra elif std is not None: assert isinstance(std, np.ndarray) and std.shape[0] in [1, 3] - if mean is not None and std is not None: + if mean is not None or std is not None: auglist.append(ColorNormalizeAug(mean, std)) return auglist @@ -552,8 +964,14 @@ def __init__(self, batch_size, data_shape, label_width=1, data_name='data', label_name='softmax_label', **kwargs): super(ImageIter, self).__init__() assert path_imgrec or path_imglist or (isinstance(imglist, list)) + num_threads = os.environ.get('MXNET_CPU_WORKER_NTHREADS', 1) + logging.info('Using %s threads for decoding...', str(num_threads)) + logging.info('Set enviroment variable MXNET_CPU_WORKER_NTHREADS to a' + ' larger number to use more threads.') + class_name = self.__class__.__name__ if path_imgrec: - print('loading recordio...') + logging.info('%s: loading recordio %s...', + class_name, path_imgrec) if path_imgidx: self.imgrec = recordio.MXIndexedRecordIO(path_imgidx, path_imgrec, 'r') # pylint: disable=redefined-variable-type self.imgidx = list(self.imgrec.keys) @@ -564,7 +982,7 @@ def __init__(self, batch_size, data_shape, label_width=1, self.imgrec = None if path_imglist: - print('loading image list...') + logging.info('%s: loading image list %s...', class_name, path_imglist) with open(path_imglist) as fin: imglist = {} imgkeys = [] @@ -576,18 +994,20 @@ def __init__(self, batch_size, data_shape, label_width=1, imgkeys.append(key) self.imglist = imglist elif isinstance(imglist, list): - print('loading image list...') + logging.info('%s: loading image list...', class_name) result = {} imgkeys = [] index = 1 for img in imglist: key = str(index) # pylint: disable=redefined-variable-type index += 1 - if isinstance(img[0], numeric_types): + if len(img) > 2: + label = nd.array(img[:-1]) + elif isinstance(img[0], numeric_types): label = nd.array([img[0]]) else: label = nd.array(img[0]) - result[key] = (label, img[1]) + result[key] = (label, img[-1]) imgkeys.append(str(key)) self.imglist = result else: @@ -616,7 +1036,7 @@ def __init__(self, batch_size, data_shape, label_width=1, if num_parts > 1: assert part_index < num_parts N = len(self.seq) - C = N / num_parts + C = N // num_parts self.seq = self.seq[part_index * C:(part_index + 1) * C] if aug_list is None: self.auglist = CreateAugmenter(data_shape, **kwargs) diff --git a/python/mxnet/test_utils.py b/python/mxnet/test_utils.py index 0666e46d930f..ae5abdd2be54 100644 --- a/python/mxnet/test_utils.py +++ b/python/mxnet/test_utils.py @@ -945,9 +945,6 @@ def download(url, fname=None, dirname=None, overwrite=False): """ if fname is None: fname = url.split('/')[-1] - if not overwrite and os.path.exists(fname): - logging.info("%s exists, skipping download", fname) - return fname if dirname is None: dirname = os.path.dirname(fname) @@ -962,6 +959,10 @@ def download(url, fname=None, dirname=None, overwrite=False): if exc.errno != errno.EEXIST: raise OSError('failed to create ' + dirname) + if not overwrite and os.path.exists(fname): + logging.info("%s exists, skipping download", fname) + return fname + r = requests.get(url, stream=True) assert r.status_code == 200, "failed to open %s" % url with open(fname, 'wb') as f: diff --git a/src/io/image_io.cc b/src/io/image_io.cc index 1ef1df1b74bd..64fd2dde1908 100644 --- a/src/io/image_io.cc +++ b/src/io/image_io.cc @@ -13,6 +13,7 @@ #include #include #include +#include #include "../operator/elemwise_op_common.h" @@ -210,6 +211,7 @@ struct MakeBorderParam : public dmlc::Parameter { int top, bot, left, right; int type; double value; + nnvm::Tuple values; DMLC_DECLARE_PARAMETER(MakeBorderParam) { DMLC_DECLARE_FIELD(top) .describe("Top margin."); @@ -224,7 +226,10 @@ struct MakeBorderParam : public dmlc::Parameter { .describe("Filling type (default=cv2.BORDER_CONSTANT)."); DMLC_DECLARE_FIELD(value) .set_default(0.0) - .describe("Fill with value."); + .describe("(Deprecated! Use ``values`` instead.) Fill with single value."); + DMLC_DECLARE_FIELD(values) + .set_default({}) + .describe("Fill with value(RGB[A] or gray), up to 4 channels."); } }; DMLC_REGISTER_PARAMETER(MakeBorderParam); @@ -255,9 +260,11 @@ inline void copyMakeBorder(const nnvm::NodeAttrs& attrs, const auto& param = nnvm::get(attrs.parsed); cv::Mat buf(inputs[0].shape_[0], inputs[0].shape_[1], cv_type, inputs[0].dptr_); cv::Mat dst(outputs[0].shape_[0], outputs[0].shape_[1], cv_type, outputs[0].dptr_); - cv::copyMakeBorder(buf, dst, - param.top, param.bot, param.left, param.right, - param.type, cv::Scalar(param.value)); + cv::Scalar color(param.value, param.value, param.value); + if (param.values.ndim() > 0) { + color = cv::Scalar(cv::Vec(param.values.begin())); + } + cv::copyMakeBorder(buf, dst, param.top, param.bot, param.left, param.right, param.type, color); CHECK(!dst.empty()); CHECK_EQ(static_cast(dst.ptr()), outputs[0].dptr_); #else @@ -300,5 +307,3 @@ NNVM_REGISTER_OP(_cvcopyMakeBorder) } // namespace io } // namespace mxnet - - diff --git a/tests/python/unittest/test_image.py b/tests/python/unittest/test_image.py new file mode 100644 index 000000000000..638dbf06a683 --- /dev/null +++ b/tests/python/unittest/test_image.py @@ -0,0 +1,167 @@ +import mxnet as mx +import numpy as np +from mxnet.test_utils import * +from common import assertRaises + + +def _get_data(url, dirname): + import os, tarfile + download(url, dirname=dirname, overwrite=False) + fname = os.path.join(dirname, url.split('/')[-1]) + tar = tarfile.open(fname) + source_images = [os.path.join(dirname, x.name) for x in tar.getmembers() if x.isfile()] + if len(source_images) < 1 or not os.path.isfile(source_images[0]): + # skip extracting if exists + tar.extractall(path=dirname) + tar.close() + return source_images + +def _get_images(): + return _get_data("http://data.mxnet.io/data/test_images.tar.gz", './data') + +def test_init(): + _get_images() + +def test_imdecode(): + try: + import cv2 + except ImportError: + return + sources = _get_images() + for img in sources: + with open(img, 'rb') as fp: + str_image = fp.read() + image = mx.image.imdecode(str_image, to_rgb=0) + cv_image = cv2.imread(img) + assert_almost_equal(image.asnumpy(), cv_image) + +def test_scale_down(): + assert mx.image.scale_down((640, 480), (720, 120)) == (640, 106) + assert mx.image.scale_down((360, 1000), (480, 500)) == (360, 375) + assert mx.image.scale_down((300, 400), (0, 0)) == (0, 0) + +def test_resize_short(): + try: + import cv2 + except ImportError: + return + sources = _get_images() + for img in sources: + cv_img = cv2.imread(img) + mx_img = mx.nd.array(cv_img[:, :, (2, 1, 0)]) + h, w, _ = cv_img.shape + for _ in range(3): + new_size = np.random.randint(1, 1000) + if h > w: + new_h, new_w = new_size * h / w, new_size + else: + new_h, new_w = new_size, new_size * w / h + for interp in range(0, 2): + # area-based/lanczos don't match with cv2? + cv_resized = cv2.resize(cv_img, (new_w, new_h), interpolation=interp) + mx_resized = mx.image.resize_short(mx_img, new_size, interp) + assert_almost_equal(mx_resized.asnumpy()[:, :, (2, 1, 0)], cv_resized, atol=3) + +def test_color_normalize(): + for _ in range(10): + mean = np.random.rand(3) * 255 + std = np.random.rand(3) + 1 + width = np.random.randint(100, 500) + height = np.random.randint(100, 500) + src = np.random.rand(height, width, 3) * 255. + mx_result = mx.image.color_normalize(mx.nd.array(src), + mx.nd.array(mean), mx.nd.array(std)) + assert_almost_equal(mx_result.asnumpy(), (src - mean) / std, atol=1e-3) + + +def test_imageiter(): + sources = _get_images() + im_list = [[np.random.randint(0, 5), x] for x in sources] + test_iter = mx.image.ImageIter(2, (3, 224, 224), label_width=1, imglist=im_list, + path_root='') + for _ in range(3): + for batch in test_iter: + pass + test_iter.reset() + + # test with list file + fname = './data/test_imageiter.lst' + file_list = ['\t'.join([str(k), str(np.random.randint(0, 5)), x]) \ + for k, x in enumerate(sources)] + with open(fname, 'w') as f: + for line in file_list: + f.write(line + '\n') + + test_iter = mx.image.ImageIter(2, (3, 224, 224), label_width=1, path_imglist=fname, + path_root='') + for batch in test_iter: + pass + + +def test_augmenters(): + # only test if all augmenters will work + # TODO(Joshua Zhang): verify the augmenter outputs + sources = _get_images() + im_list = [[0, x] for x in sources] + test_iter = mx.image.ImageIter(2, (3, 224, 224), label_width=1, imglist=im_list, + resize=640, rand_crop=True, rand_resize=True, rand_mirror=True, mean=True, + std=np.array([1.1, 1.03, 1.05]), brightness=0.1, contrast=0.1, saturation=0.1, + hue=0.1, pca_noise=0.1, rand_gray=0.2, inter_method=10, path_root='', shuffle=True) + for batch in test_iter: + pass + +def _generate_objects(): + num = np.random.randint(1, 10) + xy = np.random.rand(num, 2) + wh = np.random.rand(num, 2) / 2 + left = (xy[:, 0] - wh[:, 0])[:, np.newaxis] + right = (xy[:, 0] + wh[:, 0])[:, np.newaxis] + top = (xy[:, 1] - wh[:, 1])[:, np.newaxis] + bot = (xy[:, 1] + wh[:, 1])[:, np.newaxis] + boxes = np.maximum(0., np.minimum(1., np.hstack((left, top, right, bot)))) + cid = np.random.randint(0, 20, size=num) + label = np.hstack((cid[:, np.newaxis], boxes)).ravel().tolist() + return [2, 5] + label + +def test_image_detiter(): + sources = _get_images() + im_list = [_generate_objects() + [x] for x in sources] + det_iter = mx.image.ImageDetIter(2, (3, 300, 300), imglist=im_list, path_root='') + for _ in range(3): + for batch in det_iter: + pass + det_iter.reset() + + val_iter = mx.image.ImageDetIter(2, (3, 300, 300), imglist=im_list, path_root='') + det_iter = val_iter.sync_label_shape(det_iter) + + # test file list + fname = './data/test_imagedetiter.lst' + im_list = [[k] + _generate_objects() + [x] for k, x in enumerate(sources)] + with open(fname, 'w') as f: + for line in im_list: + line = '\t'.join([str(k) for k in line]) + f.write(line + '\n') + + det_iter = mx.image.ImageDetIter(2, (3, 400, 400), path_imglist=fname, + path_root='') + for batch in det_iter: + pass + +def test_det_augmenters(): + # only test if all augmenters will work + # TODO(Joshua Zhang): verify the augmenter outputs + sources = _get_images() + im_list = [_generate_objects() + [x] for x in sources] + det_iter = mx.image.ImageDetIter(2, (3, 300, 300), imglist=im_list, path_root='', + resize=640, rand_crop=1, rand_pad=1, rand_gray=0.1, rand_mirror=True, mean=True, + std=np.array([1.1, 1.03, 1.05]), brightness=0.1, contrast=0.1, saturation=0.1, + pca_noise=0.1, hue=0.1, inter_method=10, min_object_covered=0.5, + aspect_ratio_range=(0.2, 5), area_range=(0.1, 4.0), min_eject_coverage=0.5, + max_attempts=50) + for batch in det_iter: + pass + +if __name__ == '__main__': + import nose + nose.runmodule() diff --git a/tools/caffe_converter/caffe_parser.py b/tools/caffe_converter/caffe_parser.py index 45efe4715f03..d4abc8eac614 100644 --- a/tools/caffe_converter/caffe_parser.py +++ b/tools/caffe_converter/caffe_parser.py @@ -12,7 +12,7 @@ raise ImportError('You used to compile with protoc --python_out=./ ./caffe.proto') use_caffe = False -from google.protobuf import text_format +from google.protobuf import text_format # pylint: disable=relative-import def read_prototxt(fname): """Return a caffe_pb2.NetParameter object that defined in a prototxt file diff --git a/tools/caffe_converter/caffe_proto_utils.py b/tools/caffe_converter/caffe_proto_utils.py index 4404f39b8698..940e8a47a1d6 100644 --- a/tools/caffe_converter/caffe_proto_utils.py +++ b/tools/caffe_converter/caffe_proto_utils.py @@ -112,7 +112,7 @@ def read_network_dag(processed_deploy_prototxt): """ from caffe.proto import caffe_pb2 - from google.protobuf import text_format + from google.protobuf import text_format # pylint: disable=relative-import from collections import OrderedDict # load prototxt file From f34cd23e60e35e828efa75e3d536688d1b6e18a9 Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Sun, 23 Jul 2017 16:23:50 -0700 Subject: [PATCH 252/834] remove where= from find_packages() (#7167) --- python/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/setup.py b/python/setup.py index d4b132014c84..f5bd55de6cf6 100644 --- a/python/setup.py +++ b/python/setup.py @@ -74,7 +74,7 @@ def config_cython(): setup(name='mxnet', version=__version__, description=open(os.path.join(CURRENT_DIR, 'README.md')).read(), - packages=find_packages(where=CURRENT_DIR), + packages=find_packages(), data_files=[('mxnet', [LIB_PATH[0]])], url='https://github.com/dmlc/mxnet', ext_modules=config_cython(), From 83828cbf0fab1a2759b42a3758d26ea7299b3ae0 Mon Sep 17 00:00:00 2001 From: moin Date: Mon, 24 Jul 2017 22:09:16 +0200 Subject: [PATCH 253/834] bugfixes for linear algebra operators (#7175) --- src/operator/tensor/la_op.cc | 6 +++--- src/operator/tensor/la_op_inline.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/operator/tensor/la_op.cc b/src/operator/tensor/la_op.cc index 47582d6a8bfe..859e761c7409 100644 --- a/src/operator/tensor/la_op.cc +++ b/src/operator/tensor/la_op.cc @@ -67,7 +67,7 @@ NNVM_REGISTER_OP(_backward_linalg_gemm) .set_num_outputs(3) .set_attr_parser(ParamParser) .set_attr("FInplaceOption", [](const NodeAttrs& attrs) - { return std::vector >{{1, 0}, {2, 1}, {3, 2}}; }) + { return std::vector >{{2, 1}, {3, 2}}; }) .set_attr("FResourceRequest", [](const NodeAttrs& attrs) { return std::vector{ResourceRequest::kTempSpace}; }) .set_attr("TIsBackward", true) @@ -122,7 +122,7 @@ NNVM_REGISTER_OP(_backward_linalg_gemm2) .set_num_outputs(2) .set_attr_parser(ParamParser) .set_attr("FInplaceOption", [](const NodeAttrs& attrs) - { return std::vector >{{1, 0}, {2, 1}}; }) + { return std::vector >{{2, 1}}; }) .set_attr("FResourceRequest", [](const NodeAttrs& attrs) { return std::vector{ResourceRequest::kTempSpace}; }) .set_attr("TIsBackward", true) @@ -349,7 +349,7 @@ NNVM_REGISTER_OP(_backward_linalg_trsm) .set_num_outputs(2) .set_attr_parser(ParamParser) .set_attr("FInplaceOption", [](const NodeAttrs& attrs) - { return std::vector >{{0, 1}}; }) + { return std::vector >{{0, 1}, {1, 0}}; }) .set_attr("FResourceRequest", [](const NodeAttrs& attrs) { return std::vector{ResourceRequest::kTempSpace}; }) .set_attr("TIsBackward", true) diff --git a/src/operator/tensor/la_op_inline.h b/src/operator/tensor/la_op_inline.h index efa6c538cfff..b83bf81e6e2d 100644 --- a/src/operator/tensor/la_op_inline.h +++ b/src/operator/tensor/la_op_inline.h @@ -132,7 +132,7 @@ void potri::op(const Tensor& L, const Tensor void potri::op(const Tensor& A, const Tensor& L, const nnvm::NodeAttrs& attrs) { - if ( A.dptr_ != L.dptr_ ) Copy(L, A); + if ( A.dptr_ != L.dptr_ ) Copy(A, L); FUNC_SIGNATURE_1(dpotri, A); CopyLowerToUpper(A.dptr_, A.size(0)); } From c0377a54b59070fce86d543aaa2bba475bbbc300 Mon Sep 17 00:00:00 2001 From: Nan Zhu Date: Mon, 24 Jul 2017 19:25:12 -0700 Subject: [PATCH 254/834] [scala-package] improve the readability of Spark-Mxnet implementation (#7141) * cleanup spark * stylistic fix --- .../scala/ml/dmlc/mxnet/spark/MXNet.scala | 132 +++++++++++------- 1 file changed, 79 insertions(+), 53 deletions(-) diff --git a/scala-package/spark/src/main/scala/ml/dmlc/mxnet/spark/MXNet.scala b/scala-package/spark/src/main/scala/ml/dmlc/mxnet/spark/MXNet.scala index 17bed7e19b2e..27dd99f07233 100644 --- a/scala-package/spark/src/main/scala/ml/dmlc/mxnet/spark/MXNet.scala +++ b/scala-package/spark/src/main/scala/ml/dmlc/mxnet/spark/MXNet.scala @@ -20,9 +20,12 @@ package ml.dmlc.mxnet.spark import ml.dmlc.mxnet._ import ml.dmlc.mxnet.optimizer.SGD import ml.dmlc.mxnet.spark.io.LabeledPointIter + +import org.slf4j.{Logger, LoggerFactory} + import org.apache.spark.mllib.regression.LabeledPoint import org.apache.spark.rdd.RDD -import org.slf4j.{Logger, LoggerFactory} +import org.apache.spark.SparkContext /** * MXNet Training On Spark @@ -102,25 +105,10 @@ class MXNet extends Serializable { this } - def fit(data: RDD[LabeledPoint]): MXNetModel = { - val sc = data.context - // distribute native jars - params.jars.foreach(jar => sc.addFile(jar)) - - val trainData = { - if (params.numWorker > data.partitions.length) { - logger.info("repartitioning training set to {} partitions", params.numWorker) - data.repartition(params.numWorker) - } else if (params.numWorker < data.partitions.length) { - logger.info("repartitioning training set to {} partitions", params.numWorker) - data.coalesce(params.numWorker) - } else { - data - } - } - - val schedulerIP = utils.Network.ipAddress - val schedulerPort = utils.Network.availablePort + private def startParameterServers( + schedulerIP: String, + schedulerPort: Int, + sc: SparkContext): ParameterServer = { // TODO: check ip & port available logger.info("Starting scheduler on {}:{}", schedulerIP, schedulerPort) val scheduler = new ParameterServer(params.runtimeClasspath, role = "scheduler", @@ -140,14 +128,58 @@ class MXNet extends Serializable { java = params.javabin) require(server.startProcess(), "Failed to start ps server process") } + scheduler + } + + private def setFeedForwardModel( + optimizer: Optimizer, + numExamples: Int, + kv: KVStore, + inputInPartition: LabeledPointIter): FeedForward = { + logger.debug("Define model") + val model = new FeedForward(ctx = params.context, + symbol = params.getNetwork, + numEpoch = params.numEpoch, + optimizer = optimizer, + initializer = new Xavier(factorType = "in", magnitude = 2.34f), + argParams = null, + auxParams = null, + beginEpoch = 0, + epochSize = numExamples / params.batchSize / kv.numWorkers) + logger.info("Start training ...") + model.fit(trainData = inputInPartition, + evalData = null, + evalMetric = new Accuracy(), + kvStore = kv) + model + } + + private def setupKVStore(schedulerIP: String, schedulerPort: Int): KVStore = { + KVStoreServer.init(ParameterServer.buildEnv(role = "worker", + rootUri = schedulerIP, rootPort = schedulerPort, + numServer = params.numServer, + numWorker = params.numWorker)) + val kv = KVStore.create("dist_async") + kv.setBarrierBeforeExit(false) + kv + } + + private def reclaimResources(dataIter: LabeledPointIter, kv: KVStore): Unit = { + dataIter.dispose() + kv.setBarrierBeforeExit(true) + kv.dispose() + } + private def trainModel( + trainData: RDD[LabeledPoint], + schedulerIP: String, + schedulerPort: Int): MXNetModel = { val job = trainData.mapPartitions { partition => val dataIter = new LabeledPointIter( partition, params.dimension, params.batchSize, dataName = params.dataName, labelName = params.labelName) - // TODO: more nature way to get the # of examples? var numExamples = 0 while (dataIter.hasNext) { @@ -161,46 +193,40 @@ class MXNet extends Serializable { logger.info("Batch {}", params.batchSize) // give enough time for ps-lite to detect the dead nodes Thread.sleep(20000) - KVStoreServer.init(ParameterServer.buildEnv(role = "worker", - rootUri = schedulerIP, rootPort = schedulerPort, - numServer = params.numServer, - numWorker = params.numWorker)) - val kv = KVStore.create("dist_async") - kv.setBarrierBeforeExit(false) - - val optimizer: Optimizer = new SGD(learningRate = 0.01f, - momentum = 0.9f, wd = 0.00001f) - - logger.debug("Define model") - val model = new FeedForward(ctx = params.context, - symbol = params.getNetwork, - numEpoch = params.numEpoch, - optimizer = optimizer, - initializer = new Xavier(factorType = "in", magnitude = 2.34f), - argParams = null, - auxParams = null, - beginEpoch = 0, - epochSize = numExamples / params.batchSize / kv.numWorkers) - logger.info("Start training ...") - model.fit(trainData = dataIter, - evalData = null, - evalMetric = new Accuracy(), - kvStore = kv) - + val kv = setupKVStore(schedulerIP, schedulerPort) + val optimizer = new SGD(learningRate = 0.01f, momentum = 0.9f, wd = 0.00001f) + val model = setFeedForwardModel(optimizer, numExamples, kv, dataIter) logger.info("Training finished, waiting for other workers ...") - dataIter.dispose() - kv.setBarrierBeforeExit(true) - kv.dispose() + reclaimResources(dataIter, kv) Iterator(new MXNetModel( model, params.dimension, params.batchSize, dataName = params.dataName, labelName = params.labelName)) }.cache() - // force job to run job.foreachPartition(() => _) - // simply the first model - val mxModel = job.first() + job.first() + } + def fit(data: RDD[LabeledPoint]): MXNetModel = { + val sc = data.context + // distribute native jars + params.jars.foreach(jar => sc.addFile(jar)) + val trainData = { + if (params.numWorker > data.partitions.length) { + logger.info("repartitioning training set to {} partitions", params.numWorker) + data.repartition(params.numWorker) + } else if (params.numWorker < data.partitions.length) { + logger.info("repartitioning training set to {} partitions", params.numWorker) + data.coalesce(params.numWorker) + } else { + data + } + } + val schedulerIP = utils.Network.ipAddress + val schedulerPort = utils.Network.availablePort + val scheduler = startParameterServers(schedulerIP, schedulerPort, sc) + // simply the first model + val mxModel = trainModel(trainData, schedulerIP, schedulerPort) logger.info("Waiting for scheduler ...") scheduler.waitFor() mxModel From 84ce29bcbbe0b17240d1976f49871eab0560238c Mon Sep 17 00:00:00 2001 From: Eric Junyuan Xie Date: Mon, 24 Jul 2017 20:39:50 -0700 Subject: [PATCH 255/834] Gluon data pipeline (#7155) * add data pipeline to gluon * add cifar * fix * fix * fix --- example/gluon/actor_critic.py | 2 +- example/gluon/dcgan.py | 339 ++++++++++++----------- example/gluon/mnist.py | 35 ++- example/gluon/super_resolution.py | 7 +- python/mxnet/gluon/.gitignore | 1 + python/mxnet/gluon/__init__.py | 2 + python/mxnet/gluon/data/__init__.py | 11 + python/mxnet/gluon/data/dataloader.py | 70 +++++ python/mxnet/gluon/data/dataset.py | 89 ++++++ python/mxnet/gluon/data/sampler.py | 120 ++++++++ python/mxnet/gluon/data/vision.py | 125 +++++++++ python/mxnet/gluon/utils.py | 50 ++++ python/mxnet/image/image.py | 4 +- python/mxnet/ndarray.py | 12 +- src/operator/elemwise_op_common.h | 16 +- src/operator/tensor/elemwise_sum.cc | 20 +- src/operator/tensor/matrix_op-inl.h | 109 ++++++++ src/operator/tensor/matrix_op.cc | 52 ++++ src/operator/tensor/matrix_op.cu | 6 + tests/python/unittest/test_gluon_data.py | 53 ++++ tests/python/unittest/test_operator.py | 17 ++ 21 files changed, 933 insertions(+), 207 deletions(-) create mode 100644 python/mxnet/gluon/.gitignore create mode 100644 python/mxnet/gluon/data/__init__.py create mode 100644 python/mxnet/gluon/data/dataloader.py create mode 100644 python/mxnet/gluon/data/dataset.py create mode 100644 python/mxnet/gluon/data/sampler.py create mode 100644 python/mxnet/gluon/data/vision.py create mode 100644 tests/python/unittest/test_gluon_data.py diff --git a/example/gluon/actor_critic.py b/example/gluon/actor_critic.py index 7910c73030e1..9c475ce15017 100644 --- a/example/gluon/actor_critic.py +++ b/example/gluon/actor_critic.py @@ -43,7 +43,7 @@ def forward(self, x): return F.softmax(probs), values net = Policy() -net.collect_params().initialize(mx.init.Uniform(0.02)) +net.initialize(mx.init.Uniform(0.02)) trainer = gluon.Trainer(net.collect_params(), 'adam', {'learning_rate': 3e-2}) loss = gluon.loss.L1Loss() diff --git a/example/gluon/dcgan.py b/example/gluon/dcgan.py index 17d02e7fbede..7f644cba5962 100644 --- a/example/gluon/dcgan.py +++ b/example/gluon/dcgan.py @@ -7,10 +7,8 @@ from mxnet import gluon from mxnet.gluon import nn from mxnet import autograd -from data import cifar10_iterator import numpy as np import logging -import cv2 from datetime import datetime import os import time @@ -32,173 +30,190 @@ def visual(title, X, name): buff = np.zeros((int(n*X.shape[1]), int(n*X.shape[2]), int(X.shape[3])), dtype=np.uint8) for i, img in enumerate(X): fill_buf(buff, i, img, X.shape[1:3]) - buff = cv2.cvtColor(buff, cv2.COLOR_BGR2RGB) + buff = buff[:,:,::-1] plt.imshow(buff) plt.title(title) plt.savefig(name) - return None -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('--dataset', type=str, default='cifar10', help='dataset to use. options are cifar10 and imagenet.') - parser.add_argument('--batchSize', type=int, default=64, help='input batch size') - parser.add_argument('--imageSize', type=int, default=64, help='the height / width of the input image to network') - parser.add_argument('--nz', type=int, default=100, help='size of the latent z vector') - parser.add_argument('--ngf', type=int, default=64) - parser.add_argument('--ndf', type=int, default=64) - parser.add_argument('--niter', type=int, default=25, help='number of epochs to train for') - parser.add_argument('--lr', type=float, default=0.0002, help='learning rate, default=0.0002') - parser.add_argument('--beta1', type=float, default=0.5, help='beta1 for adam. default=0.5') - parser.add_argument('--cuda', action='store_true', help='enables cuda') - parser.add_argument('--ngpu', type=int, default=1, help='number of GPUs to use') - parser.add_argument('--netG', default='', help="path to netG (to continue training)") - parser.add_argument('--netD', default='', help="path to netD (to continue training)") - parser.add_argument('--outf', default='./results', help='folder to output images and model checkpoints') - parser.add_argument('--manualSeed', type=int, help='manual seed') - parser.add_argument('--check_point', default=True, help="save results at each epoch or not") - - opt = parser.parse_args() - print(opt) - - logging.basicConfig(level=logging.DEBUG) - ngpu = int(opt.ngpu) - nz = int(opt.nz) - ngf = int(opt.ngf) - ndf = int(opt.ndf) - nc = 3 + +parser = argparse.ArgumentParser() +parser.add_argument('--dataset', type=str, default='cifar10', help='dataset to use. options are cifar10 and imagenet.') +parser.add_argument('--batch-size', type=int, default=64, help='input batch size') +parser.add_argument('--nz', type=int, default=100, help='size of the latent z vector') +parser.add_argument('--ngf', type=int, default=64) +parser.add_argument('--ndf', type=int, default=64) +parser.add_argument('--nepoch', type=int, default=25, help='number of epochs to train for') +parser.add_argument('--lr', type=float, default=0.0002, help='learning rate, default=0.0002') +parser.add_argument('--beta1', type=float, default=0.5, help='beta1 for adam. default=0.5') +parser.add_argument('--cuda', action='store_true', help='enables cuda') +parser.add_argument('--ngpu', type=int, default=1, help='number of GPUs to use') +parser.add_argument('--netG', default='', help="path to netG (to continue training)") +parser.add_argument('--netD', default='', help="path to netD (to continue training)") +parser.add_argument('--outf', default='./results', help='folder to output images and model checkpoints') +parser.add_argument('--check-point', default=True, help="save results at each epoch or not") + +opt = parser.parse_args() +print(opt) + +logging.basicConfig(level=logging.DEBUG) +ngpu = int(opt.ngpu) +nz = int(opt.nz) +ngf = int(opt.ngf) +ndf = int(opt.ndf) +nc = 3 +if opt.cuda: ctx = mx.gpu(0) - check_point = bool(opt.check_point) - outf = opt.outf - - if not os.path.exists(outf): - os.makedirs(outf) - - if opt.dataset == 'cifar10': - train_iter, val_iter = cifar10_iterator(opt.batchSize, (3, 64, 64), 64) - - # build the generator - netG = nn.Sequential() - with netG.name_scope(): - # input is Z, going into a convolution - netG.add(nn.Conv2DTranspose(ngf * 8, 4, 1, 0, use_bias=False)) - netG.add(nn.BatchNorm()) - netG.add(nn.Activation('relu')) - # state size. (ngf*8) x 4 x 4 - netG.add(nn.Conv2DTranspose(ngf * 4, 4, 2, 1, use_bias=False)) - netG.add(nn.BatchNorm()) - netG.add(nn.Activation('relu')) - # state size. (ngf*8) x 8 x 8 - netG.add(nn.Conv2DTranspose(ngf * 2, 4, 2, 1, use_bias=False)) - netG.add(nn.BatchNorm()) - netG.add(nn.Activation('relu')) - # state size. (ngf*8) x 16 x 16 - netG.add(nn.Conv2DTranspose(ngf, 4, 2, 1, use_bias=False)) - netG.add(nn.BatchNorm()) - netG.add(nn.Activation('relu')) - # state size. (ngf*8) x 32 x 32 - netG.add(nn.Conv2DTranspose(nc, 4, 2, 1, use_bias=False)) - netG.add(nn.Activation('tanh')) - # state size. (nc) x 64 x 64 - - # build the discriminator - netD = nn.Sequential() - with netD.name_scope(): - # input is (nc) x 64 x 64 - netD.add(nn.Conv2D(ndf, 4, 2, 1, use_bias=False)) - netD.add(nn.LeakyReLU(0.2)) - # state size. (ndf) x 32 x 32 - netD.add(nn.Conv2D(ndf * 2, 4, 2, 1, use_bias=False)) - netD.add(nn.BatchNorm()) - netD.add(nn.LeakyReLU(0.2)) - # state size. (ndf) x 16 x 16 - netD.add(nn.Conv2D(ndf * 4, 4, 2, 1, use_bias=False)) - netD.add(nn.BatchNorm()) - netD.add(nn.LeakyReLU(0.2)) - # state size. (ndf) x 8 x 8 - netD.add(nn.Conv2D(ndf * 8, 4, 2, 1, use_bias=False)) - netD.add(nn.BatchNorm()) - netD.add(nn.LeakyReLU(0.2)) - # state size. (ndf) x 4 x 4 - netD.add(nn.Conv2D(2, 4, 1, 0, use_bias=False)) - - # loss - loss = gluon.loss.SoftmaxCrossEntropyLoss() - - # initialize the generator and the discriminator - netG.collect_params().initialize(mx.init.Normal(0.02), ctx=ctx) - netD.collect_params().initialize(mx.init.Normal(0.02), ctx=ctx) - - # trainer for the generator and the discriminator - trainerG = gluon.Trainer(netG.collect_params(), 'adam', {'learning_rate': opt.lr, 'beta1': opt.beta1}) - trainerD = gluon.Trainer(netD.collect_params(), 'adam', {'learning_rate': opt.lr, 'beta1': opt.beta1}) - - # ============printing============== - real_label = mx.nd.ones((opt.batchSize,), ctx=ctx) - fake_label = mx.nd.zeros((opt.batchSize,), ctx=ctx) - - metric = mx.metric.Accuracy() - print('Training... ') - stamp = datetime.now().strftime('%Y_%m_%d-%H_%M') - - iter = 0 - for epoch in range(opt.niter): - tic = time.time() - train_iter.reset() - btic = time.time() - for batch in train_iter: - ############################ - # (1) Update D network: maximize log(D(x)) + log(1 - D(G(z))) - ########################### - # train with real_t - data = batch.data[0].copyto(ctx) - noise = mx.nd.random_normal(0, 1, shape=(opt.batchSize, nz, 1, 1), ctx=ctx) - - with autograd.record(): - output = netD(data) - output = output.reshape((opt.batchSize, 2)) - errD_real = loss(output, real_label) - metric.update([real_label,], [output,]) - - fake = netG(noise) - output = netD(fake.detach()) - output = output.reshape((opt.batchSize, 2)) - errD_fake = loss(output, fake_label) - errD = errD_real + errD_fake - errD.backward() - metric.update([fake_label,], [output,]) - - trainerD.step(opt.batchSize) - - ############################ - # (2) Update G network: maximize log(D(G(z))) - ########################### - with autograd.record(): - output = netD(fake) - output = output.reshape((opt.batchSize, 2)) - errG = loss(output, real_label) - errG.backward() - - trainerG.step(opt.batchSize) - - name, acc = metric.get() - # logging.info('speed: {} samples/s'.format(opt.batchSize / (time.time() - btic))) - logging.info('discriminator loss = %f, generator loss = %f, binary training acc = %f at iter %d epoch %d' %(mx.nd.mean(errD).asscalar(), mx.nd.mean(errG).asscalar(), acc, iter, epoch)) - if iter % 200 == 0: - visual('gout', fake.asnumpy(), name=os.path.join(outf,'fake_img_iter_%d.png' %iter)) - visual('data', batch.data[0].asnumpy(), name=os.path.join(outf,'real_img_iter_%d.png' %iter)) - - iter = iter + 1 - btic = time.time() +else: + ctx = mx.cpu() +check_point = bool(opt.check_point) +outf = opt.outf + +if not os.path.exists(outf): + os.makedirs(outf) + + +def transformer(data, label): + # resize to 64x64 + data = mx.image.imresize(data, 64, 64) + # transpose from (64, 64, 3) to (3, 64, 64) + data = mx.nd.transpose(data, (2,0,1)) + # normalize to [-1, 1] + data = data.astype(np.float32)/128 - 1 + # if image is greyscale, repeat 3 times to get RGB image. + if data.shape[0] == 1: + data = mx.nd.tile(data, (3, 1, 1)) + return data, label + +train_data = gluon.data.DataLoader( + gluon.data.vision.MNIST('./data', train=True, transform=transformer), + batch_size=opt.batch_size, shuffle=True, last_batch='discard') + +val_data = gluon.data.DataLoader( + gluon.data.vision.MNIST('./data', train=False, transform=transformer), + batch_size=opt.batch_size, shuffle=False) + + +# build the generator +netG = nn.Sequential() +with netG.name_scope(): + # input is Z, going into a convolution + netG.add(nn.Conv2DTranspose(ngf * 8, 4, 1, 0, use_bias=False)) + netG.add(nn.BatchNorm()) + netG.add(nn.Activation('relu')) + # state size. (ngf*8) x 4 x 4 + netG.add(nn.Conv2DTranspose(ngf * 4, 4, 2, 1, use_bias=False)) + netG.add(nn.BatchNorm()) + netG.add(nn.Activation('relu')) + # state size. (ngf*8) x 8 x 8 + netG.add(nn.Conv2DTranspose(ngf * 2, 4, 2, 1, use_bias=False)) + netG.add(nn.BatchNorm()) + netG.add(nn.Activation('relu')) + # state size. (ngf*8) x 16 x 16 + netG.add(nn.Conv2DTranspose(ngf, 4, 2, 1, use_bias=False)) + netG.add(nn.BatchNorm()) + netG.add(nn.Activation('relu')) + # state size. (ngf*8) x 32 x 32 + netG.add(nn.Conv2DTranspose(nc, 4, 2, 1, use_bias=False)) + netG.add(nn.Activation('tanh')) + # state size. (nc) x 64 x 64 + +# build the discriminator +netD = nn.Sequential() +with netD.name_scope(): + # input is (nc) x 64 x 64 + netD.add(nn.Conv2D(ndf, 4, 2, 1, use_bias=False)) + netD.add(nn.LeakyReLU(0.2)) + # state size. (ndf) x 32 x 32 + netD.add(nn.Conv2D(ndf * 2, 4, 2, 1, use_bias=False)) + netD.add(nn.BatchNorm()) + netD.add(nn.LeakyReLU(0.2)) + # state size. (ndf) x 16 x 16 + netD.add(nn.Conv2D(ndf * 4, 4, 2, 1, use_bias=False)) + netD.add(nn.BatchNorm()) + netD.add(nn.LeakyReLU(0.2)) + # state size. (ndf) x 8 x 8 + netD.add(nn.Conv2D(ndf * 8, 4, 2, 1, use_bias=False)) + netD.add(nn.BatchNorm()) + netD.add(nn.LeakyReLU(0.2)) + # state size. (ndf) x 4 x 4 + netD.add(nn.Conv2D(2, 4, 1, 0, use_bias=False)) + +# loss +loss = gluon.loss.SoftmaxCrossEntropyLoss() + +# initialize the generator and the discriminator +netG.initialize(mx.init.Normal(0.02), ctx=ctx) +netD.initialize(mx.init.Normal(0.02), ctx=ctx) + +# trainer for the generator and the discriminator +trainerG = gluon.Trainer(netG.collect_params(), 'adam', {'learning_rate': opt.lr, 'beta1': opt.beta1}) +trainerD = gluon.Trainer(netD.collect_params(), 'adam', {'learning_rate': opt.lr, 'beta1': opt.beta1}) + +# ============printing============== +real_label = mx.nd.ones((opt.batch_size,), ctx=ctx) +fake_label = mx.nd.zeros((opt.batch_size,), ctx=ctx) + +metric = mx.metric.Accuracy() +print('Training... ') +stamp = datetime.now().strftime('%Y_%m_%d-%H_%M') + +iter = 0 +for epoch in range(opt.nepoch): + tic = time.time() + btic = time.time() + for data, _ in train_data: + ############################ + # (1) Update D network: maximize log(D(x)) + log(1 - D(G(z))) + ########################### + # train with real_t + data = data.as_in_context(ctx) + noise = mx.nd.random_normal(0, 1, shape=(opt.batch_size, nz, 1, 1), ctx=ctx) + + with autograd.record(): + output = netD(data) + output = output.reshape((opt.batch_size, 2)) + errD_real = loss(output, real_label) + metric.update([real_label,], [output,]) + + fake = netG(noise) + output = netD(fake.detach()) + output = output.reshape((opt.batch_size, 2)) + errD_fake = loss(output, fake_label) + errD = errD_real + errD_fake + errD.backward() + metric.update([fake_label,], [output,]) + + trainerD.step(opt.batch_size) + + ############################ + # (2) Update G network: maximize log(D(G(z))) + ########################### + with autograd.record(): + output = netD(fake) + output = output.reshape((-1, 2)) + errG = loss(output, real_label) + errG.backward() + + trainerG.step(opt.batch_size) name, acc = metric.get() - metric.reset() - logging.info('\nbinary training acc at epoch %d: %s=%f' % (epoch, name, acc)) - logging.info('time: %f' % (time.time() - tic)) + # logging.info('speed: {} samples/s'.format(opt.batch_size / (time.time() - btic))) + logging.info('discriminator loss = %f, generator loss = %f, binary training acc = %f at iter %d epoch %d' %(mx.nd.mean(errD).asscalar(), mx.nd.mean(errG).asscalar(), acc, iter, epoch)) + if iter % 1 == 0: + visual('gout', fake.asnumpy(), name=os.path.join(outf,'fake_img_iter_%d.png' %iter)) + visual('data', data.asnumpy(), name=os.path.join(outf,'real_img_iter_%d.png' %iter)) + + iter = iter + 1 + btic = time.time() - if check_point: - netG.collect_params().save(os.path.join(outf,'generator_epoch_%d.params' %epoch)) - netD.collect_params().save(os.path.join(outf,'discriminator_epoch_%d.params' % epoch)) + name, acc = metric.get() + metric.reset() + logging.info('\nbinary training acc at epoch %d: %s=%f' % (epoch, name, acc)) + logging.info('time: %f' % (time.time() - tic)) - netG.collect_params().save(os.path.join(outf, 'generator.params')) - netD.collect_params().save(os.path.join(outf, 'discriminator.params')) + if check_point: + netG.save_params(os.path.join(outf,'generator_epoch_%d.params' %epoch)) + netD.save_params(os.path.join(outf,'discriminator_epoch_%d.params' % epoch)) +netG.save_params(os.path.join(outf, 'generator.params')) +netD.save_params(os.path.join(outf, 'discriminator.params')) diff --git a/example/gluon/mnist.py b/example/gluon/mnist.py index 99ac2a9a8d48..9d567d5011cb 100644 --- a/example/gluon/mnist.py +++ b/example/gluon/mnist.py @@ -10,9 +10,6 @@ from mxnet import gluon, autograd from mxnet.gluon import nn -from data import mnist_iterator - - # Parse CLI arguments parser = argparse.ArgumentParser(description='MXNet Gluon MNIST Example') @@ -41,16 +38,25 @@ # data -train_data, val_data = mnist_iterator(batch_size=opt.batch_size, input_shape=(28*28,)) +def transformer(data, label): + data = data.reshape((-1,)).astype(np.float32)/255 + return data, label + +train_data = gluon.data.DataLoader( + gluon.data.vision.MNIST('./data', train=True, transform=transformer), + batch_size=opt.batch_size, shuffle=True, last_batch='discard') + +val_data = gluon.data.DataLoader( + gluon.data.vision.MNIST('./data', train=False, transform=transformer), + batch_size=opt.batch_size, shuffle=False) # train def test(ctx): metric = mx.metric.Accuracy() - val_data.reset() - for batch in val_data: - data = batch.data[0].as_in_context(ctx) - label = batch.label[0].as_in_context(ctx) + for data, label in val_data: + data = data.as_in_context(ctx) + label = label.as_in_context(ctx) output = net(data) metric.update([label], [output]) @@ -59,21 +65,20 @@ def test(ctx): def train(epochs, ctx): # Collect all parameters from net and its children, then initialize them. - net.collect_params().initialize(mx.init.Xavier(magnitude=2.24), ctx=ctx) + net.initialize(mx.init.Xavier(magnitude=2.24), ctx=ctx) # Trainer is for updating parameters with gradient. trainer = gluon.Trainer(net.collect_params(), 'sgd', - {'learning_rate': opt.lr, 'momentum': opt.momentum}) + {'learning_rate': opt.lr, 'momentum': opt.momentum}) metric = mx.metric.Accuracy() loss = gluon.loss.SoftmaxCrossEntropyLoss() for epoch in range(epochs): # reset data iterator and metric at begining of epoch. - train_data.reset() metric.reset() - for i, batch in enumerate(train_data): + for i, (data, label) in enumerate(train_data): # Copy data to ctx if necessary - data = batch.data[0].as_in_context(ctx) - label = batch.label[0].as_in_context(ctx) + data = data.as_in_context(ctx) + label = label.as_in_context(ctx) # Start recording computation graph with record() section. # Recorded graphs can then be differentiated with backward. with autograd.record(): @@ -95,7 +100,7 @@ def train(epochs, ctx): name, val_acc = test(ctx) print('[Epoch %d] Validation: %s=%f'%(epoch, name, val_acc)) - net.collect_params().save('mnist.params') + net.save_params('mnist.params') if __name__ == '__main__': diff --git a/example/gluon/super_resolution.py b/example/gluon/super_resolution.py index 521c17aeb71d..d61fb160e197 100644 --- a/example/gluon/super_resolution.py +++ b/example/gluon/super_resolution.py @@ -125,7 +125,8 @@ def test(ctx): def train(epoch, ctx): if isinstance(ctx, mx.Context): ctx = [ctx] - net.collect_params().initialize(mx.init.Orthogonal(), ctx=ctx) + net.initialize(mx.init.Orthogonal(), ctx=ctx) + # re-initialize conv4's weight to be Orthogonal net.conv4.collect_params().initialize(mx.init.Orthogonal(scale=1), ctx=ctx) trainer = gluon.Trainer(net.collect_params(), 'adam', {'learning_rate': opt.lr}) loss = gluon.loss.L2Loss() @@ -150,13 +151,13 @@ def train(epoch, ctx): print('training mse at epoch %d: %s=%f'%(i, name, acc)) test(ctx) - net.collect_params().save('superres.params') + net.save_params('superres.params') def resolve(ctx): from PIL import Image if isinstance(ctx, list): ctx = [ctx[0]] - net.collect_params().load('superres.params', ctx=ctx) + net.load_params('superres.params', ctx=ctx) img = Image.open(opt.resolve_img).convert('YCbCr') y, cb, cr = img.split() data = mx.nd.expand_dims(mx.nd.expand_dims(mx.nd.array(y), axis=0), axis=0) diff --git a/python/mxnet/gluon/.gitignore b/python/mxnet/gluon/.gitignore new file mode 100644 index 000000000000..8436a89ff416 --- /dev/null +++ b/python/mxnet/gluon/.gitignore @@ -0,0 +1 @@ +!data diff --git a/python/mxnet/gluon/__init__.py b/python/mxnet/gluon/__init__.py index 0910fdf8ce06..c559e7af343b 100644 --- a/python/mxnet/gluon/__init__.py +++ b/python/mxnet/gluon/__init__.py @@ -15,3 +15,5 @@ from . import loss from . import utils + +from . import data diff --git a/python/mxnet/gluon/data/__init__.py b/python/mxnet/gluon/data/__init__.py new file mode 100644 index 000000000000..a0623257417c --- /dev/null +++ b/python/mxnet/gluon/data/__init__.py @@ -0,0 +1,11 @@ +# coding: utf-8 +# pylint: disable=wildcard-import +"""Dataset utilities.""" + +from .dataset import * + +from .sampler import * + +from .dataloader import * + +from . import vision diff --git a/python/mxnet/gluon/data/dataloader.py b/python/mxnet/gluon/data/dataloader.py new file mode 100644 index 000000000000..148d7cd6e53c --- /dev/null +++ b/python/mxnet/gluon/data/dataloader.py @@ -0,0 +1,70 @@ +# coding: utf-8 +# pylint: disable= +"""Dataset generator.""" + +import numpy as np + +from . import sampler as _sampler +from ... import nd + + +def _batchify(data): + """Collate data into batch.""" + if isinstance(data[0], nd.NDArray): + return nd.stack(*data) + elif isinstance(data[0], tuple): + data = zip(*data) + return [_batchify(i) for i in data] + else: + data = np.asarray(data) + return nd.array(data, dtype=data.dtype) + + +class DataLoader(object): + """Loads data from a dataset and returns mini-batches of data. + + Parameters + ---------- + dataset : Dataset + Source dataset. Note that numpy and mxnet arrays can be directly used + as a Dataset. + batch_size : int + Size of mini-batch. + shuffle : bool + Whether to shuffle the samples. + sampler : Sampler + The sampler to use. Either specify sampler or shuffle, not both. + batch_sampler : Sampler + A sampler that returns mini-batches. Do not specify batch_size, + shuffle, sampler, and last_batch if batch_sampler is specified. + """ + def __init__(self, dataset, batch_size=None, shuffle=False, sampler=None, + last_batch=None, batch_sampler=None): + self._dataset = dataset + + if batch_sampler is None: + if batch_size is None: + raise ValueError("batch_size must be specified unless " \ + "batch_sampler is specified") + if sampler is None: + if shuffle: + sampler = _sampler.RandomSampler(len(dataset)) + else: + sampler = _sampler.SequentialSampler(len(dataset)) + elif shuffle: + raise ValueError("shuffle must not be specified if sampler is specified") + + batch_sampler = _sampler.BatchSampler(sampler, batch_size, last_batch) + elif batch_size is not None or shuffle or sampler is not None or \ + last_batch is not None: + raise ValueError("batch_size, shuffle, sampler and last_batch must " \ + "not be specified if batch_sampler is specified.") + + self._batch_sampler = batch_sampler + + def __iter__(self): + for batch in self._batch_sampler: + yield _batchify([self._dataset[idx] for idx in batch]) + + def __len__(self): + return self._batch_sampler diff --git a/python/mxnet/gluon/data/dataset.py b/python/mxnet/gluon/data/dataset.py new file mode 100644 index 000000000000..aefff0af16c9 --- /dev/null +++ b/python/mxnet/gluon/data/dataset.py @@ -0,0 +1,89 @@ +# coding: utf-8 +# pylint: disable= +"""Dataset container.""" +import os + +from ... import recordio, image + +class Dataset(object): + """Abstract dataset class. All datasets should have this interface. + + Subclasses need to override `__getitem__`, which returns the i-th + element, and `__len__`, which returns the total number elements. + + .. note:: An mxnet or numpy array can be directly used as a dataset. + """ + def __getitem__(self, idx): + raise NotImplementedError + + def __len__(self): + raise NotImplementedError + + +class ArrayDataset(Dataset): + """A dataset with a data array and a label array. + + The i-th sample is `(data[i], lable[i])`. + + Parameters + ---------- + data : array-like object + The data array. Can be mxnet or numpy array. + label : array-like object + The label array. Can be mxnet or numpy array. + """ + def __init__(self, data, label): + assert len(data) == len(label) + self._data = data + self._label = label + + def __getitem__(self, idx): + return self._data[idx], self._label[idx] + + def __len__(self): + return len(self._data) + + +class RecordFileDataset(Dataset): + """A dataset wrapping over a RecordIO (.rec) file. + + Each sample is a string representing the raw content of an record. + + Parameters + ---------- + filename : str + Path to rec file. + """ + def __init__(self, filename): + idx_file = os.path.splitext(filename)[0] + '.idx' + self._record = recordio.MXIndexedRecordIO(idx_file, filename, 'r') + + def __getitem__(self, idx): + return self._record.read_idx(idx) + + def __len__(self): + return len(self._record.keys) + + +class ImageRecordDataset(RecordFileDataset): + """A dataset wrapping over a RecordIO file containing images. + + Each sample is an image and its corresponding label. + + Parameters + ---------- + filename : str + Path to rec file. + flag : {0, 1}, default 1 + If 0, always convert images to greyscale. + + If 1, always convert images to colored (RGB). + """ + def __init__(self, filename, flag=1): + super(ImageRecordDataset, self).__init__(filename) + self._flag = flag + + def __getitem__(self, idx): + record = super(ImageRecordDataset, self).__getitem__(idx) + header, img = recordio.unpack(record) + return image.imdecode(img, self._flag), header.label diff --git a/python/mxnet/gluon/data/sampler.py b/python/mxnet/gluon/data/sampler.py new file mode 100644 index 000000000000..7bfc418399f5 --- /dev/null +++ b/python/mxnet/gluon/data/sampler.py @@ -0,0 +1,120 @@ +# coding: utf-8 +# pylint: disable= +"""Dataset sampler.""" + +import random + +class Sampler(object): + """Base class for samplers. + + All samplers should subclass `Sampler` and define `__iter__` and `__len__` + methods. + """ + def __iter__(self): + raise NotImplementedError + + def __len__(self): + raise NotImplementedError + + +class SequentialSampler(Sampler): + """Samples elements from [0, length) sequentially. + + Parameters + ---------- + length : int + Length of the sequence. + """ + def __init__(self, length): + self._length = length + + def __iter__(self): + return iter(range(self._length)) + + def __len__(self): + return self._length + + +class RandomSampler(Sampler): + """Samples elements from [0, length) randomly without replacement. + + Parameters + ---------- + length : int + Length of the sequence. + """ + def __init__(self, length): + self._length = length + + def __iter__(self): + indices = range(self._length) + random.shuffle(indices) + return iter(indices) + + def __len__(self): + return self._length + + +class BatchSampler(Sampler): + """Wraps over another `Sampler` and return mini-batches of samples. + + Parameters + ---------- + sampler : Sampler + The source Sampler. + batch_size : int + Size of mini-batch. + last_batch : {'keep', 'discard', 'rollover'} + Specifies how the last batch is handled if batch_size does not evenly + divide sequence length. + + If 'keep', the last batch will be returned directly, but will contain + less element than `batch_size` requires. + + If 'discard', the last batch will be discarded. + + If 'rollover', the remaining elements will be rolled over to the next + iteration. + + Examples + -------- + >>> sampler = gluon.data.SequentialSampler(10) + >>> batch_sampler = gluon.data.BatchSampler(sampler, 3, 'keep') + >>> list(batch_sampler) + [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]] + """ + def __init__(self, sampler, batch_size, last_batch='keep'): + self._sampler = sampler + self._batch_size = batch_size + self._last_batch = last_batch + self._prev = [] + + def __iter__(self): + batch, self._prev = self._prev, [] + for i in self._sampler: + batch.append(i) + if len(batch) == self._batch_size: + yield batch + batch = [] + if batch: + if self._last_batch == 'keep': + yield batch + elif self._last_batch == 'discard': + return + elif self._last_batch == 'rollover': + self._prev = batch + else: + raise ValueError( + "last_batch must be one of 'keep', 'discard', or 'rollover', " \ + "but got %s"%self._last_batch) + + def __len__(self): + if self._last_batch == 'keep': + return (len(self._sampler) + self._batch_size - 1) // self._batch_size + if self._last_batch == 'discard': + return len(self._sampler) // self._batch_size + if self._last_batch == 'rollover': + return (len(self._prev) + len(self._sampler)) // self._batch_size + raise ValueError( + "last_batch must be one of 'keep', 'discard', or 'rollover', " \ + "but got %s"%self._last_batch) diff --git a/python/mxnet/gluon/data/vision.py b/python/mxnet/gluon/data/vision.py new file mode 100644 index 000000000000..36c4642e7665 --- /dev/null +++ b/python/mxnet/gluon/data/vision.py @@ -0,0 +1,125 @@ +# coding: utf-8 +# pylint: disable= +"""Dataset container.""" + +import os +import gzip +import tarfile +import struct +import numpy as np + +from . import dataset +from ..utils import download +from ... import nd + + +class _DownloadedDataset(dataset.Dataset): + """Base class for MNIST, cifar10, etc.""" + def __init__(self, root, train, transform): + self._root = os.path.expanduser(root) + self._train = train + self._transform = transform + self._data = None + self._label = None + + self._get_data() + + def __getitem__(self, idx): + return self._transform(self._data[idx], self._label[idx]) + + def __len__(self): + return len(self._label) + + def _get_data(self): + raise NotImplementedError + + +class MNIST(_DownloadedDataset): + """MNIST handwritten digits dataset from `http://yann.lecun.com/exdb/mnist`_. + + Each sample is an image (in 3D NDArray) with shape (28, 28, 1). + + Parameters + ---------- + root : str + Path to temp folder for storing data. + train : bool + Whether to load the training or testing set. + transform : function + A user defined callback that transforms each instance. For example:: + + transform=lambda data, label: (data.astype(np.float32)/255, label) + """ + def __init__(self, root, train=True, transform=lambda data, label: (data, label)): + super(MNIST, self).__init__(root, train, transform) + + def _get_data(self): + if not os.path.isdir(self._root): + os.makedirs(self._root) + url = 'http://data.mxnet.io/data/mnist/' + if self._train: + data_file = download(url+'train-images-idx3-ubyte.gz', self._root) + label_file = download(url+'train-labels-idx1-ubyte.gz', self._root) + else: + data_file = download(url+'t10k-images-idx3-ubyte.gz', self._root) + label_file = download(url+'t10k-labels-idx1-ubyte.gz', self._root) + + with gzip.open(label_file, 'rb') as fin: + struct.unpack(">II", fin.read(8)) + label = np.fromstring(fin.read(), dtype=np.uint8).astype(np.int32) + + with gzip.open(data_file, 'rb') as fin: + struct.unpack(">IIII", fin.read(16)) + data = np.fromstring(fin.read(), dtype=np.uint8) + data = data.reshape(len(label), 28, 28, 1) + + self._data = [nd.array(x, dtype=x.dtype) for x in data] + self._label = label + + +class CIFAR10(_DownloadedDataset): + """CIFAR10 image classification dataset from `https://www.cs.toronto.edu/~kriz/cifar.html`_. + + Each sample is an image (in 3D NDArray) with shape (32, 32, 1). + + Parameters + ---------- + root : str + Path to temp folder for storing data. + train : bool + Whether to load the training or testing set. + transform : function + A user defined callback that transforms each instance. For example:: + + transform=lambda data, label: (data.astype(np.float32)/255, label) + """ + def __init__(self, root, train=True, transform=lambda data, label: (data, label)): + super(CIFAR10, self).__init__(root, train, transform) + + def _read_batch(self, filename): + with open(filename, 'rb') as fin: + data = np.fromstring(fin.read(), dtype=np.uint8).reshape(-1, 3072+1) + + return data[:, 1:].reshape(-1, 3, 32, 32).transpose(0, 2, 3, 1), \ + data[:, 0].astype(np.int32) + + def _get_data(self): + if not os.path.isdir(self._root): + os.makedirs(self._root) + url = 'https://www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz' + filename = download(url, self._root) + + with tarfile.open(filename) as tar: + tar.extractall(self._root) + + if self._train: + filename = os.path.join(self._root, 'cifar-10-batches-bin/data_batch_%d.bin') + data, label = zip(*[self._read_batch(filename%i) for i in range(1, 6)]) + data = np.concatenate(data) + label = np.concatenate(label) + else: + filename = os.path.join(self._root, 'cifar-10-batches-bin/test_batch.bin') + data, label = self._read_batch(filename) + + self._data = [nd.array(x, dtype=x.dtype) for x in data] + self._label = label diff --git a/python/mxnet/gluon/utils.py b/python/mxnet/gluon/utils.py index 27576b55f7f9..505fbc55248c 100644 --- a/python/mxnet/gluon/utils.py +++ b/python/mxnet/gluon/utils.py @@ -1,6 +1,14 @@ # coding: utf-8 # pylint: disable= """Parallelization utility optimizer.""" +import os +try: + import requests +except ImportError: + class requests_failed_to_import(object): + pass + requests = requests_failed_to_import + import math from .. import ndarray @@ -109,3 +117,45 @@ def _indent(s_, numSpaces): s = [first] + [(numSpaces * ' ') + line for line in s] s = '\n'.join(s) return s + + +def download(url, path=None, overwrite=False): + """Download an given URL + + Parameters + ---------- + url : str + URL to download + path : str, optional + Destination path to store downloaded file. By default stores to the + current directory with same name as in url. + overwrite : bool, optional + Whether to overwrite destination file if already exists. + + Returns + ------- + str + The filename of the downloaded file. + """ + if path is None: + fname = url.split('/')[-1] + elif os.path.isdir(path): + fname = os.path.join(path, url.split('/')[-1]) + else: + fname = path + + if overwrite or not os.path.exists(fname): + dirname = os.path.dirname(os.path.abspath(os.path.expanduser(fname))) + if not os.path.exists(dirname): + os.makedirs(dirname) + + print('Downloading %s from %s...'%(fname, url)) + r = requests.get(url, stream=True) + if r.status_code != 200: + raise RuntimeError("Failed downloading url %s"%url) + with open(fname, 'wb') as f: + for chunk in r.iter_content(chunk_size=1024): + if chunk: # filter out keep-alive new chunks + f.write(chunk) + + return fname diff --git a/python/mxnet/image/image.py b/python/mxnet/image/image.py index 5bf2afd09204..32b7c4f282b9 100644 --- a/python/mxnet/image/image.py +++ b/python/mxnet/image/image.py @@ -24,7 +24,7 @@ from .. import recordio -def imdecode(buf, **kwargs): +def imdecode(buf, *args, **kwargs): """Decode an image to an NDArray. Note: `imdecode` uses OpenCV (not the CV2 Python library). @@ -75,7 +75,7 @@ def imdecode(buf, **kwargs): """ if not isinstance(buf, nd.NDArray): buf = nd.array(np.frombuffer(buf, dtype=np.uint8), dtype=np.uint8) - return _internal._cvimdecode(buf, **kwargs) + return _internal._cvimdecode(buf, *args, **kwargs) def scale_down(src_size, size): diff --git a/python/mxnet/ndarray.py b/python/mxnet/ndarray.py index 4939b6c221a5..dff4889749c0 100644 --- a/python/mxnet/ndarray.py +++ b/python/mxnet/ndarray.py @@ -53,7 +53,9 @@ np.float64 : 1, np.float16 : 2, np.uint8 : 3, - np.int32 : 4 + np.int32 : 4, + np.int8 : 5, + np.int64 : 6, } _DTYPE_MX_TO_NP = { @@ -62,7 +64,9 @@ 1 : np.float64, 2 : np.float16, 3 : np.uint8, - 4 : np.int32 + 4 : np.int32, + 5 : np.int8, + 6 : np.int64, } _GRAD_REQ_MAP = { @@ -272,6 +276,10 @@ def __bool__(self): __nonzero__ = __bool__ + def __len__(self): + """Number of element along the first axis.""" + return self.shape[0] + def __getstate__(self): handle = self.handle this = {'handle' : None} diff --git a/src/operator/elemwise_op_common.h b/src/operator/elemwise_op_common.h index aa95d2d8696a..228303c85a82 100644 --- a/src/operator/elemwise_op_common.h +++ b/src/operator/elemwise_op_common.h @@ -66,8 +66,12 @@ template inline bool ElemwiseShape(const nnvm::NodeAttrs& attrs, std::vector *in_attrs, std::vector *out_attrs) { - CHECK_EQ(in_attrs->size(), static_cast(n_in)) << " in operator " << attrs.name; - CHECK_EQ(out_attrs->size(), static_cast(n_out)) << " in operator " << attrs.name; + if (n_in != -1) { + CHECK_EQ(in_attrs->size(), static_cast(n_in)) << " in operator " << attrs.name; + } + if (n_out != -1) { + CHECK_EQ(out_attrs->size(), static_cast(n_out)) << " in operator " << attrs.name; + } return ElemwiseAttr( attrs, in_attrs, out_attrs, TShape()); } @@ -76,8 +80,12 @@ template inline bool ElemwiseType(const nnvm::NodeAttrs& attrs, std::vector *in_attrs, std::vector *out_attrs) { - CHECK_EQ(in_attrs->size(), static_cast(n_in)) << " in operator " << attrs.name; - CHECK_EQ(out_attrs->size(), static_cast(n_out)) << " in operator " << attrs.name; + if (n_in != -1) { + CHECK_EQ(in_attrs->size(), static_cast(n_in)) << " in operator " << attrs.name; + } + if (n_out != -1) { + CHECK_EQ(out_attrs->size(), static_cast(n_out)) << " in operator " << attrs.name; + } return ElemwiseAttr( attrs, in_attrs, out_attrs, -1); } diff --git a/src/operator/tensor/elemwise_sum.cc b/src/operator/tensor/elemwise_sum.cc index 3c4bf719e18f..7ae7ae97acea 100644 --- a/src/operator/tensor/elemwise_sum.cc +++ b/src/operator/tensor/elemwise_sum.cc @@ -36,22 +36,6 @@ std::vector ElementWiseSumGrad( return ret; } -bool ElementWiseSumShape(const nnvm::NodeAttrs& attrs, - std::vector *in_attrs, - std::vector *out_attrs) { - CHECK_EQ(out_attrs->size(), 1); - return ElemwiseAttr( - attrs, in_attrs, out_attrs, TShape()); -} - -bool ElementWiseSumType(const nnvm::NodeAttrs& attrs, - std::vector *in_attrs, - std::vector *out_attrs) { - CHECK_EQ(out_attrs->size(), 1); - return ElemwiseAttr( - attrs, in_attrs, out_attrs, -1); -} - NNVM_REGISTER_OP(add_n) .add_alias("ElementWiseSum") .describe(R"doc(Adds all input arguments element-wise. @@ -81,8 +65,8 @@ NNVM_REGISTER_OP(add_n) "FInplaceOption", [](const NodeAttrs& attrs) { return std::vector >{{0, 0}}; }) -.set_attr("FInferShape", ElementWiseSumShape) -.set_attr("FInferType", ElementWiseSumType) +.set_attr("FInferShape", ElemwiseShape<-1, 1>) +.set_attr("FInferType", ElemwiseType<-1, 1>) .set_attr("FGradient", CloneGradient{"_backward_add_n"}) .add_argument("args", "NDArray-or-Symbol[]", "Positional input arguments"); diff --git a/src/operator/tensor/matrix_op-inl.h b/src/operator/tensor/matrix_op-inl.h index 2e1aa6661b67..75da055d0098 100644 --- a/src/operator/tensor/matrix_op-inl.h +++ b/src/operator/tensor/matrix_op-inl.h @@ -12,6 +12,7 @@ #include #include "../mshadow_op.h" #include "../elemwise_op_common.h" +#include "../channel_op_common.h" #include "../mxnet_op.h" #include "broadcast_reduce_op.h" @@ -1775,6 +1776,114 @@ void ReverseOpForward(const nnvm::NodeAttrs& attrs, } +struct StackParam : public dmlc::Parameter { + int axis; + int num_args; + DMLC_DECLARE_PARAMETER(StackParam) { + DMLC_DECLARE_FIELD(axis) + .set_default(0) + .describe("The axis in the result array along which the input arrays are stacked."); + DMLC_DECLARE_FIELD(num_args).set_lower_bound(1) + .describe("Number of inputs to be stacked."); + } +}; + + +inline bool StackOpShape(const nnvm::NodeAttrs& attrs, + std::vector *in_attrs, + std::vector *out_attrs) { + const StackParam& param = dmlc::get(attrs.parsed); + + TShape dshape; + for (const TShape& i : (*in_attrs)) { + shape_assign(&dshape, i); + } + if (dshape.ndim() == 0) return false; + + TShape oshape(dshape.ndim() + 1); + int axis = CheckAxis(param.axis, oshape.ndim()); + for (int i = 0; i < axis; ++i) { + oshape[i] = dshape[i]; + } + oshape[axis] = param.num_args; + for (index_t i = axis + 1; i < oshape.ndim(); ++i) { + oshape[i] = dshape[i-1]; + } + SHAPE_ASSIGN_CHECK(*out_attrs, 0, oshape); + + return true; +} + + +template +void StackOpForward(const nnvm::NodeAttrs& attrs, + const OpContext& ctx, + const std::vector& inputs, + const std::vector& req, + const std::vector& outputs) { + using namespace mshadow; + using namespace mshadow::expr; + const StackParam& param = dmlc::get(attrs.parsed); + int axis = CheckAxis(param.axis, outputs[0].ndim()); + + Stream *s = ctx.get_stream(); + MSHADOW_TYPE_SWITCH(outputs[0].type_flag_, DType, { + std::vector > data(inputs.size()); + Tensor out; + size_t leading = 1, trailing = 1; + for (int i = 0; i < axis; ++i) { + leading *= outputs[0].shape_[i]; + } + for (index_t i = axis + 1; i < outputs[0].ndim(); ++i) { + trailing *= outputs[0].shape_[i]; + } + size_t mid = outputs[0].shape_[axis]; + Shape<3> oshape = Shape3(leading, mid, trailing); + out = outputs[0].get_with_shape(oshape, s); + + for (index_t i = 0; i < inputs.size(); ++i) { + Shape<3> dshape = Shape3(leading, 1, trailing); + data[i] = inputs[i].get_with_shape(dshape, s); + } + Concatenate(data, &out, 1, req[0]); + }) +} + +template +void StackOpBackward(const nnvm::NodeAttrs& attrs, + const OpContext& ctx, + const std::vector& inputs, + const std::vector& req, + const std::vector& outputs) { + using namespace mshadow; + using namespace mshadow::expr; + const StackParam& param = dmlc::get(attrs.parsed); + int axis = CheckAxis(param.axis, inputs[0].ndim()); + + Stream *s = ctx.get_stream(); + MSHADOW_TYPE_SWITCH(inputs[0].type_flag_, DType, { + std::vector > grad_in(outputs.size()); + Tensor grad; + size_t leading = 1, trailing = 1; + for (int i = 0; i < axis; ++i) { + leading *= inputs[0].shape_[i]; + } + for (index_t i = axis + 1; i < inputs[0].ndim(); ++i) { + trailing *= inputs[0].shape_[i]; + } + size_t mid = inputs[0].shape_[axis]; + Shape<3> oshape = Shape3(leading, mid, trailing); + grad = inputs[0].get_with_shape(oshape, s); + + for (index_t i = 0; i < outputs.size(); ++i) { + Shape<3> dshape = Shape3(leading, 1, trailing); + grad_in[i] = outputs[i].get_with_shape(dshape, s); + } + Split(grad, &grad_in, 1, req); + }) +} + + } // namespace op } // namespace mxnet diff --git a/src/operator/tensor/matrix_op.cc b/src/operator/tensor/matrix_op.cc index 6a51d46db25c..4832b13f56c5 100644 --- a/src/operator/tensor/matrix_op.cc +++ b/src/operator/tensor/matrix_op.cc @@ -20,6 +20,7 @@ DMLC_REGISTER_PARAMETER(DotParam); DMLC_REGISTER_PARAMETER(RepeatParam); DMLC_REGISTER_PARAMETER(TileParam); DMLC_REGISTER_PARAMETER(ReverseParam); +DMLC_REGISTER_PARAMETER(StackParam); NNVM_REGISTER_OP(Reshape) .add_alias("reshape") @@ -627,5 +628,56 @@ NNVM_REGISTER_OP(_backward_reverse) return std::vector {ResourceRequest::kTempSpace}; }) .set_attr("FCompute", ReverseOpForward); + +NNVM_REGISTER_OP(stack) +.describe(R"code(Join a sequence of arrays along a new axis. + +The axis parameter specifies the index of the new axis in the dimensions of the +result. For example, if axis=0 it will be the first dimension and if axis=-1 it +will be the last dimension. + +Examples:: + + x = [1, 2] + y = [3, 4] + + stack(x, y) = [[1, 2], + [3, 4]] + stack(x, y, axis=1) = [[1, 3], + [2, 4]] +)code") +.set_num_inputs([](const nnvm::NodeAttrs& attrs) { + const StackParam& param = dmlc::get(attrs.parsed); + return static_cast(param.num_args); + }) +.set_num_outputs(1) +.set_attr_parser(ParamParser) +.set_attr("FListInputNames", + [](const NodeAttrs& attrs) { + uint32_t num_args = dmlc::get(attrs.parsed).num_args; + std::vector ret; + for (uint32_t i = 0; i < num_args; ++i) { + ret.push_back(std::string("arg") + std::to_string(i)); + } + return ret; + }) +.set_attr("key_var_num_args", "num_args") +.set_attr("FInferShape", StackOpShape) +.set_attr("FInferType", ElemwiseType<-1, 1>) +.set_attr("FCompute", StackOpForward) +.set_attr("FGradient", ElemwiseGradUseNone{"_backward_stack"}) +.add_argument("data", "NDArray-or-Symbol[]", "List of arrays to stack") +.add_arguments(StackParam::__FIELDS__()); + +NNVM_REGISTER_OP(_backward_stack) +.set_num_inputs(1) +.set_num_outputs([](const nnvm::NodeAttrs& attrs) { + const StackParam& param = dmlc::get(attrs.parsed); + return static_cast(param.num_args); + }) +.set_attr_parser(ParamParser) +.set_attr("TIsBackward", true) +.set_attr("FCompute", StackOpBackward); + } // namespace op } // namespace mxnet diff --git a/src/operator/tensor/matrix_op.cu b/src/operator/tensor/matrix_op.cu index 96c075a7d483..8cf656e999b8 100644 --- a/src/operator/tensor/matrix_op.cu +++ b/src/operator/tensor/matrix_op.cu @@ -74,5 +74,11 @@ NNVM_REGISTER_OP(reverse) NNVM_REGISTER_OP(_backward_reverse) .set_attr("FCompute", ReverseOpForward); + +NNVM_REGISTER_OP(stack) +.set_attr("FCompute", StackOpForward); + +NNVM_REGISTER_OP(_backward_stack) +.set_attr("FCompute", StackOpBackward); } // namespace op } // namespace mxnet diff --git a/tests/python/unittest/test_gluon_data.py b/tests/python/unittest/test_gluon_data.py new file mode 100644 index 000000000000..0d25cc497d32 --- /dev/null +++ b/tests/python/unittest/test_gluon_data.py @@ -0,0 +1,53 @@ +import os +import mxnet as mx +import numpy as np +from mxnet import gluon + +def test_array_dataset(): + X = np.random.uniform(size=(10, 20)) + Y = np.random.uniform(size=(10,)) + dataset = gluon.data.ArrayDataset(X, Y) + loader = gluon.data.DataLoader(dataset, 2) + + for i, (x, y) in enumerate(loader): + assert mx.test_utils.almost_equal(x.asnumpy(), X[i*2:(i+1)*2]) + assert mx.test_utils.almost_equal(y.asnumpy(), Y[i*2:(i+1)*2]) + + +def prepare_record(): + if not os.path.isdir("data/test_images"): + os.system("wget http://data.mxnet.io/data/test_images.tar.gz -O data/test_images.tar.gz") + os.system("tar -xf data/test_images.tar.gz -C data") + imgs = os.listdir('data/test_images') + record = mx.recordio.MXIndexedRecordIO('data/test.idx', 'data/test.rec', 'w') + for i, img in enumerate(imgs): + str_img = open('data/test_images/'+img, 'rb').read() + s = mx.recordio.pack((0, i, i, 0), str_img) + record.write_idx(i, s) + return 'data/test.rec' + + +def test_recordimage_dataset(): + recfile = prepare_record() + dataset = gluon.data.ImageRecordDataset(recfile) + loader = gluon.data.DataLoader(dataset, 1) + + for i, (x, y) in enumerate(loader): + assert x.shape[0] == 1 and x.shape[3] == 3 + assert y.asscalar() == i + +def test_sampler(): + seq_sampler = gluon.data.SequentialSampler(10) + assert list(seq_sampler) == list(range(10)) + rand_sampler = gluon.data.RandomSampler(10) + assert sorted(list(rand_sampler)) == list(range(10)) + seq_batch_keep = gluon.data.BatchSampler(seq_sampler, 3, 'keep') + assert sum(list(seq_batch_keep), []) == list(range(10)) + seq_batch_discard = gluon.data.BatchSampler(seq_sampler, 3, 'discard') + assert sum(list(seq_batch_discard), []) == list(range(9)) + rand_batch_keep = gluon.data.BatchSampler(rand_sampler, 3, 'keep') + assert sorted(sum(list(rand_batch_keep), [])) == list(range(10)) + +if __name__ == '__main__': + import nose + nose.runmodule() diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py index 58d39513a4a8..2f7c3b904e01 100644 --- a/tests/python/unittest/test_operator.py +++ b/tests/python/unittest/test_operator.py @@ -3642,6 +3642,23 @@ def test_laop(): check_numeric_gradient(test_sumlogdiag, [a]) +def test_stack(): + for _ in range(100): + ndim = random.randint(1, 5) + axis = random.randint(0, ndim) + if random.randint(0, 1): + axis = axis - ndim - 1 + nin = random.randint(1, 3) + dshape = [random.randint(1, 5) for _ in range(ndim)] + inputs = [np.random.uniform(size=dshape) for _ in range(nin)] + output = np.stack(inputs, axis=axis) + sym_ins = [mx.sym.var('x%d'%i) for i in range(nin)] + out = mx.sym.stack(*sym_ins, axis=axis) + check_symbolic_forward(out, inputs, [output]) + check_numeric_gradient(out, inputs) + + + if __name__ == '__main__': import nose nose.runmodule() From 238b890333b0e1c1e68fd550dee3cbf09c691eb6 Mon Sep 17 00:00:00 2001 From: Terence Wu <2326428753@qq.com> Date: Tue, 25 Jul 2017 18:43:22 +0800 Subject: [PATCH 256/834] Fix a spelling mistake (#7187) --- python/mxnet/lr_scheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/mxnet/lr_scheduler.py b/python/mxnet/lr_scheduler.py index 98ce30867aa9..9998fc8769a1 100644 --- a/python/mxnet/lr_scheduler.py +++ b/python/mxnet/lr_scheduler.py @@ -21,7 +21,7 @@ def __call__(self, num_update): The ``num_update`` is the upper bound of the number of updates applied to every weight. - Assume the optimizer has udpated *i*-th weight by *k_i* times, namely + Assume the optimizer has updated *i*-th weight by *k_i* times, namely ``optimizer.update(i, weight_i)`` is called by *k_i* times. Then:: num_update = max([k_i for all i]) From 4fef6fd5cf9631569249da1f28e4942a61f12d8a Mon Sep 17 00:00:00 2001 From: Terence Wu <2326428753@qq.com> Date: Tue, 25 Jul 2017 18:43:50 +0800 Subject: [PATCH 257/834] Fix 2 spelling mistakes (#7186) --- python/mxnet/model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/mxnet/model.py b/python/mxnet/model.py index a476d84efd92..5780ac1b9f9d 100644 --- a/python/mxnet/model.py +++ b/python/mxnet/model.py @@ -913,7 +913,7 @@ def create(symbol, X, y=None, ctx=None, ``ceil(num_train_examples / batch_size)``. optimizer : str or Optimizer, optional The name of the chosen optimizer, or an optimizer object, used for training. - initializier : initializer function, optional + initializer : initializer function, optional The initialization scheme used. eval_data : DataIter or numpy.ndarray pair If `eval_set` is ``numpy.ndarray`` pair, it should @@ -929,7 +929,7 @@ def create(symbol, X, y=None, ctx=None, A callback that is invoked at end of each batch for print purposes. kvstore: KVStore or str, optional The KVStore or a string kvstore type: 'local', 'dist_sync', 'dis_async'. - Defaults to 'local', often no need to change for single machiine. + Defaults to 'local', often no need to change for single machine. logger : logging logger, optional When not specified, default logger will be used. work_load_list : list of float or int, optional From d1d4ea5044746490e83121b2bf582f74f359a882 Mon Sep 17 00:00:00 2001 From: Przemyslaw Tredak Date: Tue, 25 Jul 2017 12:23:52 -0700 Subject: [PATCH 258/834] Set default value for dtype for alexnet (#7193) --- example/image-classification/symbols/alexnet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/image-classification/symbols/alexnet.py b/example/image-classification/symbols/alexnet.py index 2534797a9eba..e2b512b195c8 100755 --- a/example/image-classification/symbols/alexnet.py +++ b/example/image-classification/symbols/alexnet.py @@ -6,7 +6,7 @@ import mxnet as mx import numpy as np -def get_symbol(num_classes, dtype, **kwargs): +def get_symbol(num_classes, dtype='float32', **kwargs): input_data = mx.sym.Variable(name="data") if dtype == 'float16': input_data = mx.sym.Cast(data=input_data, dtype=np.float16) From f3f8a9957c6d63947bbd5f7c8e209f99ba58a99c Mon Sep 17 00:00:00 2001 From: Eric Junyuan Xie Date: Wed, 26 Jul 2017 01:26:39 -0700 Subject: [PATCH 259/834] fix ndarray setitem (#7195) * fix ndarray setitem * fix * fix * fix --- python/mxnet/gluon/data/sampler.py | 2 +- python/mxnet/ndarray.py | 31 +++++++++++++++++++----- tests/python/unittest/test_gluon_data.py | 2 ++ tests/python/unittest/test_ndarray.py | 9 +++++++ 4 files changed, 37 insertions(+), 7 deletions(-) diff --git a/python/mxnet/gluon/data/sampler.py b/python/mxnet/gluon/data/sampler.py index 7bfc418399f5..f6cedf051727 100644 --- a/python/mxnet/gluon/data/sampler.py +++ b/python/mxnet/gluon/data/sampler.py @@ -47,7 +47,7 @@ def __init__(self, length): self._length = length def __iter__(self): - indices = range(self._length) + indices = list(range(self._length)) random.shuffle(indices) return iter(indices) diff --git a/python/mxnet/ndarray.py b/python/mxnet/ndarray.py index dff4889749c0..d02119166562 100644 --- a/python/mxnet/ndarray.py +++ b/python/mxnet/ndarray.py @@ -377,11 +377,13 @@ def __setitem__(self, key, value): len(key), len(my_shape)) begin = [0 for _ in my_shape] end = [x for x in my_shape] + expand = [] for i, slice_i in enumerate(key): if isinstance(slice_i, integer_types): assert slice_i < my_shape[i] begin[i] = slice_i end[i] = slice_i + 1 + expand.append(i) elif isinstance(slice_i, py_slice): # only support continuous slicing assert slice_i.step is None, \ @@ -397,16 +399,14 @@ def __setitem__(self, key, value): if isinstance(value, NDArray): value = value.as_in_context(self.context) - _internal._crop_assign(self, value, out=self, - begin=begin, end=end) + self._slice_assign(value, begin, end, expand) elif isinstance(value, numeric_types): _internal._crop_assign_scalar(self, out=self, begin=begin, end=end, scalar=value) elif isinstance(value, (np.ndarray, np.generic)): - value = array(value, ctx=self.context) - _internal._crop_assign(self, value, out=self, - begin=begin, end=end) + value = array(value, ctx=self.context, dtype=self.dtype) + self._slice_assign(value, begin, end, expand) else: raise TypeError( 'NDArray does not support assignment with %s of type %s'%( @@ -417,6 +417,22 @@ def __setitem__(self, key, value): str(key), str(type(key)))) # pylint: enable=too-many-branches + def _slice_assign(self, value, begin, end, expand): + vshape = list(value.shape) + if expand and len(vshape) != len(begin): + if len(expand) + len(vshape) != len(begin): + sshape = [e - b for e, b in zip(end, begin)] + for i in reversed(expand): + sshape.pop(i) + raise ValueError( + "Cannot assign NDArray with shape %s to NDArray slice with " \ + "shape %s"%(str(vshape), str(sshape))) + for i in expand: + vshape.insert(i, 1) + value = value.reshape(vshape) + _internal._crop_assign(self, value, out=self, + begin=begin, end=end) + def __getitem__(self, key): """x.__getitem__(i) <=> x[i] @@ -757,7 +773,10 @@ def size(self): >>> np.prod(x.shape) 30 """ - return np.prod(self.shape) + size = 1 + for i in self.shape: + size *= i + return size @property def context(self): diff --git a/tests/python/unittest/test_gluon_data.py b/tests/python/unittest/test_gluon_data.py index 0d25cc497d32..2407f8e25991 100644 --- a/tests/python/unittest/test_gluon_data.py +++ b/tests/python/unittest/test_gluon_data.py @@ -15,6 +15,8 @@ def test_array_dataset(): def prepare_record(): + if not os.path.isdir("data"): + os.makedirs('data') if not os.path.isdir("data/test_images"): os.system("wget http://data.mxnet.io/data/test_images.tar.gz -O data/test_images.tar.gz") os.system("tar -xf data/test_images.tar.gz -C data") diff --git a/tests/python/unittest/test_ndarray.py b/tests/python/unittest/test_ndarray.py index 8b7f8d6d7bf3..f627ab837a3e 100644 --- a/tests/python/unittest/test_ndarray.py +++ b/tests/python/unittest/test_ndarray.py @@ -76,6 +76,14 @@ def test_ndarray_setitem(): x_np[:, 1:3, 1:2] = val.asnumpy() assert same(x.asnumpy(), x_np) + # short all-dim indexing + x = mx.nd.zeros(shape) + val = mx.nd.ones((3, 2)) + x[:, 1:3, 1] = val + x_np = np.zeros(shape, dtype=x.dtype) + x_np[:, 1:3, 1] = val.asnumpy() + assert same(x.asnumpy(), x_np) + x = mx.nd.zeros(shape) x[:, 1:3, 1] = 1 x_np = np.zeros(shape, dtype=x.dtype) @@ -258,6 +266,7 @@ def test_ndarray_slice(): assert A[1,2,3,4,5].asscalar() == A2[1,2,3,4,5] + def test_ndarray_crop(): # get crop x = mx.nd.ones((2, 3, 4)) From c79ec5d76e11720f04d0a6919b540b08ce25b357 Mon Sep 17 00:00:00 2001 From: Kenji Doi Date: Wed, 26 Jul 2017 17:27:51 +0900 Subject: [PATCH 260/834] Fix incorrect reference. (sym -> symbol) (#7199) --- example/image-classification/fine-tune.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/image-classification/fine-tune.py b/example/image-classification/fine-tune.py index cfb43101542e..5a2a04d2c73b 100644 --- a/example/image-classification/fine-tune.py +++ b/example/image-classification/fine-tune.py @@ -13,7 +13,7 @@ def get_fine_tune_model(symbol, arg_params, num_classes, layer_name): num_classes: the number of classes for the fine-tune datasets layer_name: the layer name before the last fully-connected layer """ - all_layers = sym.get_internals() + all_layers = symbol.get_internals() net = all_layers[layer_name+'_output'] net = mx.symbol.FullyConnected(data=net, num_hidden=num_classes, name='fc') net = mx.symbol.SoftmaxOutput(data=net, name='softmax') From 500ea39b6ed7c1f2d1a02e41f26a7a6f2c33c16b Mon Sep 17 00:00:00 2001 From: Rahul Date: Wed, 26 Jul 2017 01:28:23 -0700 Subject: [PATCH 261/834] Compilation warnings (#7198) * add jenkins script to monitor compile warnings, and some more cast warnings fixed * update jenkins script to time compilation install package time * g++5 with timing * redirect compile output to file * only output real time * print compile output to help debug * fix typo --- src/operator/tensor/matrix_op-inl.h | 4 +- .../compilation_warnings.sh | 27 +++++++++++++ .../compilation_warnings/process_output.py | 39 +++++++++++++++++++ 3 files changed, 68 insertions(+), 2 deletions(-) create mode 100644 tests/nightly/compilation_warnings/compilation_warnings.sh create mode 100644 tests/nightly/compilation_warnings/process_output.py diff --git a/src/operator/tensor/matrix_op-inl.h b/src/operator/tensor/matrix_op-inl.h index 75da055d0098..26f409a43525 100644 --- a/src/operator/tensor/matrix_op-inl.h +++ b/src/operator/tensor/matrix_op-inl.h @@ -1834,7 +1834,7 @@ void StackOpForward(const nnvm::NodeAttrs& attrs, for (int i = 0; i < axis; ++i) { leading *= outputs[0].shape_[i]; } - for (index_t i = axis + 1; i < outputs[0].ndim(); ++i) { + for (int i = axis + 1; i < outputs[0].ndim(); ++i) { trailing *= outputs[0].shape_[i]; } size_t mid = outputs[0].shape_[axis]; @@ -1868,7 +1868,7 @@ void StackOpBackward(const nnvm::NodeAttrs& attrs, for (int i = 0; i < axis; ++i) { leading *= inputs[0].shape_[i]; } - for (index_t i = axis + 1; i < inputs[0].ndim(); ++i) { + for (int i = axis + 1; i < inputs[0].ndim(); ++i) { trailing *= inputs[0].shape_[i]; } size_t mid = inputs[0].shape_[axis]; diff --git a/tests/nightly/compilation_warnings/compilation_warnings.sh b/tests/nightly/compilation_warnings/compilation_warnings.sh new file mode 100644 index 000000000000..9c377e2dd1fb --- /dev/null +++ b/tests/nightly/compilation_warnings/compilation_warnings.sh @@ -0,0 +1,27 @@ + +runme() { + cmd=$* + echo "$cmd" + $cmd + ret=$? + if [[ ${ret} != 0 ]]; then + echo " " + echo "ERROR: Return value non-zero for: $cmd" + echo " " + exit 1 + fi +} + +sudo add-apt-repository ppa:ubuntu-toolchain-r/test +sudo apt-get update +sudo apt-get -y install time g++-5 +runme make clean >/dev/null +runme mkdir build +echo "Starting make" +cp make/config.mk . +sed -i -e 's/gcc/gcc-5/g' config.mk +sed -i -e 's/g++/g++-5/g' config.mk +runme /usr/bin/time -f "%e" make -j$(nproc) &> build/compile_output.txt +cat build/compile_output.txt +echo "Finished make. Now processing output" +python tests/nightly/compilation_warnings/process_output.py build/compile_output.txt diff --git a/tests/nightly/compilation_warnings/process_output.py b/tests/nightly/compilation_warnings/process_output.py new file mode 100644 index 000000000000..1a57d81f8dad --- /dev/null +++ b/tests/nightly/compilation_warnings/process_output.py @@ -0,0 +1,39 @@ +import re +import sys +import operator + +def process_output(command_output): + warnings = {} + regex = r"(.*):\swarning:\s(.*)" + lines = command_output.split("\n") + for line in lines[:-2]: + matches = re.finditer(regex, line) + for matchNum, match in enumerate(matches): + try: + warnings[match.group()] +=1 + except KeyError: + warnings[match.group()] =1 + time = lines[-2] + return time, warnings + +def generate_stats(warnings): + total_count = sum(warnings.values()) + sorted_warnings = sorted(warnings.items(), key=operator.itemgetter(1), reverse=True) + return sorted_warnings, total_count + +def print_summary(time, warnings): + sorted_warnings, total_count = generate_stats(warnings) + print "START - Compilation warnings count" + print total_count + print "END - Compilation warnings count" + print 'START - Compilation warnings summary' + print 'Time taken to compile:', time, 's' + print 'Total number of warnings:', total_count, '\n' + print 'Below is the list of unique warnings and the number of occurrences of that warning' + for warning, count in sorted_warnings: + print count, ': ', warning + print 'END - Compilation warnings summary' + +c_output = open(sys.argv[1],'r') +time, warnings = process_output(c_output.read()) +print_summary(time, warnings) From 1ed8b19849046bce92fd3d4a390b2adc405b584a Mon Sep 17 00:00:00 2001 From: "Qiang Kou (KK)" Date: Thu, 27 Jul 2017 01:12:06 +0000 Subject: [PATCH 262/834] [R] RNN update. close #6723 (#7211) --- R-package/R/io.R | 10 ++++------ R-package/R/metric.R | 10 ++++++++++ R-package/R/model.R | 10 ++++++++-- R-package/R/rnn_model.R | 16 +++------------- R-package/tests/testthat/test_model.R | 10 +++++----- 5 files changed, 30 insertions(+), 26 deletions(-) diff --git a/R-package/R/io.R b/R-package/R/io.R index 10298fbaf056..9f6a60702505 100644 --- a/R-package/R/io.R +++ b/R-package/R/io.R @@ -1,14 +1,12 @@ -is.MXDataIter <- function(x) { - inherits(x, "Rcpp_MXNativeDataIter") || - inherits(x, "Rcpp_MXArrayDataIter") -} - #' Judge if an object is mx.dataiter #' #' @return Logical indicator #' #' @export -is.mx.dataiter <- is.MXDataIter +is.mx.dataiter <- function(x) { + inherits(x, "Rcpp_MXNativeDataIter") || + inherits(x, "Rcpp_MXArrayDataIter") +} #' Extract a certain field from DataIter. #' diff --git a/R-package/R/metric.R b/R-package/R/metric.R index 5bf4390cd614..02572f4acdc3 100644 --- a/R-package/R/metric.R +++ b/R-package/R/metric.R @@ -78,3 +78,13 @@ mx.metric.rmsle <- mx.metric.custom("rmsle", function(label, pred) { return(res) }) +#' Perplexity metric for language model +#' +#' @export +mx.metric.Perplexity <- mx.metric.custom("Perplexity", function(label, pred) { + label_probs <- as.array(mx.nd.choose.element.0index(pred, label)) + batch <- length(label_probs) + NLL <- -sum(log(pmax(1e-15, as.array(label_probs)))) / batch + Perplexity <- exp(NLL) + return(Perplexity) +}) diff --git a/R-package/R/model.R b/R-package/R/model.R index 043d0e2433ea..64cc816f0ef4 100644 --- a/R-package/R/model.R +++ b/R-package/R/model.R @@ -279,7 +279,13 @@ mx.model.train <- function(symbol, ctx, input.shape, output.shape, return(model) } -# Initialize parameters +#' Parameter initialization +#' @param symbol The symbolic configuration of the neural network. +#' @param input.shape The shape of the input for the neural network. +#' @param output.shape The shape of the output for the neural network. It can be NULL. +#' @param initializer, initializer object. The initialization scheme for parameters. +#' @param ctx mx.context. The devices used to perform initialization. +#' @export mx.model.init.params <- function(symbol, input.shape, output.shape, initializer, ctx) { if (!is.MXSymbol(symbol)) stop("symbol need to be MXSymbol") @@ -296,7 +302,7 @@ mx.model.init.params <- function(symbol, input.shape, output.shape, initializer, # Initialize the data iter mx.model.init.iter <- function(X, y, batch.size, is.train) { - if (is.MXDataIter(X)) return(X) + if (is.mx.dataiter(X)) return(X) if (is.null(y)) { if (is.train) stop("Need to provide parameter y for training with R arrays.") shape <- dim(X) diff --git a/R-package/R/rnn_model.R b/R-package/R/rnn_model.R index b269d0722601..aa4a7d03ca9b 100644 --- a/R-package/R/rnn_model.R +++ b/R-package/R/rnn_model.R @@ -3,19 +3,9 @@ is.param.name <- function(name) { grepl('gamma$', name) || grepl('beta$', name) ) } -# Initialize parameters -mx.model.init.params.rnn <- function(symbol, input.shape, initializer, ctx) { - if (!is.mx.symbol(symbol)) stop("symbol need to be MXSymbol") - slist <- symbol$infer.shape(input.shape) - if (is.null(slist)) stop("Not enough information to get shapes") - arg.params <- mx.init.create(initializer, slist$arg.shapes, ctx, skip.unknown=TRUE) - aux.params <- mx.init.create(initializer, slist$aux.shapes, ctx, skip.unknown=FALSE) - return(list(arg.params=arg.params, aux.params=aux.params)) -} - # Initialize the data iter mx.model.init.iter.rnn <- function(X, y, batch.size, is.train) { - if (is.MXDataIter(X)) return(X) + if (is.mx.dataiter(X)) return(X) shape <- dim(X) if (is.null(shape)) { num.data <- length(X) @@ -56,11 +46,11 @@ setup.rnn.model <- function(rnn.sym, ctx, } } } - params <- mx.model.init.params.rnn(rnn.sym, input.shapes, initializer, mx.cpu()) + params <- mx.model.init.params(rnn.sym, input.shapes, NULL, initializer, mx.cpu()) args <- input.shapes args$symbol <- rnn.sym args$ctx <- ctx - args$grad.req <- "add" + args$grad.req <- "write" rnn.exec <- do.call(mx.simple.bind, args) mx.exec.update.arg.arrays(rnn.exec, params$arg.params, match.name=TRUE) diff --git a/R-package/tests/testthat/test_model.R b/R-package/tests/testthat/test_model.R index 7c5b04ac27b1..4cf2a8c8e070 100644 --- a/R-package/tests/testthat/test_model.R +++ b/R-package/tests/testthat/test_model.R @@ -127,11 +127,11 @@ test_that("Fine-tune", { new_fc <- mx.symbol.FullyConnected(data = flatten, num_hidden = 2, name = "fc1") new_soft <- mx.symbol.SoftmaxOutput(data = new_fc, name = "softmax") - arg_params_new <- mxnet:::mx.model.init.params(symbol = new_soft, - input.shape = list("data" = c(224, 224, 3, 8)), - output.shape = NULL, - initializer = mx.init.uniform(0.1), - ctx = mx.cpu())$arg.params + arg_params_new <- mx.model.init.params(symbol = new_soft, + input.shape = list("data" = c(224, 224, 3, 8)), + output.shape = NULL, + initializer = mx.init.uniform(0.1), + ctx = mx.cpu())$arg.params fc1_weights_new <- arg_params_new[["fc1_weight"]] fc1_bias_new <- arg_params_new[["fc1_bias"]] From f8d2bf829109e5bb4b8a1f0e4f30ae3d2551696a Mon Sep 17 00:00:00 2001 From: Rahul Huilgol Date: Thu, 27 Jul 2017 10:24:27 -0700 Subject: [PATCH 263/834] Jenkins tests fix (#7213) * update jenkins script for mail and faster runtime * updated mxnet-pip test to run in user mode so workspace can be deleted after job finishes --- tests/jenkins/run_test_pip_installations.sh | 2 +- tests/nightly/compilation_warnings/compilation_warnings.sh | 6 +++--- tests/nightly/compilation_warnings/process_output.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/jenkins/run_test_pip_installations.sh b/tests/jenkins/run_test_pip_installations.sh index 9122ea4e7c77..9246708f4329 100755 --- a/tests/jenkins/run_test_pip_installations.sh +++ b/tests/jenkins/run_test_pip_installations.sh @@ -49,7 +49,7 @@ for DEV in "${DEVICES[@]}"; do DOCKER_CMD="${DOCKER_CMD} pip install mxnet-cu80; python tests/python/train/test_conv.py --gpu" fi - ${DOCKER_BINARY} run --rm -v ${WORKSPACE}:${WORKSPACE} ${DOCKER_TAG} bash -c "${DOCKER_CMD}" + ${DOCKER_BINARY} run --rm -v ${WORKSPACE}:${WORKSPACE} -w ${WORKSPACE} ${DOCKER_TAG} bash -c "tests/jenkins/run_as_user.sh `id -u` `id -un` `id -g` `id -un` '${DOCKER_CMD}'" done done diff --git a/tests/nightly/compilation_warnings/compilation_warnings.sh b/tests/nightly/compilation_warnings/compilation_warnings.sh index 9c377e2dd1fb..871d849b9bae 100644 --- a/tests/nightly/compilation_warnings/compilation_warnings.sh +++ b/tests/nightly/compilation_warnings/compilation_warnings.sh @@ -1,4 +1,5 @@ - +#!/bin/bash +set -e runme() { cmd=$* echo "$cmd" @@ -21,7 +22,6 @@ echo "Starting make" cp make/config.mk . sed -i -e 's/gcc/gcc-5/g' config.mk sed -i -e 's/g++/g++-5/g' config.mk -runme /usr/bin/time -f "%e" make -j$(nproc) &> build/compile_output.txt -cat build/compile_output.txt +runme /usr/bin/time -f "%e" make -j$(nproc) 2>&1 | tee build/compile_output.txt echo "Finished make. Now processing output" python tests/nightly/compilation_warnings/process_output.py build/compile_output.txt diff --git a/tests/nightly/compilation_warnings/process_output.py b/tests/nightly/compilation_warnings/process_output.py index 1a57d81f8dad..622598d72292 100644 --- a/tests/nightly/compilation_warnings/process_output.py +++ b/tests/nightly/compilation_warnings/process_output.py @@ -24,7 +24,7 @@ def generate_stats(warnings): def print_summary(time, warnings): sorted_warnings, total_count = generate_stats(warnings) print "START - Compilation warnings count" - print total_count + print total_count, 'warnings' print "END - Compilation warnings count" print 'START - Compilation warnings summary' print 'Time taken to compile:', time, 's' From 66f9b33b6965bb0ebf73b7c5d625b021f0d2adb7 Mon Sep 17 00:00:00 2001 From: Jean-Louis Queguiner Date: Thu, 27 Jul 2017 19:34:24 +0200 Subject: [PATCH 264/834] fix misleading test_conv.py in how_to (#7208) tests/python/gpu/test_conv.py is not available anymore and is refeering to the mnist example mxnet/example/image-classification see https://github.com/dmlc/mxnet/issues/4806 and https://github.com/dmlc/mxnet/issues/7204 --- docs/how_to/cloud.md | 366 +++++++++++++++++++++---------------------- 1 file changed, 183 insertions(+), 183 deletions(-) diff --git a/docs/how_to/cloud.md b/docs/how_to/cloud.md index 47ea40cf4595..67b28f8b4338 100644 --- a/docs/how_to/cloud.md +++ b/docs/how_to/cloud.md @@ -1,183 +1,183 @@ -# MXNet on the Cloud - -Deep learning can require extremely powerful hardware, often for unpredictable durations of time. -Moreover, _MXNet_ can benefit from both multiple GPUs and multiple machines. -Accordingly, cloud computing, as offered by AWS and others, -is especially well suited to training deep learning models. -Using AWS, we can rapidly fire up multiple machines with multiple GPUs each at will -and maintain the resources for precisely the amount of time needed. - -## Set Up an AWS GPU Cluster from Scratch - -In this document, we provide a step-by-step guide that will teach you -how to set up an AWS cluster with _MXNet_. We show how to: - -- [Use Amazon S3 to host data](#use-amazon-s3-to-host-data) -- [Set up an EC2 GPU instance with all dependencies installed](#set-up-an-ec2-gpu-instance) -- [Build and run MXNet on a single computer](#build-and-run-mxnet-on-a-gpu-instance) -- [Set up an EC2 GPU cluster for distributed training](#set-up-an-ec2-gpu-cluster-for-distributed-training) - -### Use Amazon S3 to Host Data - -Amazon S3 provides distributed data storage which proves especially convenient for hosting large datasets. -To use S3, you need [AWS credentials](http://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSGettingStartedGuide/AWSCredentials.html), -including an `ACCESS_KEY_ID` and a `SECRET_ACCESS_KEY`. - -To use _MXNet_ with S3, set the environment variables `AWS_ACCESS_KEY_ID` and -`AWS_SECRET_ACCESS_KEY` by adding the following two lines in -`~/.bashrc` (replacing the strings with the correct ones): - -```bash -export AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE -export AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY -``` - -There are several ways to upload data to S3. One simple way is to use -[s3cmd](http://s3tools.org/s3cmd). For example: - -```bash -wget http://data.mxnet.io/mxnet/data/mnist.zip -unzip mnist.zip && s3cmd put t*-ubyte s3://dmlc/mnist/ -``` - -### Use Pre-installed EC2 GPU Instance -The [Deep Learning AMI](https://aws.amazon.com/marketplace/pp/B01M0AXXQB?qid=1475211685369&sr=0-1&ref_=srh_res_product_title) is an Amazon Linux image -supported and maintained by Amazon Web Services for use on Amazon Elastic Compute Cloud (Amazon EC2). -It contains [MXNet-v0.9.3 tag](https://github.com/dmlc/mxnet) and the necessary components to get going with deep learning, -including Nvidia drivers, CUDA, cuDNN, Anaconda, Python2 and Python3. -The AMI IDs are the following: - -* us-east-1: ami-e7c96af1 -* us-west-2: ami-dfb13ebf -* eu-west-1: ami-6e5d6808 - -Now you can launch _MXNet_ directly on an EC2 GPU instance. -You can also use [Jupyter](http://jupyter.org) notebook on EC2 machine. -Here is a [good tutorial](https://github.com/dmlc/mxnet-notebooks) -on how to connect to a Jupyter notebook running on an EC2 instance. - -### Set Up an EC2 GPU Instance from Scratch - -_MXNet_ requires the following libraries: - -- C++ compiler with C++11 support, such as `gcc >= 4.8` -- `CUDA` (`CUDNN` in optional) for GPU linear algebra -- `BLAS` (cblas, open-blas, atblas, mkl, or others) for CPU linear algebra -- `opencv` for image augmentations -- `curl` and `openssl` for the ability to read/write to Amazon S3 - -Installing `CUDA` on EC2 instances requires some effort. Caffe has a good -[tutorial](https://github.com/BVLC/caffe/wiki/Install-Caffe-on-EC2-from-scratch-(Ubuntu,-CUDA-7,-cuDNN-3)) -on how to install CUDA 7.0 on Ubuntu 14.04. - -***Note:*** We tried CUDA 7.5 on Nov 7, 2015, but found it problematic. - -You can install the rest using the package manager. For example, on Ubuntu: - -``` -sudo apt-get update -sudo apt-get install -y build-essential git libcurl4-openssl-dev libatlas-base-dev libopencv-dev python-numpy -``` - -The Amazon Machine Image (AMI) [ami-12fd8178](https://console.aws.amazon.com/ec2/v2/home?region=us-east-1#LaunchInstanceWizard:ami=ami-12fd8178) has the packages listed above installed. - - -### Build and Run MXNet on a GPU Instance - -The following commands build _MXNet_ with CUDA/CUDNN, Amazon S3, and distributed -training. - -```bash -git clone --recursive https://github.com/dmlc/mxnet -cd mxnet; cp make/config.mk . -echo "USE_CUDA=1" >>config.mk -echo "USE_CUDA_PATH=/usr/local/cuda" >>config.mk -echo "USE_CUDNN=1" >>config.mk -echo "USE_BLAS=atlas" >> config.mk -echo "USE_DIST_KVSTORE = 1" >>config.mk -echo "USE_S3=1" >>config.mk -make -j$(nproc) -``` - -To test whether everything is installed properly, we can try training a convolutional neural network (CNN) on the MNIST dataset using a GPU: - -```bash -python tests/python/gpu/test_conv.py -``` - -If you've placed the MNIST data on `s3://dmlc/mnist`, you can read the data stored on Amazon S3 directly with the following command: - -```bash -sed -i.bak "s!data_dir = 'data'!data_dir = 's3://dmlc/mnist'!" tests/python/gpu/test_conv.py -``` - -***Note:*** You can use `sudo ln /dev/null /dev/raw1394` to fix the opencv error `libdc1394 error: Failed to initialize libdc1394`. - -### Set Up an EC2 GPU Cluster for Distributed Training - -A cluster consists of multiple computers. -You can use one computer with _MXNet_ installed as the root computer for submitting jobs,and then launch several -slave computers to run the jobs. For example, launch multiple instances using an -AMI, e.g., -[ami-12fd8178](https://console.aws.amazon.com/ec2/v2/home?region=us-east-1#LaunchInstanceWizard:ami=ami-12fd8178), -with dependencies installed. There are two options: - -- Make all slaves' ports accessible (same for the root) by setting type: All TCP, - Source: Anywhere in Configure Security Group. - -- Use the same `pem` as the root computer to access all slave computers, and - then copy the `pem` file into the root computer's `~/.ssh/id_rsa`. If you do this, all slave computers can be accessed with SSH from the root. - -Now, run the CNN on multiple computers. Assume that we are on a working -directory of the root computer, such as `~/train`, and MXNet is built as `~/mxnet`. - -1. Pack the _MXNet_ Python library into this working directory for easy - synchronization: - - ```bash - cp -r ~/mxnet/python/mxnet . - cp ~/mxnet/lib/libmxnet.so mxnet/ - ``` - - And then copy the training program: - - ```bash - cp ~/mxnet/example/image-classification/*.py . - cp -r ~/mxnet/example/image-classification/common . - ``` - -2. Prepare a host file with all slaves private IPs. For example, `cat hosts`: - - ```bash - 172.30.0.172 - 172.30.0.171 - ``` - -3. Assuming that there are two computers, train the CNN using two workers: - - ```bash - ../../tools/launch.py -n 2 -H hosts --sync-dir /tmp/mxnet python train_mnist.py --kv-store dist_sync - ``` - -***Note:*** Sometimes the jobs linger at the slave computers even though you've pressed `Ctrl-c` -at the root node. To terminate them, use the following command: - -```bash -cat hosts | xargs -I{} ssh -o StrictHostKeyChecking=no {} 'uname -a; pgrep python | xargs kill -9' -``` - -***Note:*** The preceding example is very simple to train and therefore isn't a good -benchmark for distributed training. Consider using other [examples](https://github.com/dmlc/mxnet/tree/master/example/image-classification). - -### More Options -#### Use Multiple Data Shards -It is common to pack a dataset into multiple files, especially when working in a distributed environment. -_MXNet_ supports direct loading from multiple data shards. -Put all of the record files into a folder, and point the data path to the folder. - -#### Use YARN and SGE -Although using SSH can be simple when you don't have a cluster scheduling framework, -_MXNet_ is designed to be portable to various platforms. -We provide scripts available in [tracker](https://github.com/dmlc/dmlc-core/tree/master/tracker) -to allow running on other cluster frameworks, including Hadoop (YARN) and SGE. -We welcome contributions from the community of examples of running _MXNet_ on your favorite distributed platform. +# MXNet on the Cloud + +Deep learning can require extremely powerful hardware, often for unpredictable durations of time. +Moreover, _MXNet_ can benefit from both multiple GPUs and multiple machines. +Accordingly, cloud computing, as offered by AWS and others, +is especially well suited to training deep learning models. +Using AWS, we can rapidly fire up multiple machines with multiple GPUs each at will +and maintain the resources for precisely the amount of time needed. + +## Set Up an AWS GPU Cluster from Scratch + +In this document, we provide a step-by-step guide that will teach you +how to set up an AWS cluster with _MXNet_. We show how to: + +- [Use Amazon S3 to host data](#use-amazon-s3-to-host-data) +- [Set up an EC2 GPU instance with all dependencies installed](#set-up-an-ec2-gpu-instance) +- [Build and run MXNet on a single computer](#build-and-run-mxnet-on-a-gpu-instance) +- [Set up an EC2 GPU cluster for distributed training](#set-up-an-ec2-gpu-cluster-for-distributed-training) + +### Use Amazon S3 to Host Data + +Amazon S3 provides distributed data storage which proves especially convenient for hosting large datasets. +To use S3, you need [AWS credentials](http://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSGettingStartedGuide/AWSCredentials.html), +including an `ACCESS_KEY_ID` and a `SECRET_ACCESS_KEY`. + +To use _MXNet_ with S3, set the environment variables `AWS_ACCESS_KEY_ID` and +`AWS_SECRET_ACCESS_KEY` by adding the following two lines in +`~/.bashrc` (replacing the strings with the correct ones): + +```bash +export AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE +export AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY +``` + +There are several ways to upload data to S3. One simple way is to use +[s3cmd](http://s3tools.org/s3cmd). For example: + +```bash +wget http://data.mxnet.io/mxnet/data/mnist.zip +unzip mnist.zip && s3cmd put t*-ubyte s3://dmlc/mnist/ +``` + +### Use Pre-installed EC2 GPU Instance +The [Deep Learning AMI](https://aws.amazon.com/marketplace/pp/B01M0AXXQB?qid=1475211685369&sr=0-1&ref_=srh_res_product_title) is an Amazon Linux image +supported and maintained by Amazon Web Services for use on Amazon Elastic Compute Cloud (Amazon EC2). +It contains [MXNet-v0.9.3 tag](https://github.com/dmlc/mxnet) and the necessary components to get going with deep learning, +including Nvidia drivers, CUDA, cuDNN, Anaconda, Python2 and Python3. +The AMI IDs are the following: + +* us-east-1: ami-e7c96af1 +* us-west-2: ami-dfb13ebf +* eu-west-1: ami-6e5d6808 + +Now you can launch _MXNet_ directly on an EC2 GPU instance. +You can also use [Jupyter](http://jupyter.org) notebook on EC2 machine. +Here is a [good tutorial](https://github.com/dmlc/mxnet-notebooks) +on how to connect to a Jupyter notebook running on an EC2 instance. + +### Set Up an EC2 GPU Instance from Scratch + +_MXNet_ requires the following libraries: + +- C++ compiler with C++11 support, such as `gcc >= 4.8` +- `CUDA` (`CUDNN` in optional) for GPU linear algebra +- `BLAS` (cblas, open-blas, atblas, mkl, or others) for CPU linear algebra +- `opencv` for image augmentations +- `curl` and `openssl` for the ability to read/write to Amazon S3 + +Installing `CUDA` on EC2 instances requires some effort. Caffe has a good +[tutorial](https://github.com/BVLC/caffe/wiki/Install-Caffe-on-EC2-from-scratch-(Ubuntu,-CUDA-7,-cuDNN-3)) +on how to install CUDA 7.0 on Ubuntu 14.04. + +***Note:*** We tried CUDA 7.5 on Nov 7, 2015, but found it problematic. + +You can install the rest using the package manager. For example, on Ubuntu: + +``` +sudo apt-get update +sudo apt-get install -y build-essential git libcurl4-openssl-dev libatlas-base-dev libopencv-dev python-numpy +``` + +The Amazon Machine Image (AMI) [ami-12fd8178](https://console.aws.amazon.com/ec2/v2/home?region=us-east-1#LaunchInstanceWizard:ami=ami-12fd8178) has the packages listed above installed. + + +### Build and Run MXNet on a GPU Instance + +The following commands build _MXNet_ with CUDA/CUDNN, Amazon S3, and distributed +training. + +```bash +git clone --recursive https://github.com/dmlc/mxnet +cd mxnet; cp make/config.mk . +echo "USE_CUDA=1" >>config.mk +echo "USE_CUDA_PATH=/usr/local/cuda" >>config.mk +echo "USE_CUDNN=1" >>config.mk +echo "USE_BLAS=atlas" >> config.mk +echo "USE_DIST_KVSTORE = 1" >>config.mk +echo "USE_S3=1" >>config.mk +make -j$(nproc) +``` + +To test whether everything is installed properly, we can try training a convolutional neural network (CNN) on the MNIST dataset using a GPU: + +```bash +python example/image-classification/train_mnist.py +``` + +If you've placed the MNIST data on `s3://dmlc/mnist`, you can read the data stored on Amazon S3 directly with the following command: + +```bash +sed -i.bak "s!data_dir = 'data'!data_dir = 's3://dmlc/mnist'!" example/image-classification/train_mnist.py +``` + +***Note:*** You can use `sudo ln /dev/null /dev/raw1394` to fix the opencv error `libdc1394 error: Failed to initialize libdc1394`. + +### Set Up an EC2 GPU Cluster for Distributed Training + +A cluster consists of multiple computers. +You can use one computer with _MXNet_ installed as the root computer for submitting jobs,and then launch several +slave computers to run the jobs. For example, launch multiple instances using an +AMI, e.g., +[ami-12fd8178](https://console.aws.amazon.com/ec2/v2/home?region=us-east-1#LaunchInstanceWizard:ami=ami-12fd8178), +with dependencies installed. There are two options: + +- Make all slaves' ports accessible (same for the root) by setting type: All TCP, + Source: Anywhere in Configure Security Group. + +- Use the same `pem` as the root computer to access all slave computers, and + then copy the `pem` file into the root computer's `~/.ssh/id_rsa`. If you do this, all slave computers can be accessed with SSH from the root. + +Now, run the CNN on multiple computers. Assume that we are on a working +directory of the root computer, such as `~/train`, and MXNet is built as `~/mxnet`. + +1. Pack the _MXNet_ Python library into this working directory for easy + synchronization: + + ```bash + cp -r ~/mxnet/python/mxnet . + cp ~/mxnet/lib/libmxnet.so mxnet/ + ``` + + And then copy the training program: + + ```bash + cp ~/mxnet/example/image-classification/*.py . + cp -r ~/mxnet/example/image-classification/common . + ``` + +2. Prepare a host file with all slaves private IPs. For example, `cat hosts`: + + ```bash + 172.30.0.172 + 172.30.0.171 + ``` + +3. Assuming that there are two computers, train the CNN using two workers: + + ```bash + ../../tools/launch.py -n 2 -H hosts --sync-dir /tmp/mxnet python train_mnist.py --kv-store dist_sync + ``` + +***Note:*** Sometimes the jobs linger at the slave computers even though you've pressed `Ctrl-c` +at the root node. To terminate them, use the following command: + +```bash +cat hosts | xargs -I{} ssh -o StrictHostKeyChecking=no {} 'uname -a; pgrep python | xargs kill -9' +``` + +***Note:*** The preceding example is very simple to train and therefore isn't a good +benchmark for distributed training. Consider using other [examples](https://github.com/dmlc/mxnet/tree/master/example/image-classification). + +### More Options +#### Use Multiple Data Shards +It is common to pack a dataset into multiple files, especially when working in a distributed environment. +_MXNet_ supports direct loading from multiple data shards. +Put all of the record files into a folder, and point the data path to the folder. + +#### Use YARN and SGE +Although using SSH can be simple when you don't have a cluster scheduling framework, +_MXNet_ is designed to be portable to various platforms. +We provide scripts available in [tracker](https://github.com/dmlc/dmlc-core/tree/master/tracker) +to allow running on other cluster frameworks, including Hadoop (YARN) and SGE. +We welcome contributions from the community of examples of running _MXNet_ on your favorite distributed platform. From 72d00fd38a8c1b50ae5cd16df933e800a1f10713 Mon Sep 17 00:00:00 2001 From: Piyush Singh Date: Thu, 27 Jul 2017 19:57:42 +0100 Subject: [PATCH 265/834] Fix broken amalgamation. Script was copying a declaration from a header file but missing out the definition, causing a runtime undef symbol error. (#7222) --- amalgamation/amalgamation.py | 3 ++- amalgamation/mxnet_predict0.cc | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/amalgamation/amalgamation.py b/amalgamation/amalgamation.py index da3b60ac8399..b33b81c62b4a 100644 --- a/amalgamation/amalgamation.py +++ b/amalgamation/amalgamation.py @@ -8,7 +8,8 @@ 'kvstore_dist.h', 'mach/clock.h', 'mach/mach.h', 'malloc.h', 'mkl.h', 'mkl_cblas.h', 'mkl_vsl.h', 'mkl_vsl_functions.h', 'nvml.h', 'opencv2/opencv.hpp', 'sys/stat.h', 'sys/types.h', 'cuda.h', 'cuda_fp16.h', - 'omp.h', 'execinfo.h', 'packet/sse-inl.h', 'emmintrin.h', 'thrust/device_vector.h' + 'omp.h', 'execinfo.h', 'packet/sse-inl.h', 'emmintrin.h', 'thrust/device_vector.h', + 'cusolverDn.h' ] minimum = int(sys.argv[6]) if len(sys.argv) > 5 else 0 diff --git a/amalgamation/mxnet_predict0.cc b/amalgamation/mxnet_predict0.cc index d51deb285c99..ca1b581ce195 100644 --- a/amalgamation/mxnet_predict0.cc +++ b/amalgamation/mxnet_predict0.cc @@ -26,6 +26,7 @@ #include "src/ndarray/ndarray_function.cc" +#include "src/ndarray/autograd.cc" #include "src/ndarray/ndarray.cc" #include "src/engine/engine.cc" From b7369998e8c06c4c9ff45c5a1c13aa4adb4e786c Mon Sep 17 00:00:00 2001 From: lxn2 Date: Fri, 28 Jul 2017 12:33:41 -0700 Subject: [PATCH 266/834] Add note about about Apache migration (#7238) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index a929234dd682..6b62986d6477 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ deep learning systems, and interesting insights of DL systems for hackers. What's New ---------- +* [Apache Incubator](http://incubator.apache.org/projects/mxnet.html) - We are now an Apache Incubator project. * [Version 0.10.0 Release](https://github.com/dmlc/mxnet/releases/tag/v0.10.0) - MXNet 0.10.0 Release. * [Version 0.9.3 Release](./docs/architecture/release_note_0_9.md) - First 0.9 official release. * [Version 0.9.1 Release (NNVM refactor)](./docs/architecture/release_note_0_9.md) - NNVM branch is merged into master now. An official release will be made soon. From 5c536f7d44987ef24892ba0cb473fc43a39ef3ae Mon Sep 17 00:00:00 2001 From: Yao Wang Date: Fri, 28 Jul 2017 14:57:34 -0700 Subject: [PATCH 267/834] Add Keras Installation Link (#7241) --- docs/how_to/index.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/how_to/index.md b/docs/how_to/index.md index 1e96a66e230a..cc21aa0b8ae8 100644 --- a/docs/how_to/index.md +++ b/docs/how_to/index.md @@ -36,6 +36,8 @@ and full working examples, visit the [tutorials section](../tutorials/index.md). * [How do I run MXNet on a Raspberry Pi for computer vision?](http://mxnet.io/tutorials/embedded/wine_detector.html) +* [How do I run Keras 1.2.2 with mxnet backend?](https://github.com/dmlc/keras/wiki/Installation) + ## Extend and Contribute to MXNet * [How do I join the MXNet development discussion?](http://mxnet.io/community/mxnet_channels.html) From eb2374215de684b918df9e42a997f243a98a3707 Mon Sep 17 00:00:00 2001 From: lxn2 Date: Fri, 28 Jul 2017 16:22:31 -0700 Subject: [PATCH 268/834] Add post-build email notification & fix Windows CPU build (#7225) * Add post-build email to Jenkinsfile (#7207) Add post-build action to email if build failed. This branch will be used to test this Jenkinsfile. * Add post-build email notification & fix Windows CPU build (#7212) * Add post-build email to Jenkinsfile * Add post-build email notification & fix Windows CPU build * Add new variables to Jenkinsfile (#7220) * Add post-build email to Jenkinsfile * Add post-build email notification & fix Windows CPU build * Add variable definitions --- Jenkinsfile | 596 +++++++++++++++++++++++++++------------------------- 1 file changed, 309 insertions(+), 287 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 26a96a4843bc..613b88c5e76e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -8,6 +8,10 @@ mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, dmlc-core/libdmlc.a, nnvm/lib/libnnvm docker_run = 'tests/ci_build/ci_build.sh' // timeout in minutes max_time = 60 +// assign any caught errors here +err = null +// set build status to success by default +currentBuild.result = "SUCCESS" // initialize source codes def init_git() { @@ -38,18 +42,6 @@ def init_git_win() { } } -stage("Sanity Check") { - timeout(time: max_time, unit: 'MINUTES') { - node('mxnetlinux') { - ws('workspace/sanity') { - init_git() - make('lint', 'cpplint rcpplint jnilint') - make('lint', 'pylint') - } - } - } -} - // Run make. First try to do an incremental make from a previous workspace in hope to // accelerate the compilation. If something wrong, clean the workspace and then // build from scratch. @@ -84,134 +76,6 @@ echo ${libs} | sed -e 's/,/ /g' | xargs md5sum """ } -stage('Build') { - parallel 'CPU: Openblas': { - node('mxnetlinux') { - ws('workspace/build-cpu') { - init_git() - def flag = """ \ -DEV=1 \ -USE_PROFILER=1 \ -USE_CPP_PACKAGE=1 \ -USE_BLAS=openblas \ --j\$(nproc) -""" - make("cpu", flag) - pack_lib('cpu') - } - } - }, - 'GPU: CUDA7.5+cuDNN5': { - node('mxnetlinux') { - ws('workspace/build-gpu') { - init_git() - def flag = """ \ -DEV=1 \ -USE_PROFILER=1 \ -USE_BLAS=openblas \ -USE_CUDA=1 \ -USE_CUDA_PATH=/usr/local/cuda \ -USE_CUDNN=1 \ -USE_CPP_PACKAGE=1 \ --j\$(nproc) -""" - make('gpu', flag) - pack_lib('gpu') - stash includes: 'build/cpp-package/example/test_score', name: 'cpp_test_score' - } - } - }, - 'Amalgamation': { - node('mxnetlinux') { - ws('workspace/amalgamation') { - init_git() - make('cpu', '-C amalgamation/ USE_BLAS=openblas MIN=1') - } - } - }, - 'GPU: MKLML': { - node('mxnetlinux') { - ws('workspace/build-mklml') { - init_git() - def flag = """ \ -DEV=1 \ -USE_PROFILER=1 \ -USE_BLAS=openblas \ -USE_MKL2017=1 \ -USE_MKL2017_EXPERIMENTAL=1 \ -USE_CUDA=1 \ -USE_CUDA_PATH=/usr/local/cuda \ -USE_CUDNN=1 \ -USE_CPP_PACKAGE=1 \ --j\$(nproc) -""" - make('mklml_gpu', flag) - pack_lib('mklml') - } - } - }, - 'CPU windows':{ - node('mxnetwindows') { - ws('workspace/build-cpu') { - withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas', 'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0']) { - init_git_win() - bat """mkdir build_vc14_cpu -cd build_vc14_cpu -cmake -G \"Visual Studio 14 2015 Win64\" -DUSE_CUDA=0 -DUSE_CUDNN=0 -DUSE_NVRTC=0 -DUSE_OPENCV=1 -DUSE_OPENMP=1 -DUSE_PROFILER=1 -DUSE_BLAS=open -DUSE_LAPACK=1 -DUSE_DIST_KVSTORE=0 ${env.WORKSPACE}""" - bat 'C:\\mxnet\\build_vc14_cpu.bat' - - bat '''rmdir /s/q pkg_vc14_gpu -mkdir pkg_vc14_cpu\\lib -mkdir pkg_vc14_cpu\\python -mkdir pkg_vc14_cpu\\include -mkdir pkg_vc14_cpu\\build -copy build_vc14_cpu\\Release\\libmxnet.lib pkg_vc14_cpu\\lib -copy build_vc14_cpu\\Release\\libmxnet.dll pkg_vc14_cpu\\build -xcopy python pkg_vc14_cpu\\python /E /I /Y -xcopy include pkg_vc14_cpu\\include /E /I /Y -xcopy dmlc-core\\include pkg_vc14_cpu\\include /E /I /Y -xcopy mshadow\\mshadow pkg_vc14_cpu\\include\\mshadow /E /I /Y -xcopy nnvm\\include pkg_vc14_cpu\\nnvm\\include /E /I /Y -del /Q *.7z -7z.exe a vc14_cpu.7z pkg_vc14_cpu\\ -''' - stash includes: 'vc14_cpu.7z', name: 'vc14_cpu' - } - } - } - }, - 'GPU windows':{ - node('mxnetwindows') { - ws('workspace/build-gpu') { - withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas', 'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0']) { - init_git_win() - bat """mkdir build_vc14_gpu -call "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\x86_amd64\\vcvarsx86_amd64.bat" -cd build_vc14_gpu -cmake -G \"NMake Makefiles JOM\" -DUSE_CUDA=1 -DUSE_CUDNN=1 -DUSE_NVRTC=1 -DUSE_OPENCV=1 -DUSE_OPENMP=1 -DUSE_PROFILER=1 -DUSE_BLAS=open -DUSE_LAPACK=1 -DUSE_DIST_KVSTORE=0 -DCUDA_ARCH_NAME=All -DCMAKE_CXX_FLAGS_RELEASE="/FS /MD /O2 /Ob2 /DNDEBUG" -DCMAKE_BUILD_TYPE=Release ${env.WORKSPACE}""" - bat 'C:\\mxnet\\build_vc14_gpu.bat' - bat '''rmdir /s/q pkg_vc14_gpu -mkdir pkg_vc14_gpu\\lib -mkdir pkg_vc14_gpu\\python -mkdir pkg_vc14_gpu\\include -mkdir pkg_vc14_gpu\\build -copy build_vc14_gpu\\libmxnet.lib pkg_vc14_gpu\\lib -copy build_vc14_gpu\\libmxnet.dll pkg_vc14_gpu\\build -xcopy python pkg_vc14_gpu\\python /E /I /Y -xcopy include pkg_vc14_gpu\\include /E /I /Y -xcopy dmlc-core\\include pkg_vc14_gpu\\include /E /I /Y -xcopy mshadow\\mshadow pkg_vc14_gpu\\include\\mshadow /E /I /Y -xcopy nnvm\\include pkg_vc14_gpu\\nnvm\\include /E /I /Y -del /Q *.7z -7z.exe a vc14_gpu.7z pkg_vc14_gpu\\ -''' - stash includes: 'vc14_gpu.7z', name: 'vc14_gpu' - } - } - } - } -} - // Python unittest for CPU def python_ut(docker_type) { timeout(time: max_time, unit: 'MINUTES') { @@ -234,168 +98,326 @@ def python_gpu_ut(docker_type) { } } -stage('Unit Test') { - parallel 'Python2/3: CPU': { - node('mxnetlinux') { - ws('workspace/ut-python-cpu') { - init_git() - unpack_lib('cpu') - python_ut('cpu') - } - } - }, - 'Python2/3: GPU': { - node('mxnetlinux') { - ws('workspace/ut-python-gpu') { - init_git() - unpack_lib('gpu', mx_lib) - python_gpu_ut('gpu') - } - } - }, - 'Python2/3: MKLML': { - node('mxnetlinux') { - ws('workspace/ut-python-mklml') { - init_git() - unpack_lib('mklml') - python_ut('mklml_gpu') - python_gpu_ut('mklml_gpu') - } - } - }, - 'Scala: CPU': { - node('mxnetlinux') { - ws('workspace/ut-scala-cpu') { - init_git() - unpack_lib('cpu') - timeout(time: max_time, unit: 'MINUTES') { - sh "${docker_run} cpu make scalapkg USE_BLAS=openblas" - sh "${docker_run} cpu make scalatest USE_BLAS=openblas" +try { + stage("Sanity Check") { + timeout(time: max_time, unit: 'MINUTES') { + node('mxnetlinux') { + ws('workspace/sanity') { + init_git() + make('lint', 'cpplint rcpplint jnilint') + make('lint', 'pylint') + } } } } - }, - 'R: CPU': { - node('mxnetlinux') { - ws('workspace/ut-r-cpu') { - init_git() - unpack_lib('cpu') - timeout(time: max_time, unit: 'MINUTES') { - sh "${docker_run} cpu rm -rf .Renviron" - sh "${docker_run} cpu mkdir -p /workspace/ut-r-cpu/site-library" - sh "${docker_run} cpu make rpkg USE_BLAS=openblas R_LIBS=/workspace/ut-r-cpu/site-library" - sh "${docker_run} cpu R CMD INSTALL --library=/workspace/ut-r-cpu/site-library mxnet_current_r.tar.gz" - sh "${docker_run} cpu make rpkgtest R_LIBS=/workspace/ut-r-cpu/site-library" + + stage('Build') { + parallel 'CPU: Openblas': { + node('mxnetlinux') { + ws('workspace/build-cpu') { + init_git() + def flag = """ \ + DEV=1 \ + USE_PROFILER=1 \ + USE_CPP_PACKAGE=1 \ + USE_BLAS=openblas \ + -j\$(nproc) + """ + make("cpu", flag) + pack_lib('cpu') + } } - } - } - }, - 'R: GPU': { - node('mxnetlinux') { - ws('workspace/ut-r-gpu') { - init_git() - unpack_lib('gpu') - timeout(time: max_time, unit: 'MINUTES') { - sh "${docker_run} cpu rm -rf .Renviron" - sh "${docker_run} gpu mkdir -p /workspace/ut-r-gpu/site-library" - sh "${docker_run} gpu make rpkg USE_BLAS=openblas R_LIBS=/workspace/ut-r-gpu/site-library" - sh "${docker_run} gpu R CMD INSTALL --library=/workspace/ut-r-gpu/site-library mxnet_current_r.tar.gz" - sh "${docker_run} gpu make rpkgtest R_LIBS=/workspace/ut-r-gpu/site-library" + }, + 'GPU: CUDA7.5+cuDNN5': { + node('mxnetlinux') { + ws('workspace/build-gpu') { + init_git() + def flag = """ \ + DEV=1 \ + USE_PROFILER=1 \ + USE_BLAS=openblas \ + USE_CUDA=1 \ + USE_CUDA_PATH=/usr/local/cuda \ + USE_CUDNN=1 \ + USE_CPP_PACKAGE=1 \ + -j\$(nproc) + """ + make('gpu', flag) + pack_lib('gpu') + stash includes: 'build/cpp-package/example/test_score', name: 'cpp_test_score' + } + } + }, + 'Amalgamation': { + node('mxnetlinux') { + ws('workspace/amalgamation') { + init_git() + make('cpu', '-C amalgamation/ USE_BLAS=openblas MIN=1') + } } + }, + 'GPU: MKLML': { + node('mxnetlinux') { + ws('workspace/build-mklml') { + init_git() + def flag = """ \ + DEV=1 \ + USE_PROFILER=1 \ + USE_BLAS=openblas \ + USE_MKL2017=1 \ + USE_MKL2017_EXPERIMENTAL=1 \ + USE_CUDA=1 \ + USE_CUDA_PATH=/usr/local/cuda \ + USE_CUDNN=1 \ + USE_CPP_PACKAGE=1 \ + -j\$(nproc) + """ + make('mklml_gpu', flag) + pack_lib('mklml') + } + } + }, + 'CPU windows':{ + node('mxnetwindows') { + ws('workspace/build-cpu') { + withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas', 'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0']) { + init_git_win() + bat """mkdir build_vc14_cpu + cd build_vc14_cpu + cmake -G \"Visual Studio 14 2015 Win64\" -DUSE_CUDA=0 -DUSE_CUDNN=0 -DUSE_NVRTC=0 -DUSE_OPENCV=1 -DUSE_OPENMP=1 -DUSE_PROFILER=1 -DUSE_BLAS=open -DUSE_LAPACK=1 -DUSE_DIST_KVSTORE=0 ${env.WORKSPACE}""" + bat 'C:\\mxnet\\build_vc14_cpu.bat' + + bat '''rmdir /s/q pkg_vc14_cpu + mkdir pkg_vc14_cpu\\lib + mkdir pkg_vc14_cpu\\python + mkdir pkg_vc14_cpu\\include + mkdir pkg_vc14_cpu\\build + copy build_vc14_cpu\\Release\\libmxnet.lib pkg_vc14_cpu\\lib + copy build_vc14_cpu\\Release\\libmxnet.dll pkg_vc14_cpu\\build + xcopy python pkg_vc14_cpu\\python /E /I /Y + xcopy include pkg_vc14_cpu\\include /E /I /Y + xcopy dmlc-core\\include pkg_vc14_cpu\\include /E /I /Y + xcopy mshadow\\mshadow pkg_vc14_cpu\\include\\mshadow /E /I /Y + xcopy nnvm\\include pkg_vc14_cpu\\nnvm\\include /E /I /Y + del /Q *.7z + 7z.exe a vc14_cpu.7z pkg_vc14_cpu\\ + ''' + stash includes: 'vc14_cpu.7z', name: 'vc14_cpu' + } + } + } + }, + 'GPU windows':{ + node('mxnetwindows') { + ws('workspace/build-gpu') { + withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas', 'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0']) { + init_git_win() + bat """mkdir build_vc14_gpu + call "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\x86_amd64\\vcvarsx86_amd64.bat" + cd build_vc14_gpu + cmake -G \"NMake Makefiles JOM\" -DUSE_CUDA=1 -DUSE_CUDNN=1 -DUSE_NVRTC=1 -DUSE_OPENCV=1 -DUSE_OPENMP=1 -DUSE_PROFILER=1 -DUSE_BLAS=open -DUSE_LAPACK=1 -DUSE_DIST_KVSTORE=0 -DCUDA_ARCH_NAME=All -DCMAKE_CXX_FLAGS_RELEASE="/FS /MD /O2 /Ob2 /DNDEBUG" -DCMAKE_BUILD_TYPE=Release ${env.WORKSPACE}""" + bat 'C:\\mxnet\\build_vc14_gpu.bat' + bat '''rmdir /s/q pkg_vc14_gpu + mkdir pkg_vc14_gpu\\lib + mkdir pkg_vc14_gpu\\python + mkdir pkg_vc14_gpu\\include + mkdir pkg_vc14_gpu\\build + copy build_vc14_gpu\\libmxnet.lib pkg_vc14_gpu\\lib + copy build_vc14_gpu\\libmxnet.dll pkg_vc14_gpu\\build + xcopy python pkg_vc14_gpu\\python /E /I /Y + xcopy include pkg_vc14_gpu\\include /E /I /Y + xcopy dmlc-core\\include pkg_vc14_gpu\\include /E /I /Y + xcopy mshadow\\mshadow pkg_vc14_gpu\\include\\mshadow /E /I /Y + xcopy nnvm\\include pkg_vc14_gpu\\nnvm\\include /E /I /Y + del /Q *.7z + 7z.exe a vc14_gpu.7z pkg_vc14_gpu\\ + ''' + stash includes: 'vc14_gpu.7z', name: 'vc14_gpu' + } + } + } } } - }, - 'Python2/3: CPU Win':{ - node('mxnetwindows') { - ws('workspace/ut-python-cpu') { - init_git_win() - unstash 'vc14_cpu' - bat '''rmdir /s/q pkg_vc14_cpu -7z x -y vc14_cpu.7z''' - bat """xcopy C:\\mxnet\\data data /E /I /Y -xcopy C:\\mxnet\\model model /E /I /Y -call activate py3 -set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_cpu\\python -del /S /Q ${env.WORKSPACE}\\pkg_vc14_cpu\\python\\*.pyc -C:\\mxnet\\test_cpu.bat""" - bat """xcopy C:\\mxnet\\data data /E /I /Y -xcopy C:\\mxnet\\model model /E /I /Y -call activate py2 -set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_cpu\\python -del /S /Q ${env.WORKSPACE}\\pkg_vc14_cpu\\python\\*.pyc -C:\\mxnet\\test_cpu.bat""" - } - } - }, - 'Python2/3: GPU Win':{ - node('mxnetwindows') { - ws('workspace/ut-python-gpu') { - init_git_win() - unstash 'vc14_gpu' - bat '''rmdir /s/q pkg_vc14_gpu -7z x -y vc14_gpu.7z''' - bat """xcopy C:\\mxnet\\data data /E /I /Y -xcopy C:\\mxnet\\model model /E /I /Y -call activate py3 -set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_gpu\\python -del /S /Q ${env.WORKSPACE}\\pkg_vc14_gpu\\python\\*.pyc -C:\\mxnet\\test_gpu.bat""" - bat """xcopy C:\\mxnet\\data data /E /I /Y -xcopy C:\\mxnet\\model model /E /I /Y -call activate py2 -set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_gpu\\python -del /S /Q ${env.WORKSPACE}\\pkg_vc14_gpu\\python\\*.pyc -C:\\mxnet\\test_gpu.bat""" - } - } - } -} - -stage('Integration Test') { - parallel 'Python': { - node('mxnetlinux') { - ws('workspace/it-python-gpu') { - init_git() - unpack_lib('gpu') - timeout(time: max_time, unit: 'MINUTES') { - sh "${docker_run} gpu PYTHONPATH=./python/ python example/image-classification/test_score.py" + stage('Unit Test') { + parallel 'Python2/3: CPU': { + node('mxnetlinux') { + ws('workspace/ut-python-cpu') { + init_git() + unpack_lib('cpu') + python_ut('cpu') + } } - } - } - }, - 'Caffe': { - node('mxnetlinux') { - ws('workspace/it-caffe') { - init_git() - unpack_lib('gpu') - timeout(time: max_time, unit: 'MINUTES') { - sh "${docker_run} caffe_gpu PYTHONPATH=/caffe/python:./python python tools/caffe_converter/test_converter.py" + }, + 'Python2/3: GPU': { + node('mxnetlinux') { + ws('workspace/ut-python-gpu') { + init_git() + unpack_lib('gpu', mx_lib) + python_gpu_ut('gpu') + } } - } + }, + 'Python2/3: MKLML': { + node('mxnetlinux') { + ws('workspace/ut-python-mklml') { + init_git() + unpack_lib('mklml') + python_ut('mklml_gpu') + python_gpu_ut('mklml_gpu') + } + } + }, + 'Scala: CPU': { + node('mxnetlinux') { + ws('workspace/ut-scala-cpu') { + init_git() + unpack_lib('cpu') + timeout(time: max_time, unit: 'MINUTES') { + sh "${docker_run} cpu make scalapkg USE_BLAS=openblas" + sh "${docker_run} cpu make scalatest USE_BLAS=openblas" + } + } + } + }, + 'R: CPU': { + node('mxnetlinux') { + ws('workspace/ut-r-cpu') { + init_git() + unpack_lib('cpu') + timeout(time: max_time, unit: 'MINUTES') { + sh "${docker_run} cpu rm -rf .Renviron" + sh "${docker_run} cpu mkdir -p /workspace/ut-r-cpu/site-library" + sh "${docker_run} cpu make rpkg USE_BLAS=openblas R_LIBS=/workspace/ut-r-cpu/site-library" + sh "${docker_run} cpu R CMD INSTALL --library=/workspace/ut-r-cpu/site-library mxnet_current_r.tar.gz" + sh "${docker_run} cpu make rpkgtest R_LIBS=/workspace/ut-r-cpu/site-library" + } + } + } + }, + 'R: GPU': { + node('mxnetlinux') { + ws('workspace/ut-r-gpu') { + init_git() + unpack_lib('gpu') + timeout(time: max_time, unit: 'MINUTES') { + sh "${docker_run} cpu rm -rf .Renviron" + sh "${docker_run} gpu mkdir -p /workspace/ut-r-gpu/site-library" + sh "${docker_run} gpu make rpkg USE_BLAS=openblas R_LIBS=/workspace/ut-r-gpu/site-library" + sh "${docker_run} gpu R CMD INSTALL --library=/workspace/ut-r-gpu/site-library mxnet_current_r.tar.gz" + sh "${docker_run} gpu make rpkgtest R_LIBS=/workspace/ut-r-gpu/site-library" + } + } + } + }, + 'Python2/3: CPU Win':{ + node('mxnetwindows') { + ws('workspace/ut-python-cpu') { + init_git_win() + unstash 'vc14_cpu' + bat '''rmdir /s/q pkg_vc14_cpu + 7z x -y vc14_cpu.7z''' + bat """xcopy C:\\mxnet\\data data /E /I /Y + xcopy C:\\mxnet\\model model /E /I /Y + call activate py3 + set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_cpu\\python + del /S /Q ${env.WORKSPACE}\\pkg_vc14_cpu\\python\\*.pyc + C:\\mxnet\\test_cpu.bat""" + bat """xcopy C:\\mxnet\\data data /E /I /Y + xcopy C:\\mxnet\\model model /E /I /Y + call activate py2 + set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_cpu\\python + del /S /Q ${env.WORKSPACE}\\pkg_vc14_cpu\\python\\*.pyc + C:\\mxnet\\test_cpu.bat""" + } + } + }, + 'Python2/3: GPU Win':{ + node('mxnetwindows') { + ws('workspace/ut-python-gpu') { + init_git_win() + unstash 'vc14_gpu' + bat '''rmdir /s/q pkg_vc14_gpu + 7z x -y vc14_gpu.7z''' + bat """xcopy C:\\mxnet\\data data /E /I /Y + xcopy C:\\mxnet\\model model /E /I /Y + call activate py3 + set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_gpu\\python + del /S /Q ${env.WORKSPACE}\\pkg_vc14_gpu\\python\\*.pyc + C:\\mxnet\\test_gpu.bat""" + bat """xcopy C:\\mxnet\\data data /E /I /Y + xcopy C:\\mxnet\\model model /E /I /Y + call activate py2 + set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_gpu\\python + del /S /Q ${env.WORKSPACE}\\pkg_vc14_gpu\\python\\*.pyc + C:\\mxnet\\test_gpu.bat""" + } + } + } } - }, - 'cpp-package': { - node('mxnetlinux') { - ws('workspace/it-cpp-package') { - init_git() - unpack_lib('gpu') - unstash 'cpp_test_score' - timeout(time: max_time, unit: 'MINUTES') { - sh "${docker_run} gpu cpp-package/tests/ci_test.sh" + + stage('Integration Test') { + parallel 'Python': { + node('mxnetlinux') { + ws('workspace/it-python-gpu') { + init_git() + unpack_lib('gpu') + timeout(time: max_time, unit: 'MINUTES') { + sh "${docker_run} gpu PYTHONPATH=./python/ python example/image-classification/test_score.py" + } + } + } + }, + 'Caffe': { + node('mxnetlinux') { + ws('workspace/it-caffe') { + init_git() + unpack_lib('gpu') + timeout(time: max_time, unit: 'MINUTES') { + sh "${docker_run} caffe_gpu PYTHONPATH=/caffe/python:./python python tools/caffe_converter/test_converter.py" + } + } + } + }, + 'cpp-package': { + node('mxnetlinux') { + ws('workspace/it-cpp-package') { + init_git() + unpack_lib('gpu') + unstash 'cpp_test_score' + timeout(time: max_time, unit: 'MINUTES') { + sh "${docker_run} gpu cpp-package/tests/ci_test.sh" + } + } } } } - } -} -stage('Deploy') { - node('mxnetlinux') { - ws('workspace/docs') { - if (env.BRANCH_NAME == "master") { - init_git() - sh "make docs" + stage('Deploy') { + node('mxnetlinux') { + ws('workspace/docs') { + if (env.BRANCH_NAME == "master") { + init_git() + sh "make docs" + } + } } } - } +} catch (caughtError) { + node("mxnetlinux") { + sh "echo caught error" + err = caughtError + currentBuild.result = "FAILURE" + } +} finally { + node("mxnetlinux") { + // Only send email if master failed + if (currentBuild.result == "FAILURE" && env.BRANCH_NAME == "master") { + emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}' + } + // Remember to rethrow so the build is marked as failing + if (err) { + throw err + } + } } From 959ac3947cff8e93504ce0d5e57c6728d8284dc9 Mon Sep 17 00:00:00 2001 From: lxn2 Date: Fri, 28 Jul 2017 16:26:28 -0700 Subject: [PATCH 269/834] Use different array comparison function for useful output (#7205) * Increase relative tolerance to avoid test failure * Use different array comparison function --- tests/python/gpu/test_operator_gpu.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/python/gpu/test_operator_gpu.py b/tests/python/gpu/test_operator_gpu.py index 487197f2ad7e..3c319f84bf29 100644 --- a/tests/python/gpu/test_operator_gpu.py +++ b/tests/python/gpu/test_operator_gpu.py @@ -3,7 +3,7 @@ import time import mxnet as mx import numpy as np -from mxnet.test_utils import check_consistency, set_default_context +from mxnet.test_utils import check_consistency, set_default_context, assert_almost_equal from numpy.testing import assert_allclose curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__))) @@ -1289,9 +1289,9 @@ def check_rnn_layer(layer): states = layer.begin_state(16) co, cs = layer(x, states) - assert_allclose(go.asnumpy(), co.asnumpy(), rtol=1e-2) + assert_almost_equal(go.asnumpy(), co.asnumpy(), rtol=1e-2, atol=1e-8) for g, c in zip(gs, cs): - assert_allclose(g.asnumpy(), c.asnumpy(), rtol=1e-2) + assert_almost_equal(g.asnumpy(), c.asnumpy(), rtol=1e-2, atol=1e-8) def test_rnn_layer(): From 424143ac47ab3a38ae8aedaeb3319379887de0bc Mon Sep 17 00:00:00 2001 From: "Qiang Kou (KK)" Date: Fri, 28 Jul 2017 23:57:52 +0000 Subject: [PATCH 270/834] [R] allow users to use other names than "label". close #7126 (#7232) --- R-package/R/model.R | 31 +++++++++++++++------------ R-package/tests/testthat/test_model.R | 11 +++++----- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/R-package/R/model.R b/R-package/R/model.R index 64cc816f0ef4..2ee66242d805 100644 --- a/R-package/R/model.R +++ b/R-package/R/model.R @@ -116,15 +116,16 @@ mx.model.train <- function(symbol, ctx, input.shape, output.shape, ndevice <- length(ctx) if(verbose) message(paste0("Start training with ", ndevice, " devices")) # create the executors - sliceinfo <- mx.model.slice.shape(input.shape, ndevice) - sliceinfo2 <- mx.model.slice.shape(output.shape, ndevice) + input_slice <- mx.model.slice.shape(input.shape, ndevice) + output_slice <- mx.model.slice.shape(output.shape, ndevice) arg_names <- arguments(symbol) - label_name <- arg_names[endsWith(arg_names, "label")] + output.names <- names(output.shape) + #label_name <- arg_names[endsWith(arg_names, "label")] train.execs <- lapply(1:ndevice, function(i) { arg_lst <- list(symbol = symbol, ctx = ctx[[i]], grad.req = "write") - arg_lst <- append(arg_lst, sliceinfo[[i]]$shape) - arg_lst <- append(arg_lst, sliceinfo2[[i]]$shape) + arg_lst <- append(arg_lst, input_slice[[i]]$shape) + arg_lst <- append(arg_lst, output_slice[[i]]$shape) arg_lst[["fixed.param"]] = fixed.param do.call(mx.simple.bind, arg_lst) }) @@ -152,9 +153,6 @@ mx.model.train <- function(symbol, ctx, input.shape, output.shape, kvstore$init(params.index, train.execs[[1]]$ref.arg.arrays[params.index]) } # Get the input names - # input.names <- mx.model.check.arguments(symbol) - arg_names <- arguments(symbol) - label_name <- arg_names[endsWith(arg_names, "label")] for (iteration in begin.round:end.round) { nbatch <- 0 @@ -165,14 +163,16 @@ mx.model.train <- function(symbol, ctx, input.shape, output.shape, # Get input data slice dlist <- train.data$value() slices <- lapply(1:ndevice, function(i) { - s <- sliceinfo[[i]] + s <- input_slice[[i]] ret <- sapply(names(dlist), function(n) {mx.nd.slice(dlist[[n]], s$begin, s$end)}) return(ret) }) # copy data to executor for (i in 1:ndevice) { s <- slices[[i]] - names(s)[endsWith(names(s), "label")] = label_name + if (endsWith(output.names, "label")) { + names(s)[endsWith(names(s), "label")] = output.names + } mx.exec.update.arg.arrays(train.execs[[i]], s, match.name=TRUE) } for (texec in train.execs) { @@ -186,6 +186,7 @@ mx.model.train <- function(symbol, ctx, input.shape, output.shape, for (texec in train.execs) { mx.exec.backward(texec) } + if (!is.null(kvstore)) { # push the gradient kvstore$push(params.index, lapply(train.execs, function(texec) { @@ -214,7 +215,7 @@ mx.model.train <- function(symbol, ctx, input.shape, output.shape, # Update the evaluation metrics if (!is.null(metric)) { for (i in 1 : ndevice) { - train.metric <- metric$update(slices[[i]]$label, out.preds[[i]], train.metric) + train.metric <- metric$update(slices[[i]][[length(slices[[i]])]], out.preds[[i]], train.metric) } } nbatch <- nbatch + 1 @@ -235,13 +236,15 @@ mx.model.train <- function(symbol, ctx, input.shape, output.shape, while (eval.data$iter.next()) { dlist <- eval.data$value() slices <- lapply(1:ndevice, function(i) { - s <- sliceinfo[[i]] + s <- input_slice[[i]] ret <- sapply(names(dlist), function(n) {mx.nd.slice(dlist[[n]], s$begin, s$end)}) return(ret) }) for (i in 1:ndevice) { s <- slices[[i]] - names(s)[endsWith(names(s), "label")] = label_name + if (endsWith(output.names, "label")) { + names(s)[endsWith(names(s), "label")] = output.names + } mx.exec.update.arg.arrays(train.execs[[i]], s, match.name=TRUE) } for (texec in train.execs) { @@ -252,7 +255,7 @@ mx.model.train <- function(symbol, ctx, input.shape, output.shape, }) if (!is.null(metric)) { for (i in 1 : ndevice) { - eval.metric <- metric$update(slices[[i]]$label, out.preds[[i]], eval.metric) + eval.metric <- metric$update(slices[[i]][[length(slices[[i]])]] , out.preds[[i]], eval.metric) } } } diff --git a/R-package/tests/testthat/test_model.R b/R-package/tests/testthat/test_model.R index 4cf2a8c8e070..73a212714af8 100644 --- a/R-package/tests/testthat/test_model.R +++ b/R-package/tests/testthat/test_model.R @@ -162,12 +162,11 @@ test_that("Matrix Factorization", { k <- 64 user <- mx.symbol.Variable("user") item <- mx.symbol.Variable("item") - score <- mx.symbol.Variable("label") + score <- mx.symbol.Variable("score") user1 <- mx.symbol.Embedding(data = mx.symbol.BlockGrad(user), input_dim = max_user, output_dim = k, name = "user1") item1 <- mx.symbol.Embedding(data = mx.symbol.BlockGrad(item), input_dim = max_item, - output_dim = k, name = "item1" - ) + output_dim = k, name = "item1") pred <- user1 * item1 pred1 <- mx.symbol.sum_axis(pred, axis = 1, name = "pred1") pred2 <- mx.symbol.Flatten(pred1, name = "pred2") @@ -188,10 +187,10 @@ test_that("Matrix Factorization", { value = function() { user <- .self$iter1$value()$data item <- .self$iter2$value()$data - label <- .self$iter1$value()$label + score <- .self$iter1$value()$label list(user = user, item = item, - label = label) + score = score) }, iter.next = function() { .self$iter1$iter.next() @@ -224,5 +223,5 @@ test_that("Matrix Factorization", { momentum = 0.9, epoch.end.callback = mx.callback.log.train.metric(1), input.names = c("user", "item"), - output.names = "label") + output.names = "score") }) From b996dd18ac2e82939003add11745196243681500 Mon Sep 17 00:00:00 2001 From: "Qiang Kou (KK)" Date: Sat, 29 Jul 2017 00:42:18 +0000 Subject: [PATCH 271/834] [R] lstm bucketing example (#7237) --- .../bucket_R/aclImdb_lstm_classification.R | 76 +++++++ example/rnn/bucket_R/data_preprocessing.R | 170 +++++++++++++++ example/rnn/bucket_R/gru.cell.R | 54 +++++ example/rnn/bucket_R/lstm.cell.R | 41 ++++ example/rnn/bucket_R/mx.io.bucket.iter.R | 94 ++++++++ example/rnn/bucket_R/rnn.R | 202 ++++++++++++++++++ example/rnn/bucket_R/rnn.infer.R | 79 +++++++ example/rnn/bucket_R/rnn.train.R | 120 +++++++++++ 8 files changed, 836 insertions(+) create mode 100644 example/rnn/bucket_R/aclImdb_lstm_classification.R create mode 100644 example/rnn/bucket_R/data_preprocessing.R create mode 100644 example/rnn/bucket_R/gru.cell.R create mode 100644 example/rnn/bucket_R/lstm.cell.R create mode 100644 example/rnn/bucket_R/mx.io.bucket.iter.R create mode 100644 example/rnn/bucket_R/rnn.R create mode 100644 example/rnn/bucket_R/rnn.infer.R create mode 100644 example/rnn/bucket_R/rnn.train.R diff --git a/example/rnn/bucket_R/aclImdb_lstm_classification.R b/example/rnn/bucket_R/aclImdb_lstm_classification.R new file mode 100644 index 000000000000..aaa6d388fda0 --- /dev/null +++ b/example/rnn/bucket_R/aclImdb_lstm_classification.R @@ -0,0 +1,76 @@ +require("mxnet") + +source("mx.io.bucket.iter.R") +source("rnn.train.R") + +corpus_bucketed_train <- readRDS(file = "corpus_bucketed_train_100_200_300_500_800_left.rds") +corpus_bucketed_test <- readRDS(file = "corpus_bucketed_test_100_200_300_500_800_left.rds") + +vocab <- length(corpus_bucketed_test$dic) + +### Create iterators +batch.size <- 64 + +train.data <- mx.io.bucket.iter(buckets = corpus_bucketed_train$buckets, + batch.size = batch.size, + data.mask.element = 0, + shuffle = TRUE) + +eval.data <- mx.io.bucket.iter(buckets = corpus_bucketed_test$buckets, + batch.size = batch.size, + data.mask.element = 0, + shuffle = FALSE) + +mx.set.seed(0) + +end.round <- 16 + +optimizer <- mx.opt.create("adadelta", + rho = 0.92, + epsilon = 1e-06, + wd = 2e-04, + clip_gradient = NULL, + rescale.grad = 1/batch.size) + +model_sentiment_lstm <- mx.rnn.buckets(train.data = train.data, + begin.round = 1, + end.round = end.round, + ctx = mx.cpu(), + metric = mx.metric.accuracy, + optimizer = optimizer, + num.rnn.layer = 2, + num.embed = 16, + num.hidden = 24, + num.label = 2, + input.size = vocab, + initializer = mx.init.Xavier(rnd_type = "gaussian", + factor_type = "in", + magnitude = 2), + dropout = 0.25, + config = "seq-to-one", + batch.end.callback = mx.callback.log.train.metric(period = 50), + verbose = TRUE) + +mx.model.save(model_sentiment_lstm, prefix = "model_sentiment_lstm", iteration = end.round) + +source("rnn.infer.R") + +model <- mx.model.load("model_sentiment_lstm", iteration = end.round) + +pred <- mx.rnn.infer.buckets(infer_iter = eval.data, model, "seq-to-one", ctx = mx.cpu()) + +ypred <- max.col(t(as.array(pred)), tie = "first") - 1 + +packer <- mxnet:::mx.nd.arraypacker() + +eval.data$reset() + +while (eval.data$iter.next()) { + packer$push(eval.data$value()$label) +} + +ylabel <- as.array(packer$get()) + +acc <- sum(ylabel == ypred)/length(ylabel) + +message(paste("Acc:", acc)) diff --git a/example/rnn/bucket_R/data_preprocessing.R b/example/rnn/bucket_R/data_preprocessing.R new file mode 100644 index 000000000000..c91e3fb5eb49 --- /dev/null +++ b/example/rnn/bucket_R/data_preprocessing.R @@ -0,0 +1,170 @@ +# download the IMDB dataset +if (!file.exists("aclImdb_v1.tar.gz")) { + download.file("http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz", + "aclImdb_v1.tar.gz") + untar("aclImdb_v1.tar.gz") +} + +# install required packages +list.of.packages <- c("readr", "dplyr", "stringr", "stringi") +new.packages <- list.of.packages[!(list.of.packages %in% installed.packages()[, "Package"])] +if (length(new.packages)) install.packages(new.packages) + +require("readr") +require("dplyr") +require("stringr") +require("stringi") + +negative_train_list <- list.files("./aclImdb/train/neg/", full.names = T) +positive_train_list <- list.files("./aclImdb/train/pos/", full.names = T) + +negative_test_list <- list.files("./aclImdb/test/neg/", full.names = T) +positive_test_list <- list.files("./aclImdb/test/pos/", full.names = T) + +file_import <- function(file_list) { + import <- sapply(file_list, read_file) + return(import) +} + +negative_train_raw <- file_import(negative_train_list) +positive_train_raw <- file_import(positive_train_list) + +negative_test_raw <- file_import(negative_test_list) +positive_test_raw <- file_import(positive_test_list) + +train_raw <- c(negative_train_raw, positive_train_raw) +test_raw <- c(negative_test_raw, positive_test_raw) + +saveRDS(train_raw, file = "train_raw.rds") +saveRDS(test_raw, file = "test_raw.rds") + +################################################################ Pre-process a corpus composed of a vector of sequences Build a dictionnary +################################################################ removing too rare words +text_pre_process <- function(corpus, count_threshold = 10, dic = NULL) { + raw_vec <- corpus + raw_vec <- stri_enc_toascii(str = raw_vec) + + ### remove non-printable characters + raw_vec <- str_replace_all(string = raw_vec, pattern = "[^[:print:]]", replacement = "") + raw_vec <- str_to_lower(string = raw_vec) + raw_vec <- str_replace_all(string = raw_vec, pattern = "_", replacement = " ") + raw_vec <- str_replace_all(string = raw_vec, pattern = "\\bbr\\b", replacement = "") + raw_vec <- str_replace_all(string = raw_vec, pattern = "\\s+", replacement = " ") + raw_vec <- str_trim(string = raw_vec) + + ### Split raw sequence vectors into lists of word vectors (one list element per + ### sequence) + word_vec_list <- stri_split_boundaries(raw_vec, type = "word", skip_word_none = T, + skip_word_number = F, simplify = F) + + ### Build vocabulary + if (is.null(dic)) { + word_vec_unlist <- unlist(word_vec_list) + word_vec_table <- sort(table(word_vec_unlist), decreasing = T) + word_cutoff <- which.max(word_vec_table < count_threshold) + word_keep <- names(word_vec_table)[1:(word_cutoff - 1)] + stopwords <- c(letters, "an", "the", "br") + word_keep <- setdiff(word_keep, stopwords) + } else word_keep <- names(dic)[!dic == 0] + + ### Clean the sentences to keep only the curated list of words + word_vec_list <- lapply(word_vec_list, function(x) x[x %in% word_keep]) + + # sentence_vec<- stri_split_boundaries(raw_vec, type='sentence', simplify = T) + word_vec_length <- lapply(word_vec_list, length) %>% unlist() + + ### Build dictionnary + dic <- 1:length(word_keep) + names(dic) <- word_keep + dic <- c(`¤` = 0, dic) + + ### reverse dictionnary + rev_dic <- names(dic) + names(rev_dic) <- dic + + return(list(word_vec_list = word_vec_list, dic = dic, rev_dic = rev_dic)) +} + +################################################################ +make_bucket_data <- function(word_vec_list, labels, dic, seq_len = c(225), right_pad = T) { + ### Trunc sequence to max bucket length + word_vec_list <- lapply(word_vec_list, head, n = max(seq_len)) + + word_vec_length <- lapply(word_vec_list, length) %>% unlist() + bucketID <- cut(word_vec_length, breaks = c(0, seq_len, Inf), include.lowest = T, + labels = F) + # table(bucketID) + + ### Right or Left side Padding Pad sequences to their bucket length with + ### dictionnary 0-label + word_vec_list_pad <- lapply(1:length(word_vec_list), function(x) { + length(word_vec_list[[x]]) <- seq_len[bucketID[x]] + word_vec_list[[x]][is.na(word_vec_list[[x]])] <- names(dic[1]) + if (right_pad == F) + word_vec_list[[x]] <- rev(word_vec_list[[x]]) + return(word_vec_list[[x]]) + }) + + ### Assign sequences to buckets and unroll them in order to be reshaped into arrays + unrolled_arrays <- lapply(1:length(seq_len), function(x) unlist(word_vec_list_pad[bucketID == + x])) + + ### Assign labels to their buckets + bucketed_labels <- lapply(1:length(seq_len), function(x) labels[bucketID == x]) + names(bucketed_labels) <- as.character(seq_len) + + ### Assign the dictionnary to each bucket terms + unrolled_arrays_dic <- lapply(1:length(seq_len), function(x) dic[unrolled_arrays[[x]]]) + + # length(splitted_arrays_dic[[1]]) Reshape into arrays having each sequence into + # a column + features_arrays <- lapply(1:length(seq_len), function(x) array(unrolled_arrays_dic[[x]], + dim = c(seq_len[x], length(unrolled_arrays_dic[[x]])/seq_len[x]))) + + features <- lapply(1:length(seq_len), function(x) features_arrays[[x]][1:seq_len[x], + ]) + names(features) <- as.character(seq_len) + + ### Combine data and labels into buckets + buckets <- lapply(1:length(seq_len), function(x) c(list(data = features[[x]]), + list(label = bucketed_labels[[x]]))) + names(buckets) <- as.character(seq_len) + + ### reverse dictionnary + rev_dic <- names(dic) + names(rev_dic) <- dic + + return(list(buckets = buckets, dic = dic, rev_dic = rev_dic)) +} + + +corpus_preprocessed_train <- text_pre_process(corpus = train_raw, count_threshold = 10, + dic = NULL) + +# length(corpus_preprocessed_train$dic) + +corpus_preprocessed_test <- text_pre_process(corpus = test_raw, dic = corpus_preprocessed_train$dic) + +saveRDS(corpus_preprocessed_train, file = "corpus_preprocessed_train_10.rds") +saveRDS(corpus_preprocessed_test, file = "corpus_preprocessed_test_10.rds") + +corpus_preprocessed_train <- readRDS(file = "corpus_preprocessed_train_10.rds") +corpus_preprocessed_test <- readRDS(file = "corpus_preprocessed_test_10.rds") + + +corpus_bucketed_train <- make_bucket_data(word_vec_list = corpus_preprocessed_train$word_vec_list, + labels = rep(0:1, each = 12500), dic = corpus_preprocessed_train$dic, seq_len = c(100, + 200, 300, 500, 800), right_pad = F) + +# lapply(corpus_bucketed_train$buckets, function(x) length(x[[2]])) + + +corpus_bucketed_test <- make_bucket_data(word_vec_list = corpus_preprocessed_test$word_vec_list, + labels = rep(0:1, each = 12500), dic = corpus_preprocessed_test$dic, seq_len = c(100, + 200, 300, 500, 800), right_pad = F) + +# lapply(corpus_bucketed_test$buckets, function(x) length(x[[2]])) + + +saveRDS(corpus_bucketed_train, file = "corpus_bucketed_train_100_200_300_500_800_left.rds") +saveRDS(corpus_bucketed_test, file = "corpus_bucketed_test_100_200_300_500_800_left.rds") diff --git a/example/rnn/bucket_R/gru.cell.R b/example/rnn/bucket_R/gru.cell.R new file mode 100644 index 000000000000..5932cdf17efa --- /dev/null +++ b/example/rnn/bucket_R/gru.cell.R @@ -0,0 +1,54 @@ +# GRU cell symbol +gru.cell <- function(num.hidden, indata, prev.state, param, seqidx, layeridx, dropout = 0, + data_masking) { + i2h <- mx.symbol.FullyConnected(data = indata, weight = param$gates.i2h.weight, + bias = param$gates.i2h.bias, num.hidden = num.hidden * 2, name = paste0("t", + seqidx, ".l", layeridx, ".gates.i2h")) + + if (dropout > 0) + i2h <- mx.symbol.Dropout(data = i2h, p = dropout) + + if (!is.null(prev.state)) { + h2h <- mx.symbol.FullyConnected(data = prev.state$h, weight = param$gates.h2h.weight, + bias = param$gates.h2h.bias, num.hidden = num.hidden * 2, name = paste0("t", + seqidx, ".l", layeridx, ".gates.h2h")) + gates <- i2h + h2h + } else { + gates <- i2h + } + + split.gates <- mx.symbol.split(gates, num.outputs = 2, axis = 1, squeeze.axis = F, + name = paste0("t", seqidx, ".l", layeridx, ".split")) + + update.gate <- mx.symbol.Activation(split.gates[[1]], act.type = "sigmoid") + reset.gate <- mx.symbol.Activation(split.gates[[2]], act.type = "sigmoid") + + htrans.i2h <- mx.symbol.FullyConnected(data = indata, weight = param$trans.i2h.weight, + bias = param$trans.i2h.bias, num.hidden = num.hidden, name = paste0("t", + seqidx, ".l", layeridx, ".trans.i2h")) + + if (is.null(prev.state)) { + h.after.reset <- reset.gate * 0 + } else { + h.after.reset <- prev.state$h * reset.gate + } + + htrans.h2h <- mx.symbol.FullyConnected(data = h.after.reset, weight = param$trans.h2h.weight, + bias = param$trans.h2h.bias, num.hidden = num.hidden, name = paste0("t", + seqidx, ".l", layeridx, ".trans.h2h")) + + h.trans <- htrans.i2h + htrans.h2h + h.trans.active <- mx.symbol.Activation(h.trans, act.type = "tanh") + + if (is.null(prev.state)) { + next.h <- update.gate * h.trans.active + } else { + next.h <- prev.state$h + update.gate * (h.trans.active - prev.state$h) + } + + ### Add a mask - using the mask_array approach + data_mask_expand <- mx.symbol.Reshape(data = data_masking, shape = c(1, -2)) + next.h <- mx.symbol.broadcast_mul(lhs = next.h, rhs = data_mask_expand) + + return(list(h = next.h)) +} diff --git a/example/rnn/bucket_R/lstm.cell.R b/example/rnn/bucket_R/lstm.cell.R new file mode 100644 index 000000000000..3c7b0e456d20 --- /dev/null +++ b/example/rnn/bucket_R/lstm.cell.R @@ -0,0 +1,41 @@ +# LSTM cell symbol +lstm.cell <- function(num.hidden, indata, prev.state, param, seqidx, layeridx, dropout = 0, + data_masking) { + i2h <- mx.symbol.FullyConnected(data = indata, weight = param$i2h.weight, bias = param$i2h.bias, + num.hidden = num.hidden * 4, name = paste0("t", seqidx, ".l", layeridx, ".i2h")) + + if (dropout > 0) + i2h <- mx.symbol.Dropout(data = i2h, p = dropout) + + if (!is.null(prev.state)) { + h2h <- mx.symbol.FullyConnected(data = prev.state$h, weight = param$h2h.weight, + bias = param$h2h.bias, num.hidden = num.hidden * 4, name = paste0("t", + seqidx, ".l", layeridx, ".h2h")) + gates <- i2h + h2h + } else { + gates <- i2h + } + + split.gates <- mx.symbol.split(gates, num.outputs = 4, axis = 1, squeeze.axis = F, + name = paste0("t", seqidx, ".l", layeridx, ".slice")) + + in.gate <- mx.symbol.Activation(split.gates[[1]], act.type = "sigmoid") + in.transform <- mx.symbol.Activation(split.gates[[2]], act.type = "tanh") + forget.gate <- mx.symbol.Activation(split.gates[[3]], act.type = "sigmoid") + out.gate <- mx.symbol.Activation(split.gates[[4]], act.type = "sigmoid") + + if (is.null(prev.state)) { + next.c <- in.gate * in.transform + } else { + next.c <- (forget.gate * prev.state$c) + (in.gate * in.transform) + } + + next.h <- out.gate * mx.symbol.Activation(next.c, act.type = "tanh") + + ### Add a mask - using the mask_array approach + data_mask_expand <- mx.symbol.Reshape(data = data_masking, shape = c(1, -2)) + next.c <- mx.symbol.broadcast_mul(lhs = next.c, rhs = data_mask_expand) + next.h <- mx.symbol.broadcast_mul(lhs = next.h, rhs = data_mask_expand) + + return(list(c = next.c, h = next.h)) +} diff --git a/example/rnn/bucket_R/mx.io.bucket.iter.R b/example/rnn/bucket_R/mx.io.bucket.iter.R new file mode 100644 index 000000000000..887247a320bf --- /dev/null +++ b/example/rnn/bucket_R/mx.io.bucket.iter.R @@ -0,0 +1,94 @@ +BucketIter <- setRefClass("BucketIter", fields = c("buckets", "bucket.names", "batch.size", + "data.mask.element", "shuffle", "bucket.plan", "bucketID", "epoch", "batch", + "batch.per.epoch", "seed"), contains = "Rcpp_MXArrayDataIter", methods = list(initialize = function(buckets, + batch.size, data.mask.element = 0, shuffle = FALSE, seed = 123) { + .self$buckets <- buckets + .self$bucket.names <- names(.self$buckets) + .self$batch.size <- batch.size + .self$data.mask.element <- data.mask.element + .self$epoch <- 0 + .self$batch <- 0 + .self$shuffle <- shuffle + .self$batch.per.epoch <- 0 + .self$bucket.plan <- NULL + .self$bucketID <- NULL + .self$seed <- seed + .self +}, reset = function() { + buckets_nb <- length(bucket.names) + buckets_id <- 1:buckets_nb + buckets_size <- sapply(.self$buckets, function(x) { + dim(x$data)[length(dim(x$data))] + }) + batch_per_bucket <- floor(buckets_size/.self$batch.size) + # Number of batches per epoch given the batch_size + .self$batch.per.epoch <- sum(batch_per_bucket) + .self$epoch <- .self$epoch + 1 + .self$batch <- 0 + + if (.self$shuffle) { + set.seed(.self$seed) + bucket_plan_names <- sample(rep(names(batch_per_bucket), times = batch_per_bucket)) + .self$bucket.plan <- ave(bucket_plan_names == bucket_plan_names, bucket_plan_names, + FUN = cumsum) + names(.self$bucket.plan) <- bucket_plan_names + ### Return first BucketID at reset for initialization of the model + .self$bucketID <- .self$bucket.plan[1] + + .self$buckets <- lapply(.self$buckets, function(x) { + shuffle_id <- sample(ncol(x$data)) + if (length(dim(x$label)) == 0) { + list(data = x$data[, shuffle_id], label = x$label[shuffle_id]) + } else { + list(data = x$data[, shuffle_id], label = x$label[, shuffle_id]) + } + }) + } else { + bucket_plan_names <- rep(names(batch_per_bucket), times = batch_per_bucket) + .self$bucket.plan <- ave(bucket_plan_names == bucket_plan_names, bucket_plan_names, + FUN = cumsum) + names(.self$bucket.plan) <- bucket_plan_names + } +}, iter.next = function() { + .self$batch <- .self$batch + 1 + .self$bucketID <- .self$bucket.plan[batch] + if (.self$batch > .self$batch.per.epoch) { + return(FALSE) + } else { + return(TRUE) + } +}, value = function() { + # bucketID is a named integer: the integer indicates the batch id for the given + # bucket (used to fetch appropriate samples within the bucket) the name is the a + # character containing the sequence length of the bucket (used to unroll the rnn + # to appropriate sequence length) + idx <- (.self$bucketID - 1) * (.self$batch.size) + (1:batch.size) + data <- .self$buckets[[names(.self$bucketID)]]$data[, idx, drop = F] + data_mask <- as.integer(names(.self$bucketID)) - apply(data == .self$data.mask.element, + 2, sum) + data_mask_array <- (!data == 0) + if (length(dim(.self$buckets[[names(.self$bucketID)]]$label)) == 0) { + label <- .self$buckets[[names(.self$bucketID)]]$label[idx] + } else { + label <- .self$buckets[[names(.self$bucketID)]]$label[, idx, drop = F] + } + return(list(data = mx.nd.array(data), label = mx.nd.array(label), data.mask = mx.nd.array(data_mask), + data.mask.array = mx.nd.array(data_mask_array))) +}, finalize = function() { +})) + +# +#' Create Bucket Iter +#' +#' @param buckets The data array. +#' @param batch.size The batch size used to pack the array. +#' @param data.mask.element The element to mask +#' @param shuffle Whether shuffle the data +#' @param seed The random seed +#' +#' @export +mx.io.bucket.iter <- function(buckets, batch.size, data.mask.element = 0, shuffle = FALSE, + seed = 123) { + return(BucketIter$new(buckets = buckets, batch.size = batch.size, data.mask.element = data.mask.element, + shuffle = shuffle, seed = seed)) +} diff --git a/example/rnn/bucket_R/rnn.R b/example/rnn/bucket_R/rnn.R new file mode 100644 index 000000000000..f55272f29459 --- /dev/null +++ b/example/rnn/bucket_R/rnn.R @@ -0,0 +1,202 @@ +library(mxnet) + +source("lstm.cell.R") +source("gru.cell.R") + +# unrolled RNN network +rnn.unroll <- function(num.rnn.layer, seq.len, input.size, num.embed, num.hidden, + num.label, dropout = 0, ignore_label = 0, init.state = NULL, config, cell.type = "lstm", + output_last_state = F) { + embed.weight <- mx.symbol.Variable("embed.weight") + cls.weight <- mx.symbol.Variable("cls.weight") + cls.bias <- mx.symbol.Variable("cls.bias") + + param.cells <- lapply(1:num.rnn.layer, function(i) { + if (cell.type == "lstm") { + cell <- list(i2h.weight = mx.symbol.Variable(paste0("l", i, ".i2h.weight")), + i2h.bias = mx.symbol.Variable(paste0("l", i, ".i2h.bias")), h2h.weight = mx.symbol.Variable(paste0("l", + i, ".h2h.weight")), h2h.bias = mx.symbol.Variable(paste0("l", i, + ".h2h.bias"))) + } else if (cell.type == "gru") { + cell <- list(gates.i2h.weight = mx.symbol.Variable(paste0("l", i, ".gates.i2h.weight")), + gates.i2h.bias = mx.symbol.Variable(paste0("l", i, ".gates.i2h.bias")), + gates.h2h.weight = mx.symbol.Variable(paste0("l", i, ".gates.h2h.weight")), + gates.h2h.bias = mx.symbol.Variable(paste0("l", i, ".gates.h2h.bias")), + trans.i2h.weight = mx.symbol.Variable(paste0("l", i, ".trans.i2h.weight")), + trans.i2h.bias = mx.symbol.Variable(paste0("l", i, ".trans.i2h.bias")), + trans.h2h.weight = mx.symbol.Variable(paste0("l", i, ".trans.h2h.weight")), + trans.h2h.bias = mx.symbol.Variable(paste0("l", i, ".trans.h2h.bias"))) + } + return(cell) + }) + + # embeding layer + label <- mx.symbol.Variable("label") + data <- mx.symbol.Variable("data") + data_mask <- mx.symbol.Variable("data.mask") + data_mask_array <- mx.symbol.Variable("data.mask.array") + data_mask_array <- mx.symbol.stop_gradient(data_mask_array, name = "data.mask.array") + + embed <- mx.symbol.Embedding(data = data, input_dim = input.size, weight = embed.weight, + output_dim = num.embed, name = "embed") + + wordvec <- mx.symbol.split(data = embed, axis = 1, num.outputs = seq.len, squeeze_axis = T) + data_mask_split <- mx.symbol.split(data = data_mask_array, axis = 1, num.outputs = seq.len, + squeeze_axis = T) + + last.hidden <- list() + last.states <- list() + decode <- list() + softmax <- list() + fc <- list() + + for (seqidx in 1:seq.len) { + hidden <- wordvec[[seqidx]] + + for (i in 1:num.rnn.layer) { + if (seqidx == 1) { + prev.state <- init.state[[i]] + } else { + prev.state <- last.states[[i]] + } + + if (cell.type == "lstm") { + cell.symbol <- lstm.cell + } else if (cell.type == "gru") { + cell.symbol <- gru.cell + } + + next.state <- cell.symbol(num.hidden = num.hidden, indata = hidden, prev.state = prev.state, + param = param.cells[[i]], seqidx = seqidx, layeridx = i, dropout = dropout, + data_masking = data_mask_split[[seqidx]]) + hidden <- next.state$h + # if (dropout > 0) hidden <- mx.symbol.Dropout(data=hidden, p=dropout) + last.states[[i]] <- next.state + } + + # Decoding + if (config == "one-to-one") { + last.hidden <- c(last.hidden, hidden) + } + } + + if (config == "seq-to-one") { + fc <- mx.symbol.FullyConnected(data = hidden, weight = cls.weight, bias = cls.bias, + num.hidden = num.label) + + loss <- mx.symbol.SoftmaxOutput(data = fc, name = "sm", label = label, ignore_label = ignore_label) + + } else if (config == "one-to-one") { + last.hidden_expand <- lapply(last.hidden, function(i) mx.symbol.expand_dims(i, + axis = 1)) + concat <- mx.symbol.concat(last.hidden_expand, num.args = seq.len, dim = 1) + reshape <- mx.symbol.Reshape(concat, shape = c(num.hidden, -1)) + + fc <- mx.symbol.FullyConnected(data = reshape, weight = cls.weight, bias = cls.bias, + num.hidden = num.label) + + label <- mx.symbol.reshape(data = label, shape = c(-1)) + loss <- mx.symbol.SoftmaxOutput(data = fc, name = "sm", label = label, ignore_label = ignore_label) + + } + + if (output_last_state) { + group <- mx.symbol.Group(c(unlist(last.states), loss)) + return(group) + } else { + return(loss) + } +} + +########################################### mx.rnn.buckets +mx.rnn.buckets <- function(train.data, eval.data = NULL, num.rnn.layer, num.hidden, + num.embed, num.label, input.size, ctx = NULL, num.round = 1, initializer = mx.init.uniform(0.01), + dropout = 0, config = "one-to-one", optimizer = "sgd", batch.end.callback = NULL, + epoch.end.callback = NULL, begin.round = 1, end.round = 1, metric = mx.metric.rmse, + cell.type = "lstm", verbose = FALSE) { + + if (!train.data$iter.next()) { + train.data$reset() + if (!train.data$iter.next()) + stop("Empty train.data") + } + + if (!is.null(eval.data)) { + if (!eval.data$iter.next()) { + eval.data$reset() + if (!eval.data$iter.next()) + stop("Empty eval.data") + } + } + + if (is.null(ctx)) + ctx <- mx.ctx.default() + if (!is.mx.context(ctx)) + stop("ctx must be mx.context") + if (is.character(optimizer)) { + if (is.numeric(input.shape)) { + ndim <- length(input.shape) + batchsize <- input.shape[[ndim]] + } else { + ndim <- length(input.shape[[1]]) + batchsize <- input.shape[[1]][[ndim]] + } + optimizer <- mx.opt.create(optimizer, rescale.grad = (1/batchsize), ...) + } + + # get unrolled lstm symbol + sym_list <- sapply(train.data$bucket.names, function(x) { + rnn.unroll(num.rnn.layer = num.rnn.layer, num.hidden = num.hidden, seq.len = as.integer(x), + input.size = input.size, num.embed = num.embed, num.label = num.label, + dropout = dropout, cell.type = cell.type, config = config) + }, simplify = F, USE.NAMES = T) + + # setup lstm model + symbol <- sym_list[[names(train.data$bucketID)]] + + arg.names <- symbol$arguments + input.shape <- lapply(train.data$value(), dim) + input.shape <- input.shape[names(input.shape) %in% arg.names] + + params <- mx.model.init.params(symbol, input.shape, NULL, initializer, mx.cpu()) + + ### Execute training - rnn.model.R + model <- mx.model.train.rnn.buckets(sym_list = sym_list, input.shape = input.shape, + arg.params = params$arg.params, aux.params = params$aux.params, optimizer = optimizer, + train.data = train.data, eval.data = eval.data, verbose = verbose, begin.round = begin.round, + end.round = end.round, metric = metric, ctx = ctx, batch.end.callback = batch.end.callback, + epoch.end.callback = epoch.end.callback) + + return(model) +} + + +# get the argument name of data and label +mx.model.check.arguments <- function(symbol) { + data <- NULL + label <- NULL + for (nm in arguments(symbol)) { + if (mx.util.str.endswith(nm, "data")) { + if (!is.null(data)) { + stop("Multiple fields contains suffix data") + } else { + data <- nm + } + } + if (mx.util.str.endswith(nm, "label")) { + if (!is.null(label)) { + stop("Multiple fields contains suffix label") + } else { + label <- nm + } + } + } + return(c(data, label)) +} + +# filter out null, keep the names +mx.util.filter.null <- function(lst) { + lst[!sapply(lst, is.null)] +} + + diff --git a/example/rnn/bucket_R/rnn.infer.R b/example/rnn/bucket_R/rnn.infer.R new file mode 100644 index 000000000000..41488aac898e --- /dev/null +++ b/example/rnn/bucket_R/rnn.infer.R @@ -0,0 +1,79 @@ +library(mxnet) + +source("rnn.R") + +mx.rnn.infer.buckets <- function(infer_iter, model, config, ctx = mx.cpu(), output_last_state = FALSE, + init.state = NULL, cell.type = "lstm") { + ### Infer parameters from model + if (cell.type == "lstm") { + num.rnn.layer <- round((length(model$arg.params) - 3)/4) + num.hidden <- dim(model$arg.params$l1.h2h.weight)[1] + } else if (cell.type == "gru") { + num.rnn.layer <- round((length(model$arg.params) - 3)/8) + num.hidden <- dim(model$arg.params$l1.gates.h2h.weight)[1] + } + + input.size <- dim(model$arg.params$embed.weight)[2] + num.embed <- dim(model$arg.params$embed.weight)[1] + num.label <- dim(model$arg.params$cls.bias) + + ### Initialise the iterator + infer_iter$reset() + infer_iter$iter.next() + batch_size <- infer_iter$batch.size + + # get unrolled lstm symbol + sym_list <- sapply(infer_iter$bucket.names, function(x) { + rnn.unroll(num.rnn.layer = num.rnn.layer, num.hidden = num.hidden, seq.len = as.integer(x), + input.size = input.size, num.embed = num.embed, num.label = num.label, + config = config, dropout = 0, init.state = init.state, cell.type = cell.type, + output_last_state = output_last_state) + }, simplify = F, USE.NAMES = T) + + symbol <- sym_list[[names(infer_iter$bucketID)]] + + input.shape <- lapply(infer_iter$value(), dim) + input.shape <- input.shape[names(input.shape) %in% arguments(symbol)] + + infer_shapes <- symbol$infer.shape(input.shape) + arg.params <- model$arg.params + aux.params <- model$aux.params + + input.names <- names(input.shape) + arg.names <- names(arg.params) + + # Grad request + grad_req <- rep("null", length(symbol$arguments)) + + # Arg array order + update_names <- c(input.names, arg.names) + arg_update_idx <- match(symbol$arguments, update_names) + + # Initial input shapes - need to be adapted for multi-devices - divide highest + # dimension by device nb + s <- sapply(input.shape, function(shape) { + mx.nd.zeros(shape = shape, ctx = mx.cpu()) + }) + + train.execs <- mxnet:::mx.symbol.bind(symbol = symbol, arg.arrays = c(s, arg.params)[arg_update_idx], + aux.arrays = aux.params, ctx = ctx, grad.req = grad_req) + + packer <- mxnet:::mx.nd.arraypacker() + infer_iter$reset() + while (infer_iter$iter.next()) { + # Get input data slice + dlist <- infer_iter$value()[input.names] + + symbol <- sym_list[[names(infer_iter$bucketID)]] + + texec <- mxnet:::mx.symbol.bind(symbol = symbol, arg.arrays = c(dlist, train.execs$arg.arrays[arg.names])[arg_update_idx], + aux.arrays = train.execs$aux.arrays, ctx = ctx, grad.req = grad_req) + + mx.exec.forward(texec, is.train = FALSE) + + out.preds <- mx.nd.copyto(texec$ref.outputs[[1]], mx.cpu()) + packer$push(out.preds) + } + infer_iter$reset() + return(packer$get()) +} diff --git a/example/rnn/bucket_R/rnn.train.R b/example/rnn/bucket_R/rnn.train.R new file mode 100644 index 000000000000..962430c1a297 --- /dev/null +++ b/example/rnn/bucket_R/rnn.train.R @@ -0,0 +1,120 @@ +library(mxnet) + +source("rnn.R") + +# Internal function to do multiple device training on RNN +mx.model.train.rnn.buckets <- function(ctx, sym_list, arg.params, aux.params, input.shape, + begin.round, end.round, optimizer, train.data, eval.data, metric, epoch.end.callback, + batch.end.callback, verbose = TRUE) { + symbol <- sym_list[[names(train.data$bucketID)]] + + input.names <- names(input.shape) + arg.names <- names(arg.params) + + # Grad request + grad_req <- rep("write", length(symbol$arguments)) + grad_null_idx <- match(input.names, symbol$arguments) + grad_req[grad_null_idx] <- "null" + + # Arg array order + update_names <- c(input.names, arg.names) + arg_update_idx <- match(symbol$arguments, update_names) + + s <- sapply(input.shape, function(shape) { + mx.nd.zeros(shape = shape, ctx = mx.cpu()) + }) + + train.exec <- mxnet:::mx.symbol.bind(symbol = symbol, arg.arrays = c(s, arg.params)[arg_update_idx], + aux.arrays = aux.params, ctx = ctx, grad.req = grad_req) + + updaters <- mx.opt.get.updater(optimizer, train.exec$ref.arg.arrays) + + for (iteration in begin.round:end.round) { + nbatch <- 0 + if (!is.null(metric)) { + train.metric <- metric$init() + } + train.data$reset() + while (train.data$iter.next()) { + dlist <- train.data$value()[input.names] + symbol <- sym_list[[names(train.data$bucketID)]] + + train.exec <- mxnet:::mx.symbol.bind(symbol = symbol, arg.arrays = c(dlist, + train.exec$arg.arrays[arg.names])[arg_update_idx], aux.arrays = train.exec$aux.arrays, + ctx = ctx, grad.req = grad_req) + + mx.exec.forward(train.exec, is.train = TRUE) + + # copy outputs to CPU + out.preds <- mx.nd.copyto(train.exec$ref.outputs[[1]], mx.cpu()) + + mx.exec.backward(train.exec) + + arg.blocks <- updaters(train.exec$ref.arg.arrays, train.exec$ref.grad.arrays) + mx.exec.update.arg.arrays(train.exec, arg.blocks, skip.null = TRUE) + + # Update the evaluation metrics + if (!is.null(metric)) { + train.metric <- metric$update(dlist$label, out.preds, train.metric) + } + + nbatch <- nbatch + 1 + + if (!is.null(batch.end.callback)) { + batch.end.callback(iteration, nbatch, environment()) + } + } + + if (!is.null(metric)) { + result <- metric$get(train.metric) + if (verbose) + message(paste0("[", iteration, "] Train-", result$name, "=", result$value)) + } + + if (!is.null(eval.data)) { + if (!is.null(metric)) { + eval.metric <- metric$init() + } + eval.data$reset() + while (eval.data$iter.next()) { + # Get input data slice + dlist <- eval.data$value()[input.names] + symbol <- sym_list[[names(eval.data$bucketID)]] + train.exec <- mxnet:::mx.symbol.bind(symbol = symbol, arg.arrays = c(dlist, + train.exec$arg.arrays[arg.names])[arg_update_idx], aux.arrays = train.exec$aux.arrays, + ctx = ctx, grad.req = grad_req) + + mx.exec.forward(train.exec, is.train = FALSE) + + # copy outputs to CPU + out.preds <- mx.nd.copyto(train.exec$ref.outputs[[1]], mx.cpu()) + + if (!is.null(metric)) { + eval.metric <- metric$update(dlist$label, out.preds, eval.metric) + } + } + + if (!is.null(metric)) { + result <- metric$get(eval.metric) + if (verbose) { + message(paste0("[", iteration, "] Validation-", result$name, "=", + result$value)) + } + } + } else { + eval.metric <- NULL + } + # get the model out + model <- mxnet:::mx.model.extract.model(symbol, list(train.exec)) + + epoch_continue <- TRUE + if (!is.null(epoch.end.callback)) { + epoch_continue <- epoch.end.callback(iteration, 0, environment(), verbose = verbose) + } + + if (!epoch_continue) { + break + } + } + return(model) +} From 14ba07ec5be7dbaab9a3743c3d4d771408b528d4 Mon Sep 17 00:00:00 2001 From: Yao Wang Date: Sat, 29 Jul 2017 11:59:35 -0700 Subject: [PATCH 272/834] Modify announcement (#7242) * Modify announcement * Remove cheat sheet and reorder --- docs/_static/mxnet-theme/index.html | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/_static/mxnet-theme/index.html b/docs/_static/mxnet-theme/index.html index fafb92ae64c3..c4f3c48fcaad 100644 --- a/docs/_static/mxnet-theme/index.html +++ b/docs/_static/mxnet-theme/index.html @@ -17,6 +17,12 @@
    +
    +

    Introducing - Gluon

    +

    We’re happy to introduce a new elegant, easy to use, imperative interface for MXNet. +

    + Learn More +

    MXNet 0.10.0 Released

    We're excited to announce the release of MXNet 0.10.0! Check out the release notes for latest updates.

    @@ -28,12 +34,6 @@

    MXNet Joining Apache

    Learn More
    -
    -

    MXNet in AWS re:Invent 2016

    -

    Learn how to use MXNet to build neural network models for recommendation systems. -

    - Watch Video -
    From cac9c25535c47c3fe076d84e5542004948ae281b Mon Sep 17 00:00:00 2001 From: Dick Carter Date: Sat, 29 Jul 2017 18:44:23 -0700 Subject: [PATCH 273/834] Quiet regression stacktrace pr (#7156) * Quieted the expected, but confusing, exception output from unittest/test_symbol.py:test_zero_prop2. * Quieted the expected, but confusing, exception output from some unittests. * Fixed pylint error. * Empty commit to retrigger testing. * Empty commit again to test CI failure repeatability. --- python/mxnet/test_utils.py | 21 +++++++++++++++++++ tests/python/unittest/test_autograd.py | 17 ++++++++------- .../python/unittest/test_contrib_autograd.py | 17 ++++++++------- tests/python/unittest/test_symbol.py | 14 ++++++++----- 4 files changed, 50 insertions(+), 19 deletions(-) diff --git a/python/mxnet/test_utils.py b/python/mxnet/test_utils.py index ae5abdd2be54..83c773f53fe2 100644 --- a/python/mxnet/test_utils.py +++ b/python/mxnet/test_utils.py @@ -7,9 +7,11 @@ import traceback import numbers import subprocess +import sys import os import errno import logging +from contextlib import contextmanager import numpy as np import numpy.testing as npt import mxnet as mx @@ -1044,3 +1046,22 @@ def same_array(array1, array2): return False array1[:] -= 1 return same(array1.asnumpy(), array2.asnumpy()) + +@contextmanager +def discard_stderr(): + """ + Discards error output of a routine if invoked as: + + with discard_stderr(): + ... + """ + + try: + stderr_fileno = sys.stderr.fileno() + old_stderr = os.dup(stderr_fileno) + bit_bucket = open(os.devnull, 'w') + os.dup2(bit_bucket.fileno(), stderr_fileno) + yield + finally: + os.dup2(old_stderr, stderr_fileno) + bit_bucket.close() diff --git a/tests/python/unittest/test_autograd.py b/tests/python/unittest/test_autograd.py index 3fa4a743cc25..8dea04da6abc 100644 --- a/tests/python/unittest/test_autograd.py +++ b/tests/python/unittest/test_autograd.py @@ -222,13 +222,16 @@ def test_retain_grad(): y.backward(retain_graph=False) assert (dx.asnumpy() == 2).all() - try: - with record(): - y = x + 1 - y.backward() - y.backward() - except Exception: - return + # The following sequence should throw an exception. We discard the expected + # stderr stack trace output for this operation to keep the test logs clean. + with discard_stderr(): + try: + with record(): + y = x + 1 + y.backward() + y.backward() + except Exception: + return raise AssertionError( "differentiating the same graph twice without retain_graph should fail") diff --git a/tests/python/unittest/test_contrib_autograd.py b/tests/python/unittest/test_contrib_autograd.py index 24b417afc233..e7b0ce3af752 100644 --- a/tests/python/unittest/test_contrib_autograd.py +++ b/tests/python/unittest/test_contrib_autograd.py @@ -150,13 +150,16 @@ def test_retain_grad(): y.backward(retain_graph=False) assert (dx.asnumpy() == 2).all() - try: - with train_section(): - y = x + 1 - y.backward() - y.backward() - except Exception: - return + # The following sequence should throw an exception. We discard the expected + # stderr stack trace output for this operation to keep the test logs clean. + with discard_stderr(): + try: + with train_section(): + y = x + 1 + y.backward() + y.backward() + except Exception: + return raise AssertionError( "differentiating the same graph twice without retain_graph should fail") diff --git a/tests/python/unittest/test_symbol.py b/tests/python/unittest/test_symbol.py index 093a8f3a40e0..ee9e9dcf3a55 100644 --- a/tests/python/unittest/test_symbol.py +++ b/tests/python/unittest/test_symbol.py @@ -4,6 +4,7 @@ import mxnet as mx import numpy as np from common import models +from mxnet.test_utils import discard_stderr import pickle as pkl def test_symbol_basic(): @@ -216,11 +217,14 @@ def test_zero_prop2(): exe.forward() exe.backward() - try: - y.simple_bind(ctx=mx.cpu(), x=(10, 10), idx=(10,), - type_dict={'x': np.float32, 'idx': np.int32}) - except: - return + # The following bind() should throw an exception. We discard the expected stderr + # output for this operation only in order to keep the test logs clean. + with discard_stderr(): + try: + y.simple_bind(ctx=mx.cpu(), x=(10, 10), idx=(10,), + type_dict={'x': np.float32, 'idx': np.int32}) + except: + return assert False From bcdde36cff9c475fd8187b2dfd27ff1b4014f60e Mon Sep 17 00:00:00 2001 From: Kai Li <1196594711@qq.com> Date: Sun, 30 Jul 2017 09:44:46 +0800 Subject: [PATCH 274/834] Update README.md (#7248) --- example/image-classification/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/image-classification/README.md b/example/image-classification/README.md index 25050f652c9f..a008b23472f6 100644 --- a/example/image-classification/README.md +++ b/example/image-classification/README.md @@ -39,7 +39,7 @@ commonly used options are listed as following: | Argument | Comments | | ----------------------------- | ---------------------------------------- | -| `network` | The network to train, which is defined in [symbol/](https://github.com/dmlc/mxnet/tree/master/example/image-classification/symbol). Some networks may accept additional arguments, such as `--num-layers` is used to specify the number of layers in ResNet. | +| `network`                     | The network to train, which is defined in [symbol/](https://github.com/dmlc/mxnet/tree/master/example/image-classification/symbols). Some networks may accept additional arguments, such as `--num-layers` is used to specify the number of layers in ResNet. | | `data-train`, `data-val` | The data for training and validation. It can be either a filename or a directory. For the latter, all files in the directory will be used. But if `--benchmark 1` is used, then there two arguments will be ignored. | | `gpus` | The list of GPUs to use, such as `0` or `0,3,4,7`. If an empty string `''` is given, then we will use CPU. | | `batch-size` | The batch size for SGD training. It specifies the number of examples used for each SGD iteration. If we use *k* GPUs, then each GPU will compute *batch_size/k* examples in each time. | From 66df7c819d51059f30f9002c479f33324e9c72a5 Mon Sep 17 00:00:00 2001 From: Shuai Yuan Date: Sun, 30 Jul 2017 01:18:36 -0500 Subject: [PATCH 275/834] Correct Python Docs about tensorboard path (#7250) The path of `tensorboard` in `MXNet` is wrong: Not `mxnet.tensorboard` but `mxnet.contrib.tensorboard`. --- python/mxnet/contrib/tensorboard.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/mxnet/contrib/tensorboard.py b/python/mxnet/contrib/tensorboard.py index 5bcc3440842c..40e3104409a5 100644 --- a/python/mxnet/contrib/tensorboard.py +++ b/python/mxnet/contrib/tensorboard.py @@ -28,8 +28,8 @@ class LogMetricsCallback(object): >>> evaluation_log = 'logs/eval' >>> # in this case, each training and evaluation metric pairs has same name, >>> # you can add a prefix to make it separate. - >>> batch_end_callbacks = [mx.tensorboard.LogMetricsCallback(training_log)] - >>> eval_end_callbacks = [mx.tensorboard.LogMetricsCallback(evaluation_log)] + >>> batch_end_callbacks = [mx.contrib.tensorboard.LogMetricsCallback(training_log)] + >>> eval_end_callbacks = [mx.contrib.tensorboard.LogMetricsCallback(evaluation_log)] >>> # run >>> model.fit(train, >>> ... From 13bcb5a92ded56063a31ea7b7990dbb9634cca35 Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Sun, 30 Jul 2017 12:03:14 -0700 Subject: [PATCH 276/834] gluon models (#7023) * refactor vision model * update per comments * add default to alexnet classes * rename parameters * update model store * fix lint --- example/gluon/image_classification.py | 152 ++++++ example/gluon/resnet.py | 398 -------------- python/mxnet/gluon/__init__.py | 2 + python/mxnet/gluon/model_zoo/__init__.py | 6 + python/mxnet/gluon/model_zoo/custom_layers.py | 63 +++ python/mxnet/gluon/model_zoo/model_store.py | 110 ++++ .../mxnet/gluon/model_zoo/vision/__init__.py | 106 ++++ .../mxnet/gluon/model_zoo/vision/alexnet.py | 67 +++ .../mxnet/gluon/model_zoo/vision/densenet.py | 175 ++++++ .../mxnet/gluon/model_zoo/vision/inception.py | 200 +++++++ python/mxnet/gluon/model_zoo/vision/resnet.py | 496 ++++++++++++++++++ .../gluon/model_zoo/vision/squeezenet.py | 142 +++++ python/mxnet/gluon/model_zoo/vision/vgg.py | 209 ++++++++ python/mxnet/gluon/nn/basic_layers.py | 4 +- python/mxnet/gluon/nn/conv_layers.py | 3 + python/mxnet/symbol.py | 22 + src/operator/batch_norm-inl.h | 4 +- src/operator/cudnn_batch_norm-inl.h | 4 +- tests/python/unittest/test_gluon_model_zoo.py | 33 ++ 19 files changed, 1792 insertions(+), 404 deletions(-) create mode 100644 example/gluon/image_classification.py delete mode 100644 example/gluon/resnet.py create mode 100644 python/mxnet/gluon/model_zoo/__init__.py create mode 100644 python/mxnet/gluon/model_zoo/custom_layers.py create mode 100644 python/mxnet/gluon/model_zoo/model_store.py create mode 100644 python/mxnet/gluon/model_zoo/vision/__init__.py create mode 100644 python/mxnet/gluon/model_zoo/vision/alexnet.py create mode 100644 python/mxnet/gluon/model_zoo/vision/densenet.py create mode 100644 python/mxnet/gluon/model_zoo/vision/inception.py create mode 100644 python/mxnet/gluon/model_zoo/vision/resnet.py create mode 100644 python/mxnet/gluon/model_zoo/vision/squeezenet.py create mode 100644 python/mxnet/gluon/model_zoo/vision/vgg.py create mode 100644 tests/python/unittest/test_gluon_model_zoo.py diff --git a/example/gluon/image_classification.py b/example/gluon/image_classification.py new file mode 100644 index 000000000000..bb1fa8da8b2a --- /dev/null +++ b/example/gluon/image_classification.py @@ -0,0 +1,152 @@ +from __future__ import division + +import argparse, time +import logging +logging.basicConfig(level=logging.INFO) + +import mxnet as mx +from mxnet import gluon +from mxnet.gluon import nn +from mxnet.gluon.model_zoo import vision as models +from mxnet import autograd as ag + +from data import * + +# CLI +parser = argparse.ArgumentParser(description='Train a model for image classification.') +parser.add_argument('--dataset', type=str, default='mnist', + help='dataset to use. options are mnist, cifar10, and dummy.') +parser.add_argument('--batch-size', type=int, default=32, + help='training batch size per device (CPU/GPU).') +parser.add_argument('--gpus', type=int, default=0, + help='number of gpus to use.') +parser.add_argument('--epochs', type=int, default=3, + help='number of training epochs.') +parser.add_argument('--lr', type=float, default=0.01, + help='learning rate. default is 0.01.') +parser.add_argument('--wd', type=float, default=0.0001, + help='weight decay rate. default is 0.0001.') +parser.add_argument('--seed', type=int, default=123, + help='random seed to use. Default=123.') +parser.add_argument('--benchmark', action='store_true', + help='whether to run benchmark.') +parser.add_argument('--mode', type=str, + help='mode in which to train the model. options are symbolic, imperative, hybrid') +parser.add_argument('--model', type=str, required=True, + help='type of model to use. see vision_model for options.') +parser.add_argument('--use_thumbnail', action='store_true', + help='use thumbnail or not in resnet. default is false.') +parser.add_argument('--batch-norm', action='store_true', + help='enable batch normalization or not in vgg. default is false.') +parser.add_argument('--use-pretrained', action='store_true', + help='enable using pretrained model from gluon.') +parser.add_argument('--log-interval', type=int, default=50, help='Number of batches to wait before logging.') +opt = parser.parse_args() + +print(opt) + +mx.random.seed(opt.seed) + +dataset_classes = {'mnist': 10, 'cifar10': 10, 'imagenet': 1000, 'dummy': 1000} + +batch_size, dataset, classes = opt.batch_size, opt.dataset, dataset_classes[opt.dataset] + +gpus = opt.gpus + +if opt.benchmark: + batch_size = 32 + dataset = 'dummy' + classes = 1000 + +batch_size *= max(1, gpus) +context = [mx.gpu(i) for i in range(gpus)] if gpus > 0 else [mx.cpu()] + +model_name = opt.model + +kwargs = {'ctx': context, 'pretrained': opt.use_pretrained, 'classes': classes} +if model_name.startswith('resnet'): + kwargs['thumbnail'] = opt.use_thumbnail +elif model_name.startswith('vgg'): + kwargs['batch_norm'] = opt.batch_norm + +net = models.get_model(opt.model, **kwargs) + +# get dataset iterators +if dataset == 'mnist': + train_data, val_data = mnist_iterator(batch_size, (1, 32, 32)) +elif dataset == 'cifar10': + train_data, val_data = cifar10_iterator(batch_size, (3, 32, 32)) +elif dataset == 'dummy': + if model_name == 'inceptionv3': + train_data, val_data = dummy_iterator(batch_size, (3, 299, 299)) + else: + train_data, val_data = dummy_iterator(batch_size, (3, 224, 224)) + +def test(ctx): + metric = mx.metric.Accuracy() + val_data.reset() + for batch in val_data: + data = gluon.utils.split_and_load(batch.data[0], ctx_list=ctx, batch_axis=0) + label = gluon.utils.split_and_load(batch.label[0], ctx_list=ctx, batch_axis=0) + outputs = [] + for x in data: + outputs.append(net(x)) + metric.update(label, outputs) + return metric.get() + + +def train(epochs, ctx): + if isinstance(ctx, mx.Context): + ctx = [ctx] + net.initialize(mx.init.Xavier(magnitude=2.24), ctx=ctx) + trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': opt.lr, 'wd': opt.wd}) + metric = mx.metric.Accuracy() + loss = gluon.loss.SoftmaxCrossEntropyLoss() + + for epoch in range(epochs): + tic = time.time() + train_data.reset() + metric.reset() + btic = time.time() + for i, batch in enumerate(train_data): + data = gluon.utils.split_and_load(batch.data[0], ctx_list=ctx, batch_axis=0) + label = gluon.utils.split_and_load(batch.label[0], ctx_list=ctx, batch_axis=0) + outputs = [] + Ls = [] + with ag.record(): + for x, y in zip(data, label): + z = net(x) + L = loss(z, y) + # store the loss and do backward after we have done forward + # on all GPUs for better speed on multiple GPUs. + Ls.append(L) + outputs.append(z) + for L in Ls: + L.backward() + trainer.step(batch.data[0].shape[0]) + metric.update(label, outputs) + if opt.log_interval and not (i+1)%opt.log_interval: + name, acc = metric.get() + logging.info('[Epoch %d Batch %d] speed: %f samples/s, training: %s=%f'%( + epoch, i, batch_size/(time.time()-btic), name, acc)) + btic = time.time() + + name, acc = metric.get() + logging.info('[Epoch %d] training: %s=%f'%(epoch, name, acc)) + logging.info('[Epoch %d] time cost: %f'%(epoch, time.time()-tic)) + name, val_acc = test(ctx) + logging.info('[Epoch %d] validation: %s=%f'%(epoch, name, val_acc)) + + net.save_params('image-classifier-%s-%d.params'%(opt.model, epochs)) + +if __name__ == '__main__': + if opt.mode == 'symbolic': + data = mx.sym.var('data') + out = net(data) + softmax = mx.sym.SoftmaxOutput(out, name='softmax') + mod = mx.mod.Module(softmax, context=[mx.gpu(i) for i in range(gpus)] if gpus > 0 else [mx.cpu()]) + mod.fit(train_data, num_epoch=opt.epochs, batch_end_callback = mx.callback.Speedometer(batch_size, 1)) + else: + if opt.mode == 'hybrid': + net.hybridize() + train(opt.epochs, context) diff --git a/example/gluon/resnet.py b/example/gluon/resnet.py deleted file mode 100644 index 44517eaf15f2..000000000000 --- a/example/gluon/resnet.py +++ /dev/null @@ -1,398 +0,0 @@ -from __future__ import division, print_function - -import argparse, time -import logging -logging.basicConfig(level=logging.INFO) - -import mxnet as mx -from mxnet import gluon -from mxnet.gluon import nn -from mxnet import autograd as ag - -from data import * - -# CLI -parser = argparse.ArgumentParser(description='Train a resnet model for image classification.') -parser.add_argument('--dataset', type=str, default='cifar10', - help='dataset to use. options are mnist, cifar10, and dummy.') -parser.add_argument('--batch-size', type=int, default=32, - help='training batch size per device (CPU/GPU).') -parser.add_argument('--resnet-version', type=int, default=1, - help='whether to use ResnetV1 or ResnetV2. default is 1.') -parser.add_argument('--resnet-layers', type=int, default=50, - help='layers of resnet to use. options are 18, 50. default is 50.') -parser.add_argument('--gpus', type=int, default=0, - help='number of gpus to use.') -parser.add_argument('--epochs', type=int, default=3, - help='number of training epochs.') -parser.add_argument('--lr', type=float, default=0.01, - help='learning Rate. default is 0.01.') -parser.add_argument('--seed', type=int, default=123, - help='random seed to use. Default=123.') -parser.add_argument('--thumbnail', action='store_true', default=False, - help='use thumbnail or not. default is false.') -parser.add_argument('--benchmark', action='store_true', default=False, - help='whether to run benchmark.') -parser.add_argument('--symbolic', action='store_true', default=False, - help='whether to train in symbolic way with module.') -parser.add_argument('--log-interval', type=int, default=100, - help='Number of batches to wait before logging.') -opt = parser.parse_args() - -print(opt) - - -# Define network - -def conv3x3(filters, stride, in_channels): - return nn.Conv2D(filters, kernel_size=3, strides=stride, padding=1, - use_bias=False, in_channels=in_channels) - -class BasicBlockV1(gluon.HybridBlock): - def __init__(self, filters, stride, downsample=False, in_channels=0, **kwargs): - super(BasicBlockV1, self).__init__(**kwargs) - with self.name_scope(): - self.conv1 = conv3x3(filters, stride, in_channels) - self.bn1 = nn.BatchNorm(in_channels=in_channels) - self.conv2 = conv3x3(filters, 1, filters) - self.bn2 = nn.BatchNorm(in_channels=filters) - if downsample: - self.conv_ds = nn.Conv2D(filters, kernel_size=1, strides=stride, use_bias=False, in_channels=in_channels) - self.bn_ds = nn.BatchNorm(in_channels=filters) - self.downsample = downsample - - def hybrid_forward(self, F, x): - residual = x - - out = self.conv1(x) - out = self.bn1(out) - out = F.Activation(x, act_type='relu') - - out = self.conv2(out) - out = self.bn2(out) - - if self.downsample: - residual = self.conv_ds(x) - residual = self.bn_ds(residual) - - out = residual + out - out = F.Activation(out, act_type='relu') - - return out - - -class BottleneckV1(gluon.HybridBlock): - def __init__(self, filters, stride, downsample=False, in_channels=0, **kwargs): - super(BottleneckV1, self).__init__(**kwargs) - with self.name_scope(): - self.conv1 = nn.Conv2D(filters//4, kernel_size=1, strides=1, in_channels=in_channels) - self.bn1 = nn.BatchNorm(in_channels=filters//4) - self.conv2 = conv3x3(filters//4, stride, filters//4) - self.bn2 = nn.BatchNorm(in_channels=filters//4) - self.conv3 = nn.Conv2D(filters, kernel_size=1, strides=1, in_channels=filters//4) - self.bn3 = nn.BatchNorm(in_channels=filters) - if downsample: - self.conv_ds = nn.Conv2D(filters, kernel_size=1, strides=stride, use_bias=False, in_channels=in_channels) - self.bn_ds = nn.BatchNorm(in_channels=filters) - self.downsample = downsample - - def hybrid_forward(self, F, x): - residual = x - - out = self.conv1(x) - out = self.bn1(out) - out = F.Activation(out, act_type='relu') - - out = self.conv2(out) - out = self.bn2(out) - out = F.Activation(out, act_type='relu') - - out = self.conv3(out) - out = self.bn3(out) - - if self.downsample: - residual = self.conv_ds(x) - residual = self.bn_ds(residual) - - out = out + residual - - out = F.Activation(out, act_type='relu') - return out - - -class ResnetV1(gluon.HybridBlock): - def __init__(self, block, classes, layers, filters, thumbnail=False, **kwargs): - super(ResnetV1, self).__init__(**kwargs) - with self.name_scope(): - assert len(layers) == len(filters) - 1 - self._thumbnail = thumbnail - if thumbnail: - self.conv0 = conv3x3(filters[0], 1, 3) - else: - self.conv0 = nn.Conv2D(filters[0], 7, 2, 3, use_bias=False, - in_channels=3) - self.bn0 = nn.BatchNorm(in_channels=filters[0]) - self.pool0 = nn.MaxPool2D(3, 2, 1) - - self.body = nn.HybridSequential() - in_channels = filters[0] - for i in range(len(layers)): - stride = 1 if i == 0 else 2 - self.body.add(self._make_layer(block, layers[i], filters[i+1], - stride, in_channels=filters[i])) - in_channels = filters[i+1] - - self.pool1 = nn.GlobalAvgPool2D() - self.dense1 = nn.Dense(classes, in_units=filters[-1]) - - def _make_layer(self, block, layers, filters, stride, in_channels=0): - layer = nn.HybridSequential() - layer.add(block(filters, stride, True, in_channels=in_channels)) - for i in range(layers-1): - layer.add(block(filters, 1, False, in_channels=filters)) - return layer - - def hybrid_forward(self, F, x): - x = self.conv0(x) - if not self._thumbnail: - x = self.bn0(x) - x = F.Activation(x, act_type='relu') - x = self.pool0(x) - - x = self.body(x) - - x = self.pool1(x) - x = x.reshape((0, -1)) - x = self.dense1(x) - - return x - - -class BasicBlockV2(gluon.HybridBlock): - def __init__(self, filters, stride, downsample=False, in_channels=0, **kwargs): - super(BasicBlockV2, self).__init__(**kwargs) - with self.name_scope(): - self.bn1 = nn.BatchNorm(in_channels=in_channels) - self.conv1 = conv3x3(filters, stride, in_channels) - self.bn2 = nn.BatchNorm(in_channels=filters) - self.conv2 = conv3x3(filters, 1, filters) - if downsample: - self.downsample = nn.Conv2D(filters, 1, stride, use_bias=False, - in_channels=in_channels) - else: - self.downsample = None - - def hybrid_forward(self, F, x): - if not self.downsample: - residual = x - x = self.bn1(x) - x = F.Activation(x, act_type='relu') - if self.downsample: - residual = self.downsample(x) - x = self.conv1(x) - - x = self.bn2(x) - x = F.Activation(x, act_type='relu') - x = self.conv2(x) - - return x + residual - - -class BottleneckV2(gluon.HybridBlock): - def __init__(self, filters, stride, downsample=False, in_channels=0, **kwargs): - super(BottleneckV2, self).__init__(**kwargs) - with self.name_scope(): - self.bn1 = nn.BatchNorm(in_channels=in_channels) - self.conv1 = conv3x3(filters//4, 1, in_channels) - self.bn2 = nn.BatchNorm(in_channels=filters//4) - self.conv2 = conv3x3(filters//4, stride, filters//4) - self.bn3 = nn.BatchNorm(in_channels=filters//4) - self.conv3 = conv3x3(filters, 1, filters//4) - if downsample: - self.downsample = nn.Conv2D(filters, 1, stride, use_bias=False, - in_channels=in_channels) - else: - self.downsample = None - - def hybrid_forward(self, F, x): - if not self.downsample: - residual = x - x = self.bn1(x) - x = F.Activation(x, act_type='relu') - if self.downsample: - residual = self.downsample(x) - x = self.conv1(x) - - x = self.bn2(x) - x = F.Activation(x, act_type='relu') - x = self.conv2(x) - - x = self.bn3(x) - x = F.Activation(x, act_type='relu') - x = self.conv3(x) - - return x + residual - -class ResnetV2(gluon.HybridBlock): - def __init__(self, block, classes, layers, filters, thumbnail=False, **kwargs): - super(ResnetV2, self).__init__(**kwargs) - with self.name_scope(): - assert len(layers) == len(filters) - 1 - self._thumbnail = thumbnail - self.bn_data = nn.BatchNorm(in_channels=3, scale=False, center=False) - if thumbnail: - self.conv0 = conv3x3(filters[0], 1, 3) - else: - self.conv0 = nn.Conv2D(filters[0], 7, 2, 3, use_bias=False, - in_channels=3) - self.bn0 = nn.BatchNorm(in_channels=filters[0]) - self.pool0 = nn.MaxPool2D(3, 2, 1) - - self.body = nn.HybridSequential() - in_channels = filters[0] - for i in range(len(layers)): - stride = 1 if i == 0 else 2 - self.body.add(self._make_layer(block, layers[i], filters[i+1], - stride, in_channels=in_channels)) - in_channels = filters[i+1] - - self.bn1 = nn.BatchNorm(in_channels=in_channels) - self.pool1 = nn.GlobalAvgPool2D() - self.dense1 = nn.Dense(classes, in_units=in_channels) - - def _make_layer(self, block, layers, filters, stride, in_channels=0): - layer = nn.HybridSequential() - layer.add(block(filters, stride, True, in_channels=in_channels)) - for i in range(layers-1): - layer.add(block(filters, 1, False, in_channels=filters)) - return layer - - def hybrid_forward(self, F, x): - x = self.bn_data(x) - x = self.conv0(x) - if not self._thumbnail: - x = self.bn0(x) - x = F.Activation(x, act_type='relu') - x = self.pool0(x) - - x = self.body(x) - - x = self.bn1(x) - x = F.Activation(x, act_type='relu') - x = self.pool1(x) - x = x.reshape((0, -1)) - x = self.dense1(x) - - return x - -# construct net -resnet_spec = { 18: ('basic_block', [2, 2, 2], [16, 16, 32, 64]), - 34: ('basic_block', [3, 4, 6, 3], [16, 16, 32, 64]), - 50: ('bottle_neck', [3, 4, 6, 3], [64, 256, 512, 1024, 2048]), - 101: ('bottle_neck', [3, 4, 23, 3], [64, 256, 512, 1024, 2048]), - 152: ('bottle_neck', [3, 8, 36, 3], [64, 256, 512, 1024, 2048]) } - -resnet_net_versions = [ResnetV1, ResnetV2] -resnet_block_versions = [{'basic_block': BasicBlockV1, 'bottle_neck': BottleneckV1}, - {'basic_block': BasicBlockV2, 'bottle_neck': BottleneckV2}] - -def get_resnet(version, num_layers, classes, use_thumbnail): - block_type, layers, filters = resnet_spec[num_layers] - resnet = resnet_net_versions[version] - block = resnet_block_versions[version][block_type] - return resnet(block, classes, layers, filters, use_thumbnail) - -dataset_classes = {'mnist': 10, 'cifar10': 10, 'imagenet': 1000, 'dummy': 1000} - -batch_size, dataset, classes = opt.batch_size, opt.dataset, dataset_classes[opt.dataset] - -gpus, version = opt.gpus, opt.resnet_version-1 - -if opt.benchmark: - batch_size = 32 - dataset = 'dummy' - classes = 1000 - version = 0 - - -net = get_resnet(version, opt.resnet_layers, classes, opt.thumbnail) - -batch_size *= max(1, gpus) - -# get dataset iterators -if dataset == 'mnist': - train_data, val_data = mnist_iterator(batch_size, (1, 32, 32)) -elif dataset == 'cifar10': - train_data, val_data = cifar10_iterator(batch_size, (3, 32, 32)) -elif dataset == 'dummy': - train_data, val_data = dummy_iterator(batch_size, (3, 224, 224)) - -def test(ctx): - metric = mx.metric.Accuracy() - val_data.reset() - for batch in val_data: - data = gluon.utils.split_and_load(batch.data[0], ctx_list=ctx, batch_axis=0) - label = gluon.utils.split_and_load(batch.label[0], ctx_list=ctx, batch_axis=0) - outputs = [] - for x in data: - outputs.append(net(x)) - metric.update(label, outputs) - return metric.get() - - -def train(epoch, ctx): - if isinstance(ctx, mx.Context): - ctx = [ctx] - net.collect_params().initialize(mx.init.Xavier(magnitude=2.24), ctx=ctx) - trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': 0.1}) - metric = mx.metric.Accuracy() - loss = gluon.loss.SoftmaxCrossEntropyLoss() - - for epoch in range(epoch): - tic = time.time() - train_data.reset() - metric.reset() - btic = time.time() - for i, batch in enumerate(train_data): - data = gluon.utils.split_and_load(batch.data[0], ctx_list=ctx, batch_axis=0) - label = gluon.utils.split_and_load(batch.label[0], ctx_list=ctx, batch_axis=0) - outputs = [] - Ls = [] - with ag.record(): - for x, y in zip(data, label): - z = net(x) - L = loss(z, y) - # store the loss and do backward after we have done forward - # on all GPUs for better speed on multiple GPUs. - Ls.append(L) - outputs.append(z) - for L in Ls: - L.backward() - trainer.step(batch.data[0].shape[0]) - metric.update(label, outputs) - if opt.log_interval: - name, acc = metric.get() - print('[Epoch %d Batch %d] speed: %f samples/s, training: %s=%f'%( - epoch, i, batch_size/(time.time()-btic), name, acc)) - btic = time.time() - - name, acc = metric.get() - print('[Epoch %d] training: %s=%f'%(epoch, name, acc)) - print('[Epoch %d] time cost: %f'%(epoch, time.time()-tic)) - - name, val_acc = test(ctx) - print('[Epoch %d] validation: %s=%f'%(epoch, name, val_acc)) - - net.collect_params().save('resnet.params') - - -if __name__ == '__main__': - if opt.symbolic: - data = mx.sym.var('data') - out = net(data) - softmax = mx.sym.SoftmaxOutput(out, name='softmax') - mod = mx.mod.Module(softmax, context=[mx.gpu(i) for i in range(gpus)] if gpus > 0 else [mx.cpu()]) - mod.fit(train_data, num_epoch=opt.epochs, batch_end_callback = mx.callback.Speedometer(batch_size, 1)) - else: - net.hybridize() - train(opt.epochs, [mx.gpu(i) for i in range(gpus)] if gpus > 0 else [mx.cpu()]) diff --git a/python/mxnet/gluon/__init__.py b/python/mxnet/gluon/__init__.py index c559e7af343b..e3c341e97f02 100644 --- a/python/mxnet/gluon/__init__.py +++ b/python/mxnet/gluon/__init__.py @@ -17,3 +17,5 @@ from . import utils from . import data + +from . import model_zoo diff --git a/python/mxnet/gluon/model_zoo/__init__.py b/python/mxnet/gluon/model_zoo/__init__.py new file mode 100644 index 000000000000..aa5f148513bd --- /dev/null +++ b/python/mxnet/gluon/model_zoo/__init__.py @@ -0,0 +1,6 @@ +# coding: utf-8 +"""Predefined and pretrained models.""" + +from . import model_store + +from . import vision diff --git a/python/mxnet/gluon/model_zoo/custom_layers.py b/python/mxnet/gluon/model_zoo/custom_layers.py new file mode 100644 index 000000000000..871cfb061e23 --- /dev/null +++ b/python/mxnet/gluon/model_zoo/custom_layers.py @@ -0,0 +1,63 @@ +# coding: utf-8 +# pylint: disable= arguments-differ +"""Custom neural network layers in model_zoo.""" + +from ..block import Block, HybridBlock + +class HybridConcurrent(HybridBlock): + """Lays `HybridBlock`s concurrently. + + Example:: + + net = HybridConcurrent() + # use net's name_scope to give child Blocks appropriate names. + with net.name_scope(): + net.add(nn.Dense(10, activation='relu')) + net.add(nn.Dense(20)) + net.add(Identity()) + """ + def __init__(self, concat_dim, prefix=None, params=None): + super(HybridConcurrent, self).__init__(prefix=prefix, params=params) + self.concat_dim = concat_dim + + def add(self, block): + """Adds block on top of the stack.""" + self.register_child(block) + + def hybrid_forward(self, F, x): + out = [] + for block in self._children: + out.append(block(x)) + out = F.concat(*out, dim=self.concat_dim) + return out + + def __repr__(self): + s = '{name}(\n{modstr}\n)' + modstr = '\n'.join([' ({key}): {block}'.format(key=key, + block=_indent(block.__repr__(), 2)) + for key, block in enumerate(self._children) + if isinstance(block, Block)]) + return s.format(name=self.__class__.__name__, + modstr=modstr) + + +class Identity(HybridBlock): + """Block that passes through the input directly. + + This layer is often used in conjunction with HybridConcurrent + block for residual connection. + + Example:: + + net = HybridConcurrent() + # use net's name_scope to give child Blocks appropriate names. + with net.name_scope(): + net.add(nn.Dense(10, activation='relu')) + net.add(nn.Dense(20)) + net.add(Identity()) + """ + def __init__(self, prefix=None, params=None): + super(Identity, self).__init__(prefix=prefix, params=params) + + def hybrid_forward(self, F, x): + return x diff --git a/python/mxnet/gluon/model_zoo/model_store.py b/python/mxnet/gluon/model_zoo/model_store.py new file mode 100644 index 000000000000..6a11626795f9 --- /dev/null +++ b/python/mxnet/gluon/model_zoo/model_store.py @@ -0,0 +1,110 @@ +# coding: utf-8 +"""Model zoo for pre-trained models.""" +from __future__ import print_function +__all__ = ['get_model_file', 'purge'] +import hashlib +import os +import zipfile + +from ...test_utils import download + +_model_sha1 = {name: checksum for checksum, name in [ + ('44335d1f0046b328243b32a26a4fbd62d9057b45', 'alexnet'), + ('f27dbf2dbd5ce9a80b102d89c7483342cd33cb31', 'densenet121'), + ('b6c8a95717e3e761bd88d145f4d0a214aaa515dc', 'densenet161'), + ('2603f878403c6aa5a71a124c4a3307143d6820e9', 'densenet169'), + ('1cdbc116bc3a1b65832b18cf53e1cb8e7da017eb', 'densenet201'), + ('ed47ec45a937b656fcc94dabde85495bbef5ba1f', 'inceptionv3'), + ('d2b128fa89477c2e20061607a53a8d9f66ce239d', 'resnet101_v1'), + ('6562166cd597a6328a32a0ce47bb651df80b3bbb', 'resnet152_v1'), + ('38d6d423c22828718ec3397924b8e116a03e6ac0', 'resnet18_v1'), + ('4dc2c2390a7c7990e0ca1e53aeebb1d1a08592d1', 'resnet34_v1'), + ('2a903ab21260c85673a78fe65037819a843a1f43', 'resnet50_v1'), + ('264ba4970a0cc87a4f15c96e25246a1307caf523', 'squeezenet1.0'), + ('33ba0f93753c83d86e1eb397f38a667eaf2e9376', 'squeezenet1.1'), + ('dd221b160977f36a53f464cb54648d227c707a05', 'vgg11'), + ('ee79a8098a91fbe05b7a973fed2017a6117723a8', 'vgg11_bn'), + ('6bc5de58a05a5e2e7f493e2d75a580d83efde38c', 'vgg13'), + ('7d97a06c3c7a1aecc88b6e7385c2b373a249e95e', 'vgg13_bn'), + ('649467530119c0f78c4859999e264e7bf14471a9', 'vgg16'), + ('6b9dbe6194e5bfed30fd7a7c9a71f7e5a276cb14', 'vgg16_bn'), + ('f713436691eee9a20d70a145ce0d53ed24bf7399', 'vgg19'), + ('9730961c9cea43fd7eeefb00d792e386c45847d6', 'vgg19_bn')]} + +_url_format = 'https://{bucket}.s3.amazonaws.com/gluon/models/{file_name}.zip' +bucket = 'apache-mxnet' + +def short_hash(name): + if name not in _model_sha1: + raise ValueError('Pretrained model for {name} is not available.'.format(name=name)) + return _model_sha1[name][:8] + +def verified(file_path, name): + sha1 = hashlib.sha1() + with open(file_path, 'rb') as f: + while True: + data = f.read(1048576) + if not data: + break + sha1.update(data) + + return sha1.hexdigest() == _model_sha1[name] + +def get_model_file(name, local_dir=os.path.expanduser('~/.mxnet/models/')): + r"""Return location for the pretrained on local file system. + + This function will download from online model zoo when model cannot be found or has mismatch. + + Parameters + ---------- + name : str + Name of the model. + local_dir : str, default '~/.mxnet/models' + Location for keeping the model parameters. + + Returns + ------- + file_path + Path to the requested pretrained model file. + """ + file_name = '{name}-{short_hash}'.format(name=name, + short_hash=short_hash(name)) + file_path = os.path.join(local_dir, file_name+'.params') + if os.path.exists(file_path): + if verified(file_path, name): + return file_path + else: + print('Mismatch in the content of model file detected. Downloading again.') + else: + print('Model file is not found. Downloading.') + + if not os.path.exists(local_dir): + os.makedirs(local_dir) + + download(_url_format.format(bucket=bucket, + file_name=file_name), + fname=file_name+'.zip', + dirname=local_dir, + overwrite=True) + zip_file_path = os.path.join(local_dir, file_name+'.zip') + with zipfile.ZipFile(zip_file_path) as zf: + zf.extractall(local_dir) + os.remove(zip_file_path) + + if verified(file_path, name): + return file_path + else: + raise ValueError('Downloaded file has different hash. Please try again.') + +def purge(local_dir=os.path.expanduser('~/.mxnet/models/')): + r"""Purge all pretrained model files in local file store. + + Parameters + ---------- + local_dir : str, default '~/.mxnet/models' + Location for keeping the model parameters. + """ + files = os.listdir(local_dir) + for f in files: + if f.endswith(".params"): + os.remove(os.path.join(local_dir, f)) diff --git a/python/mxnet/gluon/model_zoo/vision/__init__.py b/python/mxnet/gluon/model_zoo/vision/__init__.py new file mode 100644 index 000000000000..e0498dcc6bca --- /dev/null +++ b/python/mxnet/gluon/model_zoo/vision/__init__.py @@ -0,0 +1,106 @@ +# coding: utf-8 +# pylint: disable=wildcard-import, arguments-differ +r"""Module for pre-defined neural network models. +This module contains definitions for the following model architectures: +- `AlexNet`_ +- `DenseNet`_ +- `Inception V3`_ +- `ResNet V1`_ +- `ResNet V2`_ +- `SqueezeNet`_ +- `VGG`_ +You can construct a model with random weights by calling its constructor: +.. code:: python + import mxnet.gluon.models as models + resnet18 = models.resnet18_v1() + alexnet = models.alexnet() + squeezenet = models.squeezenet1_0() + densenet = models.densenet_161() +We provide pre-trained models for all the models except ResNet V2. +These can constructed by passing +``pretrained=True``: +.. code:: python + import mxnet.gluon.models as models + resnet18 = models.resnet18_v1(pretrained=True) + alexnet = models.alexnet(pretrained=True) +Pretrained model is converted from torchvision. +All pre-trained models expect input images normalized in the same way, +i.e. mini-batches of 3-channel RGB images of shape (N x 3 x H x W), +where N is the batch size, and H and W are expected to be at least 224. +The images have to be loaded in to a range of [0, 1] and then normalized +using ``mean = [0.485, 0.456, 0.406]`` and ``std = [0.229, 0.224, 0.225]``. +The transformation should preferrably happen at preprocessing. You can use +``mx.image.color_normalize`` for such transformation:: + image = image/255 + normalized = mx.image.color_normalize(image, + mean=mx.nd.array([0.485, 0.456, 0.406]), + std=mx.nd.array([0.229, 0.224, 0.225])) + +.. _AlexNet: https://arxiv.org/abs/1404.5997 +.. _DenseNet: https://arxiv.org/abs/1608.06993 +.. _Inception V3: http://arxiv.org/abs/1512.00567 +.. _ResNet V1: https://arxiv.org/abs/1512.03385 +.. _ResNet V2: https://arxiv.org/abs/1512.03385 +.. _SqueezeNet: https://arxiv.org/abs/1602.07360 +.. _VGG: https://arxiv.org/abs/1409.1556 +""" + +from .alexnet import * + +from .densenet import * + +from .inception import * + +from .resnet import * + +from .squeezenet import * + +from .vgg import * + +def get_model(name, **kwargs): + """Returns a pre-defined model by name + + Parameters + ---------- + name : str + Name of the model. + pretrained : bool + Whether to load the pretrained weights for model. + classes : int + Number of classes for the output layer. + + Returns + ------- + HybridBlock + The model. + """ + models = {'resnet18_v1': resnet18_v1, + 'resnet34_v1': resnet34_v1, + 'resnet50_v1': resnet50_v1, + 'resnet101_v1': resnet101_v1, + 'resnet152_v1': resnet152_v1, + 'resnet18_v2': resnet18_v2, + 'resnet34_v2': resnet34_v2, + 'resnet50_v2': resnet50_v2, + 'resnet101_v2': resnet101_v2, + 'resnet152_v2': resnet152_v2, + 'vgg11': vgg11, + 'vgg13': vgg13, + 'vgg16': vgg16, + 'vgg19': vgg19, + 'vgg11_bn': vgg11_bn, + 'vgg13_bn': vgg13_bn, + 'vgg16_bn': vgg16_bn, + 'vgg19_bn': vgg19_bn, + 'alexnet': alexnet, + 'densenet121': densenet121, + 'densenet161': densenet161, + 'densenet169': densenet169, + 'densenet201': densenet201, + 'squeezenet1.0': squeezenet1_0, + 'squeezenet1.1': squeezenet1_1, + 'inceptionv3': inception_v3, + } + name = name.lower() + assert name in models, 'Model %s is not supported'%name + return models[name](**kwargs) diff --git a/python/mxnet/gluon/model_zoo/vision/alexnet.py b/python/mxnet/gluon/model_zoo/vision/alexnet.py new file mode 100644 index 000000000000..dd5104d062f2 --- /dev/null +++ b/python/mxnet/gluon/model_zoo/vision/alexnet.py @@ -0,0 +1,67 @@ +# coding: utf-8 +# pylint: disable= arguments-differ +"""Alexnet, implemented in Gluon.""" +__all__ = ['AlexNet', 'alexnet'] + +from ....context import cpu +from ...block import HybridBlock +from ... import nn + +# Net +class AlexNet(HybridBlock): + r"""AlexNet model from the `"One weird trick..." `_ paper. + + Parameters + ---------- + classes : int, default 1000 + Number of classes for the output layer. + """ + def __init__(self, classes=1000, **kwargs): + super(AlexNet, self).__init__(**kwargs) + with self.name_scope(): + self.features = nn.HybridSequential(prefix='') + with self.features.name_scope(): + self.features.add(nn.Conv2D(64, kernel_size=11, strides=4, padding=2)) + self.features.add(nn.Activation('relu')) + self.features.add(nn.MaxPool2D(pool_size=3, strides=2)) + self.features.add(nn.Conv2D(192, kernel_size=5, padding=2)) + self.features.add(nn.Activation('relu')) + self.features.add(nn.MaxPool2D(pool_size=3, strides=2)) + self.features.add(nn.Conv2D(384, kernel_size=3, padding=1)) + self.features.add(nn.Activation('relu')) + self.features.add(nn.Conv2D(256, kernel_size=3, padding=1)) + self.features.add(nn.Activation('relu')) + self.features.add(nn.Conv2D(256, kernel_size=3, padding=1)) + self.features.add(nn.Activation('relu')) + self.features.add(nn.MaxPool2D(pool_size=3, strides=2)) + self.features.add(nn.Flatten()) + + self.classifier = nn.HybridSequential(prefix='') + with self.classifier.name_scope(): + self.classifier.add(nn.Dropout(0.5)) + self.classifier.add(nn.Dense(4096, activation='relu')) + self.classifier.add(nn.Dropout(0.5)) + self.classifier.add(nn.Dense(4096, activation='relu')) + self.classifier.add(nn.Dense(classes)) + + def hybrid_forward(self, F, x): + x = self.features(x) + x = self.classifier(x) + return x + +# Constructor +def alexnet(pretrained=False, ctx=cpu(), **kwargs): + r"""AlexNet model from the `"One weird trick..." `_ paper. + + Parameters + ---------- + pretrained : bool, default False + Whether to load the pretrained weights for model. + ctx : Context, default CPU + The context in which to load the pretrained weights. + """ + net = AlexNet(**kwargs) + if pretrained: + from ..model_store import get_model_file + net.load_params(get_model_file('alexnet'), ctx=ctx) + return net diff --git a/python/mxnet/gluon/model_zoo/vision/densenet.py b/python/mxnet/gluon/model_zoo/vision/densenet.py new file mode 100644 index 000000000000..9e1ff7799892 --- /dev/null +++ b/python/mxnet/gluon/model_zoo/vision/densenet.py @@ -0,0 +1,175 @@ +# coding: utf-8 +# pylint: disable= arguments-differ +"""DenseNet, implemented in Gluon.""" +__all__ = ['DenseNet', 'densenet121', 'densenet161', 'densenet169', 'densenet201'] + +from ....context import cpu +from ...block import HybridBlock +from ... import nn +from ..custom_layers import HybridConcurrent, Identity + +# Helpers +def _make_dense_block(num_layers, bn_size, growth_rate, dropout, stage_index): + out = nn.HybridSequential(prefix='stage%d_'%stage_index) + with out.name_scope(): + for _ in range(num_layers): + out.add(_make_dense_layer(growth_rate, bn_size, dropout)) + return out + +def _make_dense_layer(growth_rate, bn_size, dropout): + new_features = nn.HybridSequential(prefix='') + new_features.add(nn.BatchNorm()) + new_features.add(nn.Activation('relu')) + new_features.add(nn.Conv2D(bn_size * growth_rate, kernel_size=1, use_bias=False)) + new_features.add(nn.BatchNorm()) + new_features.add(nn.Activation('relu')) + new_features.add(nn.Conv2D(growth_rate, kernel_size=3, padding=1, use_bias=False)) + if dropout: + new_features.add(nn.Dropout(dropout)) + + out = HybridConcurrent(concat_dim=1, prefix='') + out.add(Identity()) + out.add(new_features) + + return out + +def _make_transition(num_output_features): + out = nn.HybridSequential(prefix='') + out.add(nn.BatchNorm()) + out.add(nn.Activation('relu')) + out.add(nn.Conv2D(num_output_features, kernel_size=1, use_bias=False)) + out.add(nn.AvgPool2D(pool_size=2, strides=2)) + return out + +# Net +class DenseNet(HybridBlock): + r"""Densenet-BC model from the + `"Densely Connected Convolutional Networks" `_ paper. + + Parameters + ---------- + num_init_features : int + Number of filters to learn in the first convolution layer. + growth_rate : int + Number of filters to add each layer (`k` in the paper). + block_config : list of int + List of integers for numbers of layers in each pooling block. + bn_size : int, default 4 + Multiplicative factor for number of bottle neck layers. + (i.e. bn_size * k features in the bottleneck layer) + dropout : float, default 0 + Rate of dropout after each dense layer. + classes : int, default 1000 + Number of classification classes. + """ + def __init__(self, num_init_features, growth_rate, block_config, + bn_size=4, dropout=0, classes=1000, **kwargs): + + super(DenseNet, self).__init__(**kwargs) + with self.name_scope(): + self.features = nn.HybridSequential(prefix='') + self.features.add(nn.Conv2D(num_init_features, kernel_size=7, + strides=2, padding=3, use_bias=False)) + self.features.add(nn.BatchNorm()) + self.features.add(nn.Activation('relu')) + self.features.add(nn.MaxPool2D(pool_size=3, strides=2, padding=1)) + # Add dense blocks + num_features = num_init_features + for i, num_layers in enumerate(block_config): + self.features.add(_make_dense_block(num_layers, bn_size, growth_rate, dropout, i+1)) + num_features = num_features + num_layers * growth_rate + if i != len(block_config) - 1: + self.features.add(_make_transition(num_features // 2)) + num_features = num_features // 2 + self.features.add(nn.BatchNorm()) + self.features.add(nn.Activation('relu')) + self.features.add(nn.AvgPool2D(pool_size=7)) + self.features.add(nn.Flatten()) + + self.classifier = nn.Dense(classes) + + def hybrid_forward(self, F, x): + x = self.features(x) + x = self.classifier(x) + return x + + +# Specification +densenet_spec = {121: (64, 32, [6, 12, 24, 16]), + 161: (96, 48, [6, 12, 36, 24]), + 169: (64, 32, [6, 12, 32, 32]), + 201: (64, 32, [6, 12, 48, 32])} + + +# Constructor +def get_densenet(num_layers, pretrained=False, ctx=cpu(), **kwargs): + r"""Densenet-BC model from the + `"Densely Connected Convolutional Networks" `_ paper. + + Parameters + ---------- + num_layers : int + Number of layers for the variant of densenet. Options are 121, 161, 169, 201. + pretrained : bool, default False + Whether to load the pretrained weights for model. + ctx : Context, default CPU + The context in which to load the pretrained weights. + """ + num_init_features, growth_rate, block_config = densenet_spec[num_layers] + net = DenseNet(num_init_features, growth_rate, block_config, **kwargs) + if pretrained: + from ..model_store import get_model_file + net.load_params(get_model_file('densenet%d'%(num_layers)), ctx=ctx) + return net + +def densenet121(**kwargs): + r"""Densenet-BC 121-layer model from the + `"Densely Connected Convolutional Networks" `_ paper. + + Parameters + ---------- + pretrained : bool, default False + Whether to load the pretrained weights for model. + ctx : Context, default CPU + The context in which to load the pretrained weights. + """ + return get_densenet(121, **kwargs) + +def densenet161(**kwargs): + r"""Densenet-BC 161-layer model from the + `"Densely Connected Convolutional Networks" `_ paper. + + Parameters + ---------- + pretrained : bool, default False + Whether to load the pretrained weights for model. + ctx : Context, default CPU + The context in which to load the pretrained weights. + """ + return get_densenet(161, **kwargs) + +def densenet169(**kwargs): + r"""Densenet-BC 169-layer model from the + `"Densely Connected Convolutional Networks" `_ paper. + + Parameters + ---------- + pretrained : bool, default False + Whether to load the pretrained weights for model. + ctx : Context, default CPU + The context in which to load the pretrained weights. + """ + return get_densenet(169, **kwargs) + +def densenet201(**kwargs): + r"""Densenet-BC 201-layer model from the + `"Densely Connected Convolutional Networks" `_ paper. + + Parameters + ---------- + pretrained : bool, default False + Whether to load the pretrained weights for model. + ctx : Context, default CPU + The context in which to load the pretrained weights. + """ + return get_densenet(201, **kwargs) diff --git a/python/mxnet/gluon/model_zoo/vision/inception.py b/python/mxnet/gluon/model_zoo/vision/inception.py new file mode 100644 index 000000000000..8a28666d56cb --- /dev/null +++ b/python/mxnet/gluon/model_zoo/vision/inception.py @@ -0,0 +1,200 @@ +# coding: utf-8 +# pylint: disable= arguments-differ +"""Inception, implemented in Gluon.""" +__all__ = ['Inception3', 'inception_v3'] + +from ....context import cpu +from ...block import HybridBlock +from ... import nn +from ..custom_layers import HybridConcurrent + +# Helpers +def _make_basic_conv(**kwargs): + out = nn.HybridSequential(prefix='') + out.add(nn.Conv2D(use_bias=False, **kwargs)) + out.add(nn.BatchNorm(epsilon=0.001)) + out.add(nn.Activation('relu')) + return out + +def _make_branch(use_pool, *conv_settings): + out = nn.HybridSequential(prefix='') + if use_pool == 'avg': + out.add(nn.AvgPool2D(pool_size=3, strides=1, padding=1)) + elif use_pool == 'max': + out.add(nn.MaxPool2D(pool_size=3, strides=2)) + setting_names = ['channels', 'kernel_size', 'strides', 'padding'] + for setting in conv_settings: + kwargs = {} + for i, value in enumerate(setting): + if value is not None: + kwargs[setting_names[i]] = value + out.add(_make_basic_conv(**kwargs)) + return out + +def _make_A(pool_features, prefix): + out = HybridConcurrent(concat_dim=1, prefix=prefix) + with out.name_scope(): + out.add(_make_branch(None, + (64, 1, None, None))) + out.add(_make_branch(None, + (48, 1, None, None), + (64, 5, None, 2))) + out.add(_make_branch(None, + (64, 1, None, None), + (96, 3, None, 1), + (96, 3, None, 1))) + out.add(_make_branch('avg', + (pool_features, 1, None, None))) + return out + +def _make_B(prefix): + out = HybridConcurrent(concat_dim=1, prefix=prefix) + with out.name_scope(): + out.add(_make_branch(None, + (384, 3, 2, None))) + out.add(_make_branch(None, + (64, 1, None, None), + (96, 3, None, 1), + (96, 3, 2, None))) + out.add(_make_branch('max')) + return out + +def _make_C(channels_7x7, prefix): + out = HybridConcurrent(concat_dim=1, prefix=prefix) + with out.name_scope(): + out.add(_make_branch(None, + (192, 1, None, None))) + out.add(_make_branch(None, + (channels_7x7, 1, None, None), + (channels_7x7, (1, 7), None, (0, 3)), + (192, (7, 1), None, (3, 0)))) + out.add(_make_branch(None, + (channels_7x7, 1, None, None), + (channels_7x7, (7, 1), None, (3, 0)), + (channels_7x7, (1, 7), None, (0, 3)), + (channels_7x7, (7, 1), None, (3, 0)), + (192, (1, 7), None, (0, 3)))) + out.add(_make_branch('avg', + (192, 1, None, None))) + return out + +def _make_D(prefix): + out = HybridConcurrent(concat_dim=1, prefix=prefix) + with out.name_scope(): + out.add(_make_branch(None, + (192, 1, None, None), + (320, 3, 2, None))) + out.add(_make_branch(None, + (192, 1, None, None), + (192, (1, 7), None, (0, 3)), + (192, (7, 1), None, (3, 0)), + (192, 3, 2, None))) + out.add(_make_branch('max')) + return out + +def _make_E(prefix): + out = HybridConcurrent(concat_dim=1, prefix=prefix) + with out.name_scope(): + out.add(_make_branch(None, + (320, 1, None, None))) + + branch_3x3 = nn.HybridSequential(prefix='') + out.add(branch_3x3) + branch_3x3.add(_make_branch(None, + (384, 1, None, None))) + branch_3x3_split = HybridConcurrent(concat_dim=1, prefix='') + branch_3x3_split.add(_make_branch(None, + (384, (1, 3), None, (0, 1)))) + branch_3x3_split.add(_make_branch(None, + (384, (3, 1), None, (1, 0)))) + branch_3x3.add(branch_3x3_split) + + branch_3x3dbl = nn.HybridSequential(prefix='') + out.add(branch_3x3dbl) + branch_3x3dbl.add(_make_branch(None, + (448, 1, None, None), + (384, 3, None, 1))) + branch_3x3dbl_split = HybridConcurrent(concat_dim=1, prefix='') + branch_3x3dbl.add(branch_3x3dbl_split) + branch_3x3dbl_split.add(_make_branch(None, + (384, (1, 3), None, (0, 1)))) + branch_3x3dbl_split.add(_make_branch(None, + (384, (3, 1), None, (1, 0)))) + + out.add(_make_branch('avg', + (192, 1, None, None))) + return out + +def make_aux(classes): + out = nn.HybridSequential(prefix='') + out.add(nn.AvgPool2D(pool_size=5, strides=3)) + out.add(_make_basic_conv(channels=128, kernel_size=1)) + out.add(_make_basic_conv(channels=768, kernel_size=5)) + out.add(nn.Flatten()) + out.add(nn.Dense(classes)) + return out + +# Net +class Inception3(HybridBlock): + r"""Inception v3 model from + `"Rethinking the Inception Architecture for Computer Vision" + `_ paper. + + Parameters + ---------- + classes : int, default 1000 + Number of classification classes. + """ + def __init__(self, classes=1000, **kwargs): + super(Inception3, self).__init__(**kwargs) + # self.use_aux_logits = use_aux_logits + with self.name_scope(): + self.features = nn.HybridSequential(prefix='') + self.features.add(_make_basic_conv(channels=32, kernel_size=3, strides=2)) + self.features.add(_make_basic_conv(channels=32, kernel_size=3)) + self.features.add(_make_basic_conv(channels=64, kernel_size=3, padding=1)) + self.features.add(nn.MaxPool2D(pool_size=3, strides=2)) + self.features.add(_make_basic_conv(channels=80, kernel_size=1)) + self.features.add(_make_basic_conv(channels=192, kernel_size=3)) + self.features.add(nn.MaxPool2D(pool_size=3, strides=2)) + self.features.add(_make_A(32, 'A1_')) + self.features.add(_make_A(64, 'A2_')) + self.features.add(_make_A(64, 'A3_')) + self.features.add(_make_B('B_')) + self.features.add(_make_C(128, 'C1_')) + self.features.add(_make_C(160, 'C2_')) + self.features.add(_make_C(160, 'C3_')) + self.features.add(_make_C(192, 'C4_')) + + self.classifier = nn.HybridSequential(prefix='') + self.classifier.add(_make_D('D_')) + self.classifier.add(_make_E('E1_')) + self.classifier.add(_make_E('E2_')) + self.classifier.add(nn.AvgPool2D(pool_size=8)) + self.classifier.add(nn.Dropout(0.5)) + self.classifier.add(nn.Flatten()) + self.classifier.add(nn.Dense(classes)) + + def hybrid_forward(self, F, x): + x = self.features(x) + x = self.classifier(x) + return x + +# Constructor +def inception_v3(pretrained=False, ctx=cpu(), **kwargs): + r"""Inception v3 model from + `"Rethinking the Inception Architecture for Computer Vision" + `_ paper. + + Parameters + ---------- + pretrained : bool, default False + Whether to load the pretrained weights for model. + ctx : Context, default CPU + The context in which to load the pretrained weights. + """ + net = Inception3(**kwargs) + if pretrained: + from ..model_store import get_model_file + net.load_params(get_model_file('inceptionv3'), ctx=ctx) + return net diff --git a/python/mxnet/gluon/model_zoo/vision/resnet.py b/python/mxnet/gluon/model_zoo/vision/resnet.py new file mode 100644 index 000000000000..2870911aa5cf --- /dev/null +++ b/python/mxnet/gluon/model_zoo/vision/resnet.py @@ -0,0 +1,496 @@ +# coding: utf-8 +# pylint: disable= arguments-differ +"""ResNets, implemented in Gluon.""" +from __future__ import division + +__all__ = ['ResNetV1', 'ResNetV2', + 'resnet18_v1', 'resnet34_v1', 'resnet50_v1', 'resnet101_v1', 'resnet152_v1', + 'resnet18_v2', 'resnet34_v2', 'resnet50_v2', 'resnet101_v2', 'resnet152_v2', + 'get_resnet'] + +from ....context import cpu +from ...block import HybridBlock +from ... import nn + +# Helpers +def _conv3x3(channels, stride, in_channels): + return nn.Conv2D(channels, kernel_size=3, strides=stride, padding=1, + use_bias=False, in_channels=in_channels) + + +# Blocks +class BasicBlockV1(HybridBlock): + r"""BasicBlock V1 from `"Deep Residual Learning for Image Recognition" + `_ paper. + This is used for ResNet V1 for 18, 34 layers. + + Parameters + ---------- + channels : int + Number of output channels. + stride : int + Stride size. + downsample : bool, default False + Whether to downsample the input. + in_channels : int, default 0 + Number of input channels. Default is 0, to infer from the graph. + """ + def __init__(self, channels, stride, downsample=False, in_channels=0, **kwargs): + super(BasicBlockV1, self).__init__(**kwargs) + self.body = nn.HybridSequential(prefix='') + self.body.add(_conv3x3(channels, stride, in_channels)) + self.body.add(nn.BatchNorm()) + self.body.add(nn.Activation('relu')) + self.body.add(_conv3x3(channels, 1, channels)) + self.body.add(nn.BatchNorm()) + if downsample: + self.downsample = nn.HybridSequential(prefix='') + self.downsample.add(nn.Conv2D(channels, kernel_size=1, strides=stride, + use_bias=False, in_channels=in_channels)) + self.downsample.add(nn.BatchNorm()) + else: + self.downsample = None + + def hybrid_forward(self, F, x): + residual = x + + x = self.body(x) + + if self.downsample: + residual = self.downsample(residual) + + x = F.Activation(residual+x, act_type='relu') + + return x + + +class BottleneckV1(HybridBlock): + r"""Bottleneck V1 from `"Deep Residual Learning for Image Recognition" + `_ paper. + This is used for ResNet V1 for 50, 101, 152 layers. + + Parameters + ---------- + channels : int + Number of output channels. + stride : int + Stride size. + downsample : bool, default False + Whether to downsample the input. + in_channels : int, default 0 + Number of input channels. Default is 0, to infer from the graph. + """ + def __init__(self, channels, stride, downsample=False, in_channels=0, **kwargs): + super(BottleneckV1, self).__init__(**kwargs) + self.body = nn.HybridSequential(prefix='') + self.body.add(nn.Conv2D(channels//4, kernel_size=1, strides=1)) + self.body.add(nn.BatchNorm()) + self.body.add(nn.Activation('relu')) + self.body.add(_conv3x3(channels//4, stride, channels//4)) + self.body.add(nn.BatchNorm()) + self.body.add(nn.Activation('relu')) + self.body.add(nn.Conv2D(channels, kernel_size=1, strides=1)) + self.body.add(nn.BatchNorm()) + if downsample: + self.downsample = nn.HybridSequential(prefix='') + self.downsample.add(nn.Conv2D(channels, kernel_size=1, strides=stride, + use_bias=False, in_channels=in_channels)) + self.downsample.add(nn.BatchNorm()) + else: + self.downsample = None + + def hybrid_forward(self, F, x): + residual = x + + x = self.body(x) + + if self.downsample: + residual = self.downsample(residual) + + x = F.Activation(x + residual, act_type='relu') + return x + + +class BasicBlockV2(HybridBlock): + r"""BasicBlock V2 from + `"Identity Mappings in Deep Residual Networks" + `_ paper. + This is used for ResNet V2 for 18, 34 layers. + + Parameters + ---------- + channels : int + Number of output channels. + stride : int + Stride size. + downsample : bool, default False + Whether to downsample the input. + in_channels : int, default 0 + Number of input channels. Default is 0, to infer from the graph. + """ + def __init__(self, channels, stride, downsample=False, in_channels=0, **kwargs): + super(BasicBlockV2, self).__init__(**kwargs) + self.bn1 = nn.BatchNorm() + self.conv1 = _conv3x3(channels, stride, in_channels) + self.bn2 = nn.BatchNorm() + self.conv2 = _conv3x3(channels, 1, channels) + if downsample: + self.downsample = nn.Conv2D(channels, 1, stride, use_bias=False, + in_channels=in_channels) + else: + self.downsample = None + + def hybrid_forward(self, F, x): + residual = x + x = self.bn1(x) + x = F.Activation(x, act_type='relu') + if self.downsample: + residual = self.downsample(x) + x = self.conv1(x) + + x = self.bn2(x) + x = F.Activation(x, act_type='relu') + x = self.conv2(x) + + return x + residual + + +class BottleneckV2(HybridBlock): + r"""Bottleneck V2 from + `"Identity Mappings in Deep Residual Networks" + `_ paper. + This is used for ResNet V2 for 50, 101, 152 layers. + + Parameters + ---------- + channels : int + Number of output channels. + stride : int + Stride size. + downsample : bool, default False + Whether to downsample the input. + in_channels : int, default 0 + Number of input channels. Default is 0, to infer from the graph. + """ + def __init__(self, channels, stride, downsample=False, in_channels=0, **kwargs): + super(BottleneckV2, self).__init__(**kwargs) + self.bn1 = nn.BatchNorm() + self.conv1 = _conv3x3(channels//4, 1, in_channels) + self.bn2 = nn.BatchNorm() + self.conv2 = _conv3x3(channels//4, stride, channels//4) + self.bn3 = nn.BatchNorm() + self.conv3 = _conv3x3(channels, 1, channels//4) + if downsample: + self.downsample = nn.Conv2D(channels, 1, stride, use_bias=False, + in_channels=in_channels) + else: + self.downsample = None + + def hybrid_forward(self, F, x): + residual = x + x = self.bn1(x) + x = F.Activation(x, act_type='relu') + if self.downsample: + residual = self.downsample(x) + x = self.conv1(x) + + x = self.bn2(x) + x = F.Activation(x, act_type='relu') + x = self.conv2(x) + + x = self.bn3(x) + x = F.Activation(x, act_type='relu') + x = self.conv3(x) + + return x + residual + + +# Nets +class ResNetV1(HybridBlock): + r"""ResNet V1 model from + `"Deep Residual Learning for Image Recognition" + `_ paper. + + Parameters + ---------- + block : HybridBlock + Class for the residual block. Options are BasicBlockV1, BottleneckV1. + layers : list of int + Numbers of layers in each block + channels : list of int + Numbers of channels in each block. Length should be one larger than layers list. + classes : int, default 1000 + Number of classification classes. + thumbnail : bool, default False + Enable thumbnail. + """ + def __init__(self, block, layers, channels, classes=1000, thumbnail=False, **kwargs): + super(ResNetV1, self).__init__(**kwargs) + assert len(layers) == len(channels) - 1 + with self.name_scope(): + self.features = nn.HybridSequential(prefix='') + if thumbnail: + self.features.add(_conv3x3(channels[0], 1, 3)) + else: + self.features.add(nn.Conv2D(channels[0], 7, 2, 3, use_bias=False, + in_channels=3)) + self.features.add(nn.BatchNorm()) + self.features.add(nn.Activation('relu')) + self.features.add(nn.MaxPool2D(3, 2, 1)) + + for i, num_layer in enumerate(layers): + stride = 1 if i == 0 else 2 + self.features.add(self._make_layer(block, num_layer, channels[i+1], + stride, i+1, in_channels=channels[i])) + + self.classifier = nn.HybridSequential(prefix='') + self.classifier.add(nn.GlobalAvgPool2D()) + self.classifier.add(nn.Flatten()) + self.classifier.add(nn.Dense(classes, in_units=channels[-1])) + + def _make_layer(self, block, layers, channels, stride, stage_index, in_channels=0): + layer = nn.HybridSequential(prefix='stage%d_'%stage_index) + with layer.name_scope(): + layer.add(block(channels, stride, channels != in_channels, in_channels=in_channels, + prefix='')) + for _ in range(layers-1): + layer.add(block(channels, 1, False, in_channels=channels, prefix='')) + return layer + + def hybrid_forward(self, F, x): + x = self.features(x) + x = self.classifier(x) + + return x + + +class ResNetV2(HybridBlock): + r"""ResNet V2 model from + `"Identity Mappings in Deep Residual Networks" + `_ paper. + + Parameters + ---------- + block : HybridBlock + Class for the residual block. Options are BasicBlockV1, BottleneckV1. + layers : list of int + Numbers of layers in each block + channels : list of int + Numbers of channels in each block. Length should be one larger than layers list. + classes : int, default 1000 + Number of classification classes. + thumbnail : bool, default False + Enable thumbnail. + """ + def __init__(self, block, layers, channels, classes=1000, thumbnail=False, **kwargs): + super(ResNetV2, self).__init__(**kwargs) + assert len(layers) == len(channels) - 1 + with self.name_scope(): + self.features = nn.HybridSequential(prefix='') + self.features.add(nn.BatchNorm(scale=False, center=False)) + if thumbnail: + self.features.add(_conv3x3(channels[0], 1, 3)) + else: + self.features.add(nn.Conv2D(channels[0], 7, 2, 3, use_bias=False, + in_channels=3)) + self.features.add(nn.BatchNorm()) + self.features.add(nn.Activation('relu')) + self.features.add(nn.MaxPool2D(3, 2, 1)) + + in_channels = channels[0] + for i, num_layer in enumerate(layers): + stride = 1 if i == 0 else 2 + self.features.add(self._make_layer(block, num_layer, channels[i+1], + stride, i+1, in_channels=in_channels)) + in_channels = channels[i+1] + + self.classifier = nn.HybridSequential(prefix='') + self.classifier.add(nn.BatchNorm()) + self.classifier.add(nn.Activation('relu')) + self.classifier.add(nn.GlobalAvgPool2D()) + self.classifier.add(nn.Flatten()) + self.classifier.add(nn.Dense(classes, in_units=in_channels)) + + def _make_layer(self, block, layers, channels, stride, stage_index, in_channels=0): + layer = nn.HybridSequential(prefix='stage%d_'%stage_index) + with layer.name_scope(): + layer.add(block(channels, stride, channels != in_channels, in_channels=in_channels, + prefix='')) + for _ in range(layers-1): + layer.add(block(channels, 1, False, in_channels=channels, prefix='')) + return layer + + def hybrid_forward(self, F, x): + x = self.features(x) + x = self.classifier(x) + return x + + +# Specification +resnet_spec = {18: ('basic_block', [2, 2, 2, 2], [64, 64, 128, 256, 512]), + 34: ('basic_block', [3, 4, 6, 3], [64, 64, 128, 256, 512]), + 50: ('bottle_neck', [3, 4, 6, 3], [64, 256, 512, 1024, 2048]), + 101: ('bottle_neck', [3, 4, 23, 3], [64, 256, 512, 1024, 2048]), + 152: ('bottle_neck', [3, 8, 36, 3], [64, 256, 512, 1024, 2048])} + +resnet_net_versions = [ResNetV1, ResNetV2] +resnet_block_versions = [{'basic_block': BasicBlockV1, 'bottle_neck': BottleneckV1}, + {'basic_block': BasicBlockV2, 'bottle_neck': BottleneckV2}] + + +# Constructor +def get_resnet(version, num_layers, pretrained=False, ctx=cpu(), **kwargs): + r"""ResNet V1 model from `"Deep Residual Learning for Image Recognition" + `_ paper. + ResNet V2 model from `"Identity Mappings in Deep Residual Networks" + `_ paper. + + Parameters + ---------- + version : int + Version of ResNet. Options are 1, 2. + num_layers : int + Numbers of layers. Options are 18, 34, 50, 101, 152. + pretrained : bool, default False + Whether to load the pretrained weights for model. + ctx : Context, default CPU + The context in which to load the pretrained weights. + """ + block_type, layers, channels = resnet_spec[num_layers] + resnet_class = resnet_net_versions[version-1] + block_class = resnet_block_versions[version-1][block_type] + net = resnet_class(block_class, layers, channels, **kwargs) + if pretrained: + from ..model_store import get_model_file + net.load_params(get_model_file('resnet%d_v%d'%(num_layers, version)), ctx=ctx) + return net + +def resnet18_v1(**kwargs): + r"""ResNet-18 V1 model from `"Deep Residual Learning for Image Recognition" + `_ paper. + + Parameters + ---------- + pretrained : bool, default False + Whether to load the pretrained weights for model. + ctx : Context, default CPU + The context in which to load the pretrained weights. + """ + return get_resnet(1, 18, **kwargs) + +def resnet34_v1(**kwargs): + r"""ResNet-34 V1 model from `"Deep Residual Learning for Image Recognition" + `_ paper. + + Parameters + ---------- + pretrained : bool, default False + Whether to load the pretrained weights for model. + ctx : Context, default CPU + The context in which to load the pretrained weights. + """ + return get_resnet(1, 34, **kwargs) + +def resnet50_v1(**kwargs): + r"""ResNet-50 V1 model from `"Deep Residual Learning for Image Recognition" + `_ paper. + + Parameters + ---------- + pretrained : bool, default False + Whether to load the pretrained weights for model. + ctx : Context, default CPU + The context in which to load the pretrained weights. + """ + return get_resnet(1, 50, **kwargs) + +def resnet101_v1(**kwargs): + r"""ResNet-101 V1 model from `"Deep Residual Learning for Image Recognition" + `_ paper. + + Parameters + ---------- + pretrained : bool, default False + Whether to load the pretrained weights for model. + ctx : Context, default CPU + The context in which to load the pretrained weights. + """ + return get_resnet(1, 101, **kwargs) + +def resnet152_v1(**kwargs): + r"""ResNet-152 V1 model from `"Deep Residual Learning for Image Recognition" + `_ paper. + + Parameters + ---------- + pretrained : bool, default False + Whether to load the pretrained weights for model. + ctx : Context, default CPU + The context in which to load the pretrained weights. + """ + return get_resnet(1, 152, **kwargs) + +def resnet18_v2(**kwargs): + r"""ResNet-18 V2 model from `"Identity Mappings in Deep Residual Networks" + `_ paper. + + Parameters + ---------- + pretrained : bool, default False + Whether to load the pretrained weights for model. + ctx : Context, default CPU + The context in which to load the pretrained weights. + """ + return get_resnet(2, 18, **kwargs) + +def resnet34_v2(**kwargs): + r"""ResNet-34 V2 model from `"Identity Mappings in Deep Residual Networks" + `_ paper. + + Parameters + ---------- + pretrained : bool, default False + Whether to load the pretrained weights for model. + ctx : Context, default CPU + The context in which to load the pretrained weights. + """ + return get_resnet(2, 34, **kwargs) + +def resnet50_v2(**kwargs): + r"""ResNet-50 V2 model from `"Identity Mappings in Deep Residual Networks" + `_ paper. + + Parameters + ---------- + pretrained : bool, default False + Whether to load the pretrained weights for model. + ctx : Context, default CPU + The context in which to load the pretrained weights. + """ + return get_resnet(2, 50, **kwargs) + +def resnet101_v2(**kwargs): + r"""ResNet-101 V2 model from `"Identity Mappings in Deep Residual Networks" + `_ paper. + + Parameters + ---------- + pretrained : bool, default False + Whether to load the pretrained weights for model. + ctx : Context, default CPU + The context in which to load the pretrained weights. + """ + return get_resnet(2, 101, **kwargs) + +def resnet152_v2(**kwargs): + r"""ResNet-152 V2 model from `"Identity Mappings in Deep Residual Networks" + `_ paper. + + Parameters + ---------- + pretrained : bool, default False + Whether to load the pretrained weights for model. + ctx : Context, default CPU + The context in which to load the pretrained weights. + """ + return get_resnet(2, 152, **kwargs) diff --git a/python/mxnet/gluon/model_zoo/vision/squeezenet.py b/python/mxnet/gluon/model_zoo/vision/squeezenet.py new file mode 100644 index 000000000000..bfcb5cbc9bae --- /dev/null +++ b/python/mxnet/gluon/model_zoo/vision/squeezenet.py @@ -0,0 +1,142 @@ +# coding: utf-8 +# pylint: disable= arguments-differ +"""SqueezeNet, implemented in Gluon.""" +__all__ = ['SqueezeNet', 'squeezenet1_0', 'squeezenet1_1'] + +from ....context import cpu +from ...block import HybridBlock +from ... import nn +from ..custom_layers import HybridConcurrent + +# Helpers +def _make_fire(squeeze_channels, expand1x1_channels, expand3x3_channels): + out = nn.HybridSequential(prefix='') + out.add(_make_fire_conv(squeeze_channels, 1)) + + paths = HybridConcurrent(concat_dim=1, prefix='') + paths.add(_make_fire_conv(expand1x1_channels, 1)) + paths.add(_make_fire_conv(expand3x3_channels, 3, 1)) + out.add(paths) + + return out + +def _make_fire_conv(channels, kernel_size, padding=0): + out = nn.HybridSequential(prefix='') + out.add(nn.Conv2D(channels, kernel_size, padding=padding)) + out.add(nn.Activation('relu')) + return out + +# Net +class SqueezeNet(HybridBlock): + r"""SqueezeNet model from the `"SqueezeNet: AlexNet-level accuracy with 50x fewer parameters + and <0.5MB model size" `_ paper. + SqueezeNet 1.1 model from the `official SqueezeNet repo + `_. + SqueezeNet 1.1 has 2.4x less computation and slightly fewer parameters + than SqueezeNet 1.0, without sacrificing accuracy. + + Parameters + ---------- + version : str + Version of squeezenet. Options are '1.0', '1.1'. + classes : int, default 1000 + Number of classification classes. + """ + def __init__(self, version, classes=1000, **kwargs): + super(SqueezeNet, self).__init__(**kwargs) + assert version in ['1.0', '1.1'], ("Unsupported SqueezeNet version {version}:" + "1.0 or 1.1 expected".format(version=version)) + with self.name_scope(): + self.features = nn.HybridSequential(prefix='') + if version == '1.0': + self.features.add(nn.Conv2D(96, kernel_size=7, strides=2)) + self.features.add(nn.Activation('relu')) + self.features.add(nn.MaxPool2D(pool_size=3, strides=2, ceil_mode=True)) + self.features.add(_make_fire(16, 64, 64)) + self.features.add(_make_fire(16, 64, 64)) + self.features.add(_make_fire(32, 128, 128)) + self.features.add(nn.MaxPool2D(pool_size=3, strides=2, ceil_mode=True)) + self.features.add(_make_fire(32, 128, 128)) + self.features.add(_make_fire(48, 192, 192)) + self.features.add(_make_fire(48, 192, 192)) + self.features.add(_make_fire(64, 256, 256)) + self.features.add(nn.MaxPool2D(pool_size=3, strides=2, ceil_mode=True)) + self.features.add(_make_fire(64, 256, 256)) + else: + self.features.add(nn.Conv2D(64, kernel_size=3, strides=2)) + self.features.add(nn.Activation('relu')) + self.features.add(nn.MaxPool2D(pool_size=3, strides=2, ceil_mode=True)) + self.features.add(_make_fire(16, 64, 64)) + self.features.add(_make_fire(16, 64, 64)) + self.features.add(nn.MaxPool2D(pool_size=3, strides=2, ceil_mode=True)) + self.features.add(_make_fire(32, 128, 128)) + self.features.add(_make_fire(32, 128, 128)) + self.features.add(nn.MaxPool2D(pool_size=3, strides=2, ceil_mode=True)) + self.features.add(_make_fire(48, 192, 192)) + self.features.add(_make_fire(48, 192, 192)) + self.features.add(_make_fire(64, 256, 256)) + self.features.add(_make_fire(64, 256, 256)) + + self.classifier = nn.HybridSequential(prefix='') + self.classifier.add(nn.Dropout(0.5)) + self.classifier.add(nn.Conv2D(classes, kernel_size=1)) + self.classifier.add(nn.Activation('relu')) + self.classifier.add(nn.AvgPool2D(13)) + self.classifier.add(nn.Flatten()) + + def hybrid_forward(self, F, x): + x = self.features(x) + x = self.classifier(x) + return x + +# Constructor +def get_squeezenet(version, pretrained=False, ctx=cpu(), **kwargs): + r"""SqueezeNet model from the `"SqueezeNet: AlexNet-level accuracy with 50x fewer parameters + and <0.5MB model size" `_ paper. + SqueezeNet 1.1 model from the `official SqueezeNet repo + `_. + SqueezeNet 1.1 has 2.4x less computation and slightly fewer parameters + than SqueezeNet 1.0, without sacrificing accuracy. + + Parameters + ---------- + version : str + Version of squeezenet. Options are '1.0', '1.1'. + pretrained : bool, default False + Whether to load the pretrained weights for model. + ctx : Context, default CPU + The context in which to load the pretrained weights. + """ + net = SqueezeNet(version, **kwargs) + if pretrained: + from ..model_store import get_model_file + net.load_params(get_model_file('squeezenet%s'%version), ctx=ctx) + return net + +def squeezenet1_0(**kwargs): + r"""SqueezeNet 1.0 model from the `"SqueezeNet: AlexNet-level accuracy with 50x fewer parameters + and <0.5MB model size" `_ paper. + + Parameters + ---------- + pretrained : bool, default False + Whether to load the pretrained weights for model. + ctx : Context, default CPU + The context in which to load the pretrained weights. + """ + return get_squeezenet('1.0', **kwargs) + +def squeezenet1_1(**kwargs): + r"""SqueezeNet 1.1 model from the `official SqueezeNet repo + `_. + SqueezeNet 1.1 has 2.4x less computation and slightly fewer parameters + than SqueezeNet 1.0, without sacrificing accuracy. + + Parameters + ---------- + pretrained : bool, default False + Whether to load the pretrained weights for model. + ctx : Context, default CPU + The context in which to load the pretrained weights. + """ + return get_squeezenet('1.1', **kwargs) diff --git a/python/mxnet/gluon/model_zoo/vision/vgg.py b/python/mxnet/gluon/model_zoo/vision/vgg.py new file mode 100644 index 000000000000..96a4fa1f3700 --- /dev/null +++ b/python/mxnet/gluon/model_zoo/vision/vgg.py @@ -0,0 +1,209 @@ +# coding: utf-8 +# pylint: disable= arguments-differ +"""VGG, implemented in Gluon.""" +from __future__ import division +__all__ = ['VGG', + 'vgg11', 'vgg13', 'vgg16', 'vgg19', + 'vgg11_bn', 'vgg13_bn', 'vgg16_bn', 'vgg19_bn', + 'get_vgg'] + +from ....context import cpu +from ....initializer import Xavier +from ...block import HybridBlock +from ... import nn + + +class VGG(HybridBlock): + r"""VGG model from the `"Very Deep Convolutional Networks for Large-Scale Image Recognition" + `_ paper. + + Parameters + ---------- + layers : list of int + Numbers of layers in each feature block. + filters : list of int + Numbers of filters in each feature block. List length should match the layers. + classes : int, default 1000 + Number of classification classes. + batch_norm : bool, default False + Use batch normalization. + """ + def __init__(self, layers, filters, classes=1000, batch_norm=False, **kwargs): + super(VGG, self).__init__(**kwargs) + assert len(layers) == len(filters) + with self.name_scope(): + self.features = self._make_features(layers, filters, batch_norm) + self.classifier = nn.HybridSequential(prefix='') + self.classifier.add(nn.Dense(4096, activation='relu', + weight_initializer='normal', + bias_initializer='zeros')) + self.classifier.add(nn.Dropout(rate=0.5)) + self.classifier.add(nn.Dense(4096, activation='relu', + weight_initializer='normal', + bias_initializer='zeros')) + self.classifier.add(nn.Dropout(rate=0.5)) + self.classifier.add(nn.Dense(classes, + weight_initializer='normal', + bias_initializer='zeros')) + + def _make_features(self, layers, filters, batch_norm): + featurizer = nn.HybridSequential(prefix='') + for i, num in enumerate(layers): + for _ in range(num): + featurizer.add(nn.Conv2D(filters[i], kernel_size=3, padding=1, + weight_initializer=Xavier(rnd_type='gaussian', + factor_type='out', + magnitude=2), + bias_initializer='zeros')) + if batch_norm: + featurizer.add(nn.BatchNorm()) + featurizer.add(nn.Activation('relu')) + featurizer.add(nn.MaxPool2D(strides=2)) + return featurizer + + def hybrid_forward(self, F, x): + x = self.features(x) + x = self.classifier(x) + return x + + +# Specification +vgg_spec = {11: ([1, 1, 2, 2, 2], [64, 128, 256, 512, 512]), + 13: ([2, 2, 2, 2, 2], [64, 128, 256, 512, 512]), + 16: ([2, 2, 3, 3, 3], [64, 128, 256, 512, 512]), + 19: ([2, 2, 4, 4, 4], [64, 128, 256, 512, 512])} + + +# Constructors +def get_vgg(num_layers, pretrained=False, ctx=cpu(), **kwargs): + r"""VGG model from the `"Very Deep Convolutional Networks for Large-Scale Image Recognition" + `_ paper. + + Parameters + ---------- + num_layers : int + Number of layers for the variant of densenet. Options are 11, 13, 16, 19. + pretrained : bool, default False + Whether to load the pretrained weights for model. + ctx : Context, default CPU + The context in which to load the pretrained weights. + """ + layers, filters = vgg_spec[num_layers] + net = VGG(layers, filters, **kwargs) + if pretrained: + from ..model_store import get_model_file + batch_norm_suffix = '_bn' if kwargs.get('batch_norm') else '' + net.load_params(get_model_file('vgg%d%s'%(num_layers, batch_norm_suffix)), ctx=ctx) + return net + +def vgg11(**kwargs): + r"""VGG-11 model from the `"Very Deep Convolutional Networks for Large-Scale Image Recognition" + `_ paper. + + Parameters + ---------- + pretrained : bool, default False + Whether to load the pretrained weights for model. + ctx : Context, default CPU + The context in which to load the pretrained weights. + """ + return get_vgg(11, **kwargs) + +def vgg13(**kwargs): + r"""VGG-13 model from the `"Very Deep Convolutional Networks for Large-Scale Image Recognition" + `_ paper. + + Parameters + ---------- + pretrained : bool, default False + Whether to load the pretrained weights for model. + ctx : Context, default CPU + The context in which to load the pretrained weights. + """ + return get_vgg(13, **kwargs) + +def vgg16(**kwargs): + r"""VGG-16 model from the `"Very Deep Convolutional Networks for Large-Scale Image Recognition" + `_ paper. + + Parameters + ---------- + pretrained : bool, default False + Whether to load the pretrained weights for model. + ctx : Context, default CPU + The context in which to load the pretrained weights. + """ + return get_vgg(16, **kwargs) + +def vgg19(**kwargs): + r"""VGG-19 model from the `"Very Deep Convolutional Networks for Large-Scale Image Recognition" + `_ paper. + + Parameters + ---------- + pretrained : bool, default False + Whether to load the pretrained weights for model. + ctx : Context, default CPU + The context in which to load the pretrained weights. + """ + return get_vgg(19, **kwargs) + +def vgg11_bn(**kwargs): + r"""VGG-11 model with batch normalization from the + `"Very Deep Convolutional Networks for Large-Scale Image Recognition" + `_ paper. + + Parameters + ---------- + pretrained : bool, default False + Whether to load the pretrained weights for model. + ctx : Context, default CPU + The context in which to load the pretrained weights. + """ + kwargs['batch_norm'] = True + return get_vgg(11, **kwargs) + +def vgg13_bn(**kwargs): + r"""VGG-13 model with batch normalization from the + `"Very Deep Convolutional Networks for Large-Scale Image Recognition" + `_ paper. + + Parameters + ---------- + pretrained : bool, default False + Whether to load the pretrained weights for model. + ctx : Context, default CPU + The context in which to load the pretrained weights. + """ + kwargs['batch_norm'] = True + return get_vgg(13, **kwargs) + +def vgg16_bn(**kwargs): + r"""VGG-16 model with batch normalization from the + `"Very Deep Convolutional Networks for Large-Scale Image Recognition" + `_ paper. + + Parameters + ---------- + pretrained : bool, default False + Whether to load the pretrained weights for model. + ctx : Context, default CPU + The context in which to load the pretrained weights. + """ + kwargs['batch_norm'] = True + return get_vgg(16, **kwargs) + +def vgg19_bn(**kwargs): + r"""VGG-19 model with batch normalization from the + `"Very Deep Convolutional Networks for Large-Scale Image Recognition" + `_ paper. + + Parameters + ---------- + pretrained : bool, default False + Whether to load the pretrained weights for model. + ctx : Context, default CPU + The context in which to load the pretrained weights. + """ + kwargs['batch_norm'] = True + return get_vgg(19, **kwargs) diff --git a/python/mxnet/gluon/nn/basic_layers.py b/python/mxnet/gluon/nn/basic_layers.py index 069baf94079d..6a2000e418dc 100644 --- a/python/mxnet/gluon/nn/basic_layers.py +++ b/python/mxnet/gluon/nn/basic_layers.py @@ -235,7 +235,7 @@ class BatchNorm(HybridBlock): set `axis=1` in `BatchNorm`. If `layout='NHWC'`, then set `axis=3`. momentum: float, default 0.9 Momentum for the moving average. - epsilon: float, default 1e-3 + epsilon: float, default 1e-5 Small float added to variance to avoid dividing by zero. center: bool, default True If True, add offset of `beta` to normalized tensor. @@ -265,7 +265,7 @@ class BatchNorm(HybridBlock): Output shape: Same shape as input. """ - def __init__(self, axis=1, momentum=0.9, epsilon=1e-3, center=True, scale=True, + def __init__(self, axis=1, momentum=0.9, epsilon=1e-5, center=True, scale=True, beta_initializer='zeros', gamma_initializer='ones', running_mean_initializer='zeros', running_variance_initializer='ones', in_channels=0, **kwargs): diff --git a/python/mxnet/gluon/nn/conv_layers.py b/python/mxnet/gluon/nn/conv_layers.py index caa2dd87eb5f..bb2ffea96c11 100644 --- a/python/mxnet/gluon/nn/conv_layers.py +++ b/python/mxnet/gluon/nn/conv_layers.py @@ -111,6 +111,9 @@ def hybrid_forward(self, F, x, weight, bias=None): act = self.act(act) return act + def _alias(self): + return 'conv' + def __repr__(self): s = '{name}({mapping}, kernel_size={kernel}, stride={stride}' len_kernel_size = len(self._kwargs['kernel']) diff --git a/python/mxnet/symbol.py b/python/mxnet/symbol.py index 4a9a3f4550c8..56ab27b15dd7 100644 --- a/python/mxnet/symbol.py +++ b/python/mxnet/symbol.py @@ -2117,6 +2117,28 @@ def ones(shape, dtype=None, **kwargs): return _internal._ones(shape=shape, dtype=dtype, **kwargs) +def full(shape, val, dtype=None, **kwargs): + """Returns a new array of given shape and type, filled with the given value `val`. + + Parameters + ---------- + shape : int or sequence of ints + Shape of the new array. + val : scalar + Fill value. + dtype : str or numpy.dtype, optional + The value type of the inner value, default to ``np.float32``. + + Returns + ------- + out : Symbol + The created Symbol + """ + if dtype is None: + dtype = _numpy.float32 + return _internal._MulScalar(ones(shape=shape, dtype=dtype, **kwargs), scalar=val) + + def arange(start, stop=None, step=1.0, repeat=1, name=None, dtype=None): """Returns evenly spaced values within a given interval. diff --git a/src/operator/batch_norm-inl.h b/src/operator/batch_norm-inl.h index 1c735c4abff8..2d9c96be0cb9 100644 --- a/src/operator/batch_norm-inl.h +++ b/src/operator/batch_norm-inl.h @@ -38,7 +38,7 @@ constexpr int DEFAULT_AXIS = 1; /*! \brief Parameters for BatchNoram operator */ struct BatchNormParam : public dmlc::Parameter { - float eps; + double eps; float momentum; bool fix_gamma; bool use_global_stats; @@ -48,7 +48,7 @@ struct BatchNormParam : public dmlc::Parameter { DMLC_DECLARE_PARAMETER(BatchNormParam) { DMLC_DECLARE_FIELD(eps).set_default(1e-3f) .describe("Epsilon to prevent div 0. " - "Must be bigger than CUDNN_BN_MIN_EPSILON " + "Must be no less than CUDNN_BN_MIN_EPSILON " "defined in cudnn.h when using cudnn (usually 1e-5)"); DMLC_DECLARE_FIELD(momentum).set_default(0.9f) .describe("Momentum for moving average"); diff --git a/src/operator/cudnn_batch_norm-inl.h b/src/operator/cudnn_batch_norm-inl.h index 5c4179057294..6005b0f58b12 100644 --- a/src/operator/cudnn_batch_norm-inl.h +++ b/src/operator/cudnn_batch_norm-inl.h @@ -28,8 +28,8 @@ class CuDNNBatchNormOp : public Operator { public: explicit CuDNNBatchNormOp(BatchNormParam param) { using namespace mshadow; - CHECK_GT(param.eps, CUDNN_BN_MIN_EPSILON) - << "CuDNN requires eps to be greater than " << CUDNN_BN_MIN_EPSILON; + CHECK_GE(param.eps, CUDNN_BN_MIN_EPSILON) + << "CuDNN requires eps to be no less than " << CUDNN_BN_MIN_EPSILON; this->param_ = param; init_cudnn_ = false; dtype_ = DataType::kCudnnFlag; diff --git a/tests/python/unittest/test_gluon_model_zoo.py b/tests/python/unittest/test_gluon_model_zoo.py new file mode 100644 index 000000000000..7a38d606d2bc --- /dev/null +++ b/tests/python/unittest/test_gluon_model_zoo.py @@ -0,0 +1,33 @@ +import mxnet as mx +from mxnet.gluon import nn +from mxnet.gluon.model_zoo.custom_layers import HybridConcurrent, Identity + + +def test_concurrent(): + model = HybridConcurrent(concat_dim=1) + model.add(nn.Dense(128, activation='tanh', in_units=10)) + model.add(nn.Dense(64, activation='tanh', in_units=10)) + model.add(nn.Dense(32, in_units=10)) + + # symbol + x = mx.sym.var('data') + y = model(x) + assert len(y.list_arguments()) == 7 + + # ndarray + model.collect_params().initialize(mx.init.Xavier(magnitude=2.24)) + x = model(mx.nd.zeros((32, 10))) + assert x.shape == (32, 224) + x.wait_to_read() + + +def test_identity(): + model = Identity() + x = mx.nd.random_uniform(shape=(128, 33, 64)) + mx.test_utils.assert_almost_equal(model(x).asnumpy(), + x.asnumpy()) + + +if __name__ == '__main__': + import nose + nose.runmodule() From 37e40be8f9a80f57a54eba29836b1cabf4d8cb50 Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Sun, 30 Jul 2017 14:43:12 -0700 Subject: [PATCH 277/834] fix pretty print (#7254) * fix pretty print * add tests --- python/mxnet/gluon/model_zoo/custom_layers.py | 1 + tests/python/unittest/test_gluon_model_zoo.py | 22 +++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/python/mxnet/gluon/model_zoo/custom_layers.py b/python/mxnet/gluon/model_zoo/custom_layers.py index 871cfb061e23..da1ca868dd76 100644 --- a/python/mxnet/gluon/model_zoo/custom_layers.py +++ b/python/mxnet/gluon/model_zoo/custom_layers.py @@ -3,6 +3,7 @@ """Custom neural network layers in model_zoo.""" from ..block import Block, HybridBlock +from ..utils import _indent class HybridConcurrent(HybridBlock): """Lays `HybridBlock`s concurrently. diff --git a/tests/python/unittest/test_gluon_model_zoo.py b/tests/python/unittest/test_gluon_model_zoo.py index 7a38d606d2bc..200037c067e2 100644 --- a/tests/python/unittest/test_gluon_model_zoo.py +++ b/tests/python/unittest/test_gluon_model_zoo.py @@ -1,6 +1,8 @@ +from __future__ import print_function import mxnet as mx from mxnet.gluon import nn from mxnet.gluon.model_zoo.custom_layers import HybridConcurrent, Identity +from mxnet.gluon.model_zoo.vision import get_model def test_concurrent(): @@ -28,6 +30,26 @@ def test_identity(): x.asnumpy()) +def test_models(): + all_models = ['resnet18_v1', 'resnet34_v1', 'resnet50_v1', 'resnet101_v1', 'resnet152_v1', + 'resnet18_v2', 'resnet34_v2', 'resnet50_v2', 'resnet101_v2', 'resnet152_v2', + 'vgg11', 'vgg13', 'vgg16', 'vgg19', + 'vgg11_bn', 'vgg13_bn', 'vgg16_bn', 'vgg19_bn', + 'alexnet', 'inceptionv3', + 'densenet121', 'densenet161', 'densenet169', 'densenet201', + 'squeezenet1.0', 'squeezenet1.1'] + pretrained_to_test = set(['squeezenet1.1']) + + for model_name in all_models: + test_pretrain = model_name in pretrained_to_test + model = get_model(model_name, pretrained=test_pretrain) + data_shape = (7, 3, 224, 224) if 'inception' not in model_name else (7, 3, 299, 299) + print(model) + if not test_pretrain: + model.collect_params().initialize() + model(mx.nd.random_uniform(shape=data_shape)) + + if __name__ == '__main__': import nose nose.runmodule() From dcceb2fae6db0fe91ba12430303a0f7d279a8a5b Mon Sep 17 00:00:00 2001 From: Kai Li <1196594711@qq.com> Date: Mon, 31 Jul 2017 08:18:16 +0800 Subject: [PATCH 278/834] Update README.md (#7249) --- example/image-classification/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/example/image-classification/README.md b/example/image-classification/README.md index a008b23472f6..3f514e2a391f 100644 --- a/example/image-classification/README.md +++ b/example/image-classification/README.md @@ -2,7 +2,7 @@ This fold contains examples for image classification. The goal of image classifcation is to identify the objects contained in images. The following -[example](http://mxnet.io/tutorials/python/predict_imagenet.html) shows +[example](http://mxnet.io/tutorials/python/predict_image.html) shows recognized object classes with corresponding probabilities using a pre-traind model. @@ -102,7 +102,7 @@ We provide multiple pre-trained models on various datasets. Use [common/modelzone.py](https://github.com/dmlc/mxnet/blob/master/example/image-classification/common/modelzoo.py) to download these models. These models can be used in any front-end language MXNet supports. For example, -[the tutorial](http://mxnet.io/tutorials/python/predict_imagenet.html) shows how +[the tutorial](http://mxnet.io/tutorials/python/predict_image.html) shows how to classify an image with jupyter notebook. ### ImageNet 1K From 42544eda02d5e7ff527704b42d62c0f81faa17f6 Mon Sep 17 00:00:00 2001 From: Eric Junyuan Xie Date: Sun, 30 Jul 2017 19:55:55 -0700 Subject: [PATCH 279/834] add reset_ctx (#7221) * add reset_ctx * add paramdict * add symbolBlock * rename blocks * rename * fix --- nnvm | 2 +- python/mxnet/gluon/block.py | 156 +++++++++++--- .../mxnet/gluon/model_zoo/vision/alexnet.py | 22 +- python/mxnet/gluon/nn/basic_layers.py | 25 ++- python/mxnet/gluon/nn/conv_layers.py | 11 +- python/mxnet/gluon/parameter.py | 199 +++++++++++------- python/mxnet/gluon/rnn/rnn_cell.py | 81 ++++--- python/mxnet/symbol.py | 16 +- src/ndarray/autograd.cc | 3 +- tests/python/unittest/test_nn.py | 24 +++ 10 files changed, 361 insertions(+), 178 deletions(-) diff --git a/nnvm b/nnvm index c96dd0e126a7..0a45136fae47 160000 --- a/nnvm +++ b/nnvm @@ -1 +1 @@ -Subproject commit c96dd0e126a788089fe700cf6effe4e87bc40e05 +Subproject commit 0a45136fae475a8313dc66b6bebd87a722f20e7f diff --git a/python/mxnet/gluon/block.py b/python/mxnet/gluon/block.py index e8ec12be570b..cfc5e5744338 100644 --- a/python/mxnet/gluon/block.py +++ b/python/mxnet/gluon/block.py @@ -2,6 +2,8 @@ # pylint: disable= arguments-differ """Base container class for all neural network models.""" +import copy + from .. import symbol, ndarray, initializer from ..symbol import Symbol from ..ndarray import NDArray @@ -18,6 +20,7 @@ def __init__(self, block): self._block = block self._counter = {} self._old_scope = None + self._name_scope = None @staticmethod def create(prefix, params, hint): @@ -46,9 +49,13 @@ def create(prefix, params, hint): def __enter__(self): self._old_scope = _BlockScope._current _BlockScope._current = self + self._name_scope = _name.Prefix(self._block.prefix) + self._name_scope.__enter__() return self def __exit__(self, ptype, value, trace): + self._name_scope.__exit__(ptype, value, trace) + self._name_scope = None _BlockScope._current = self._old_scope @@ -134,6 +141,7 @@ def forward(self, x): """ def __init__(self, prefix=None, params=None): self._prefix, self._params = _BlockScope.create(prefix, params, self._alias()) + self._name = self._prefix[:-1] if self._prefix.endswith('_') else self._prefix self._scope = _BlockScope(self) self._children = [] @@ -162,9 +170,7 @@ def prefix(self): @property def name(self): """Name of this `Block`, without '_' in the end.""" - if self.prefix.endswith('_'): - return self.prefix[:-1] - return self.prefix + return self._name def name_scope(self): """Returns a name space object managing a child `Block` and parameter @@ -309,26 +315,26 @@ def hybridize(self, active=True): super(HybridBlock, self).hybridize(active) def _get_graph(self, *args): - if self._cached_graph: - return self._cached_graph + if not self._cached_graph: + args, self._in_format = _flatten(args) + inputs = [symbol.var('input_%d'%i) for i in range(len(args))] + grouped_inputs = _regroup(inputs, self._in_format)[0] - args, self._in_format = _flatten(args) - syms = [symbol.var(str(i)) for i in range(len(args))] - sym_args = _regroup(syms, self._in_format)[0] + params = {i: j.var() for i, j in self._reg_params.items()} + with self.name_scope(): + out = self.hybrid_forward(symbol, *grouped_inputs, **params) # pylint: disable=no-value-for-parameter + out, self._out_format = _flatten(out) - params = {i: j.var() for i, j in self._reg_params.items()} - out = self.hybrid_forward(symbol, *sym_args, **params) # pylint: disable=no-value-for-parameter - out, self._out_format = _flatten(out) + self._cached_graph = inputs, symbol.Group(out) - self._cached_graph = syms, symbol.Group(out) return self._cached_graph def infer_shape(self, *args): """Infers shape of Parameters from inputs.""" - syms, out = self._get_graph(*args) - args, _, = _flatten(args) + inputs, out = self._get_graph(*args) + args, _ = _flatten(args) arg_shapes, _, aux_shapes = out.infer_shape( - **{i.name: j.shape for i, j in zip(syms, args)}) + **{i.name: j.shape for i, j in zip(inputs, args)}) sdict = {i: j for i, j in zip(out.list_arguments(), arg_shapes)} sdict.update({name : shape for name, shape in \ zip(out.list_auxiliary_states(), aux_shapes)}) @@ -336,21 +342,33 @@ def infer_shape(self, *args): i.shape = sdict[i.name] def _build_cache(self, *args): - self.infer_shape(*args) - for i in self.collect_params().values(): - i._finish_deferred_init() - - _, out = self._get_graph(*args) + inputs, out = self._get_graph(*args) self._cached_op = ndarray.CachedOp(out) + params = dict(self.collect_params().items()) self._cached_params = [params.get(name, None) for name in out.list_inputs()] - self._in_idx = [(i, int(name)) for i, name in enumerate(out.list_inputs()) + assert len(params) + len(self._cached_graph[0]) == len(out.list_inputs()), \ + "Wrong number of inputs." + + name2pos = {var.name: i for i, var in enumerate(inputs)} + self._in_idx = [(i, name2pos[name]) for i, name in enumerate(out.list_inputs()) if name not in params] def _call_cached_op(self, *args): + if self._cached_op is None: + self._build_cache(*args) + + try: + cargs = [i.data() if i else None for i in self._cached_params] + except DeferredInitializationError: + self.infer_shape(*args) + for i in self._cached_params: + if i is not None: + i._finish_deferred_init() + cargs = [i.data() if i else None for i in self._cached_params] + args, fmt = _flatten(args) assert fmt == self._in_format, "Invalid input format" - cargs = [i.data() if i else None for i in self._cached_params] for i, j in self._in_idx: cargs[i] = args[j] out = self._cached_op(*cargs) @@ -362,9 +380,6 @@ def forward(self, x, *args): """Defines the forward computation. Arguments can be either `NDArray` or `Symbol`.""" if isinstance(x, NDArray): - if self._active and self._cached_op is None: - self._build_cache(x, *args) - with x.context as ctx: if self._active: return self._call_cached_op(x, *args) @@ -376,11 +391,12 @@ def forward(self, x, *args): i._finish_deferred_init() params = {i: j.data(ctx) for i, j in self._reg_params.items()} return self.hybrid_forward(ndarray, x, *args, **params) - else: - assert isinstance(x, Symbol), \ - "HybridBlock requires the first argument to forward be either " \ - "Symbol or NDArray, but got %s"%type(x) - params = {i: j.var() for i, j in self._reg_params.items()} + + assert isinstance(x, Symbol), \ + "HybridBlock requires the first argument to forward be either " \ + "Symbol or NDArray, but got %s"%type(x) + params = {i: j.var() for i, j in self._reg_params.items()} + with self.name_scope(): return self.hybrid_forward(symbol, x, *args, **params) def hybrid_forward(self, F, x, *args, **kwargs): @@ -395,3 +411,83 @@ def hybrid_forward(self, F, x, *args, **kwargs): """ # pylint: disable= invalid-name raise NotImplementedError + + +class SymbolBlock(HybridBlock): + """Construct block from symbol. This is useful for using pre-trained models + as feature extractors. For example, you may want to extract get the output + from fc2 layer in AlexNet. + + Parameters + ---------- + outputs : Symbol or list of Symbol + The desired output for SymbolBlock. + inputs : Symbol or list of Symbol + The Variables in output's argument that should be used as inputs. + params : ParameterDict + Parameter dictionary for arguments and auxililary states of outputs + that are not inputs. + + Examples + -------- + >>> # To extract the feature from fc1 and fc2 layers of AlexNet: + >>> alexnet = gluon.model_zoo.vision.alexnet(pretrained=True, ctx=mx.cpu(), + prefix='model_') + >>> inputs = mx.sym.var('data') + >>> out = alexnet(inputs) + >>> internals = out.get_internals() + >>> print(internals.list_outputs()) + ['data', ..., 'model_dense0_relu_fwd_output', ..., 'model_dense1_relu_fwd_output', ...] + >>> outputs = [internals['model_dense0_relu_fwd_output'], + internals['model_dense1_relu_fwd_output']] + >>> # Create SymbolBlock that shares parameters with alexnet + >>> feat_model = gluon.SymbolBlock(outputs, inputs, params=alexnet.collect_params()) + >>> x = mx.nd.random_normal(shape=(16, 3, 224, 224)) + >>> print(feat_model(x)) + """ + def __init__(self, outputs, inputs, params=None): + super(SymbolBlock, self).__init__(prefix=None, params=None) + self._prefix = '' + self._params = ParameterDict('', params) + if isinstance(inputs, symbol.Symbol) and len(inputs.list_outputs()) == 1: + inputs = [inputs] + if isinstance(outputs, symbol.Symbol) and len(outputs.list_outputs()) == 1: + outputs = [outputs] + + syms, self._in_format = _flatten(inputs) + out, self._out_format = _flatten(outputs) + out = symbol.Group(out) + + input_names = set() + for i in syms: + assert len(i.get_internals().list_outputs()) == 1, \ + "Input symbols must be variable, but %s is an output of operators"%str(i) + input_names.add(i.name) + + for i in out.list_arguments(): + if i not in input_names: + self.params.get(i, allow_deferred_init=True) + + for i in out.list_auxiliary_states(): + if i not in input_names: + self.params.get(i, grad_req='null', allow_deferred_init=True) + + self._cached_graph = syms, out + self._build_cache() + + def forward(self, x, *args): + if isinstance(x, NDArray): + with x.context: + return self._call_cached_op(x, *args) + + assert isinstance(x, Symbol), \ + "HybridBlock requires the first argument to forward be either " \ + "Symbol or NDArray, but got %s"%type(x) + args, in_fmt = _flatten([x] + list(args)) + assert in_fmt == self._in_format, "Invalid input format" + ret = copy.copy(self._cached_graph[1]) + ret._compose(**{k.name: v for k, v in zip(self._cached_graph[0], args)}) + return _regroup(ret, self._out_format)[0] + + def hybrid_forward(self, F, x, *args, **kwargs): + raise NotImplementedError diff --git a/python/mxnet/gluon/model_zoo/vision/alexnet.py b/python/mxnet/gluon/model_zoo/vision/alexnet.py index dd5104d062f2..86ff9324baca 100644 --- a/python/mxnet/gluon/model_zoo/vision/alexnet.py +++ b/python/mxnet/gluon/model_zoo/vision/alexnet.py @@ -21,27 +21,27 @@ def __init__(self, classes=1000, **kwargs): with self.name_scope(): self.features = nn.HybridSequential(prefix='') with self.features.name_scope(): - self.features.add(nn.Conv2D(64, kernel_size=11, strides=4, padding=2)) - self.features.add(nn.Activation('relu')) + self.features.add(nn.Conv2D(64, kernel_size=11, strides=4, + padding=2, activation='relu')) self.features.add(nn.MaxPool2D(pool_size=3, strides=2)) - self.features.add(nn.Conv2D(192, kernel_size=5, padding=2)) - self.features.add(nn.Activation('relu')) + self.features.add(nn.Conv2D(192, kernel_size=5, padding=2, + activation='relu')) self.features.add(nn.MaxPool2D(pool_size=3, strides=2)) - self.features.add(nn.Conv2D(384, kernel_size=3, padding=1)) - self.features.add(nn.Activation('relu')) - self.features.add(nn.Conv2D(256, kernel_size=3, padding=1)) - self.features.add(nn.Activation('relu')) - self.features.add(nn.Conv2D(256, kernel_size=3, padding=1)) - self.features.add(nn.Activation('relu')) + self.features.add(nn.Conv2D(384, kernel_size=3, padding=1, + activation='relu')) + self.features.add(nn.Conv2D(256, kernel_size=3, padding=1, + activation='relu')) + self.features.add(nn.Conv2D(256, kernel_size=3, padding=1, + activation='relu')) self.features.add(nn.MaxPool2D(pool_size=3, strides=2)) self.features.add(nn.Flatten()) self.classifier = nn.HybridSequential(prefix='') with self.classifier.name_scope(): - self.classifier.add(nn.Dropout(0.5)) self.classifier.add(nn.Dense(4096, activation='relu')) self.classifier.add(nn.Dropout(0.5)) self.classifier.add(nn.Dense(4096, activation='relu')) + self.classifier.add(nn.Dropout(0.5)) self.classifier.add(nn.Dense(classes)) def hybrid_forward(self, F, x): diff --git a/python/mxnet/gluon/nn/basic_layers.py b/python/mxnet/gluon/nn/basic_layers.py index 6a2000e418dc..063deb4ba069 100644 --- a/python/mxnet/gluon/nn/basic_layers.py +++ b/python/mxnet/gluon/nn/basic_layers.py @@ -132,15 +132,17 @@ def __init__(self, units, activation=None, use_bias=True, else: self.bias = None if activation is not None: - self.act = Activation(activation) + self.act = Activation(activation, prefix=activation+'_') else: self.act = None def hybrid_forward(self, F, x, weight, bias=None): if bias is None: - act = F.FullyConnected(x, weight, no_bias=True, num_hidden=self._units) + act = F.FullyConnected(x, weight, no_bias=True, num_hidden=self._units, + name='fwd') else: - act = F.FullyConnected(x, weight, bias, num_hidden=self._units) + act = F.FullyConnected(x, weight, bias, num_hidden=self._units, + name='fwd') if self.act is not None: act = self.act(act) return act @@ -177,7 +179,7 @@ def _alias(self): return self._act_type def hybrid_forward(self, F, x): - return F.Activation(x, act_type=self._act_type) + return F.Activation(x, act_type=self._act_type, name='fwd') def __repr__(self): s = '{name}({_act_type})' @@ -213,7 +215,7 @@ def __init__(self, rate, **kwargs): self._rate = rate def hybrid_forward(self, F, x): - return F.Dropout(x, p=self._rate) + return F.Dropout(x, p=self._rate, name='fwd') def __repr__(self): s = '{name}(p = {_rate})' @@ -271,7 +273,7 @@ def __init__(self, axis=1, momentum=0.9, epsilon=1e-5, center=True, scale=True, in_channels=0, **kwargs): super(BatchNorm, self).__init__(**kwargs) self._kwargs = {'axis': axis, 'eps': epsilon, 'momentum': momentum, - 'fix_gamma': not center} + 'fix_gamma': not scale} if in_channels != 0: self.in_channels = in_channels @@ -291,7 +293,8 @@ def __init__(self, axis=1, momentum=0.9, epsilon=1e-5, center=True, scale=True, allow_deferred_init=True) def hybrid_forward(self, F, x, gamma, beta, running_mean, running_var): - return F.BatchNorm(x, gamma, beta, running_mean, running_var, **self._kwargs) + return F.BatchNorm(x, gamma, beta, running_mean, running_var, + name='fwd', **self._kwargs) def __repr__(self): s = '{name}({content}' @@ -328,7 +331,7 @@ def __init__(self, alpha, **kwargs): self._alpha = alpha def hybrid_forward(self, F, x): - return F.LeakyReLU(x, act_type='leaky', slope=self._alpha) + return F.LeakyReLU(x, act_type='leaky', slope=self._alpha, name='fwd') def __repr__(self): s = '{name}({alpha})' @@ -369,11 +372,11 @@ def __init__(self, input_dim, output_dim, dtype='float32', allow_deferred_init=True) def hybrid_forward(self, F, x, weight): - return F.Embedding(x, weight, **self._kwargs) + return F.Embedding(x, weight, name='fwd', **self._kwargs) def __repr__(self): - s = '{name}({input_dim} -> {output_dim}, {dtype})' - return s.format(name=self.__class__.__name__, + s = '{block_name}({input_dim} -> {output_dim}, {dtype})' + return s.format(block_name=self.__class__.__name__, **self._kwargs) diff --git a/python/mxnet/gluon/nn/conv_layers.py b/python/mxnet/gluon/nn/conv_layers.py index bb2ffea96c11..d9608a151220 100644 --- a/python/mxnet/gluon/nn/conv_layers.py +++ b/python/mxnet/gluon/nn/conv_layers.py @@ -98,15 +98,15 @@ def __init__(self, channels, kernel_size, strides, padding, dilation, self.bias = None if activation is not None: - self.act = Activation(activation) + self.act = Activation(activation, prefix=activation+'_') else: self.act = None def hybrid_forward(self, F, x, weight, bias=None): if bias is None: - act = getattr(F, self._op_name)(x, weight, **self._kwargs) + act = getattr(F, self._op_name)(x, weight, name='fwd', **self._kwargs) else: - act = getattr(F, self._op_name)(x, weight, bias, **self._kwargs) + act = getattr(F, self._op_name)(x, weight, bias, name='fwd', **self._kwargs) if self.act is not None: act = self.act(act) return act @@ -644,8 +644,11 @@ def __init__(self, pool_size, strides, padding, ceil_mode, global_pool, 'global_pool': global_pool, 'pool_type': pool_type, 'pooling_convention': 'full' if ceil_mode else 'valid'} + def _alias(self): + return 'pool' + def hybrid_forward(self, F, x): - return F.Pooling(x, **self._kwargs) + return F.Pooling(x, name='fwd', **self._kwargs) def __repr__(self): s = '{name}(size={kernel}, stride={stride}, padding={pad}, ceil_mode={ceil_mode})' diff --git a/python/mxnet/gluon/parameter.py b/python/mxnet/gluon/parameter.py index 981b78b721e7..657981cbd6c3 100644 --- a/python/mxnet/gluon/parameter.py +++ b/python/mxnet/gluon/parameter.py @@ -3,6 +3,7 @@ """Neural network parameter.""" from collections import OrderedDict +import warnings import numpy as np @@ -82,59 +83,22 @@ def __repr__(self): s = 'Parameter {name} (shape={shape}, dtype={dtype})' return s.format(**self.__dict__) - def initialize(self, init=None, ctx=None, default_init=initializer.Uniform()): - """Initializes parameter and gradient arrays. Only used for `NDArray` API. - - Parameters - ---------- - init : Initializer - The initializer to use. Overrides `Parameter.init` and default_init. - ctx : Context or list of Context, defaults to `context.current_context()`. - Initialize Parameter on given context. If ctx is a list of Context, a - copy will be made for each context. - - .. note:: Copies are independent arrays. User is responsible for keeping - their values consistent when updating. Normally `gluon.Trainer` does this for you. - default_init : Initializer - Default initializer is used when both `init` and `Parameter.init` are `None`. - - Examples - -------- - >>> weight = mx.gluon.Parameter('weight', shape=(2, 2)) - >>> weight.initialize(ctx=mx.cpu(0)) - >>> weight.data() - [[-0.01068833 0.01729892] - [ 0.02042518 -0.01618656]] - - >>> weight.grad() - [[ 0. 0.] - [ 0. 0.]] - - >>> weight.initialize(ctx=[mx.gpu(0), mx.gpu(1)]) - >>> weight.data(mx.gpu(0)) - [[-0.00873779 -0.02834515] - [ 0.05484822 -0.06206018]] - - >>> weight.data(mx.gpu(1)) - [[-0.00873779 -0.02834515] - [ 0.05484822 -0.06206018]] - - """ - if ctx is None: - ctx = [context.current_context()] - if isinstance(ctx, Context): - ctx = [ctx] - if init is None: - init = default_init if self.init is None else self.init - if not self.shape or np.prod(self.shape) <= 0: - if self.allow_deferred_init: - self._defered_init = (init, ctx, default_init) - return - raise ValueError("Cannot initialize Parameter %s because it has " \ - "invalid shape: %s."%(self.name, str(self.shape))) - - self._defered_init = (init, ctx, default_init) - self._finish_deferred_init() + def _check_initialized(self, ctx=None): + if self._data is not None: + if ctx is not None and ctx not in self._data: + raise RuntimeError( + "Parameter %s was not initialized on context %s. " + "It was only initialized on %s."%( + self.name, str(ctx), str(self.list_ctx()))) + return + if self._defered_init: + raise DeferredInitializationError + raise RuntimeError( + "Parameter %s has not been initialized. Note that " \ + "you should initialize parameters and create Trainer " \ + "with Block.collect_params() instead of Block.params " \ + "because the later does not include Parameters of " \ + "nested child Blocks"%(self.name)) def _load_init(self, data, ctx): """(Re)initializes by loading from data.""" @@ -202,6 +166,98 @@ def _init_impl(self, data, ctx): autograd.mark_variables(self.list_data(), self.list_grad(), self.grad_req) + def _reduce(self): + """Reduce data from multiple context.""" + block = self.list_data() + data = ndarray.add_n(*(w.copyto(context.cpu()) for w in block)) / len(block) + return data + + def initialize(self, init=None, ctx=None, default_init=initializer.Uniform(), + force_reinit=False): + """Initializes parameter and gradient arrays. Only used for `NDArray` API. + + Parameters + ---------- + init : Initializer + The initializer to use. Overrides `Parameter.init` and default_init. + ctx : Context or list of Context, defaults to `context.current_context()`. + Initialize Parameter on given context. If ctx is a list of Context, a + copy will be made for each context. + + .. note:: Copies are independent arrays. User is responsible for keeping + their values consistent when updating. Normally `gluon.Trainer` does this for you. + default_init : Initializer + Default initializer is used when both `init` and `Parameter.init` are `None`. + force_reinit : bool, default False + Whether to force re-initialization if parameter is already initialized. + + Examples + -------- + >>> weight = mx.gluon.Parameter('weight', shape=(2, 2)) + >>> weight.initialize(ctx=mx.cpu(0)) + >>> weight.data() + [[-0.01068833 0.01729892] + [ 0.02042518 -0.01618656]] + + >>> weight.grad() + [[ 0. 0.] + [ 0. 0.]] + + >>> weight.initialize(ctx=[mx.gpu(0), mx.gpu(1)]) + >>> weight.data(mx.gpu(0)) + [[-0.00873779 -0.02834515] + [ 0.05484822 -0.06206018]] + + >>> weight.data(mx.gpu(1)) + [[-0.00873779 -0.02834515] + [ 0.05484822 -0.06206018]] + + """ + if self._data is not None and not force_reinit: + warnings.warn("Parameter %s is already initialized, ignoring. " \ + "Set force_reinit=True to re-initialize."%self.name) + return + self._data = self._grad = None + + if ctx is None: + ctx = [context.current_context()] + if isinstance(ctx, Context): + ctx = [ctx] + if init is None: + init = default_init if self.init is None else self.init + if not self.shape or np.prod(self.shape) <= 0: + if self.allow_deferred_init: + self._defered_init = (init, ctx, default_init) + return + raise ValueError("Cannot initialize Parameter %s because it has " \ + "invalid shape: %s."%(self.name, str(self.shape))) + + self._defered_init = (init, ctx, default_init) + self._finish_deferred_init() + + def reset_ctx(self, ctx): + """Re-assign Parameter to other contexts. + + ctx : Context or list of Context, default `context.current_context()`. + Assign Parameter to given context. If ctx is a list of Context, a + copy will be made for each context. + """ + if ctx is None: + ctx = [context.current_context()] + if isinstance(ctx, Context): + ctx = [ctx] + if self._data: + data = self._reduce() + with autograd.pause(): + self._init_impl(data, ctx) + elif self._defered_init: + init, _, default_init = self._defered_init + self._defered_init = (init, ctx, default_init) + else: + raise ValueError("Cannot reset context for Parameter %s because it " + "has not been initialized."%self.name) + + def set_data(self, data): """Sets this parameter's value on all contexts to data.""" assert self._data is not None, \ @@ -209,23 +265,6 @@ def set_data(self, data): for arr in self.list_data(): arr[:] = data - def _check_initialized(self, ctx=None): - if self._data is not None: - if ctx is not None and ctx not in self._data: - raise RuntimeError( - "Parameter %s was not initialized on context %s. " - "It was only initialized on %s."%( - self.name, str(ctx), str(self.list_ctx()))) - return - if self._defered_init: - raise DeferredInitializationError - raise RuntimeError( - "Parameter %s has not been initialized. Note that " \ - "you should initialize parameters and create Trainer " \ - "with Block.collect_params() instead of Block.params " \ - "because the later does not include Parameters of " \ - "nested child Blocks"%(self.name)) - def data(self, ctx=None): """Returns a copy of this parameter on one context. Must have been initialized on this context before. @@ -404,7 +443,8 @@ def update(self, other): else: self._params[k] = v - def initialize(self, init=initializer.Uniform(), ctx=None, verbose=False): + def initialize(self, init=initializer.Uniform(), ctx=None, verbose=False, + force_reinit=False): """Initializes all Parameters managed by this dictionary to be used for `NDArray` API. It has no effect when using `Symbol` API. @@ -415,17 +455,29 @@ def initialize(self, init=initializer.Uniform(), ctx=None, verbose=False): Otherwise, `Parameter.init` takes precedence. ctx : Context or list of Context Keeps a copy of Parameters on one or many context(s). + force_reinit : bool, default False + Whether to force re-initialization if parameter is already initialized. """ if verbose: init.set_verbosity(verbose=verbose) for _, v in self.items(): - v.initialize(None, ctx, init) + v.initialize(None, ctx, init, force_reinit=force_reinit) def zero_grad(self): """Sets all Parameters' gradient buffer to 0.""" for i in self.values(): i.zero_grad() + def reset_ctx(self, ctx): + """Re-assign all Parameters to other contexts. + + ctx : Context or list of Context, default `context.current_context()`. + Assign Parameter to given context. If ctx is a list of Context, a + copy will be made for each context. + """ + for i in self.values(): + i.reset_ctx(ctx) + def save(self, filename, strip_prefix=''): """Save parameters to file. @@ -436,8 +488,7 @@ def save(self, filename, strip_prefix=''): """ arg_dict = {} for param in self.values(): - block = param.list_data() - weight = sum(w.copyto(context.cpu()) for w in block) / len(block) + weight = param._reduce() if not param.name.startswith(strip_prefix): raise ValueError( "Prefix %s is to be striped before saving, but Parameter " \ diff --git a/python/mxnet/gluon/rnn/rnn_cell.py b/python/mxnet/gluon/rnn/rnn_cell.py index e06599cc89d3..7315a2783223 100644 --- a/python/mxnet/gluon/rnn/rnn_cell.py +++ b/python/mxnet/gluon/rnn/rnn_cell.py @@ -108,10 +108,6 @@ def state_info(self, batch_size=0): """shape and layout information of states""" raise NotImplementedError() - @property - def _curr_prefix(self): - return '%st%d_'%(self.prefix, self._counter) - def begin_state(self, batch_size=0, func=ndarray.zeros, **kwargs): """Initial state for this cell. @@ -313,15 +309,15 @@ def _alias(self): def hybrid_forward(self, F, inputs, states, i2h_weight, h2h_weight, i2h_bias, h2h_bias): - name = self._curr_prefix + prefix = 't%d_'%self._counter i2h = F.FullyConnected(data=inputs, weight=i2h_weight, bias=i2h_bias, num_hidden=self._hidden_size, - name='%si2h'%name) + name=prefix+'i2h') h2h = F.FullyConnected(data=states[0], weight=h2h_weight, bias=h2h_bias, num_hidden=self._hidden_size, - name='%sh2h'%name) + name=prefix+'h2h') output = self._get_activation(F, i2h + h2h, self._activation, - name='%sout'%name) + name=prefix+'out') return output, [output] @@ -382,28 +378,21 @@ def _alias(self): def hybrid_forward(self, F, inputs, states, i2h_weight, h2h_weight, i2h_bias, h2h_bias): - name = self._curr_prefix + prefix = 't%d_'%self._counter i2h = F.FullyConnected(data=inputs, weight=i2h_weight, bias=i2h_bias, - num_hidden=self._hidden_size*4, - name='%si2h'%name) + num_hidden=self._hidden_size*4, name=prefix+'i2h') h2h = F.FullyConnected(data=states[0], weight=h2h_weight, bias=h2h_bias, - num_hidden=self._hidden_size*4, - name='%sh2h'%name) + num_hidden=self._hidden_size*4, name=prefix+'h2h') gates = i2h + h2h - slice_gates = F.SliceChannel(gates, num_outputs=4, - name="%sslice"%name) - in_gate = F.Activation(slice_gates[0], act_type="sigmoid", - name='%si'%name) - forget_gate = F.Activation(slice_gates[1], act_type="sigmoid", - name='%sf'%name) - in_transform = F.Activation(slice_gates[2], act_type="tanh", - name='%sc'%name) - out_gate = F.Activation(slice_gates[3], act_type="sigmoid", - name='%so'%name) + slice_gates = F.SliceChannel(gates, num_outputs=4, name=prefix+'slice') + in_gate = F.Activation(slice_gates[0], act_type="sigmoid", name=prefix+'i') + forget_gate = F.Activation(slice_gates[1], act_type="sigmoid", name=prefix+'f') + in_transform = F.Activation(slice_gates[2], act_type="tanh", name=prefix+'c') + out_gate = F.Activation(slice_gates[3], act_type="sigmoid", name=prefix+'o') next_c = F._internal._plus(forget_gate * states[1], in_gate * in_transform, - name='%sstate'%name) + name=prefix+'state') next_h = F._internal._mul(out_gate, F.Activation(next_c, act_type="tanh"), - name='%sout'%name) + name=prefix+'out') return next_h, [next_h, next_c] @@ -463,32 +452,34 @@ def _alias(self): def hybrid_forward(self, F, inputs, states, i2h_weight, h2h_weight, i2h_bias, h2h_bias): # pylint: disable=too-many-locals - name = self._curr_prefix + prefix = 't%d_'%self._counter prev_state_h = states[0] i2h = F.FullyConnected(data=inputs, weight=i2h_weight, bias=i2h_bias, num_hidden=self._hidden_size * 3, - name="%si2h" % name) + name=prefix+'i2h') h2h = F.FullyConnected(data=prev_state_h, weight=h2h_weight, bias=h2h_bias, num_hidden=self._hidden_size * 3, - name="%sh2h" % name) + name=prefix+'h2h') - i2h_r, i2h_z, i2h = F.SliceChannel(i2h, num_outputs=3, name="%si2h_slice" % name) - h2h_r, h2h_z, h2h = F.SliceChannel(h2h, num_outputs=3, name="%sh2h_slice" % name) + i2h_r, i2h_z, i2h = F.SliceChannel(i2h, num_outputs=3, + name=prefix+'i2h_slice') + h2h_r, h2h_z, h2h = F.SliceChannel(h2h, num_outputs=3, + name=prefix+'h2h_slice') reset_gate = F.Activation(i2h_r + h2h_r, act_type="sigmoid", - name="%sr_act" % name) + name=prefix+'r_act') update_gate = F.Activation(i2h_z + h2h_z, act_type="sigmoid", - name="%sz_act" % name) + name=prefix+'z_act') next_h_tmp = F.Activation(i2h + reset_gate * h2h, act_type="tanh", - name="%sh_act" % name) + name=prefix+'h_act') next_h = F._internal._plus((1. - update_gate) * next_h_tmp, update_gate * prev_state_h, - name='%sout' % name) + name=prefix+'out') return next_h, [next_h] @@ -563,17 +554,17 @@ class DropoutCell(HybridRecurrentCell): Parameters ---------- - dropout : float + rate : float Percentage of elements to drop out, which is 1 - percentage to retain. """ - def __init__(self, dropout, prefix=None, params=None): + def __init__(self, rate, prefix=None, params=None): super(DropoutCell, self).__init__(prefix, params) - assert isinstance(dropout, numeric_types), "dropout probability must be a number" - self.dropout = dropout + assert isinstance(rate, numeric_types), "rate must be a number" + self.rate = rate def __repr__(self): - s = '{name}(p = {dropout})' + s = '{name}(rate = {rate})' return s.format(name=self.__class__.__name__, **self.__dict__) @@ -584,8 +575,8 @@ def _alias(self): return 'dropout' def hybrid_forward(self, F, inputs, states): - if self.dropout > 0: - inputs = F.Dropout(data=inputs, p=self.dropout) + if self.rate > 0: + inputs = F.Dropout(data=inputs, p=self.rate, name='t%d_fwd'%self._counter) return inputs, states def unroll(self, length, inputs, begin_state=None, layout='NTC', merge_outputs=None): @@ -610,13 +601,15 @@ class ModifierCell(HybridRecurrentCell): should be used instead. """ def __init__(self, base_cell): - super(ModifierCell, self).__init__(prefix=None, params=None) + assert not base_cell._modified, \ + "Cell %s is already modified. One cell cannot be modified twice"%base_cell.name base_cell._modified = True + super(ModifierCell, self).__init__(prefix=base_cell.prefix+self._alias(), + params=None) self.base_cell = base_cell @property def params(self): - self._own_params = False return self.base_cell.params def state_info(self, batch_size=0): @@ -697,7 +690,7 @@ def __init__(self, base_cell): def hybrid_forward(self, F, inputs, states): output, states = self.base_cell(inputs, states) - output = F.elemwise_add(output, inputs, name="%s_plus_residual" % output.name) + output = F.elemwise_add(output, inputs, name='t%d_fwd'%self._counter) return output, states def unroll(self, length, inputs, begin_state=None, layout='NTC', merge_outputs=None): diff --git a/python/mxnet/symbol.py b/python/mxnet/symbol.py index 56ab27b15dd7..f467f9c860c5 100644 --- a/python/mxnet/symbol.py +++ b/python/mxnet/symbol.py @@ -3,6 +3,10 @@ # pylint: disable=import-error, no-name-in-module """Symbolic configuration API of MXNet.""" from __future__ import absolute_import as _abs +try: + from __builtin__ import slice as py_slice +except ImportError: + from builtins import slice as py_slice import ctypes import warnings @@ -484,9 +488,16 @@ def __getitem__(self, index): Indexing key """ + output_names = self.list_outputs() + if isinstance(index, py_slice): + start = 0 if index.start is None else index.start + stop = len(output_names) if index.stop is None else index.stop + step = 1 if index.step is None else index.step + return Group([self[i] for i in range(start, stop, step)]) + if isinstance(index, string_types): idx = None - for i, name in enumerate(self.list_outputs()): + for i, name in enumerate(output_names): if name == index: if idx is not None: raise ValueError('There are multiple outputs with name \"%s\"' % index) @@ -494,9 +505,10 @@ def __getitem__(self, index): if idx is None: raise ValueError('Cannot find output that matches name \"%s\"' % index) index = idx + if not isinstance(index, int): raise TypeError('Symbol only support integer index to fetch i-th output') - if index >= (len(self.list_outputs())): + if index >= len(output_names): # Important, python determines the end by this exception raise IndexError handle = SymbolHandle() diff --git a/src/ndarray/autograd.cc b/src/ndarray/autograd.cc index b606a4dcdaa8..f990ee2973fd 100644 --- a/src/ndarray/autograd.cc +++ b/src/ndarray/autograd.cc @@ -133,7 +133,8 @@ AGNodePtr AutogradRuntime::RecordOp(const nnvm::Op* op, for (uint32_t i = 0; i < outputs.size(); ++i) { CHECK(outputs[i].entry_.is_none()) - << "Inplace operation is not supported when recording with autograd. " + << "Inplace operations (+=, -=, x[:]=, etc) are not supported when " + << "recording with autograd. " << "Assigning to NDArrays that are already in a computational graph " << "will cause undefined behavior when evaluating gradients. " << "Please call backward first to clear the graph or do this out side of " diff --git a/tests/python/unittest/test_nn.py b/tests/python/unittest/test_nn.py index 58839785b9f2..d4514e2eb77e 100644 --- a/tests/python/unittest/test_nn.py +++ b/tests/python/unittest/test_nn.py @@ -13,6 +13,9 @@ def test_parameter(): assert p.data(mx.cpu(0)).shape == (10, 10) assert p.var().name == 'weight' + p.reset_ctx(ctx=[mx.cpu(1), mx.cpu(2)]) + assert p.list_ctx() == [mx.cpu(1), mx.cpu(2)] + def test_paramdict(): params = gluon.ParameterDict('net_') @@ -65,6 +68,27 @@ def test_basic(): x.wait_to_read() +def test_symbol_block(): + model = nn.HybridSequential() + model.add(nn.Dense(128, activation='tanh')) + model.add(nn.Dropout(0.5)) + model.add(nn.Dense(64, activation='tanh')) + model.add(nn.Dense(32, in_units=64)) + model.add(nn.Activation('relu')) + + model.initialize() + + inputs = mx.sym.var('data') + outputs = model(inputs).get_internals() + + smodel = gluon.SymbolBlock(outputs, inputs, params=model.collect_params()) + + assert len(smodel(mx.nd.zeros((16, 10)))) == 14 + + out = smodel(mx.sym.var('in')) + assert len(out.get_internals().list_outputs()) == len(outputs.list_outputs()) + + def check_layer_forward(layer, dshape): layer.collect_params().initialize() with mx.autograd.record(): From 1ed5f9fea5ba399bfdf3e3808123f1b153397c44 Mon Sep 17 00:00:00 2001 From: Leonard Lausen Date: Mon, 31 Jul 2017 11:57:27 +0900 Subject: [PATCH 280/834] Fixes for gluon RNN [WIP] (#7258) * Fix documentation on recurrent state shape for gluon * Fix state info for gluon rnn_layers * Revert "Fix state info for gluon rnn_layers" This reverts commit 5f791d6e8b643f08dfde6475bef396d9abddb875. --- python/mxnet/gluon/rnn/rnn_layer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/mxnet/gluon/rnn/rnn_layer.py b/python/mxnet/gluon/rnn/rnn_layer.py index a22cb0138bcb..d5673690e587 100644 --- a/python/mxnet/gluon/rnn/rnn_layer.py +++ b/python/mxnet/gluon/rnn/rnn_layer.py @@ -268,7 +268,7 @@ class RNN(_RNNLayer): Recurrent state shape: The recurrent state's shape is `(num_layers, batch_size, num_hidden)`. If `bidirectional` is True, state shape will instead be - `(num_layers, batch_size, 2*num_hidden)` + `(2*num_layers, batch_size, num_hidden)` Examples @@ -366,7 +366,7 @@ class LSTM(_RNNLayer): The recurrent state is a list of two NDArrays. Both has shape `(num_layers, batch_size, num_hidden)`. If `bidirectional` is True, state shape will instead be - `(num_layers, batch_size, 2*num_hidden)`. + `(2*num_layers, batch_size, num_hidden)`. Examples @@ -460,7 +460,7 @@ class GRU(_RNNLayer): Recurrent state shape: The recurrent state's shape is `(num_layers, batch_size, num_hidden)`. If `bidirectional` is True, state shape will instead be - `(num_layers, batch_size, 2*num_hidden)` + `(2*num_layers, batch_size, num_hidden)` Examples From 414a96c277b57bab850295c764f8ed9ae95d3eec Mon Sep 17 00:00:00 2001 From: Terence Wu <2326428753@qq.com> Date: Tue, 1 Aug 2017 00:12:03 +0800 Subject: [PATCH 281/834] Fix a spelling mistake (#7266) --- python/mxnet/symbol.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/mxnet/symbol.py b/python/mxnet/symbol.py index f467f9c860c5..a6c2b0ab689d 100644 --- a/python/mxnet/symbol.py +++ b/python/mxnet/symbol.py @@ -2180,7 +2180,7 @@ def arange(start, stop=None, step=1.0, repeat=1, name=None, dtype=None): def _make_atomic_symbol_function(handle, name): - """Create an atomic symbol function by handle and funciton name.""" + """Create an atomic symbol function by handle and function name.""" real_name = ctypes.c_char_p() desc = ctypes.c_char_p() num_args = mx_uint() From 024525f051fdebab3276db91f7000a911323fd00 Mon Sep 17 00:00:00 2001 From: Leonard Lausen Date: Tue, 1 Aug 2017 01:14:53 +0900 Subject: [PATCH 282/834] Fix gluon zero grad (#7263) --- python/mxnet/gluon/parameter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/mxnet/gluon/parameter.py b/python/mxnet/gluon/parameter.py index 657981cbd6c3..1c311ef687e3 100644 --- a/python/mxnet/gluon/parameter.py +++ b/python/mxnet/gluon/parameter.py @@ -335,7 +335,7 @@ def zero_grad(self): parameter is uninitialized or doesn't require gradient.""" if self._grad is None: return - for i in self._grad: + for i in self._grad.values(): i[:] = 0 def var(self): From e8301791498473e4b9fc3dd48e6c177a93513091 Mon Sep 17 00:00:00 2001 From: Mu Li Date: Mon, 31 Jul 2017 14:34:42 -0700 Subject: [PATCH 283/834] [Doc] Add the new doc link to to hybrid.md (#7271) --- docs/tutorials/gluon/hybrid.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/tutorials/gluon/hybrid.md b/docs/tutorials/gluon/hybrid.md index e128ff586e76..7e043c816402 100644 --- a/docs/tutorials/gluon/hybrid.md +++ b/docs/tutorials/gluon/hybrid.md @@ -1,5 +1,7 @@ # Hybrid - Faster training and easy deployment +*Note: a newer version is available [here](http://gluon.mxnet.io/P14-C05-hybridize.html).* + Deep learning frameworks can be roughly divided into two categories: declarative and imperative. With declarative frameworks (including Tensorflow, Theano, etc) users first declare a fixed computation graph and then execute it end-to-end. From c521fa5d12d88f3b8ad0917ed8ad66093d9631b9 Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Mon, 31 Jul 2017 15:58:22 -0700 Subject: [PATCH 284/834] add getitem to container layers (#7265) * add getitem to container layers * add __len__ * move __len__ to container classes --- python/mxnet/gluon/nn/basic_layers.py | 12 ++++++++++++ python/mxnet/gluon/rnn/rnn_cell.py | 6 ++++++ 2 files changed, 18 insertions(+) diff --git a/python/mxnet/gluon/nn/basic_layers.py b/python/mxnet/gluon/nn/basic_layers.py index 063deb4ba069..afe2df7b5f4c 100644 --- a/python/mxnet/gluon/nn/basic_layers.py +++ b/python/mxnet/gluon/nn/basic_layers.py @@ -38,6 +38,12 @@ def __repr__(self): return s.format(name=self.__class__.__name__, modstr=modstr) + def __getitem__(self, i): + return self._children[i] + + def __len__(self): + return len(self._children) + class HybridSequential(HybridBlock): """Stacks `HybridBlock`s sequentially. @@ -71,6 +77,12 @@ def __repr__(self): return s.format(name=self.__class__.__name__, modstr=modstr) + def __getitem__(self, i): + return self._children[i] + + def __len__(self): + return len(self._children) + class Dense(HybridBlock): """Just your regular densely-connected NN layer. diff --git a/python/mxnet/gluon/rnn/rnn_cell.py b/python/mxnet/gluon/rnn/rnn_cell.py index 7315a2783223..e6ce65b31df0 100644 --- a/python/mxnet/gluon/rnn/rnn_cell.py +++ b/python/mxnet/gluon/rnn/rnn_cell.py @@ -545,6 +545,12 @@ def unroll(self, length, inputs, begin_state=None, layout='NTC', merge_outputs=N return inputs, next_states + def __getitem__(self, i): + return self._children[i] + + def __len__(self): + return len(self._children) + def hybrid_forward(self, *args, **kwargs): raise NotImplementedError From 92a93b803e58d0df3011584ae6d04684f928f759 Mon Sep 17 00:00:00 2001 From: "Qiang Kou (KK)" Date: Tue, 1 Aug 2017 01:09:14 +0000 Subject: [PATCH 285/834] [R][DOC] make sure all code in vignettes can run without error (#7274) --- .../vignettes/CallbackFunctionTutorial.Rmd | 32 +++++------- ...sDogsFinetune.rmd => CatsDogsFinetune.Rmd} | 18 +++---- R-package/vignettes/CharRnnModel.Rmd | 52 +++++++++++++------ .../classifyRealImageWithPretrainedModel.Rmd | 17 +++--- .../vignettes/fiveMinutesNeuralNetwork.Rmd | 1 - 5 files changed, 67 insertions(+), 53 deletions(-) rename R-package/vignettes/{CatsDogsFinetune.rmd => CatsDogsFinetune.Rmd} (93%) diff --git a/R-package/vignettes/CallbackFunctionTutorial.Rmd b/R-package/vignettes/CallbackFunctionTutorial.Rmd index 97b6ce3161a0..91b4c096ec18 100644 --- a/R-package/vignettes/CallbackFunctionTutorial.Rmd +++ b/R-package/vignettes/CallbackFunctionTutorial.Rmd @@ -1,14 +1,10 @@ -MXNet R Tutorial on Callback Function +MXNet R Tutorial for Callback Function ====================================== This vignette gives users a guideline for using and writing callback functions, -which can very useful in model training. +which can be very useful in model training. -This tutorial is written in Rmarkdown. - -- You can directly view the hosted version of the tutorial from [MXNet R Document](http://mxnet.readthedocs.io/en/latest/packages/r/CallbackFunctionTutorial.html) - -- You can find the Rmarkdown source from [here](https://github.com/dmlc/mxnet/blob/master/R-package/vignettes/CallbackFunctionTutorial.Rmd) +This tutorial is written in Rmarkdown. You can find the Rmarkdown source from [here](https://github.com/dmlc/mxnet/blob/master/R-package/vignettes/CallbackFunctionTutorial.Rmd) Model training example ---------- @@ -50,11 +46,12 @@ model <- mx.model.FeedForward.create( ctx=mx.cpu(), num.round=10, array.batch.size=20, learning.rate=2e-6, momentum=0.9, eval.metric=mx.metric.rmse, epoch.end.callback = mx.callback.save.checkpoint("boston")) +list.files(pattern = "^boston") ``` -- `mx.callback.log.train.metric` is used to log training metric each period. You can use it either as a `batch.end.callback` or a -`epoch.end.callback`. +- `mx.callback.log.train.metric` is used to log training metric each period. +You can use it either as a `batch.end.callback` or a `epoch.end.callback`. ```{r} model <- mx.model.FeedForward.create( @@ -97,26 +94,25 @@ The `mx.callback.save.checkpoint` function below is stateless. It just get the m ```{r, eval=FALSE} mx.callback.save.checkpoint <- function(prefix, period=1) { - function(iteration, nbatch, env, verbose) { + function(iteration, nbatch, env, verbose=TRUE) { if (iteration %% period == 0) { mx.model.save(env$model, prefix, iteration) - if(verbose) cat(sprintf("Model checkpoint saved to %s-%04d.params\n", prefix, iteration)) + if(verbose) message(sprintf("Model checkpoint saved to %s-%04d.params\n", prefix, iteration)) } return(TRUE) } } ``` -The `mx.callback.log.train.metric` is a little more complex. It will hold a reference class and update it during the training -process. +The `mx.callback.log.train.metric` is a little more complex. It holds a reference class and update it during the training process. ```{r, eval=FALSE} mx.callback.log.train.metric <- function(period, logger=NULL) { - function(iteration, nbatch, env, verbose) { + function(iteration, nbatch, env, verbose=TRUE) { if (nbatch %% period == 0 && !is.null(env$metric)) { result <- env$metric$get(env$train.metric) - if (nbatch != 0) - if(verbose) cat(paste0("Batch [", nbatch, "] Train-", result$name, "=", result$value, "\n")) + if (nbatch != 0 & verbose) + message(paste0("Batch [", nbatch, "] Train-", result$name, "=", result$value)) if (!is.null(logger)) { if (class(logger) != "mx.metric.logger") { stop("Invalid mx.metric.logger.") @@ -124,8 +120,8 @@ mx.callback.log.train.metric <- function(period, logger=NULL) { logger$train <- c(logger$train, result$value) if (!is.null(env$eval.metric)) { result <- env$metric$get(env$eval.metric) - if (nbatch != 0) - cat(paste0("Batch [", nbatch, "] Validation-", result$name, "=", result$value, "\n")) + if (nbatch != 0 & verbose) + message(paste0("Batch [", nbatch, "] Validation-", result$name, "=", result$value)) logger$eval <- c(logger$eval, result$value) } } diff --git a/R-package/vignettes/CatsDogsFinetune.rmd b/R-package/vignettes/CatsDogsFinetune.Rmd similarity index 93% rename from R-package/vignettes/CatsDogsFinetune.rmd rename to R-package/vignettes/CatsDogsFinetune.Rmd index c137ee8b7cef..e30b5137a2f1 100644 --- a/R-package/vignettes/CatsDogsFinetune.rmd +++ b/R-package/vignettes/CatsDogsFinetune.Rmd @@ -30,7 +30,7 @@ library(abind) ### Renaming train files ```{r} -files <- list.files("./train") +files <- list.files("./train/") old_names <- sapply(files, strsplit, split = ".", fixed = TRUE) max_length <- max(sapply(old_names, function(x) nchar(x[[2]]))) zeros <- max_length - sapply(old_names, function(x) nchar(x[[2]])) @@ -51,7 +51,7 @@ Map(function(x, y) file.rename(from = x, to = y), files, new_names) ### Training images: 224x224, padded with empty space ```{r} -files <- list.files("./train", recursive = TRUE) +files <- list.files("./train/", recursive = TRUE) new_names <- paste0("./train_pad_224x224/", files) files <- paste0("./train/", files) dir.create("./train_pad_224x224/") @@ -77,7 +77,7 @@ Map(function(x, y) { ### Renaming test files ```{r} -files <- list.files("./test") +files <- list.files("./test/") max_length <- max(sapply(files, nchar)) zeros <- max_length - sapply(files, nchar) zeros <- sapply(zeros, function(x) paste(rep(0, x), collapse = "")) @@ -92,7 +92,7 @@ Map(function(x, y) file.rename(from = x, to = y), files, newnames) ### Test images: 224x224, padded with empty space ```{r} -files <- list.files("./test") +files <- list.files("./test/") new_names <- paste0("./test_pad_224x224/", files) files <- paste0("./test/", files) dir.create("./test_pad_224x224/") @@ -168,11 +168,11 @@ new_soft <- mx.symbol.SoftmaxOutput(data = new_fc, name = "softmax") # set name to original name in symbol$arguments -arg_params_new <- mxnet:::mx.model.init.params(symbol = new_soft, - input.shape = list("data" = c(224, 224, 3, 8)), - output.shape = NULL, - initializer = mx.init.uniform(0.1), - ctx = mx.cpu())$arg.params +arg_params_new <- mx.model.init.params(symbol = new_soft, + input.shape = list("data" = c(224, 224, 3, 8)), + output.shape = NULL, + initializer = mx.init.uniform(0.1), + ctx = mx.cpu())$arg.params fc1_weights_new <- arg_params_new[["fc1_weight"]] fc1_bias_new <- arg_params_new[["fc1_bias"]] diff --git a/R-package/vignettes/CharRnnModel.Rmd b/R-package/vignettes/CharRnnModel.Rmd index 2cb4b00ec1ac..9dc00a39d96b 100644 --- a/R-package/vignettes/CharRnnModel.Rmd +++ b/R-package/vignettes/CharRnnModel.Rmd @@ -1,23 +1,20 @@ Char RNN Example ============================================= -This example aims to show how to use lstm model to build a char level language model, and generate text from it. We use a tiny shakespeare text for demo purpose. - -Data can be found at [here](https://github.com/dmlc/web-data/tree/master/mxnet/tinyshakespeare) - -Preface -------- -This tutorial is written in Rmarkdown. -- You can directly view the hosted version of the tutorial from [MXNet R Document](http://mxnet.readthedocs.io/en/latest/packages/r/CharRnnModel.html) -- You can find the download the Rmarkdown source from [here](https://github.com/dmlc/mxnet/blob/master/R-package/vignettes/CharRnnModel.Rmd) +This example aims to show how to use the LSTM model to build a char-level language model, and generate text from it. We use a tiny shakespeare text for demo purpose. Data can be found at [here](https://github.com/dmlc/web-data/tree/master/mxnet/tinyshakespeare). +This tutorial is written in Rmarkdown. You can find the Rmarkdown source from [here](https://github.com/dmlc/mxnet/blob/master/R-package/vignettes/CharRnnModel.Rmd). Load Data --------- + First of all, load in the data and preprocess it. + ```{r} require(mxnet) ``` + Set basic network parameters. + ```{r} batch.size = 32 seq.len = 32 @@ -30,7 +27,9 @@ wd=0.00001 clip_gradient=1 update.period = 1 ``` -download the data. + +Download the data. + ```{r} download.data <- function(data_dir) { dir.create(data_dir, showWarnings = FALSE) @@ -40,7 +39,9 @@ download.data <- function(data_dir) { } } ``` + Make dictionary from text. + ```{r} make.dict <- function(text, max.vocab=10000) { text <- strsplit(text, '') @@ -58,7 +59,9 @@ make.dict <- function(text, max.vocab=10000) { return (dic) } ``` + Transfer text into data feature. + ```{r} make.data <- function(file.path, seq.len=32, max.vocab=10000, dic=NULL) { fi <- file(file.path, "r") @@ -91,7 +94,9 @@ make.data <- function(file.path, seq.len=32, max.vocab=10000, dic=NULL) { return (list(data=data, dic=dic, lookup.table=lookup.table)) } ``` + Move tail text. + ```{r} drop.tail <- function(X, batch.size) { shape <- dim(X) @@ -99,7 +104,9 @@ drop.tail <- function(X, batch.size) { return (X[, 1:(nstep * batch.size)]) } ``` -get the label of X + +Get the label of X + ```{r} get.label <- function(X) { label <- array(0, dim=dim(X)) @@ -113,7 +120,9 @@ get.label <- function(X) { return (label) } ``` -get training data and eval data + +Get training data and eval data + ```{r} download.data("./data/") ret <- make.data("./data/input.txt", seq.len=seq.len) @@ -141,6 +150,7 @@ X.val <- list(data=X.val.data, label=X.val.label) Training Model -------------- + In `mxnet`, we have a function called `mx.lstm` so that users can build a general lstm model. ```{r} @@ -164,7 +174,9 @@ model <- mx.lstm(X.train, X.val, Inference from model -------------------- -helper function for random sample. + +Some helper functions for random sample. + ```{r} cdf <- function(weights) { total <- sum(weights) @@ -190,6 +202,7 @@ search.val <- function(cdf, x) { } return (l) } + choice <- function(weights) { cdf.vals <- cdf(as.array(weights)) x <- runif(1) @@ -197,7 +210,9 @@ choice <- function(weights) { return (idx) } ``` -we can use random output or fixed output by choosing largest probability. + +We can use random output or fixed output by choosing largest probability. + ```{r} make.output <- function(prob, sample=FALSE) { if (!sample) { @@ -212,7 +227,7 @@ make.output <- function(prob, sample=FALSE) { ``` In `mxnet`, we have a function called `mx.lstm.inference` so that users can build a inference from lstm model and then use function `mx.lstm.forward` to get forward output from the inference. -Build inference from model. + ```{r} infer.model <- mx.lstm.inference(num.lstm.layer=num.lstm.layer, input.size=vocab, @@ -222,7 +237,9 @@ infer.model <- mx.lstm.inference(num.lstm.layer=num.lstm.layer, arg.params=model$arg.params, ctx=mx.cpu()) ``` -generate a sequence of 75 chars using function `mx.lstm.forward`. + +Generate a sequence of 75 chars using function `mx.lstm.forward`. + ```{r} start <- 'a' seq.len <- 75 @@ -238,7 +255,7 @@ for (i in (1:(seq.len-1))) { last.id <- make.output(prob, random.sample) out <- paste0(out, lookup.table[[last.id]]) } -cat (paste0(out, "\n")) +message(out) ``` The result: ``` @@ -250,4 +267,5 @@ Other RNN models ---------------- In `mxnet`, other RNN models like custom RNN and gru is also provided. - For **custom RNN model**, you can replace `mx.lstm` with `mx.rnn` to train rnn model. Also, you can replace `mx.lstm.inference` and `mx.lstm.forward` with `mx.rnn.inference` and `mx.rnn.forward` to inference from rnn model and get forward result from the inference model. + - For **GRU model**, you can replace `mx.lstm` with `mx.gru` to train gru model. Also, you can replace `mx.lstm.inference` and `mx.lstm.forward` with `mx.gru.inference` and `mx.gru.forward` to inference from gru model and get forward result from the inference model. \ No newline at end of file diff --git a/R-package/vignettes/classifyRealImageWithPretrainedModel.Rmd b/R-package/vignettes/classifyRealImageWithPretrainedModel.Rmd index 34847fd5705c..f9d14d920b80 100644 --- a/R-package/vignettes/classifyRealImageWithPretrainedModel.Rmd +++ b/R-package/vignettes/classifyRealImageWithPretrainedModel.Rmd @@ -1,24 +1,24 @@ -Classify Real-World Images with Pre-trained Model +Classify Real-world Images with Pre-trained Model ================================================= MXNet is a flexible and efficient deep learning framework. One of the cool things that a deep learning algorithm can do is to classify real world images. -In this example we will show how to use a pretrained Inception-BatchNorm Network to predict the class of +In this example we will show how to use a pretrained Inception-BatchNorm network to predict the content of real world image. The network architecture is described in [1]. -The pre-trained Inception-BatchNorm network is able to be downloaded from [this link](http://data.mxnet.io/mxnet/data/Inception.zip) -This model gives the recent state-of-art prediction accuracy on image net dataset. +The pre-trained Inception-BatchNorm network can be downloaded from [this link](http://data.mxnet.io/mxnet/data/Inception.zip). +This model gives the recent state-of-art prediction accuracy on the image net dataset. Preface ------- -This tutorial is written in Rmarkdown. -- You can directly view the hosted version of the tutorial from [MXNet R Document](http://mxnet.readthedocs.io/en/latest/packages/r/classifyRealImageWithPretrainedModel.html) -- You can find the download the Rmarkdown source from [here](https://github.com/dmlc/mxnet/blob/master/R-package/vignettes/classifyRealImageWithPretrainedModel.Rmd) +This tutorial is written in Rmarkdown. You can find the Rmarkdown source from [here](https://github.com/dmlc/mxnet/blob/master/R-package/vignettes/classifyRealImageWithPretrainedModel.Rmd) Package Loading --------------- -To get started, we load the mxnet package by require mxnet. + +To get started, we load the `mxnet` package first. + ```{r} require(mxnet) ``` @@ -31,6 +31,7 @@ require(imager) Load the Pretrained Model ------------------------- + Make sure you unzip the pre-trained model in current folder. And we can use the model loading function to load the model into R. diff --git a/R-package/vignettes/fiveMinutesNeuralNetwork.Rmd b/R-package/vignettes/fiveMinutesNeuralNetwork.Rmd index 5cb9aafb8088..bc45c9612e0f 100644 --- a/R-package/vignettes/fiveMinutesNeuralNetwork.Rmd +++ b/R-package/vignettes/fiveMinutesNeuralNetwork.Rmd @@ -177,4 +177,3 @@ dim(test.y) ``` Congratulations! Now you have learnt the basic for using `mxnet`. Please check the other tutorials for advanced features. - From f965542da6dd334b278e9f46abc295bff0e230e6 Mon Sep 17 00:00:00 2001 From: Terence Wu <2326428753@qq.com> Date: Tue, 1 Aug 2017 11:36:23 +0800 Subject: [PATCH 286/834] Fix a spelling mistake (#7277) --- include/mxnet/kvstore.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/mxnet/kvstore.h b/include/mxnet/kvstore.h index a77f653d492c..11db28e6cd20 100644 --- a/include/mxnet/kvstore.h +++ b/include/mxnet/kvstore.h @@ -48,7 +48,7 @@ class KVStore { /*! * \brief Initialize a list of key-value pair to the store. * - * One must initalize the key before \ref Push and \ref Pull, and a key + * One must initialize the key before \ref Push and \ref Pull, and a key * should be only initialized once * * It returns after data have been initialized successfully. From f187d5bbb6479947867086f0b57c25cb4c5ed7d0 Mon Sep 17 00:00:00 2001 From: Viacheslav Kovalevskyi Date: Mon, 31 Jul 2017 20:37:13 -0700 Subject: [PATCH 287/834] Now Jenkins correctly executes clean command when incremental build failed. Fix for #7272. (#7275) --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index 613b88c5e76e..e48ecf207955 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -52,6 +52,7 @@ def make(docker_type, make_flag) { } catch (exc) { echo 'Incremental compilation failed. Fall back to build from scratch' sh "${docker_run} ${docker_type} sudo make clean" + sh "${docker_run} ${docker_type} sudo make -C amalgamation/ clean" sh "${docker_run} ${docker_type} make ${make_flag}" } } From 5393002a2299ea79ad857cc015d703d63bc641ec Mon Sep 17 00:00:00 2001 From: Leonard Lausen Date: Tue, 1 Aug 2017 12:40:01 +0900 Subject: [PATCH 288/834] Gluon RNN fixes for seqlen 1 (#7260) * Gluon RNN fixes for seqlen 1 * Use _as_list from base_module * Move _as_list to base and allow tuples --- python/mxnet/base.py | 18 ++++++++++++++++++ python/mxnet/gluon/rnn/rnn_cell.py | 7 ++++--- python/mxnet/module/base_module.py | 18 +----------------- 3 files changed, 23 insertions(+), 20 deletions(-) diff --git a/python/mxnet/base.py b/python/mxnet/base.py index f714924a2eb8..6d537529e8af 100644 --- a/python/mxnet/base.py +++ b/python/mxnet/base.py @@ -300,3 +300,21 @@ def _add_fileline(obj): _add_fileline(obj.__func__) if inspect.isclass(obj) and incursive: add_fileline_to_docstring(obj, False) + +def _as_list(obj): + """A utility function that converts the argument to a list if it is not already. + + Parameters + ---------- + obj : object + + Returns + ------- + If `obj` is a list or tuple, return it. Otherwise, return `[obj]` as a + single-element list. + + """ + if isinstance(obj, (list, tuple)): + return obj + else: + return [obj] diff --git a/python/mxnet/gluon/rnn/rnn_cell.py b/python/mxnet/gluon/rnn/rnn_cell.py index e6ce65b31df0..87c656c3020f 100644 --- a/python/mxnet/gluon/rnn/rnn_cell.py +++ b/python/mxnet/gluon/rnn/rnn_cell.py @@ -6,7 +6,7 @@ from __future__ import print_function from ... import symbol, ndarray -from ...base import string_types, numeric_types +from ...base import string_types, numeric_types, _as_list from ..block import Block, HybridBlock from ..utils import _indent from .. import tensor_types @@ -50,8 +50,9 @@ def _format_sequence(length, inputs, layout, merge, in_layout=None): batch_size = inputs.shape[batch_axis] if merge is False: assert length is None or length == inputs.shape[in_axis] - inputs = ndarray.split(inputs, axis=in_axis, num_outputs=inputs.shape[in_axis], - squeeze_axis=1) + inputs = _as_list(ndarray.split(inputs, axis=in_axis, + num_outputs=inputs.shape[in_axis], + squeeze_axis=1)) else: assert length is None or len(inputs) == length if isinstance(inputs[0], symbol.Symbol): diff --git a/python/mxnet/module/base_module.py b/python/mxnet/module/base_module.py index cb6cfccb2759..cacce25ff083 100644 --- a/python/mxnet/module/base_module.py +++ b/python/mxnet/module/base_module.py @@ -12,23 +12,7 @@ from ..model import BatchEndParam from ..initializer import Uniform from ..io import DataDesc - - -def _as_list(obj): - """A utility function that treat the argument as a list. - - Parameters - ---------- - obj : object - - Returns - ------- - If `obj` is a list, return it. Otherwise, return `[obj]` as a single-element list. - """ - if isinstance(obj, list): - return obj - else: - return [obj] +from ..base import _as_list def _check_input_names(symbol, names, typename, throw): From 43ea3554dd5efd9fc0fb3550dc25bafe86e469a1 Mon Sep 17 00:00:00 2001 From: Przemyslaw Tredak Date: Mon, 31 Jul 2017 20:40:14 -0700 Subject: [PATCH 289/834] Update cub for CUDA 9 (#7270) --- cub | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cub b/cub index 80dbf02aa36d..05eb57faa0a4 160000 --- a/cub +++ b/cub @@ -1 +1 @@ -Subproject commit 80dbf02aa36d9ef881629e2ee2c15415ba07cef5 +Subproject commit 05eb57faa0a4cac37c2a86fdf4b4dc865a95a1a3 From 1784f62bd0f3f12f77a2fe41e23a1cfb9be6dec9 Mon Sep 17 00:00:00 2001 From: "Joshua Z. Zhang" Date: Mon, 31 Jul 2017 21:51:03 -0700 Subject: [PATCH 290/834] fix random sized crop (#7173) * fix random sized crop * fix typo --- python/mxnet/image/image.py | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/python/mxnet/image/image.py b/python/mxnet/image/image.py index 32b7c4f282b9..76cbea87fa3f 100644 --- a/python/mxnet/image/image.py +++ b/python/mxnet/image/image.py @@ -399,25 +399,26 @@ def random_size_crop(src, size, min_area, ratio, interp=2): """ h, w, _ = src.shape - new_ratio = random.uniform(*ratio) - if new_ratio * h > w: - max_area = w * int(w / new_ratio) - else: - max_area = h * int(h * new_ratio) + area = h * w + for _ in range(10): + target_area = random.uniform(min_area, 1.0) * area + new_ratio = random.uniform(*ratio) - min_area *= h * w - if max_area < min_area: - return random_crop(src, size, interp) - new_area = random.uniform(min_area, max_area) - new_w = int(np.sqrt(new_area * new_ratio)) - new_h = int(np.sqrt(new_area / new_ratio)) + new_w = int(round(np.sqrt(target_area * new_ratio))) + new_h = int(round(np.sqrt(target_area / new_ratio))) - assert new_w <= w and new_h <= h - x0 = random.randint(0, w - new_w) - y0 = random.randint(0, h - new_h) + if random.random() < 0.5: + new_h, new_w = new_w, new_h - out = fixed_crop(src, x0, y0, new_w, new_h, size, interp) - return out, (x0, y0, new_w, new_h) + if new_w <= w and new_h <= h: + x0 = random.randint(0, w - new_w) + y0 = random.randint(0, h - new_h) + + out = fixed_crop(src, x0, y0, new_w, new_h, size, interp) + return out, (x0, y0, new_w, new_h) + + # fall back to center_crop + return center_crop(src, size, interp) class Augmenter(object): From 4c4aa56e2ab4022826b752ea88bc1544333bb5e0 Mon Sep 17 00:00:00 2001 From: Zack Chase Lipton Date: Tue, 1 Aug 2017 10:05:00 -0700 Subject: [PATCH 291/834] =?UTF-8?q?fixed=20dcgan=20to=20use=20integer=20di?= =?UTF-8?q?vision=20so=20it=20won't=20die=20horribly=20in=20Pytho=E2=80=A6?= =?UTF-8?q?=20(#7287)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit n3 making me want to die as well --- example/gluon/dcgan.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/example/gluon/dcgan.py b/example/gluon/dcgan.py index 7f644cba5962..f643b28f4bc1 100644 --- a/example/gluon/dcgan.py +++ b/example/gluon/dcgan.py @@ -14,11 +14,11 @@ import time def fill_buf(buf, i, img, shape): - n = buf.shape[0]/shape[1] - m = buf.shape[1]/shape[0] + n = buf.shape[0]//shape[1] + m = buf.shape[1]//shape[0] sx = (i%m)*shape[0] - sy = (i/m)*shape[1] + sy = (i//m)*shape[1] buf[sy:sy+shape[1], sx:sx+shape[0], :] = img return None From 583fd9ed9c5886704fdeb94cf354da5268b6bf4f Mon Sep 17 00:00:00 2001 From: "Joshua Z. Zhang" Date: Tue, 1 Aug 2017 10:08:41 -0700 Subject: [PATCH 292/834] change random sized crop lower bound to 0.08 (#7278) --- python/mxnet/image/image.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/mxnet/image/image.py b/python/mxnet/image/image.py index 76cbea87fa3f..cf17ecf13ffb 100644 --- a/python/mxnet/image/image.py +++ b/python/mxnet/image/image.py @@ -869,7 +869,7 @@ def CreateAugmenter(data_shape, resize=0, rand_crop=False, rand_resize=False, ra crop_size = (data_shape[2], data_shape[1]) if rand_resize: assert rand_crop - auglist.append(RandomSizedCropAug(crop_size, 0.3, (3.0 / 4.0, 4.0 / 3.0), inter_method)) + auglist.append(RandomSizedCropAug(crop_size, 0.08, (3.0 / 4.0, 4.0 / 3.0), inter_method)) elif rand_crop: auglist.append(RandomCropAug(crop_size, inter_method)) else: From 4762785470ad954cc849d4d4ed209848806a232f Mon Sep 17 00:00:00 2001 From: SmartAILM Date: Wed, 2 Aug 2017 01:09:14 +0800 Subject: [PATCH 293/834] fix dataloader length bug (#7283) --- python/mxnet/gluon/data/dataloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/mxnet/gluon/data/dataloader.py b/python/mxnet/gluon/data/dataloader.py index 148d7cd6e53c..b251deb9bb56 100644 --- a/python/mxnet/gluon/data/dataloader.py +++ b/python/mxnet/gluon/data/dataloader.py @@ -67,4 +67,4 @@ def __iter__(self): yield _batchify([self._dataset[idx] for idx in batch]) def __len__(self): - return self._batch_sampler + return len(self._batch_sampler) From 861e929cec8fa8fbab06884d9605debb74cd7217 Mon Sep 17 00:00:00 2001 From: Tobias Domhan Date: Tue, 1 Aug 2017 20:00:30 +0200 Subject: [PATCH 294/834] Properly check argument types in ndarray.save to avoid segfaults. (#6894) --- python/mxnet/ndarray.py | 10 ++++++++-- tests/python/unittest/test_ndarray.py | 11 ++++++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/python/mxnet/ndarray.py b/python/mxnet/ndarray.py index d02119166562..01399d54a54f 100644 --- a/python/mxnet/ndarray.py +++ b/python/mxnet/ndarray.py @@ -2282,6 +2282,7 @@ def negative(arr): """ return multiply(arr, -1.0) + def load(fname): """Loads an array from file. @@ -2329,7 +2330,7 @@ def save(fname, data): ---------- fname : str The filename. - data : list of ``NDArray` or dict of str to ``NDArray`` + data : ``NDArray``, list of ``NDArray` or dict of str to ``NDArray`` The data to save. Examples @@ -2343,6 +2344,8 @@ def save(fname, data): >>> mx.nd.load('my_dict') {'y': , 'x': } """ + if isinstance(data, NDArray): + data = [data] handles = [] if isinstance(data, dict): keys = [] @@ -2354,12 +2357,15 @@ def save(fname, data): keys.append(c_str(key)) handles.append(val.handle) keys = c_array(ctypes.c_char_p, keys) - else: + elif isinstance(data, list): for val in data: if not isinstance(val, NDArray): raise TypeError('save only accept dict str->NDArray or list of NDArray') handles.append(val.handle) keys = None + else: + raise ValueError("data needs to either be a NDArray, dict of str, NDArray pairs " + "or a list of NDarrays.") check_call(_LIB.MXNDArraySave(c_str(fname), mx_uint(len(handles)), c_array(NDArrayHandle, handles), diff --git a/tests/python/unittest/test_ndarray.py b/tests/python/unittest/test_ndarray.py index f627ab837a3e..79a022b17ac4 100644 --- a/tests/python/unittest/test_ndarray.py +++ b/tests/python/unittest/test_ndarray.py @@ -217,11 +217,11 @@ def test_ndarray_pickle(): def test_ndarray_saveload(): np.random.seed(0) - maxdim = 5 nrepeat = 10 fname = 'tmp_list.bin' for repeat in range(nrepeat): data = [] + # test save/load as list for i in range(10): data.append(random_ndarray(np.random.randint(1, 5))) mx.nd.save(fname, data) @@ -229,6 +229,7 @@ def test_ndarray_saveload(): assert len(data) == len(data2) for x, y in zip(data, data2): assert np.sum(x.asnumpy() != y.asnumpy()) == 0 + # test save/load as dict dmap = {'ndarray xx %s' % i : x for i, x in enumerate(data)} mx.nd.save(fname, dmap) dmap2 = mx.nd.load(fname) @@ -236,6 +237,14 @@ def test_ndarray_saveload(): for k, x in dmap.items(): y = dmap2[k] assert np.sum(x.asnumpy() != y.asnumpy()) == 0 + # test save/load as ndarray + # we expect the single ndarray to be converted into a list containing the ndarray + single_ndarray = data[0] + mx.nd.save(fname, single_ndarray) + single_ndarray_loaded = mx.nd.load(fname) + assert len(single_ndarray_loaded) == 1 + single_ndarray_loaded = single_ndarray_loaded[0] + assert np.sum(single_ndarray.asnumpy() != single_ndarray_loaded.asnumpy()) == 0 os.remove(fname) def test_ndarray_legacy_load(): From af56a7776899c42ea2502f2001da62406d3573f1 Mon Sep 17 00:00:00 2001 From: bhavinthaker Date: Tue, 1 Aug 2017 11:09:28 -0700 Subject: [PATCH 295/834] add support for port package manager on macOS (#7276) --- setup-utils/install-mxnet-osx-python.sh | 534 ++++++++++++++++++------ 1 file changed, 399 insertions(+), 135 deletions(-) diff --git a/setup-utils/install-mxnet-osx-python.sh b/setup-utils/install-mxnet-osx-python.sh index b9b1ddc9a00f..f9e4e775534e 100755 --- a/setup-utils/install-mxnet-osx-python.sh +++ b/setup-utils/install-mxnet-osx-python.sh @@ -25,37 +25,134 @@ then fi export MXNET_HOME_OLD="$HOME/mxnet_${TARIKH}" export MXNET_LOG=${MXNET_HOME}/buildMXNet_mac.log + # Insert the Homebrew directory at the top of your PATH environment variable -export PATH=/usr/local/bin:/usr/local/sbin:$PATH +export PATH="$PATH:/usr/local/bin:/usr/local/sbin" # for brew +export PATH="$PATH:/usr/bin:/opt/local/bin" # for macports + +export MACPORTS_WEB="https://guide.macports.org/chunked/installing.macports.html" + +export BREW_PKGS="pkg-config python opencv graphviz homebrew/science/openblas" +export PORT_PKGS="pkgconfig python36 opencv graphviz openblas-devel" + +# graphviz, opencv-python skipped since already installed via brew/port +export PIP_PKGS_ALL="cython numpy" +export PIP_PKGS_USER="requests jupyter" + export SLEEP_TIME=2 LINE="########################################################################" -echo $LINE -echo " " -echo "This script installs MXNet on MacOS in \${MXNET_HOME}" -echo "If not set, the default value of \${MXNET_HOME} = ~/mxnet" -echo "The current value of \${MXNET_HOME} = ${MXNET_HOME}" -echo " " -echo "If this directory is already present, it is renamed to retain earlier contents." -echo "You may want to check and delete this directory if not required." -echo " " -echo "This script has been tested on: MacOS El Capitan and Sierra" -echo " " -echo "If you face any problems with this script, please let us know at:" -echo " https://stackoverflow.com/questions/tagged/mxnet" -echo " " -echo "Typical run-time for this script is around 7 minutes." -echo "If your environment has never been setup for development (e.g. gcc), " -echo "it could take up to 30 minutes or longer." -echo " " -MACOS_VERSION=`/usr/bin/uname -r` -echo "Your macOS version is: $MACOS_VERSION" -echo " " -echo $LINE -sleep ${SLEEP_TIME} - -echo "You may have to enter your password for sudo access to install python for MXNet." -sudo ls > /dev/null +print_intro_msg() { + # + # NOTE: Please test and ensure that the message does NOT scroll + # beyond the standard 80x25 format of a terminal shell. + # + echo $LINE + echo " " + echo "MXNet is a flexible, efficient and scalable library for Deep Learning." + echo " " + echo "This script installs MXNet on MacOS in \${MXNET_HOME}" + echo "If not set, the default value of \${MXNET_HOME} = ~/mxnet" + echo "The current value of \${MXNET_HOME} = ${MXNET_HOME}" + echo " " + echo "If this directory is already present, it is renamed to retain earlier contents." + echo "You may want to check and delete this directory if not required." + echo " " + echo "This script has been tested on: MacOS El Capitan (10.11) and Sierra (10.12)" + echo " " + echo "If you face any problems with this script, please let us know at:" + echo " https://stackoverflow.com/questions/tagged/mxnet" + echo " " + echo "Typical run-time for this script is around 10 minutes." + echo "If your environment has never been setup for development (e.g. gcc), " + echo "it could take up to 30 minutes or longer." + echo " " + MACOS_VERSION=`/usr/bin/uname -r` + echo "Your macOS version is: $MACOS_VERSION" + echo " " + echo $LINE + echo " " + sleep ${SLEEP_TIME} +} # print_intro_msg() + +# wrapper routine to stop the script if the command invoked returns error +chkret() { + cmd=$* + echo "$cmd" + $cmd + ret=$? + if [[ ${ret} != 0 ]]; then + echo " " + echo "ERROR: Return value non-zero for: $cmd" + echo " " + exit 1 + fi +} # chkret() + +chk_mac_vers() { + export mac_vers=`sw_vers -productVersion | cut -d '.' -f 1,2` + if [[ $mac_vers != "10.11" && $mac_vers != "10.12" ]]; + then + echo " " + echo "ERROR: macOS version $mac_vers NOT supported." + echo " " + echo "Your macOS version is:" + sw_vers + echo " " + exit 1 + fi +} # chk_mac_vers() + +install_brew() { + echo " " + while true; do + echo "This script will install/update brew and " + echo "following dependent packages required for MXNet." + echo " Dependent brew packages: ${BREW_PKGS}" + echo " Dependent pip packages: ${PIP_PKGS_ALL} ${PIP_PKGS_USER}" + read -p "Do you want to continue? (y/n): " response + echo " " + case $response in + [Yy]* ) break;; + [Nn]* ) exit;; + * ) echo "Please answer yes or no.";; + esac + done + + echo " " + echo "BEGIN: Check/Install/Update Homebrew" + BREW_PATH=`which brew` + if [[ (-z ${BREW_PATH}) || (! -f ${BREW_PATH}) ]]; + then + yes '' | /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" + ret=$? + if [[ ${ret} != 0 ]]; then + echo " " + echo "ERROR: Return value non-zero for: homebrew installation using ruby" + echo " " + exit 1 + fi + else + chkret brew update + fi + echo "END: Check/Install/Update Homebrew" + echo $LINE + echo " " + + echo "BEGIN: Install dependent brew packages for MXNet: ${BREW_PKGS}" + + chkret brew tap homebrew/science + + # install each individually to see progress for each + for pkg in ${BREW_PKGS} + do + chkret brew_pkg_install ${pkg} + done + + echo "END: Install dependent brew packages for MXNet: ${BREW_PKGS}" + echo $LINE + echo " " +} # install_brew() brew_pkg_install () { pkg=$1 @@ -67,22 +164,175 @@ brew_pkg_install () { else echo "$pkg already installed" fi -} +} # brew_pkg_install -runme() { - cmd=$* - echo "$cmd" - $cmd - ret=$? - if [[ ${ret} != 0 ]]; then +install_port () { + echo " " + while true; do + echo "This script will install/update port and " + echo "following dependent packages required for MXNet." + echo " Dependent port packages: ${PORT_PKGS}" + echo " Dependent pip packages: ${PIP_PKGS_ALL} ${PIP_PKGS_USER}" + read -p "Do you want to continue? (y/n): " response echo " " - echo "ERROR: Return value non-zero for: $cmd" + case $response in + [Yy]* ) break;; + [Nn]* ) exit;; + * ) echo "Please answer yes or no.";; + esac + done + + echo " " + echo "BEGIN: Check/Install/Update port" + MACPORTS_PATH=`which port` + if [[ (-z ${MACPORTS_PATH}) || (! -f ${MACPORTS_PATH}) ]]; + then + echo " " + echo "ERROR: Please install port for your macOS version from:" + echo " " + echo $MACPORTS_WEB echo " " exit 1 + else + echo "NOTE: Updating port if required" + export SLEEP_TIME=2 + sudo port upgrade outdated + echo " " + echo "port version is:" + port version + echo " " fi -} + echo "END: Check/Install/Update port" + echo $LINE + echo " " + + echo "BEGIN: Install dependent port packages for MXNet: ${PORT_PKGS}" + echo " " + #sudo port install python36-readline + # install each individually to see progress for each + for pkg in ${PORT_PKGS} + do + chkret sudo port install ${pkg} + done + if [[ ! -f /opt/local/include/cblas.h ]]; + then + sudo ln -s /opt/local/include/cblas_openblas.h /opt/local/include/cblas.h + fi + #if [[ ! -f /usr/local/opt/openblas/lib/libopenblas.a ]]; + #then + # sudo mkdir -p /usr/local/opt/openblas/lib + # sudo ln -s /opt/local/lib/libopenblas.a /usr/local/opt/openblas/lib/libopenblas.a + #fi + + echo " " + echo "END: Install dependent port packages for MXNet: ${PORT_PKGS}" + echo $LINE + echo " " +} # install_port + +install_mac_pkg_manager() { + BREW_PATH=`which brew` + if [[ (-z ${BREW_PATH}) || (! -f ${BREW_PATH}) ]]; + then + echo "NOTE: brew NOT installed" + export MAC_BREW=0 + else + echo "NOTE: brew installed" + export MAC_BREW=1 + export PKG_MGR="brew" + fi + + MACPORTS_PATH=`which port` + if [[ (-z ${MACPORTS_PATH}) || (! -f ${MACPORTS_PATH}) ]]; + then + echo "NOTE: port NOT installed" + export MAC_PORT=0 + else + echo "NOTE: port installed" + export MAC_PORT=1 + export PKG_MGR="port" + fi + + if [[ $MAC_PORT -eq 1 && $MAC_BREW -eq 1 ]]; + then + echo "NOTE: Both port and brew installed" + export MAC_PKG_ASK=1 + export PKG_MGR="" + elif [[ $MAC_PORT -eq 0 && $MAC_BREW -eq 0 ]]; + then + echo "NOTE: Neither port and brew installed" + export MAC_PKG_ASK=1 + export PKG_MGR="" + else + export MAC_PKG_ASK=0 + + while true; do + echo "NOTE: Using the already installed package manager: $PKG_MGR" + read -p "Do you want to continue? (y/n): " response + echo " " + case $response in + [Yy]* ) break;; + [Nn]* ) exit;; + * ) echo "Please answer yes or no.";; + esac + done + fi + + if [[ $MAC_PKG_ASK -eq 1 ]]; + then + export MAC_BREW=0 + export MAC_PORT=0 + while true; do + echo " " + echo "NOTE: This script supports Homebrew OR Port package manager." + echo " " + read -p "Which package manager do you want to use? (b/p): " pkg_mgr + echo " " + case $pkg_mgr in + [Bb]* ) export MAC_BREW=1; break;; + [Pp]* ) export MAC_PORT=1; break;; + * ) echo "Please answer: b or p";; + esac + done + fi + + if [[ $MAC_PORT -eq 1 ]]; + then + install_port + else + install_brew + fi +} # install_mac_pkg_manager + +install_dep_pip_for_mxnet() { + echo " " + echo "BEGIN: Install dependent pip packages for MXNet: " + echo "${PIP_PKGS_ALL} ${PIP_PKGS_USER}" + echo " " + + # NOTE: sudo used here + chkret sudo easy_install pip + chkret sudo pip install --upgrade pip + for pkg in ${PIP_PKGS_ALL} + do + chkret sudo pip install ${pkg} + done + #chkret sudo pip install --upgrade numpy + + # NOTE: no sudo used here + for pkg in ${PIP_PKGS_USER} + do + chkret pip install --user ${pkg} + done + + echo "END: Install dependent pip packages for MXNet: ${PIP_PKGS_ALL} ${PIP_PKGS_USER}" + echo $LINE + echo " " +} # install_dep_pip_for_mxnet() download_mxnet() { + echo " " + echo "BEGIN: Download MXNet" if [ -d ${MXNET_HOME} ]; then mv ${MXNET_HOME} ${MXNET_HOME_OLD} echo " " @@ -100,116 +350,130 @@ download_mxnet() { echo " " sleep ${SLEEP_TIME} - runme git clone ${MXNET_GITPATH} ${MXNET_HOME} --recursive + chkret git clone ${MXNET_GITPATH} ${MXNET_HOME} --recursive sleep ${SLEEP_TIME} cd ${MXNET_HOME} echo " " #echo "Checkout tag = ${MXNET_TAG}" - #runme git checkout ${MXNET_TAG} + #chkret git checkout ${MXNET_TAG} #echo " " sleep ${SLEEP_TIME} -} + echo "END: Download MXNet" + echo $LINE + echo " " +} # download_mxnet -echo " " -echo "BEGIN: Check/Install/Update Homebrew" -BREW_PATH=`/usr/bin/which brew` -if [[ (-z ${BREW_PATH}) || (! -f ${BREW_PATH}) ]]; -then - yes '' | /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" -else - runme brew update -fi -echo "END: Check/Install/Update Homebrew" -echo $LINE -echo " " - -echo " " -echo "BEGIN: Install dependent brew packages for MXNet" - -runme brew tap homebrew/science - -runme brew_pkg_install pkg-config -runme brew_pkg_install python -runme brew_pkg_install opencv -runme brew_pkg_install numpy -runme brew_pkg_install homebrew/science/openblas - -echo "END: Install dependent brew packages for MXNet" -echo $LINE -echo " " - -echo "BEGIN: Install dependent pip packages for MXNet" -runme pip install --upgrade pip -runme pip install --user requests -runme pip install graphviz -runme pip install jupyter -runme pip install cython -runme pip install --user opencv-python -echo "END: Install dependent pip packages for MXNet" -echo $LINE -echo " " - -echo "BEGIN: Download MXNet" -download_mxnet -echo "END: Download MXNet" -sleep ${SLEEP_TIME} -echo $LINE -echo " " - -# Compile MXNet: It assumes MXNet source is in ${MXNET_HOME} -echo "BEGIN: Compile MXNet" -cd ${MXNET_HOME} -runme cp make/osx.mk ./config.mk -runme echo "USE_BLAS = openblas" >> ./config.mk -runme echo "ADD_CFLAGS += -I/usr/local/opt/openblas/include" >> ./config.mk -runme echo "ADD_LDFLAGS += -L/usr/local/opt/openblas/lib" >> ./config.mk -runme echo "ADD_LDFLAGS += -L/usr/local/lib/graphviz/" >> ./config.mk -echo " " -echo "Running Make" -echo " " -runme make -j$(sysctl -n hw.ncpu) -echo "END: Compile MXNet" -sleep ${SLEEP_TIME} -echo $LINE -echo " " - -echo "BEGIN: Install MXNet package for Python" -runme cd ${MXNET_HOME}/python -runme sudo python setup.py install -echo "END: Install MXNet package for Python" -sleep ${SLEEP_TIME} -echo $LINE -echo " " - - -echo "BEGIN: Test MXNet" -python << END > mxnet_test.log +compile_mxnet() { + # Compile MXNet: It assumes MXNet source is in ${MXNET_HOME} + echo "BEGIN: Compile MXNet" + cd ${MXNET_HOME} + chkret cp make/osx.mk ./config.mk.tmp + + touch ./config.mk + # rm any old setting of USE_BLAS, if present in config file + egrep -v "^USE_BLAS" ./config.mk.tmp >> ./config.mk + # add the new setting of USE_BLAS to the config file + echo "USE_BLAS = openblas" >> ./config.mk + + if [[ $MAC_PORT -eq 1 ]]; + then + echo "ADD_CFLAGS += -I/opt/local/lib" >> ./config.mk + echo "ADD_LDFLAGS += -L/opt/local/lib" >> ./config.mk + echo "ADD_LDFLAGS += -L/opt/local/lib/graphviz/" >> ./config.mk + else + echo "ADD_CFLAGS += -I/usr/local/opt/openblas/include" >> ./config.mk + echo "ADD_LDFLAGS += -L/usr/local/opt/openblas/lib" >> ./config.mk + echo "ADD_LDFLAGS += -L/usr/local/lib/graphviz/" >> ./config.mk + fi + echo " " + + echo "NOTE: The following compile-time configurations will be used." + echo " If you want to change any of them, edit the following file" + echo " in another terminal window and then press enter to continue." + echo " " + echo " ${MXNET_HOME}/config.mk" + echo " " + echo $LINE + # remove commented and blank lines + egrep -v "^#" ${MXNET_HOME}/config.mk | egrep -v "^$" + echo $LINE + echo " " + read -p "Press enter to continue ..." + echo " " + echo "Running Make" + echo " " + chkret make -j$(sysctl -n hw.ncpu) + echo "END: Compile MXNet" + sleep ${SLEEP_TIME} + echo $LINE + echo " " +} # compile_mxnet + +install_mxnet_python() { + echo " " + echo "BEGIN: Install MXNet package for Python" + chkret cd ${MXNET_HOME}/python + chkret sudo python setup.py install + echo "END: Install MXNet package for Python" + sleep ${SLEEP_TIME} + echo $LINE + echo " " +} # install_mxnet_python + + +test_mxnet_python() { + echo "BEGIN: Test MXNet" + rm -f mxnet_test.log + python << END > mxnet_test.log import mxnet as mx a = mx.nd.ones((2, 3)); print ((a*2).asnumpy()); END -cat << END > mxnet_test.expected + rm -f mxnet_test.expected + cat << END > mxnet_test.expected [[ 2. 2. 2.] [ 2. 2. 2.]] END -diff mxnet_test.log mxnet_test.expected -if [[ $? = 0 ]]; then - echo $LINE - echo " " - echo "SUCCESS: MXNet test passed" - echo "SUCCESS: MXNet is successfully installed and works fine!" - export MXNET_VERSION=`echo "import mxnet as mx; print(mx.__version__)" | python` - echo "SUCCESS: MXNet Version is: $MXNET_VERSION" - echo "END: Test MXNet" - echo " " - echo ":-)" - exit 0 -else - echo $LINE - echo " " - echo "ERROR: MXNet test failed" - echo "END: Test MXNet" - echo " " - echo ":-(" - exit 1 -fi + diff mxnet_test.log mxnet_test.expected + if [[ $? = 0 ]]; then + echo " " + echo "SUCCESS: MXNet test passed" + echo "SUCCESS: MXNet is successfully installed and works fine!" + export MXNET_VERSION=`echo "import mxnet as mx; print(mx.__version__)" | python` + echo "SUCCESS: MXNet Version is: $MXNET_VERSION" + echo "END: Test MXNet" + echo ":-)" + echo " " + echo "FYI : You can fine-tune MXNet run-time behavior using environment variables described at:" + echo " http://mxnet.io/how_to/env_var.html" + echo " " + echo "NEXT: Try the MNIST tutorial at: http://mxnet.io/tutorials/python/mnist.html" + echo " Try other tutorials at : http://mxnet.io/tutorials" + echo " " + echo $LINE + echo " " + rm -f mxnet_test.log mxnet_test.expected + exit 0 + else + echo " " + echo "ERROR: Following files differ: mxnet_test.log mxnet_test.expected" + echo "ERROR: MXNet test failed" + echo "END: Test MXNet" + echo " " + echo ":-(" + exit 1 + fi +} # test_mxnet_python() + +main() { + print_intro_msg + chk_mac_vers + install_mac_pkg_manager + install_dep_pip_for_mxnet + download_mxnet + compile_mxnet + install_mxnet_python + test_mxnet_python +} # main + +main From 1d1d6c21905754a500b1cdc62bcbef18518e52a2 Mon Sep 17 00:00:00 2001 From: cpansprout Date: Tue, 1 Aug 2017 11:10:15 -0700 Subject: [PATCH 296/834] =?UTF-8?q?Require=20Mouse=20v2.1.0=20with=20expli?= =?UTF-8?q?cit=20=E2=80=98v=E2=80=99=20(#7171)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ExtUtils::MakeMaker version that comes with perl 5.24.0 (7.10_01) does not like three-part versions without the initial v. So it gives me the following message: ``` Warning: prerequisite Mouse 2.1.0 not found. We have v2.4.10. ``` And then installation does not proceed smoothly. Adding the v solves this. --- perl-package/AI-MXNet/Makefile.PL | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/perl-package/AI-MXNet/Makefile.PL b/perl-package/AI-MXNet/Makefile.PL index 4f42af0d7650..990176d1b493 100644 --- a/perl-package/AI-MXNet/Makefile.PL +++ b/perl-package/AI-MXNet/Makefile.PL @@ -22,7 +22,7 @@ my %WriteMakefileArgs = ( "AI::MXNetCAPI" => "1.0101", "AI::NNVMCAPI" => "1.01", "Function::Parameters" => "1.0705", - "Mouse" => "2.1.0", + "Mouse" => "v2.1.0", "PDL" => "2.007", "GraphViz" => "2.14" }, @@ -38,7 +38,7 @@ my %FallbackPrereqs = ( "AI::MXNetCAPI" => "1.0101", "AI::NNVMCAPI" => "1.01", "Function::Parameters" => "1.0705", - "Mouse" => "2.1.0", + "Mouse" => "v2.1.0", "PDL" => "2.007", "GraphViz" => "2.14" ); From 1e2dfa80ca2eb2ca485f4e8559dc1ed60ddeb619 Mon Sep 17 00:00:00 2001 From: Aston <22279212+astonzhang@users.noreply.github.com> Date: Tue, 1 Aug 2017 16:09:57 -0700 Subject: [PATCH 297/834] Fix typoes (#7297) * Fix a typo * update --- python/mxnet/gluon/parameter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/mxnet/gluon/parameter.py b/python/mxnet/gluon/parameter.py index 1c311ef687e3..ee73774f6b0f 100644 --- a/python/mxnet/gluon/parameter.py +++ b/python/mxnet/gluon/parameter.py @@ -24,7 +24,7 @@ class DeferredInitializationError(MXNetError): class Parameter(object): """A Container holding parameters (weights) of `Block`s. - `Parameter` holds a copy of the the parameter on each `Context` after + `Parameter` holds a copy of the parameter on each `Context` after it is initialized with `Parameter.initialize(...)`. If `grad_req` is not `null`, it will also hold a gradient array on each `Context`:: @@ -353,7 +353,7 @@ class ParameterDict(object): Parameters ---------- prefix : str, default '' - The prefix to be prepended to all Parameters' name created by this dict. + The prefix to be prepended to all Parameters' names created by this dict. shared : ParameterDict or None If not `None`, when this dict's `get` method creates a new parameter, will first try to retrieve it from `shared` dict. Usually used for sharing From 202373f7331fcbad80bc677f4a7f76df6cd91175 Mon Sep 17 00:00:00 2001 From: Rahul Huilgol Date: Tue, 1 Aug 2017 16:10:17 -0700 Subject: [PATCH 298/834] fix initialization warning by g++ (#7294) --- src/io/iter_image_recordio_2.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/io/iter_image_recordio_2.cc b/src/io/iter_image_recordio_2.cc index 18d12ff7e25a..9d4ebf4b2864 100644 --- a/src/io/iter_image_recordio_2.cc +++ b/src/io/iter_image_recordio_2.cc @@ -360,10 +360,10 @@ inline void ImageRecordIOParser2::ParseChunk(dmlc::InputSplit::Blob * chu (rand_uniform(*(prnds_[tid])) * normalize_param_.max_random_illumination * 2 - normalize_param_.max_random_illumination) * normalize_param_.scale; } + DType RGBA[4] = {}; for (int i = 0; i < res.rows; ++i) { uchar* im_data = res.ptr(i); for (int j = 0; j < res.cols; ++j) { - DType RGBA[4]; for (int k = 0; k < n_channels; ++k) { RGBA[k] = im_data[swap_indices[k]]; } From 3f3d0fa288367d3afa4143d5f175709b557c8484 Mon Sep 17 00:00:00 2001 From: Viacheslav Kovalevskyi Date: Tue, 1 Aug 2017 21:50:26 -0700 Subject: [PATCH 299/834] finetune now includes short hint how to run the script on windows (#7298) * Now Jenkins correctly executes clean command when incremental build failed. Fix for #7272. * Comment added for running scipt on Windows. --- docs/how_to/finetune.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/how_to/finetune.md b/docs/how_to/finetune.md index 79d06cb5bb77..f6c164c28db9 100644 --- a/docs/how_to/finetune.md +++ b/docs/how_to/finetune.md @@ -45,6 +45,8 @@ training set, and the rest for the validation set. We resize images into 256x256 size and pack them into the rec file. The scripts to prepare the data is as following. +> In order to successfully run the following bash script on Windows please use https://cygwin.com/install.html . + ```sh wget http://www.vision.caltech.edu/Image_Datasets/Caltech256/256_ObjectCategories.tar tar -xf 256_ObjectCategories.tar From 5aab4adc638478a00055bdbb9031426c4cac7464 Mon Sep 17 00:00:00 2001 From: Eric Junyuan Xie Date: Tue, 1 Aug 2017 22:45:55 -0700 Subject: [PATCH 300/834] Update ndarray.py (#7296) --- python/mxnet/ndarray.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/mxnet/ndarray.py b/python/mxnet/ndarray.py index 01399d54a54f..fdecebbe7996 100644 --- a/python/mxnet/ndarray.py +++ b/python/mxnet/ndarray.py @@ -124,6 +124,7 @@ class NDArray(NDArrayBase): """ __slots__ = [] + __array_priority__ = 1000.0 # pylint: disable= no-member, undefined-variable def __repr__(self): """Returns a string representation of the array.""" From 63330961ef622e39870d3ea469fa4ee2e507d2fb Mon Sep 17 00:00:00 2001 From: Pedro Larroy Date: Wed, 2 Aug 2017 22:39:09 +0200 Subject: [PATCH 301/834] Fix mix of tabs and spaces (#7311) --- amalgamation/jni/predictor.cc | 126 +++++++++++++++++----------------- 1 file changed, 63 insertions(+), 63 deletions(-) diff --git a/amalgamation/jni/predictor.cc b/amalgamation/jni/predictor.cc index 2687d1d9d93e..b6cc9370b1f9 100644 --- a/amalgamation/jni/predictor.cc +++ b/amalgamation/jni/predictor.cc @@ -6,105 +6,105 @@ JNIEXPORT jlong JNICALL Java_org_dmlc_mxnet_Predictor_createPredictor (JNIEnv *env, jclass, jbyteArray jsymbol, jbyteArray jparams, jint devType, jint devId, jobjectArray jkeys, jobjectArray jshapes) { - jbyte* symbol = env->GetByteArrayElements(jsymbol, 0); - jbyte* params = env->GetByteArrayElements(jparams, 0); - jsize params_len = env->GetArrayLength(jparams); + jbyte* symbol = env->GetByteArrayElements(jsymbol, 0); + jbyte* params = env->GetByteArrayElements(jparams, 0); + jsize params_len = env->GetArrayLength(jparams); - std::vector> track; - std::vector keys; + std::vector> track; + std::vector keys; for (int i=0; iGetArrayLength(jkeys); i++) { jstring js = (jstring) env->GetObjectArrayElement(jkeys, i); const char *s = env->GetStringUTFChars(js, 0); - keys.emplace_back(s); - track.emplace_back(js, s); + keys.emplace_back(s); + track.emplace_back(js, s); } - std::vector index; - std::vector shapes; + std::vector index; + std::vector shapes; mx_uint prev = 0; index.emplace_back(prev); for (int i=0; iGetArrayLength(jshapes); i++) { jintArray jshape = (jintArray) env->GetObjectArrayElement(jshapes, i); - jsize shape_len = env->GetArrayLength(jshape); - jint *shape = env->GetIntArrayElements(jshape, 0); + jsize shape_len = env->GetArrayLength(jshape); + jint *shape = env->GetIntArrayElements(jshape, 0); prev += shape_len; - index.emplace_back(prev); - for (int j=0; jReleaseIntArrayElements(jshape, shape, 0); + index.emplace_back(prev); + for (int j=0; jReleaseIntArrayElements(jshape, shape, 0); } - PredictorHandle handle = 0; - if (MXPredCreate((const char *)symbol, (const char *)params, params_len, devType, devId, (mx_uint)keys.size(), &(keys[0]), &(index[0]), &(shapes[0]), &handle) < 0) { - jclass MxnetException = env->FindClass("org/dmlc/mxnet/MxnetException"); - env->ThrowNew(MxnetException, MXGetLastError()); - } + PredictorHandle handle = 0; + if (MXPredCreate((const char *)symbol, (const char *)params, params_len, devType, devId, (mx_uint)keys.size(), &(keys[0]), &(index[0]), &(shapes[0]), &handle) < 0) { + jclass MxnetException = env->FindClass("org/dmlc/mxnet/MxnetException"); + env->ThrowNew(MxnetException, MXGetLastError()); + } - env->ReleaseByteArrayElements(jsymbol, symbol, 0); - env->ReleaseByteArrayElements(jparams, params, 0); - for (auto& t: track) { - env->ReleaseStringUTFChars(t.first, t.second); - } + env->ReleaseByteArrayElements(jsymbol, symbol, 0); + env->ReleaseByteArrayElements(jparams, params, 0); + for (auto& t: track) { + env->ReleaseStringUTFChars(t.first, t.second); + } - return (jlong)handle; + return (jlong)handle; } JNIEXPORT void JNICALL Java_org_dmlc_mxnet_Predictor_nativeFree (JNIEnv *, jclass, jlong h) { - PredictorHandle handle = (PredictorHandle)h; - MXPredFree(handle); + PredictorHandle handle = (PredictorHandle)h; + MXPredFree(handle); } JNIEXPORT jfloatArray JNICALL Java_org_dmlc_mxnet_Predictor_nativeGetOutput (JNIEnv *env, jclass, jlong h, jint index) { - PredictorHandle handle = (PredictorHandle)h; - - mx_uint *shape = 0; - mx_uint shape_len; - if (MXPredGetOutputShape(handle, index, &shape, &shape_len) < 0) { - jclass MxnetException = env->FindClass("org/dmlc/mxnet/MxnetException"); - env->ThrowNew(MxnetException, MXGetLastError()); - } - - size_t size = 1; - for (mx_uint i=0; i data(size); - if (MXPredGetOutput(handle, index, &(data[0]), size) < 0) { - jclass MxnetException = env->FindClass("org/dmlc/mxnet/MxnetException"); - env->ThrowNew(MxnetException, MXGetLastError()); - } - - jfloatArray joutput = env->NewFloatArray(size); + PredictorHandle handle = (PredictorHandle)h; + + mx_uint *shape = 0; + mx_uint shape_len; + if (MXPredGetOutputShape(handle, index, &shape, &shape_len) < 0) { + jclass MxnetException = env->FindClass("org/dmlc/mxnet/MxnetException"); + env->ThrowNew(MxnetException, MXGetLastError()); + } + + size_t size = 1; + for (mx_uint i=0; i data(size); + if (MXPredGetOutput(handle, index, &(data[0]), size) < 0) { + jclass MxnetException = env->FindClass("org/dmlc/mxnet/MxnetException"); + env->ThrowNew(MxnetException, MXGetLastError()); + } + + jfloatArray joutput = env->NewFloatArray(size); jfloat *out = env->GetFloatArrayElements(joutput, NULL); for (int i=0; iReleaseFloatArrayElements(joutput, out, 0); - return joutput; + return joutput; } JNIEXPORT void JNICALL Java_org_dmlc_mxnet_Predictor_nativeForward (JNIEnv *env, jclass, jlong h, jstring jkey, jfloatArray jinput) { - PredictorHandle handle = (PredictorHandle)h; - const char *key = env->GetStringUTFChars(jkey, 0); - jfloat* input = env->GetFloatArrayElements(jinput, 0); - jsize input_len = env->GetArrayLength(jinput); - - if (MXPredSetInput(handle, key, input, input_len) < 0) { - jclass MxnetException = env->FindClass("org/dmlc/mxnet/MxnetException"); - env->ThrowNew(MxnetException, MXGetLastError()); - } - - env->ReleaseStringUTFChars(jkey, key); - env->ReleaseFloatArrayElements(jinput, input, 0); - if (MXPredForward(handle) < 0) { - jclass MxnetException = env->FindClass("org/dmlc/mxnet/MxnetException"); - env->ThrowNew(MxnetException, MXGetLastError()); - } + PredictorHandle handle = (PredictorHandle)h; + const char *key = env->GetStringUTFChars(jkey, 0); + jfloat* input = env->GetFloatArrayElements(jinput, 0); + jsize input_len = env->GetArrayLength(jinput); + + if (MXPredSetInput(handle, key, input, input_len) < 0) { + jclass MxnetException = env->FindClass("org/dmlc/mxnet/MxnetException"); + env->ThrowNew(MxnetException, MXGetLastError()); + } + + env->ReleaseStringUTFChars(jkey, key); + env->ReleaseFloatArrayElements(jinput, input, 0); + if (MXPredForward(handle) < 0) { + jclass MxnetException = env->FindClass("org/dmlc/mxnet/MxnetException"); + env->ThrowNew(MxnetException, MXGetLastError()); + } } From 7578e357f084b346282a9c5cda0deb324816b06b Mon Sep 17 00:00:00 2001 From: Aston <22279212+astonzhang@users.noreply.github.com> Date: Wed, 2 Aug 2017 13:41:32 -0700 Subject: [PATCH 302/834] Change variable/function names: defered -> deferred (#7301) * Fix a typo * update * change variable name from defered_init to deferred_init * change function name: test_defered_init -> test_deferred_init --- python/mxnet/gluon/parameter.py | 30 +++++++++++++++--------------- tests/python/unittest/test_nn.py | 2 +- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/python/mxnet/gluon/parameter.py b/python/mxnet/gluon/parameter.py index ee73774f6b0f..0ae829ab79ec 100644 --- a/python/mxnet/gluon/parameter.py +++ b/python/mxnet/gluon/parameter.py @@ -77,7 +77,7 @@ def __init__(self, name, grad_req='write', shape=None, dtype=mx_real_t, self._var = None self._data = None self._grad = None - self._defered_init = () + self._deferred_init = () def __repr__(self): s = 'Parameter {name} (shape={shape}, dtype={dtype})' @@ -91,7 +91,7 @@ def _check_initialized(self, ctx=None): "It was only initialized on %s."%( self.name, str(ctx), str(self.list_ctx()))) return - if self._defered_init: + if self._deferred_init: raise DeferredInitializationError raise RuntimeError( "Parameter %s has not been initialized. Note that " \ @@ -116,8 +116,8 @@ def _load_init(self, data, ctx): if isinstance(ctx, Context): ctx = [ctx] if self._data is None: - if self._defered_init: - assert set(ctx) == set(self._defered_init[1]), \ + if self._deferred_init: + assert set(ctx) == set(self._deferred_init[1]), \ "Failed to load Parameter %s on %s because it was " \ "previous initialized on %s."%( self.name, str(ctx), str(self.list_ctx())) @@ -128,14 +128,14 @@ def _load_init(self, data, ctx): "previous initialized on %s."%( self.name, str(ctx), str(self.list_ctx())) self.set_data(data) - self._defered_init = () + self._deferred_init = () def _finish_deferred_init(self): """Finishes deferred initialization.""" - if not self._defered_init: + if not self._deferred_init: return - init, ctx, default_init = self._defered_init - self._defered_init = () + init, ctx, default_init = self._deferred_init + self._deferred_init = () assert self.shape is not None and np.prod(self.shape) > 0, \ "Cannot initialize Parameter %s because it has " \ "invalid shape: %s. Please specify in_units, " \ @@ -227,12 +227,12 @@ def initialize(self, init=None, ctx=None, default_init=initializer.Uniform(), init = default_init if self.init is None else self.init if not self.shape or np.prod(self.shape) <= 0: if self.allow_deferred_init: - self._defered_init = (init, ctx, default_init) + self._deferred_init = (init, ctx, default_init) return raise ValueError("Cannot initialize Parameter %s because it has " \ "invalid shape: %s."%(self.name, str(self.shape))) - self._defered_init = (init, ctx, default_init) + self._deferred_init = (init, ctx, default_init) self._finish_deferred_init() def reset_ctx(self, ctx): @@ -250,9 +250,9 @@ def reset_ctx(self, ctx): data = self._reduce() with autograd.pause(): self._init_impl(data, ctx) - elif self._defered_init: - init, _, default_init = self._defered_init - self._defered_init = (init, ctx, default_init) + elif self._deferred_init: + init, _, default_init = self._deferred_init + self._deferred_init = (init, ctx, default_init) else: raise ValueError("Cannot reset context for Parameter %s because it " "has not been initialized."%self.name) @@ -325,8 +325,8 @@ def list_grad(self): def list_ctx(self): """Returns a list of contexts this parameter is initialized on.""" if self._data is None: - if self._defered_init: - return self._defered_init[1] + if self._deferred_init: + return self._deferred_init[1] raise RuntimeError("Parameter %s has not been initialized"%self.name) return list(self._data.keys()) diff --git a/tests/python/unittest/test_nn.py b/tests/python/unittest/test_nn.py index d4514e2eb77e..e29306326e0c 100644 --- a/tests/python/unittest/test_nn.py +++ b/tests/python/unittest/test_nn.py @@ -264,7 +264,7 @@ def test_at(): x.backward() -def test_defered_init(): +def test_deferred_init(): x = mx.nd.ones((5, 4, 10, 10)) layer = nn.Conv2D(10, 2) layer.collect_params().initialize() From 59769736a402b834b5d53bcdaae895ac4a069e12 Mon Sep 17 00:00:00 2001 From: "Qiang Kou (KK)" Date: Wed, 2 Aug 2017 21:51:34 +0000 Subject: [PATCH 303/834] [R] RNN bucketing with multiple devices. (#7315) --- .../bucket_R/aclImdb_lstm_classification.R | 55 +++---- example/rnn/bucket_R/mx.io.bucket.iter.R | 6 +- example/rnn/bucket_R/rnn.R | 44 +++--- example/rnn/bucket_R/rnn.train.R | 140 ++++++++++++++---- 4 files changed, 157 insertions(+), 88 deletions(-) diff --git a/example/rnn/bucket_R/aclImdb_lstm_classification.R b/example/rnn/bucket_R/aclImdb_lstm_classification.R index aaa6d388fda0..bb5eaacf26dd 100644 --- a/example/rnn/bucket_R/aclImdb_lstm_classification.R +++ b/example/rnn/bucket_R/aclImdb_lstm_classification.R @@ -11,51 +11,30 @@ vocab <- length(corpus_bucketed_test$dic) ### Create iterators batch.size <- 64 -train.data <- mx.io.bucket.iter(buckets = corpus_bucketed_train$buckets, - batch.size = batch.size, - data.mask.element = 0, - shuffle = TRUE) +num.round <- 16 -eval.data <- mx.io.bucket.iter(buckets = corpus_bucketed_test$buckets, - batch.size = batch.size, - data.mask.element = 0, - shuffle = FALSE) +train.data <- mx.io.bucket.iter(buckets = corpus_bucketed_train$buckets, batch.size = batch.size, + data.mask.element = 0, shuffle = TRUE) + +eval.data <- mx.io.bucket.iter(buckets = corpus_bucketed_test$buckets, batch.size = batch.size, + data.mask.element = 0, shuffle = FALSE) mx.set.seed(0) +optimizer <- mx.opt.create("adadelta", rho = 0.92, epsilon = 1e-06, wd = 2e-04, clip_gradient = NULL, + rescale.grad = 1/batch.size) + +model_sentiment_lstm <- mx.rnn.buckets(train.data = train.data, begin.round = 1, + num.round = num.round, ctx = mx.cpu(), metric = mx.metric.accuracy, optimizer = optimizer, + num.rnn.layer = 2, num.embed = 16, num.hidden = 24, num.label = 2, input.size = vocab, + initializer = mx.init.Xavier(rnd_type = "gaussian", factor_type = "in", magnitude = 2), + dropout = 0.25, config = "seq-to-one", batch.end.callback = mx.callback.log.train.metric(period = 50), + verbose = TRUE) -end.round <- 16 - -optimizer <- mx.opt.create("adadelta", - rho = 0.92, - epsilon = 1e-06, - wd = 2e-04, - clip_gradient = NULL, - rescale.grad = 1/batch.size) - -model_sentiment_lstm <- mx.rnn.buckets(train.data = train.data, - begin.round = 1, - end.round = end.round, - ctx = mx.cpu(), - metric = mx.metric.accuracy, - optimizer = optimizer, - num.rnn.layer = 2, - num.embed = 16, - num.hidden = 24, - num.label = 2, - input.size = vocab, - initializer = mx.init.Xavier(rnd_type = "gaussian", - factor_type = "in", - magnitude = 2), - dropout = 0.25, - config = "seq-to-one", - batch.end.callback = mx.callback.log.train.metric(period = 50), - verbose = TRUE) - -mx.model.save(model_sentiment_lstm, prefix = "model_sentiment_lstm", iteration = end.round) +mx.model.save(model_sentiment_lstm, prefix = "model_sentiment_lstm", iteration = num.round) source("rnn.infer.R") -model <- mx.model.load("model_sentiment_lstm", iteration = end.round) +model <- mx.model.load("model_sentiment_lstm", iteration = num.round) pred <- mx.rnn.infer.buckets(infer_iter = eval.data, model, "seq-to-one", ctx = mx.cpu()) diff --git a/example/rnn/bucket_R/mx.io.bucket.iter.R b/example/rnn/bucket_R/mx.io.bucket.iter.R index 887247a320bf..61f87957ede0 100644 --- a/example/rnn/bucket_R/mx.io.bucket.iter.R +++ b/example/rnn/bucket_R/mx.io.bucket.iter.R @@ -64,16 +64,14 @@ BucketIter <- setRefClass("BucketIter", fields = c("buckets", "bucket.names", "b # to appropriate sequence length) idx <- (.self$bucketID - 1) * (.self$batch.size) + (1:batch.size) data <- .self$buckets[[names(.self$bucketID)]]$data[, idx, drop = F] - data_mask <- as.integer(names(.self$bucketID)) - apply(data == .self$data.mask.element, - 2, sum) data_mask_array <- (!data == 0) if (length(dim(.self$buckets[[names(.self$bucketID)]]$label)) == 0) { label <- .self$buckets[[names(.self$bucketID)]]$label[idx] } else { label <- .self$buckets[[names(.self$bucketID)]]$label[, idx, drop = F] } - return(list(data = mx.nd.array(data), label = mx.nd.array(label), data.mask = mx.nd.array(data_mask), - data.mask.array = mx.nd.array(data_mask_array))) + return(list(data = mx.nd.array(data), data.mask.array = mx.nd.array(data_mask_array), + label = mx.nd.array(label))) }, finalize = function() { })) diff --git a/example/rnn/bucket_R/rnn.R b/example/rnn/bucket_R/rnn.R index f55272f29459..ea02b959a7e3 100644 --- a/example/rnn/bucket_R/rnn.R +++ b/example/rnn/bucket_R/rnn.R @@ -33,7 +33,6 @@ rnn.unroll <- function(num.rnn.layer, seq.len, input.size, num.embed, num.hidden # embeding layer label <- mx.symbol.Variable("label") data <- mx.symbol.Variable("data") - data_mask <- mx.symbol.Variable("data.mask") data_mask_array <- mx.symbol.Variable("data.mask.array") data_mask_array <- mx.symbol.stop_gradient(data_mask_array, name = "data.mask.array") @@ -112,8 +111,8 @@ rnn.unroll <- function(num.rnn.layer, seq.len, input.size, num.embed, num.hidden mx.rnn.buckets <- function(train.data, eval.data = NULL, num.rnn.layer, num.hidden, num.embed, num.label, input.size, ctx = NULL, num.round = 1, initializer = mx.init.uniform(0.01), dropout = 0, config = "one-to-one", optimizer = "sgd", batch.end.callback = NULL, - epoch.end.callback = NULL, begin.round = 1, end.round = 1, metric = mx.metric.rmse, - cell.type = "lstm", verbose = FALSE) { + epoch.end.callback = NULL, begin.round = 1, metric = mx.metric.rmse, cell.type = "lstm", + kvstore = "local", verbose = FALSE) { if (!train.data$iter.next()) { train.data$reset() @@ -131,8 +130,11 @@ mx.rnn.buckets <- function(train.data, eval.data = NULL, num.rnn.layer, num.hidd if (is.null(ctx)) ctx <- mx.ctx.default() - if (!is.mx.context(ctx)) - stop("ctx must be mx.context") + if (is.mx.context(ctx)) { + ctx <- list(ctx) + } + if (!is.list(ctx)) + stop("ctx must be mx.context or list of mx.context") if (is.character(optimizer)) { if (is.numeric(input.shape)) { ndim <- length(input.shape) @@ -155,17 +157,28 @@ mx.rnn.buckets <- function(train.data, eval.data = NULL, num.rnn.layer, num.hidd symbol <- sym_list[[names(train.data$bucketID)]] arg.names <- symbol$arguments - input.shape <- lapply(train.data$value(), dim) - input.shape <- input.shape[names(input.shape) %in% arg.names] + input.names <- c("data", "data.mask.array") + input.shape <- sapply(input.names, function(n) { + dim(train.data$value()[[n]]) + }, simplify = FALSE) + output.names <- "label" + output.shape <- sapply(output.names, function(n) { + dim(train.data$value()[[n]]) + }, simplify = FALSE) + + params <- mx.model.init.params(symbol, input.shape, output.shape, initializer, + mx.cpu()) - params <- mx.model.init.params(symbol, input.shape, NULL, initializer, mx.cpu()) + kvstore <- mxnet:::mx.model.create.kvstore(kvstore, params$arg.params, length(ctx), + verbose = verbose) ### Execute training - rnn.model.R model <- mx.model.train.rnn.buckets(sym_list = sym_list, input.shape = input.shape, - arg.params = params$arg.params, aux.params = params$aux.params, optimizer = optimizer, - train.data = train.data, eval.data = eval.data, verbose = verbose, begin.round = begin.round, - end.round = end.round, metric = metric, ctx = ctx, batch.end.callback = batch.end.callback, - epoch.end.callback = epoch.end.callback) + output.shape = output.shape, arg.params = params$arg.params, aux.params = params$aux.params, + optimizer = optimizer, train.data = train.data, eval.data = eval.data, verbose = verbose, + begin.round = begin.round, end.round = num.round, metric = metric, ctx = ctx, + batch.end.callback = batch.end.callback, epoch.end.callback = epoch.end.callback, + kvstore = kvstore) return(model) } @@ -193,10 +206,3 @@ mx.model.check.arguments <- function(symbol) { } return(c(data, label)) } - -# filter out null, keep the names -mx.util.filter.null <- function(lst) { - lst[!sapply(lst, is.null)] -} - - diff --git a/example/rnn/bucket_R/rnn.train.R b/example/rnn/bucket_R/rnn.train.R index 962430c1a297..b833b2b1d37a 100644 --- a/example/rnn/bucket_R/rnn.train.R +++ b/example/rnn/bucket_R/rnn.train.R @@ -4,30 +4,57 @@ source("rnn.R") # Internal function to do multiple device training on RNN mx.model.train.rnn.buckets <- function(ctx, sym_list, arg.params, aux.params, input.shape, - begin.round, end.round, optimizer, train.data, eval.data, metric, epoch.end.callback, - batch.end.callback, verbose = TRUE) { + output.shape, begin.round, end.round, optimizer, train.data, eval.data, metric, + epoch.end.callback, batch.end.callback, kvstore, verbose = TRUE) { symbol <- sym_list[[names(train.data$bucketID)]] input.names <- names(input.shape) + output.names <- names(output.shape) arg.names <- names(arg.params) + ndevice <- length(ctx) + if (verbose) + message(paste0("Start training with ", ndevice, " devices")) + input_slice <- mxnet:::mx.model.slice.shape(input.shape, ndevice) + output_slice <- mxnet:::mx.model.slice.shape(output.shape, ndevice) + + # Grad request grad_req <- rep("write", length(symbol$arguments)) + # grad_null_idx <- match(c(input.names, output.names), symbol$arguments) grad_null_idx <- match(input.names, symbol$arguments) grad_req[grad_null_idx] <- "null" # Arg array order - update_names <- c(input.names, arg.names) + update_names <- c(input.names, output.names, arg.names) arg_update_idx <- match(symbol$arguments, update_names) - s <- sapply(input.shape, function(shape) { - mx.nd.zeros(shape = shape, ctx = mx.cpu()) + train.execs <- lapply(1:ndevice, function(i) { + s <- sapply(append(input_slice[[i]]$shape, output_slice[[i]]$shape), function(shape) { + mx.nd.zeros(shape = shape, ctx = mx.cpu()) + }) + mxnet:::mx.symbol.bind(symbol = symbol, arg.arrays = c(s, arg.params)[arg_update_idx], + aux.arrays = aux.params, ctx = mx.cpu(), grad.req = grad_req) }) - train.exec <- mxnet:::mx.symbol.bind(symbol = symbol, arg.arrays = c(s, arg.params)[arg_update_idx], - aux.arrays = aux.params, ctx = ctx, grad.req = grad_req) + # KVStore related stuffs + params.index <- as.integer(mxnet:::mx.util.filter.null(lapply(1:length(train.execs[[1]]$ref.grad.arrays), + function(k) { + if (!is.null(train.execs[[1]]$ref.grad.arrays[[k]])) k else NULL + }))) + update.on.kvstore <- FALSE + if (!is.null(kvstore) && kvstore$update.on.kvstore) { + update.on.kvstore <- TRUE + kvstore$set.optimizer(optimizer) + } else { + updaters <- lapply(1:ndevice, function(i) { + mx.opt.get.updater(optimizer, train.execs[[i]]$ref.arg.arrays) + }) + } - updaters <- mx.opt.get.updater(optimizer, train.exec$ref.arg.arrays) + if (!is.null(kvstore)) { + kvstore$init(params.index, train.execs[[1]]$ref.arg.arrays[params.index]) + } for (iteration in begin.round:end.round) { nbatch <- 0 @@ -36,26 +63,67 @@ mx.model.train.rnn.buckets <- function(ctx, sym_list, arg.params, aux.params, in } train.data$reset() while (train.data$iter.next()) { - dlist <- train.data$value()[input.names] + dlist <- train.data$value() #[input.names] symbol <- sym_list[[names(train.data$bucketID)]] + slices <- lapply(1:ndevice, function(i) { + s <- input_slice[[i]] + ret <- sapply(names(dlist), function(n) { + mxnet:::mx.nd.slice(dlist[[n]], s$begin, s$end) + }) + return(ret) + }) - train.exec <- mxnet:::mx.symbol.bind(symbol = symbol, arg.arrays = c(dlist, - train.exec$arg.arrays[arg.names])[arg_update_idx], aux.arrays = train.exec$aux.arrays, - ctx = ctx, grad.req = grad_req) + train.execs <- lapply(1:ndevice, function(i) { + s <- slices[[i]] + mxnet:::mx.symbol.bind(symbol = symbol, arg.arrays = c(s, train.execs[[i]]$arg.arrays[arg.names])[arg_update_idx], + aux.arrays = train.execs[[i]]$aux.arrays, ctx = ctx[[i]], grad.req = grad_req) + }) - mx.exec.forward(train.exec, is.train = TRUE) + for (texec in train.execs) { + mx.exec.forward(texec, is.train = TRUE) + } - # copy outputs to CPU - out.preds <- mx.nd.copyto(train.exec$ref.outputs[[1]], mx.cpu()) + out.preds <- lapply(train.execs, function(texec) { + mx.nd.copyto(texec$ref.outputs[[1]], mx.cpu()) + }) - mx.exec.backward(train.exec) + for (texec in train.execs) { + mx.exec.backward(texec) + } - arg.blocks <- updaters(train.exec$ref.arg.arrays, train.exec$ref.grad.arrays) - mx.exec.update.arg.arrays(train.exec, arg.blocks, skip.null = TRUE) + if (!is.null(kvstore)) { + # push the gradient + kvstore$push(params.index, lapply(train.execs, function(texec) { + texec$ref.grad.arrays[params.index] + }), -params.index) + } + if (update.on.kvstore) { + # pull back weight + kvstore$pull(params.index, lapply(train.execs, function(texec) { + texec$ref.arg.arrays[params.index] + }), -params.index) + } else { + # pull back gradient sums + if (!is.null(kvstore)) { + kvstore$pull(params.index, lapply(train.execs, function(texec) { + texec$ref.grad.arrays[params.index] + }), -params.index) + } + arg.blocks <- lapply(1:ndevice, function(i) { + updaters[[i]](train.execs[[i]]$ref.arg.arrays, train.execs[[i]]$ref.grad.arrays) + }) + for (i in 1:ndevice) { + mx.exec.update.arg.arrays(train.execs[[i]], arg.blocks[[i]], skip.null = TRUE) + } + } # Update the evaluation metrics if (!is.null(metric)) { - train.metric <- metric$update(dlist$label, out.preds, train.metric) + # train.metric <- metric$update(dlist$label, out.preds, train.metric) + for (i in 1:ndevice) { + train.metric <- metric$update(slices[[i]][[length(slices[[i]])]], + out.preds[[i]], train.metric) + } } nbatch <- nbatch + 1 @@ -78,19 +146,37 @@ mx.model.train.rnn.buckets <- function(ctx, sym_list, arg.params, aux.params, in eval.data$reset() while (eval.data$iter.next()) { # Get input data slice - dlist <- eval.data$value()[input.names] + dlist <- eval.data$value() #[input.names] symbol <- sym_list[[names(eval.data$bucketID)]] - train.exec <- mxnet:::mx.symbol.bind(symbol = symbol, arg.arrays = c(dlist, - train.exec$arg.arrays[arg.names])[arg_update_idx], aux.arrays = train.exec$aux.arrays, - ctx = ctx, grad.req = grad_req) + slices <- lapply(1:ndevice, function(i) { + s <- input_slice[[i]] + ret <- sapply(names(dlist), function(n) { + mxnet:::mx.nd.slice(dlist[[n]], s$begin, s$end) + }) + return(ret) + }) - mx.exec.forward(train.exec, is.train = FALSE) + + train.execs <- lapply(1:ndevice, function(i) { + s <- slices[[i]] + mxnet:::mx.symbol.bind(symbol = symbol, arg.arrays = c(s, train.execs[[i]]$arg.arrays[arg.names])[arg_update_idx], + aux.arrays = train.execs[[i]]$aux.arrays, ctx = ctx[[i]], grad.req = grad_req) + }) + + for (texec in train.execs) { + mx.exec.forward(texec, is.train = FALSE) + } # copy outputs to CPU - out.preds <- mx.nd.copyto(train.exec$ref.outputs[[1]], mx.cpu()) + out.preds <- lapply(train.execs, function(texec) { + mx.nd.copyto(texec$ref.outputs[[1]], mx.cpu()) + }) if (!is.null(metric)) { - eval.metric <- metric$update(dlist$label, out.preds, eval.metric) + for (i in 1:ndevice) { + eval.metric <- metric$update(slices[[i]][[length(slices[[i]])]], + out.preds[[i]], eval.metric) + } } } @@ -105,7 +191,7 @@ mx.model.train.rnn.buckets <- function(ctx, sym_list, arg.params, aux.params, in eval.metric <- NULL } # get the model out - model <- mxnet:::mx.model.extract.model(symbol, list(train.exec)) + model <- mxnet:::mx.model.extract.model(symbol, train.execs) epoch_continue <- TRUE if (!is.null(epoch.end.callback)) { From e0639eb16f9fae926151a1c8a244877a535becf5 Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Wed, 2 Aug 2017 14:52:17 -0700 Subject: [PATCH 304/834] lstm crf example (#7253) --- example/gluon/lstm_crf.py | 213 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 213 insertions(+) create mode 100644 example/gluon/lstm_crf.py diff --git a/example/gluon/lstm_crf.py b/example/gluon/lstm_crf.py new file mode 100644 index 000000000000..8344789faa6e --- /dev/null +++ b/example/gluon/lstm_crf.py @@ -0,0 +1,213 @@ +import mxnet as mx +from mxnet import autograd as ag, ndarray as nd, gluon +from mxnet.gluon import Block, nn, rnn +import mxnet.optimizer as optim +import sys + +# This example demonstrates how LSTM-CRF model can be implemented in Gluon to perform +# noun-phrase chunking as a sequence labeling task. + +mx.random.seed(1) + +# Helper functions to make the code more readable. +def to_scalar(x): + return int(x.asscalar()) + +def argmax(vec): + # return the argmax as a python int + idx = nd.argmax(vec, axis=1) + return to_scalar(idx) + +def prepare_sequence(seq, word2idx): + return nd.array([word2idx[w] for w in seq]) + +# Compute log sum exp is numerically more stable than multiplying probabilities +def log_sum_exp(vec): + max_score = nd.max(vec).asscalar() + return nd.log(nd.sum(nd.exp(vec - max_score))) + max_score + +# Model +class BiLSTM_CRF(Block): + def __init__(self, vocab_size, tag2idx, embedding_dim, hidden_dim): + super(BiLSTM_CRF, self).__init__() + with self.name_scope(): + self.embedding_dim = embedding_dim + self.hidden_dim = hidden_dim + self.vocab_size = vocab_size + self.tag2idx = tag2idx + self.tagset_size = len(tag2idx) + + self.word_embeds = nn.Embedding(vocab_size, embedding_dim) + self.lstm = rnn.LSTM(hidden_dim // 2, num_layers=1, bidirectional=True) + + # Maps the output of the LSTM into tag space. + self.hidden2tag = nn.Dense(self.tagset_size) + + # Matrix of transition parameters. Entry i,j is the score of + # transitioning *to* i *from* j. + self.transitions = nd.random_normal(shape=(self.tagset_size, self.tagset_size)) + + self.hidden = self.init_hidden() + + def init_hidden(self): + return [nd.random_normal(shape=(2, 1, self.hidden_dim // 2)), + nd.random_normal(shape=(2, 1, self.hidden_dim // 2))] + + def _forward_alg(self, feats): + # Do the forward algorithm to compute the partition function + alphas = [[-10000.] * self.tagset_size] + alphas[0][self.tag2idx[START_TAG]] = 0. + alphas = nd.array(alphas) + + # Iterate through the sentence + for feat in feats: + alphas_t = [] # The forward variables at this timestep + for next_tag in range(self.tagset_size): + # broadcast the emission score: it is the same regardless of + # the previous tag + emit_score = feat[next_tag].reshape((1, -1)) + # the ith entry of trans_score is the score of transitioning to + # next_tag from i + trans_score = self.transitions[next_tag].reshape((1, -1)) + # The ith entry of next_tag_var is the value for the + # edge (i -> next_tag) before we do log-sum-exp + next_tag_var = alphas + trans_score + emit_score + # The forward variable for this tag is log-sum-exp of all the + # scores. + alphas_t.append(log_sum_exp(next_tag_var)) + alphas = nd.concat(*alphas_t, dim=0).reshape((1, -1)) + terminal_var = alphas + self.transitions[self.tag2idx[STOP_TAG]] + alpha = log_sum_exp(terminal_var) + return alpha + + def _get_lstm_features(self, sentence): + self.hidden = self.init_hidden() + length = sentence.shape[0] + embeds = self.word_embeds(sentence).reshape((length, 1, -1)) + lstm_out, self.hidden = self.lstm(embeds, self.hidden) + lstm_out = lstm_out.reshape((length, self.hidden_dim)) + lstm_feats = self.hidden2tag(lstm_out) + return nd.split(lstm_feats, num_outputs=length, axis=0, squeeze_axis=True) + + def _score_sentence(self, feats, tags): + # Gives the score of a provided tag sequence + score = nd.array([0]) + tags = nd.concat(nd.array([self.tag2idx[START_TAG]]), *tags, dim=0) + for i, feat in enumerate(feats): + score = score + \ + self.transitions[to_scalar(tags[i+1]), to_scalar(tags[i])] + feat[to_scalar(tags[i+1])] + score = score + self.transitions[self.tag2idx[STOP_TAG], + to_scalar(tags[int(tags.shape[0]-1)])] + return score + + def _viterbi_decode(self, feats): + backpointers = [] + + # Initialize the viterbi variables in log space + vvars = nd.full((1, self.tagset_size), -10000.) + vvars[0, self.tag2idx[START_TAG]] = 0 + + for feat in feats: + bptrs_t = [] # holds the backpointers for this step + viterbivars_t = [] # holds the viterbi variables for this step + + for next_tag in range(self.tagset_size): + # next_tag_var[i] holds the viterbi variable for tag i at the + # previous step, plus the score of transitioning + # from tag i to next_tag. + # We don't include the emission scores here because the max + # does not depend on them (we add them in below) + next_tag_var = vvars + self.transitions[next_tag] + best_tag_id = argmax(next_tag_var) + bptrs_t.append(best_tag_id) + viterbivars_t.append(next_tag_var[0, best_tag_id]) + # Now add in the emission scores, and assign vvars to the set + # of viterbi variables we just computed + vvars = (nd.concat(*viterbivars_t, dim=0) + feat).reshape((1, -1)) + backpointers.append(bptrs_t) + + # Transition to STOP_TAG + terminal_var = vvars + self.transitions[self.tag2idx[STOP_TAG]] + best_tag_id = argmax(terminal_var) + path_score = terminal_var[0, best_tag_id] + + # Follow the back pointers to decode the best path. + best_path = [best_tag_id] + for bptrs_t in reversed(backpointers): + best_tag_id = bptrs_t[best_tag_id] + best_path.append(best_tag_id) + # Pop off the start tag (we dont want to return that to the caller) + start = best_path.pop() + assert start == self.tag2idx[START_TAG] # Sanity check + best_path.reverse() + return path_score, best_path + + def neg_log_likelihood(self, sentence, tags): + feats = self._get_lstm_features(sentence) + forward_score = self._forward_alg(feats) + gold_score = self._score_sentence(feats, tags) + return forward_score - gold_score + + def forward(self, sentence): # dont confuse this with _forward_alg above. + # Get the emission scores from the BiLSTM + lstm_feats = self._get_lstm_features(sentence) + + # Find the best path, given the features. + score, tag_seq = self._viterbi_decode(lstm_feats) + return score, tag_seq + +# Run training +START_TAG = "" +STOP_TAG = "" +EMBEDDING_DIM = 5 +HIDDEN_DIM = 4 + +# Make up some training data +training_data = [( + "the wall street journal reported today that apple corporation made money".split(), + "B I I I O O O B I O O".split() +), ( + "georgia tech is a university in georgia".split(), + "B I O O O O B".split() +)] + +word2idx = {} +for sentence, tags in training_data: + for word in sentence: + if word not in word2idx: + word2idx[word] = len(word2idx) + +tag2idx = {"B": 0, "I": 1, "O": 2, START_TAG: 3, STOP_TAG: 4} + +model = BiLSTM_CRF(len(word2idx), tag2idx, EMBEDDING_DIM, HIDDEN_DIM) +model.collect_params().initialize(mx.init.Xavier(magnitude=2.24), ctx=mx.cpu()) +optimizer = gluon.Trainer(model.collect_params(), 'sgd', {'learning_rate': 0.01, 'wd': 1e-4}) + +# Check predictions before training +precheck_sent = prepare_sequence(training_data[0][0], word2idx) +precheck_tags = nd.array([tag2idx[t] for t in training_data[0][1]]) +print(model(precheck_sent)) + +# Make sure prepare_sequence from earlier in the LSTM section is loaded +for epoch in range(300): # again, normally you would NOT do 300 epochs, it is toy data + for sentence, tags in training_data: + # Step 1. Get our inputs ready for the network, that is, + # turn them into Variables of word indices. + # Remember to use autograd to record the calculation. + with ag.record(): + sentence_in = prepare_sequence(sentence, word2idx) + targets = nd.array([tag2idx[t] for t in tags]) + + # Step 2. Run our forward pass. + neg_log_likelihood = model.neg_log_likelihood(sentence_in, targets) + + # Step 3. Compute the loss, gradients, and update the parameters by + # calling optimizer.step() + neg_log_likelihood.backward() + optimizer.step(1) + +# Check predictions after training +precheck_sent = prepare_sequence(training_data[0][0], word2idx) +print(model(precheck_sent)) + +# Acknowledgement: this example is adopted from pytorch nlp tutorials. From 8e0f627bbcf7b8f22c35e32a9b968f41ca1439e7 Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Wed, 2 Aug 2017 16:19:41 -0700 Subject: [PATCH 305/834] add doc for gluon, sym/nd contrib (#7284) * add doc for contrib * docs for gluon, sym/nd contrib --- Makefile | 3 +- docs/api/python/gluon.md | 497 +++++++++++++++--- docs/api/python/ndarray.md | 34 ++ docs/api/python/symbol.md | 34 ++ python/mxnet/gluon/model_zoo/vision/resnet.py | 2 + 5 files changed, 492 insertions(+), 78 deletions(-) diff --git a/Makefile b/Makefile index f200b876db45..5c7f54dcd759 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,7 @@ endif ifndef DMLC_CORE DMLC_CORE = $(ROOTDIR)/dmlc-core endif +CORE_INC = $(wildcard $(DMLC_CORE)/include/*/*.h) ifndef NNVM_PATH NNVM_PATH = $(ROOTDIR)/nnvm @@ -291,7 +292,7 @@ build/plugin/%.o: plugin/%.cc $(NVCC) $(NVCCFLAGS) $(CUDA_ARCH) -Xcompiler "$(CFLAGS) -Isrc/operator" -M -MT $*_gpu.o $< >$*_gpu.d $(NVCC) -c -o $@ $(NVCCFLAGS) $(CUDA_ARCH) -Xcompiler "$(CFLAGS) -Isrc/operator" $< -%.o: %.cc +%.o: %.cc $(CORE_INC) @mkdir -p $(@D) $(CXX) -std=c++11 -c $(CFLAGS) -MMD -Isrc/operator -c $< -o $@ diff --git a/docs/api/python/gluon.md b/docs/api/python/gluon.md index cda4a07f85dd..6e213bbe05e0 100644 --- a/docs/api/python/gluon.md +++ b/docs/api/python/gluon.md @@ -21,58 +21,379 @@ in Python and then deploy with symbolic graph in C++ and Scala. ## Parameter ```eval_rst -.. currentmodule:: mxnet.gluon +.. autosummary:: + :nosignatures: + + Parameter + ParameterDict ``` +## Containers + ```eval_rst -.. currentmodule:: mxnet.gluon -.. autoclass:: mxnet.gluon.Parameter - :members: -.. autoclass:: mxnet.gluon.ParameterDict - :members: +.. autosummary:: + :nosignatures: + + Block + HybridBlock + SymbolBlock ``` +## Neural Network Layers + +```eval_rst +.. currentmodule:: mxnet.gluon.nn +``` + +### Containers + +```eval_rst +.. autosummary:: + :nosignatures: + + Sequential + HybridSequential +``` + + +### Basic Layers + + +```eval_rst +.. autosummary:: + :nosignatures: + + Dense + Activation + Dropout + BatchNorm + LeakyReLU + Embedding +``` + + +### Convolutional Layers + + +```eval_rst +.. autosummary:: + :nosignatures: + + Conv1D + Conv2D + Conv3D + Conv1DTranspose + Conv2DTranspose + Conv3DTranspose +``` + + + +### Pooling Layers -## Containers + +```eval_rst +.. autosummary:: + :nosignatures: + + MaxPool1D + MaxPool2D + MaxPool3D + AvgPool1D + AvgPool2D + AvgPool3D + GlobalMaxPool1D + GlobalMaxPool2D + GlobalMaxPool3D + GlobalAvgPool1D + GlobalAvgPool2D + GlobalAvgPool3D +``` + + + +## Recurrent Layers + +```eval_rst +.. currentmodule:: mxnet.gluon.rnn +``` + + +```eval_rst +.. autosummary:: + :nosignatures: + + RecurrentCell + RNN + LSTM + GRU + RNNCell + LSTMCell + GRUCell + SequentialRNNCell + BidirectionalCell + DropoutCell + ZoneoutCell + ResidualCell +``` + + +## Trainer ```eval_rst .. currentmodule:: mxnet.gluon -.. autoclass:: mxnet.gluon.Block - :members: - .. automethod:: forward -.. autoclass:: mxnet.gluon.HybridBlock - :members: +.. autosummary:: + :nosignatures: - .. automethod:: hybrid_forward + Trainer ``` -## Neural Network Layers + +## Loss functions ```eval_rst -.. currentmodule:: mxnet.gluon.nn +.. currentmodule:: mxnet.gluon.loss ``` -### Containers +```eval_rst +.. autosummary:: + :nosignatures: + + L2Loss + L1Loss + SoftmaxCrossEntropyLoss + KLDivLoss +``` + +## Utilities ```eval_rst -.. currentmodule:: mxnet.gluon.nn +.. currentmodule:: mxnet.gluon.utils +``` - .. automethod:: __call__ -.. autoclass:: mxnet.gluon.nn.Sequential - :members: -.. autoclass:: mxnet.gluon.nn.HybridSequential - :members: + +```eval_rst +.. autosummary:: + :nosignatures: + + split_data + split_and_load + clip_global_norm ``` +## Data -### Basic Layers +```eval_rst +.. currentmodule:: mxnet.gluon.data +``` + +```eval_rst +.. autosummary:: + :nosignatures: + + Dataset + ArrayDataset + RecordFileDataset + ImageRecordDataset +``` + +```eval_rst +.. autosummary:: + :nosignatures: + + Sampler + SequentialSampler + RandomSampler + BatchSampler +``` + +```eval_rst +.. autosummary:: + :nosignatures: + + DataLoader +``` + +### Vision + +```eval_rst +.. currentmodule:: mxnet.gluon.data.vision +``` + +```eval_rst +.. autosummary:: + :nosignatures: + + MNIST + CIFAR10 +``` + +## Model Zoo + +Model zoo provides pre-defined and pre-trained models to help bootstrap machine learning applications. + +### Vision + +```eval_rst +.. currentmodule:: mxnet.gluon.model_zoo.vision +``` + +```eval_rst +.. autosummary:: + :nosignatures: + + get_model +``` + +#### ResNet + +```eval_rst +.. autosummary:: + :nosignatures: + + resnet18_v1 + resnet34_v1 + resnet50_v1 + resnet101_v1 + resnet152_v1 + resnet18_v2 + resnet34_v2 + resnet50_v2 + resnet101_v2 + resnet152_v2 +``` + +```eval_rst +.. autosummary:: + :nosignatures: + + ResNetV1 + ResNetV2 + BasicBlockV1 + BasicBlockV2 + BottleneckV1 + BottleneckV2 + get_resnet +``` + +#### VGG + +```eval_rst +.. autosummary:: + :nosignatures: + + vgg11 + vgg13 + vgg16 + vgg19 + vgg11_bn + vgg13_bn + vgg16_bn + vgg19_bn +``` + +```eval_rst +.. autosummary:: + :nosignatures: + + VGG + get_vgg +``` +#### Alexnet ```eval_rst -.. currentmodule:: mxnet.gluon.nn +.. autosummary:: + :nosignatures: + + alexnet +``` + +```eval_rst +.. autosummary:: + :nosignatures: + + AlexNet +``` + +#### DenseNet + +```eval_rst +.. autosummary:: + :nosignatures: + + densenet121 + densenet161 + densenet169 + densenet201 +``` + +```eval_rst +.. autosummary:: + :nosignatures: + + DenseNet +``` + +#### SqueezeNet + +```eval_rst +.. autosummary:: + :nosignatures: + + squeezenet1_0 + squeezenet1_1 +``` + +```eval_rst +.. autosummary:: + :nosignatures: + + SqueezeNet +``` + +#### Inception + +```eval_rst +.. autosummary:: + :nosignatures: + + inception_v3 +``` + +```eval_rst +.. autosummary:: + :nosignatures: + + Inception3 +``` + +## API Reference + + + +```eval_rst +.. autoclass:: mxnet.gluon.Parameter + :members: +.. autoclass:: mxnet.gluon.ParameterDict + :members: + +.. autoclass:: mxnet.gluon.Block + :members: + + .. automethod:: __call__ +.. autoclass:: mxnet.gluon.HybridBlock + :members: +.. autoclass:: mxnet.gluon.SymbolBlock + :members: + +.. autoclass:: mxnet.gluon.nn.Sequential + :members: +.. autoclass:: mxnet.gluon.nn.HybridSequential + :members: .. autoclass:: mxnet.gluon.nn.Dense :members: .. autoclass:: mxnet.gluon.nn.Activation @@ -85,14 +406,6 @@ in Python and then deploy with symbolic graph in C++ and Scala. :members: .. autoclass:: mxnet.gluon.nn.Embedding :members: -``` - - -### Convolutional Layers - - -```eval_rst -.. currentmodule:: mxnet.gluon.nn .. autoclass:: mxnet.gluon.nn.Conv1D :members: .. autoclass:: mxnet.gluon.nn.Conv2D @@ -105,15 +418,6 @@ in Python and then deploy with symbolic graph in C++ and Scala. :members: .. autoclass:: mxnet.gluon.nn.Conv3DTranspose :members: -``` - - - -### Pooling Layers - - -```eval_rst -.. currentmodule:: mxnet.gluon.nn .. autoclass:: mxnet.gluon.nn.MaxPool1D :members: .. autoclass:: mxnet.gluon.nn.MaxPool2D @@ -138,18 +442,7 @@ in Python and then deploy with symbolic graph in C++ and Scala. :members: .. autoclass:: mxnet.gluon.nn.GlobalAvgPool3D :members: -``` - - -## Recurrent Layers - -```eval_rst -.. currentmodule:: mxnet.gluon.rnn -``` - - -```eval_rst .. autoclass:: mxnet.gluon.rnn.RecurrentCell :members: @@ -176,26 +469,10 @@ in Python and then deploy with symbolic graph in C++ and Scala. :members: .. autoclass:: mxnet.gluon.rnn.ResidualCell :members: -``` - -## Trainer - -```eval_rst -.. currentmodule:: mxnet.gluon -``` - - -```eval_rst .. autoclass:: mxnet.gluon.Trainer :members: -``` - - -## Loss functions -```eval_rst -.. currentmodule:: mxnet.gluon.loss .. autoclass:: mxnet.gluon.loss.L2Loss :members: .. autoclass:: mxnet.gluon.loss.L1Loss @@ -204,19 +481,85 @@ in Python and then deploy with symbolic graph in C++ and Scala. :members: .. autoclass:: mxnet.gluon.loss.KLDivLoss :members: -``` +.. automethod:: mxnet.gluon.utils.split_data -## Utilities +.. automethod:: mxnet.gluon.utils.split_and_load -```eval_rst -.. currentmodule:: mxnet.gluon.utils -``` +.. automethod:: mxnet.gluon.utils.clip_global_norm +.. autoclass:: mxnet.gluon.data.Dataset + :members: +.. autoclass:: mxnet.gluon.data.ArrayDataset + :members: +.. autoclass:: mxnet.gluon.data.RecordFileDataset + :members: +.. autoclass:: mxnet.gluon.data.ImageRecordDataset + :members: +.. autoclass:: mxnet.gluon.data.Sampler + :members: +.. autoclass:: mxnet.gluon.data.SequentialSampler + :members: +.. autoclass:: mxnet.gluon.data.RandomSampler + :members: +.. autoclass:: mxnet.gluon.data.BatchSampler + :members: +.. autoclass:: mxnet.gluon.data.DataLoader + :members: +.. automodule:: mxnet.gluon.data.vision + :members: -```eval_rst -.. automethod:: mxnet.gluon.utils.split_data -.. automethod:: mxnet.gluon.utils.split_and_load -.. automethod:: mxnet.gluon.utils.clip_global_norm +.. automodule:: mxnet.gluon.model_zoo.vision + :members: +.. automethod:: mxnet.gluon.model_zoo.vision.resnet18_v1 +.. automethod:: mxnet.gluon.model_zoo.vision.resnet34_v1 +.. automethod:: mxnet.gluon.model_zoo.vision.resnet50_v1 +.. automethod:: mxnet.gluon.model_zoo.vision.resnet101_v1 +.. automethod:: mxnet.gluon.model_zoo.vision.resnet152_v1 +.. automethod:: mxnet.gluon.model_zoo.vision.resnet18_v2 +.. automethod:: mxnet.gluon.model_zoo.vision.resnet34_v2 +.. automethod:: mxnet.gluon.model_zoo.vision.resnet50_v2 +.. automethod:: mxnet.gluon.model_zoo.vision.resnet101_v2 +.. automethod:: mxnet.gluon.model_zoo.vision.resnet152_v2 +.. automethod:: mxnet.gluon.model_zoo.vision.get_resnet +.. autoclass:: mxnet.gluon.model_zoo.vision.ResNetV1 + :members: +.. autoclass:: mxnet.gluon.model_zoo.vision.BasicBlockV1 + :members: +.. autoclass:: mxnet.gluon.model_zoo.vision.BottleneckV1 + :members: +.. autoclass:: mxnet.gluon.model_zoo.vision.ResNetV2 + :members: +.. autoclass:: mxnet.gluon.model_zoo.vision.BasicBlockV2 + :members: +.. autoclass:: mxnet.gluon.model_zoo.vision.BottleneckV2 + :members: +.. automethod:: mxnet.gluon.model_zoo.vision.vgg11 +.. automethod:: mxnet.gluon.model_zoo.vision.vgg13 +.. automethod:: mxnet.gluon.model_zoo.vision.vgg16 +.. automethod:: mxnet.gluon.model_zoo.vision.vgg19 +.. automethod:: mxnet.gluon.model_zoo.vision.vgg11_bn +.. automethod:: mxnet.gluon.model_zoo.vision.vgg13_bn +.. automethod:: mxnet.gluon.model_zoo.vision.vgg16_bn +.. automethod:: mxnet.gluon.model_zoo.vision.vgg19_bn +.. automethod:: mxnet.gluon.model_zoo.vision.get_vgg +.. autoclass:: mxnet.gluon.model_zoo.vision.VGG + :members: +.. automethod:: mxnet.gluon.model_zoo.vision.alexnet +.. autoclass:: mxnet.gluon.model_zoo.vision.AlexNet + :members: +.. automethod:: mxnet.gluon.model_zoo.vision.densenet121 +.. automethod:: mxnet.gluon.model_zoo.vision.densenet161 +.. automethod:: mxnet.gluon.model_zoo.vision.densenet169 +.. automethod:: mxnet.gluon.model_zoo.vision.densenet201 +.. autoclass:: mxnet.gluon.model_zoo.vision.DenseNet + :members: +.. automethod:: mxnet.gluon.model_zoo.vision.squeezenet1_0 +.. automethod:: mxnet.gluon.model_zoo.vision.squeezenet1_1 +.. autoclass:: mxnet.gluon.model_zoo.vision.SqueezeNet + :members: +.. automethod:: mxnet.gluon.model_zoo.vision.inception_v3 +.. autoclass:: mxnet.gluon.model_zoo.vision.Inception3 + :members: ``` diff --git a/docs/api/python/ndarray.md b/docs/api/python/ndarray.md index a782b910e656..5e9f7e1a1184 100644 --- a/docs/api/python/ndarray.md +++ b/docs/api/python/ndarray.md @@ -463,6 +463,37 @@ In the rest of this document, we first overview the methods provided by the Custom ``` +## Contrib + +```eval_rst +.. warning:: This package contains experimental APIs and may change in the near future. +``` + +The `contrib.ndarray` module contains many useful experimental APIs for new features. This is a place for the community to try out the new features, so that feature contributors can receive feedback. + +```eval_rst +.. currentmodule:: mxnet.contrib.ndarray + +.. autosummary:: + :nosignatures: + + CTCLoss + DeformableConvolution + DeformablePSROIPooling + MultiBoxDetection + MultiBoxPrior + MultiBoxTarget + MultiProposal + PSROIPooling + Proposal + count_sketch + ctc_loss + dequantize + fft + ifft + quantize +``` + ## API Reference @@ -474,6 +505,9 @@ In the rest of this document, we first overview the methods provided by the .. automodule:: mxnet.random :members: +.. automodule:: mxnet.contrib.ndarray + :members: + ``` diff --git a/docs/api/python/symbol.md b/docs/api/python/symbol.md index 0ebb869290bf..dd455eee587a 100644 --- a/docs/api/python/symbol.md +++ b/docs/api/python/symbol.md @@ -480,6 +480,37 @@ Composite multiple symbols into a new one by an operator. Custom ``` +## Contrib + +```eval_rst +.. warning:: This package contains experimental APIs and may change in the near future. +``` + +The `contrib.symbol` module contains many useful experimental APIs for new features. This is a place for the community to try out the new features, so that feature contributors can receive feedback. + +```eval_rst +.. currentmodule:: mxnet.contrib.symbol + +.. autosummary:: + :nosignatures: + + CTCLoss + DeformableConvolution + DeformablePSROIPooling + MultiBoxDetection + MultiBoxPrior + MultiBoxTarget + MultiProposal + PSROIPooling + Proposal + count_sketch + ctc_loss + dequantize + fft + ifft + quantize +``` + ## API Reference @@ -488,6 +519,9 @@ Composite multiple symbols into a new one by an operator. .. automodule:: mxnet.symbol :members: +.. automodule:: mxnet.contrib.symbol + :members: + ``` diff --git a/python/mxnet/gluon/model_zoo/vision/resnet.py b/python/mxnet/gluon/model_zoo/vision/resnet.py index 2870911aa5cf..5e2adad52781 100644 --- a/python/mxnet/gluon/model_zoo/vision/resnet.py +++ b/python/mxnet/gluon/model_zoo/vision/resnet.py @@ -4,6 +4,8 @@ from __future__ import division __all__ = ['ResNetV1', 'ResNetV2', + 'BasicBlockV1', 'BasicBlockV2', + 'BottleneckV1', 'BottleneckV2', 'resnet18_v1', 'resnet34_v1', 'resnet50_v1', 'resnet101_v1', 'resnet152_v1', 'resnet18_v2', 'resnet34_v2', 'resnet50_v2', 'resnet101_v2', 'resnet152_v2', 'get_resnet'] From e8ed54875617340d22c6f93a78bb2cd0344d237c Mon Sep 17 00:00:00 2001 From: alues Date: Thu, 3 Aug 2017 08:11:07 +0800 Subject: [PATCH 306/834] Fix install error for jupyter (#7308) --- setup-utils/install-mxnet-ubuntu-python.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/setup-utils/install-mxnet-ubuntu-python.sh b/setup-utils/install-mxnet-ubuntu-python.sh index 345b669498e8..ba060745da8f 100644 --- a/setup-utils/install-mxnet-ubuntu-python.sh +++ b/setup-utils/install-mxnet-ubuntu-python.sh @@ -19,9 +19,12 @@ make -j$(nproc) echo "Installing Numpy..." sudo apt-get install python-numpy -echo "Installing Python setuptools..." +echo "Installing Python setuptools pip..." sudo apt-get install -y python-setuptools python-pip +echo "Updating pip..." +sudo pip install -U pip + echo "Installing Python package for MXNet..." cd python; sudo python setup.py install From cffbc2c5790261a77b217e1f4cc90a2cac7aeb7f Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Wed, 2 Aug 2017 22:50:55 -0700 Subject: [PATCH 307/834] reduce model zoo test size (#7318) * reduce model zoo test size * add model print back for pretty-print test --- tests/python/unittest/test_gluon_model_zoo.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/python/unittest/test_gluon_model_zoo.py b/tests/python/unittest/test_gluon_model_zoo.py index 200037c067e2..db26fd4a96ab 100644 --- a/tests/python/unittest/test_gluon_model_zoo.py +++ b/tests/python/unittest/test_gluon_model_zoo.py @@ -3,7 +3,10 @@ from mxnet.gluon import nn from mxnet.gluon.model_zoo.custom_layers import HybridConcurrent, Identity from mxnet.gluon.model_zoo.vision import get_model +import sys +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) def test_concurrent(): model = HybridConcurrent(concat_dim=1) @@ -43,11 +46,12 @@ def test_models(): for model_name in all_models: test_pretrain = model_name in pretrained_to_test model = get_model(model_name, pretrained=test_pretrain) - data_shape = (7, 3, 224, 224) if 'inception' not in model_name else (7, 3, 299, 299) + data_shape = (2, 3, 224, 224) if 'inception' not in model_name else (2, 3, 299, 299) + eprint('testing forward for %s'%model_name) print(model) if not test_pretrain: model.collect_params().initialize() - model(mx.nd.random_uniform(shape=data_shape)) + model(mx.nd.random_uniform(shape=data_shape)).wait_to_read() if __name__ == '__main__': From e6a1139812db33bd11fcc8491915f6a51d42418b Mon Sep 17 00:00:00 2001 From: Eric Junyuan Xie Date: Thu, 3 Aug 2017 10:36:11 -0700 Subject: [PATCH 308/834] add backward(is_train=False) and always mode for dropout (#7303) * add backward(is_train=False) and always mode for dropout * fix * fix * fix slicing * fix mkl dropout --- include/mxnet/c_api.h | 36 +++++++++- include/mxnet/executor.h | 2 +- python/mxnet/autograd.py | 70 ++++++++++++++----- python/mxnet/base.py | 4 +- python/mxnet/contrib/autograd.py | 2 + python/mxnet/executor.py | 29 +++----- .../mxnet/gluon/model_zoo/vision/__init__.py | 5 +- python/mxnet/image/detection.py | 5 +- python/mxnet/ndarray.py | 17 +++-- src/c_api/c_api_executor.cc | 9 ++- src/c_api/c_api_ndarray.cc | 20 +++++- src/executor/graph_executor.cc | 4 +- src/executor/graph_executor.h | 2 +- src/ndarray/autograd.cc | 10 +-- src/ndarray/autograd.h | 14 +++- src/ndarray/ndarray.cc | 1 + src/operator/dropout-inl.h | 21 ++++-- src/operator/dropout.cc | 3 +- tests/python/unittest/test_autograd.py | 16 +++++ tests/python/unittest/test_operator.py | 36 ++++++++++ 20 files changed, 243 insertions(+), 63 deletions(-) diff --git a/include/mxnet/c_api.h b/include/mxnet/c_api.h index 47447fb37196..d9a5315c9167 100644 --- a/include/mxnet/c_api.h +++ b/include/mxnet/c_api.h @@ -551,6 +551,13 @@ MXNET_DLL int MXImperativeInvoke(AtomicSymbolCreator creator, int num_params, const char **param_keys, const char **param_vals); +/*! + * \brief set whether to record operator for autograd + * \param is_recording 1 when recording, 0 when not recording. + * \param prev returns the previous status before this set. + * \return 0 when success, -1 when failure happens + */ +MXNET_DLL int MXAutogradSetIsRecording(int is_recording, int* prev); /*! * \brief set whether to record operator for autograd * \param is_train 1 when training, 0 when testing @@ -588,6 +595,20 @@ MXNET_DLL int MXAutogradBackward(mx_uint num_output, NDArrayHandle* output_handles, NDArrayHandle* ograd_handles, int retain_graph); +/*! +* \brief compute the gradient of outputs w.r.t variabels +* \param num_output number of output NDArray +* \param output_handles output NDArrays +* \param ograd_handles head gradient for NDArrays +* \param retain_graph whether to keep the graph after backward +* \param is_train whether to do backward for training or inference +* \return 0 when success, -1 when failure happens +*/ +MXNET_DLL int MXAutogradBackwardEx(mx_uint num_output, + NDArrayHandle* output_handles, + NDArrayHandle* ograd_handles, + int retain_graph, + int is_train); /*! * \brief create cached operator */ @@ -1028,7 +1049,20 @@ MXNET_DLL int MXExecutorForward(ExecutorHandle handle, int is_train); MXNET_DLL int MXExecutorBackward(ExecutorHandle handle, mx_uint len, NDArrayHandle *head_grads); - +/*! + * \brief Excecutor run backward + * + * \param handle execute handle + * \param len lenth + * \param head_grads NDArray handle for heads' gradient + * \param is_train int value to indicate whether the backward pass is for evaluation + * + * \return 0 when success, -1 when failure happens + */ +MXNET_DLL int MXExecutorBackwardEx(ExecutorHandle handle, + mx_uint len, + NDArrayHandle *head_grads, + int is_train); /*! * \brief Get executor's head NDArray * diff --git a/include/mxnet/executor.h b/include/mxnet/executor.h index 40bd60f5f405..9308587c8d72 100644 --- a/include/mxnet/executor.h +++ b/include/mxnet/executor.h @@ -58,7 +58,7 @@ class Executor { * * \param head_grads the gradient of head nodes to be backproped. */ - virtual void Backward(const std::vector &head_grads) = 0; + virtual void Backward(const std::vector &head_grads, bool is_train = true) = 0; /*! * \brief print the execution plan info to output stream. * \param os the output stream we like to print to. diff --git a/python/mxnet/autograd.py b/python/mxnet/autograd.py index b97d350f258d..2f33052e663e 100644 --- a/python/mxnet/autograd.py +++ b/python/mxnet/autograd.py @@ -12,9 +12,7 @@ def set_recording(is_recording): """Set status to recording/not recording. When recording, graph will be constructed - for gradient computation. Operators will also run with ctx.is_train=True. For example, - Dropout will drop inputs randomly when is_train=True while simply passing through - if is_train=False. + for gradient computation. Parameters ---------- @@ -25,46 +23,77 @@ def set_recording(is_recording): previous state before this set. """ prev = ctypes.c_int() - check_call(_LIB.MXAutogradSetIsTraining( + check_call(_LIB.MXAutogradSetIsRecording( ctypes.c_int(is_recording), ctypes.byref(prev))) return bool(prev.value) +def set_training(is_train): + """Set status to training/not training. This affects ctx.is_train in operator + running context. For example, Dropout will drop inputs randomly when + is_train=True while simply passing through if is_train=False. + + Parameters + ---------- + is_train: bool + + Returns + ------- + previous state before this set. + """ + prev = ctypes.c_int() + check_call(_LIB.MXAutogradSetIsTraining( + ctypes.c_int(is_train), ctypes.byref(prev))) + return bool(prev.value) + -class TrainingStateScope(object): +class RecordingStateScope(object): """Scope for managing training state. Example:: - with TrainingStateScope(True): + with RecordingStateScope(True, True): y = model(x) backward([y]) """ - def __init__(self, enter_state): + def __init__(self, enter_state, is_train): self._enter_state = enter_state + self._enter_is_train = is_train self._prev = None + self._prev_is_train = None def __enter__(self): self._prev = set_recording(self._enter_state) + self._prev_is_train = set_training(self._enter_is_train) def __exit__(self, ptype, value, trace): if self._prev != self._enter_state: set_recording(self._prev) + if self._prev_is_train != self._enter_is_train: + set_training(self._prev_is_train) -def record(): +def record(is_train=True): """Returns a training scope context to be used in 'with' statement and captures training code. + .. note:: When forwarding with is_train=False, the corresponding backward + should also use is_train=False, otherwise gradient is undefined. + Example:: with autograd.record(): y = model(x) backward([y]) metric.update(...) optim.step(...) + + Parameters + ---------- + is_train: bool, default True + Whether to do forward for training or inference. """ - return TrainingStateScope(True) + return RecordingStateScope(True, is_train) -def pause(): +def pause(is_train=False): """Returns a testing scope context to be used in 'with' statement and captures testing code. @@ -74,8 +103,13 @@ def pause(): backward([y]) with autograd.pause(): # testing, IO, gradient updates... + + Parameters + ---------- + is_train: bool, default False + Whether to do forward for training or inference. """ - return TrainingStateScope(False) + return RecordingStateScope(False, is_train) def mark_variables(variables, gradients, grad_reqs='write'): @@ -109,7 +143,7 @@ def mark_variables(variables, gradients, grad_reqs='write'): c_array(NDArrayHandle, gradient_handles))) -def backward(heads, head_grads=None, retain_graph=False): +def backward(heads, head_grads=None, retain_graph=False, is_train=True): """Compute the gradients of heads w.r.t previously marked variables. Parameters @@ -118,6 +152,8 @@ def backward(heads, head_grads=None, retain_graph=False): Output NDArray(s) head_grads: NDArray or list of NDArray or None Gradients with respect to heads. + is_train: bool, optional + Whether to do backward for training or inference. """ if isinstance(heads, NDArray): assert head_grads is None or isinstance(head_grads, NDArray) @@ -129,11 +165,12 @@ def backward(heads, head_grads=None, retain_graph=False): output_handles.append(arr.handle) if head_grads is None: - check_call(_LIB.MXAutogradBackward( + check_call(_LIB.MXAutogradBackwardEx( len(output_handles), c_array(NDArrayHandle, output_handles), ctypes.c_void_p(0), - ctypes.c_int(retain_graph))) + ctypes.c_int(retain_graph), + ctypes.c_int(is_train))) return ograd_handles = [] @@ -145,8 +182,9 @@ def backward(heads, head_grads=None, retain_graph=False): assert len(ograd_handles) == len(output_handles), \ "heads and head_grads must have the same length" - check_call(_LIB.MXAutogradBackward( + check_call(_LIB.MXAutogradBackwardEx( len(output_handles), c_array(NDArrayHandle, output_handles), c_array(NDArrayHandle, ograd_handles), - ctypes.c_int(retain_graph))) + ctypes.c_int(retain_graph), + ctypes.c_int(is_train))) diff --git a/python/mxnet/base.py b/python/mxnet/base.py index 6d537529e8af..ddaeb6e77d54 100644 --- a/python/mxnet/base.py +++ b/python/mxnet/base.py @@ -18,14 +18,14 @@ #---------------------------- if sys.version_info[0] == 3: string_types = str, - numeric_types = (float, int, np.float32, np.int32) + numeric_types = (float, int, np.generic) integer_types = int # this function is needed for python3 # to convert ctypes.char_p .value back to python str py_str = lambda x: x.decode('utf-8') else: string_types = basestring, - numeric_types = (float, int, long, np.float32, np.int32) + numeric_types = (float, int, long, np.generic) integer_types = (int, long) py_str = lambda x: x diff --git a/python/mxnet/contrib/autograd.py b/python/mxnet/contrib/autograd.py index e56361efdb1f..9074e452c981 100644 --- a/python/mxnet/contrib/autograd.py +++ b/python/mxnet/contrib/autograd.py @@ -28,6 +28,8 @@ def set_is_training(is_train): prev = ctypes.c_int() check_call(_LIB.MXAutogradSetIsTraining( ctypes.c_int(is_train), ctypes.byref(prev))) + check_call(_LIB.MXAutogradSetIsRecording( + ctypes.c_int(is_train), ctypes.byref(prev))) return bool(prev.value) diff --git a/python/mxnet/executor.py b/python/mxnet/executor.py index 6b9aab2de6f1..d2b108cc04ed 100644 --- a/python/mxnet/executor.py +++ b/python/mxnet/executor.py @@ -5,7 +5,6 @@ import ctypes import copy -import warnings import numpy as np from .base import _LIB from .base import mx_uint, NDArrayHandle, ExecutorHandle @@ -61,7 +60,6 @@ def __init__(self, handle, symbol, ctx, grad_req, group2ctx): self._aux_dict = None self._output_dict = None self._monitor_callback = None - self._output_dirty = False self._ctx = copy.deepcopy(ctx) self._grad_req = copy.deepcopy(grad_req) self._group2ctx = copy.deepcopy(group2ctx) @@ -99,8 +97,7 @@ def forward(self, is_train=False, **kwargs): ---------- is_train: bool, optional Whether this forward is for evaluation purpose. If True, - a backward call is expected to follow. Otherwise following - backward is invalid. + a backward call is expected to follow. **kwargs Additional specification of input arguments. @@ -132,15 +129,9 @@ def forward(self, is_train=False, **kwargs): self.handle, ctypes.c_int(int(is_train)))) - if self._output_dirty: - warnings.warn( - "Calling forward the second time after forward(is_train=True) " - "without calling backward first. Is this intended?", stacklevel=2) - self._output_dirty = is_train - return self.outputs - def backward(self, out_grads=None): + def backward(self, out_grads=None, is_train=True): """Do backward pass to get the gradient of arguments. Parameters @@ -149,6 +140,11 @@ def backward(self, out_grads=None): Gradient on the outputs to be propagated back. This parameter is only needed when bind is called on outputs that are not a loss function. + is_train : bool, default True + Whether this backward is for training or inference. Note that in rare + cases you want to call backward with is_train=False to get gradient + during inference. + Examples -------- @@ -211,16 +207,11 @@ def backward(self, out_grads=None): if not isinstance(obj, NDArray): raise TypeError("inputs must be NDArray") ndarray = c_array(NDArrayHandle, [item.handle for item in out_grads]) - check_call(_LIB.MXExecutorBackward( + check_call(_LIB.MXExecutorBackwardEx( self.handle, mx_uint(len(out_grads)), - ndarray)) - - if not self._output_dirty: - warnings.warn( - "Calling backward without calling forward(is_train=True) " - "first. Behavior is undefined.", stacklevel=2) - self._output_dirty = False + ndarray, + ctypes.c_int(is_train))) def set_monitor_callback(self, callback): """Install callback for monitor. diff --git a/python/mxnet/gluon/model_zoo/vision/__init__.py b/python/mxnet/gluon/model_zoo/vision/__init__.py index e0498dcc6bca..56e46f9a0c74 100644 --- a/python/mxnet/gluon/model_zoo/vision/__init__.py +++ b/python/mxnet/gluon/model_zoo/vision/__init__.py @@ -102,5 +102,8 @@ def get_model(name, **kwargs): 'inceptionv3': inception_v3, } name = name.lower() - assert name in models, 'Model %s is not supported'%name + if name not in models: + raise ValueError( + 'Model %s is not supported. Available options are\n\t%s'%( + name, '\n\t'.join(sorted(models.keys())))) return models[name](**kwargs) diff --git a/python/mxnet/image/detection.py b/python/mxnet/image/detection.py index d5e5c1e7e691..0a16ac36fc98 100644 --- a/python/mxnet/image/detection.py +++ b/python/mxnet/image/detection.py @@ -756,8 +756,9 @@ def next(self): assert i < batch_size, 'Batch size must be multiples of augmenter output length' batch_data[i][:] = self.postprocess_data(datum) num_object = label.shape[0] - batch_label[i][0:num_object][:] = nd.array(label) - batch_label[i][num_object:][:] = -1 + batch_label[i][0:num_object] = nd.array(label) + if num_object < batch_label[i].shape[0]: + batch_label[i][num_object:] = -1 i += 1 except StopIteration: if not i: diff --git a/python/mxnet/ndarray.py b/python/mxnet/ndarray.py index fdecebbe7996..b2178a98a84e 100644 --- a/python/mxnet/ndarray.py +++ b/python/mxnet/ndarray.py @@ -124,6 +124,7 @@ class NDArray(NDArrayBase): """ __slots__ = [] + # make numpy functions return NDArray instead of numpy object array __array_priority__ = 1000.0 # pylint: disable= no-member, undefined-variable def __repr__(self): @@ -1058,22 +1059,30 @@ def detach(self): check_call(_LIB.MXNDArrayDetach(self.handle, ctypes.byref(hdl))) return NDArray(hdl) - def backward(self, out_grad=None, retain_graph=False): + def backward(self, out_grad=None, retain_graph=False, is_train=True): """Compute the gradients of this NDArray w.r.t variables. Parameters ---------- - out_grad: list of NDArray or None + out_grad : NDArray, optional + Gradient with respect to head. + retain_graph : bool, optional + Whether to retain the computaion graph for another backward + pass on the same graph. By default the computaion history + is cleared. + is_train : bool, optional + Whether to compute gradient for training or inference. """ if out_grad is None: ograd_handles = [NDArrayHandle(0)] else: ograd_handles = [out_grad.handle] - check_call(_LIB.MXAutogradBackward( + check_call(_LIB.MXAutogradBackwardEx( 1, c_array(NDArrayHandle, [self.handle]), c_array(NDArrayHandle, ograd_handles), - ctypes.c_int(retain_graph))) + ctypes.c_int(retain_graph), + ctypes.c_int(is_train))) def onehot_encode(indices, out): diff --git a/src/c_api/c_api_executor.cc b/src/c_api/c_api_executor.cc index ca49402ecf7e..3ba3154f2d97 100644 --- a/src/c_api/c_api_executor.cc +++ b/src/c_api/c_api_executor.cc @@ -35,6 +35,13 @@ int MXExecutorForward(ExecutorHandle handle, int is_train) { int MXExecutorBackward(ExecutorHandle handle, mx_uint len, NDArrayHandle *head_grads) { + return MXExecutorBackwardEx(handle, len, head_grads, true); +} + +int MXExecutorBackwardEx(ExecutorHandle handle, + mx_uint len, + NDArrayHandle *head_grads, + int is_train) { API_BEGIN(); Executor *exec = static_cast(handle); std::vector ndarrays; @@ -42,7 +49,7 @@ int MXExecutorBackward(ExecutorHandle handle, for (mx_uint i = 0; i < len; ++i) { ndarrays.push_back(*args_ptr[i]); } - exec->Backward(ndarrays); + exec->Backward(ndarrays, is_train); API_END(); } diff --git a/src/c_api/c_api_ndarray.cc b/src/c_api/c_api_ndarray.cc index 818f263cb3b7..f40139424b31 100644 --- a/src/c_api/c_api_ndarray.cc +++ b/src/c_api/c_api_ndarray.cc @@ -378,7 +378,7 @@ void ImperativeInvokeImpl(const Context& default_ctx, } if (fn) { - if (AutogradRuntime::Get()->IsTraining()) { + if (AutogradRuntime::Get()->IsRecording()) { AutogradRuntime::Get()->RecordImperativeFCompute(op, attrs, &ndinputs, &ndoutputs); } @@ -387,7 +387,7 @@ void ImperativeInvokeImpl(const Context& default_ctx, } else if (createop.count(op)) { auto state = createop[op](attrs, ctx, ret->arg_shapes, ret->arg_types); - if (AutogradRuntime::Get()->IsTraining()) { + if (AutogradRuntime::Get()->IsRecording()) { AutogradRuntime::Get()->RecordImperativeOperator(state, op, attrs, &ndinputs, &ndoutputs); } @@ -528,6 +528,12 @@ int MXAutogradSetIsTraining(int is_training, int* prev) { API_END(); } +int MXAutogradSetIsRecording(int is_recording, int* prev) { + API_BEGIN(); + *prev = AutogradRuntime::Get()->SetIsRecording(static_cast(is_recording)); + API_END(); +} + int MXAutogradMarkVariables(mx_uint num_var, NDArrayHandle *var_handles, mx_uint *reqs_array, @@ -556,6 +562,14 @@ int MXAutogradBackward(mx_uint num_output, NDArrayHandle *output_handles, NDArrayHandle *ograd_handles, int retain_graph) { + return MXAutogradBackwardEx(num_output, output_handles, ograd_handles, retain_graph, true); +} + +int MXAutogradBackwardEx(mx_uint num_output, + NDArrayHandle *output_handles, + NDArrayHandle *ograd_handles, + int retain_graph, + int is_train) { API_BEGIN(); MXAPIThreadLocalEntry *ret = MXAPIThreadLocalStore::Get(); @@ -574,6 +588,6 @@ int MXAutogradBackward(mx_uint num_output, } } - AutogradRuntime::Get()->ComputeGradient(outputs, ograds, retain_graph); + AutogradRuntime::Get()->ComputeGradient(outputs, ograds, retain_graph, is_train); API_END(); } diff --git a/src/executor/graph_executor.cc b/src/executor/graph_executor.cc index af5ec7f492dd..a17f44a7cff5 100644 --- a/src/executor/graph_executor.cc +++ b/src/executor/graph_executor.cc @@ -42,7 +42,7 @@ void GraphExecutor::PartialForward(bool is_train, int step, int *step_left) { *step_left = static_cast(num_forward_nodes_ - sstep - 1); } -void GraphExecutor::Backward(const std::vector& head_grads) { +void GraphExecutor::Backward(const std::vector& head_grads, bool is_train) { const auto& idx = graph_.indexed_graph(); if (num_forward_inputs_ != idx.input_nodes().size()) { for (size_t i = 0; i < head_grad_array_.size(); ++i) { @@ -57,7 +57,7 @@ void GraphExecutor::Backward(const std::vector& head_grads) { } } } - RunOps(true, num_forward_nodes_, idx.num_nodes()); + RunOps(is_train, num_forward_nodes_, idx.num_nodes()); } void GraphExecutor::Print(std::ostream &os) const { // NOLINT(*) diff --git a/src/executor/graph_executor.h b/src/executor/graph_executor.h index 5b6fa395b242..0efb8ae09f4a 100644 --- a/src/executor/graph_executor.h +++ b/src/executor/graph_executor.h @@ -44,7 +44,7 @@ class GraphExecutor : public Executor { virtual ~GraphExecutor(); void Forward(bool is_train) override; void PartialForward(bool is_train, int step, int *step_left) override; - void Backward(const std::vector &head_grads) override; + void Backward(const std::vector &head_grads, bool is_train = true) override; const std::vector& outputs() const override; const std::unordered_map& in_arg_map() const override; const std::unordered_map& arg_grad_map() const override; diff --git a/src/ndarray/autograd.cc b/src/ndarray/autograd.cc index f990ee2973fd..efb6bc9dbf8d 100644 --- a/src/ndarray/autograd.cc +++ b/src/ndarray/autograd.cc @@ -23,9 +23,11 @@ using nnvm::NodeEntryMap; using exec::GraphExecutor; #if DMLC_CXX11_THREAD_LOCAL -thread_local bool AutogradRuntime::is_train_; +thread_local bool AutogradRuntime::is_train_ = false; +thread_local bool AutogradRuntime::is_recording_ = false; #else -MX_THREAD_LOCAL bool AutogradRuntime::is_train_; +MX_THREAD_LOCAL bool AutogradRuntime::is_train_ = false; +MX_THREAD_LOCAL bool AutogradRuntime::is_recording_ = false; #endif template @@ -149,7 +151,7 @@ AGNodePtr AutogradRuntime::RecordOp(const nnvm::Op* op, void AutogradRuntime::ComputeGradient(const std::vector& outputs, const std::vector& ograds, - bool retain_graph) { + bool retain_graph, bool is_train) { static auto& fmutate_inputs = nnvm::Op::GetAttr("FMutateInputs"); std::vector heads; Symbol sym; @@ -233,7 +235,7 @@ void AutogradRuntime::ComputeGradient(const std::vector& outputs, } } - exec->Backward(head_grads); + exec->Backward(head_grads, is_train); delete exec; } diff --git a/src/ndarray/autograd.h b/src/ndarray/autograd.h index baf843dbd4e1..474864009688 100644 --- a/src/ndarray/autograd.h +++ b/src/ndarray/autograd.h @@ -63,6 +63,16 @@ class AutogradRuntime { bool IsTraining() const { return is_train_; } + /*! \brief turn on or turn off operator recording for autograd. */ + bool SetIsRecording(bool is_recording) { + bool old = is_recording_; + is_recording_ = is_recording; + return old; + } + /*! \brief whether operator recording is on. */ + bool IsRecording() const { + return is_recording_; + } /*! \brief mark variables for computing gradients. */ void MarkVariables(const std::vector& variables, const std::vector& grad_reqs, @@ -81,7 +91,7 @@ class AutogradRuntime { /*! \brief compute the gradient of outputs w.r.t variables. */ void ComputeGradient(const std::vector& outputs, const std::vector& ograds, - bool retain_graph); + bool retain_graph, bool is_train); /*! \return AutogradRuntime singleton */ static AutogradRuntime* Get(); /*! \brief Get shared pointer reference to AutogradRuntime singleton. @@ -109,8 +119,10 @@ class AutogradRuntime { /*! \brief indicate whether is training. */ #if DMLC_CXX11_THREAD_LOCAL static thread_local bool is_train_; + static thread_local bool is_recording_; #else static MX_THREAD_LOCAL bool is_train_; + static MX_THREAD_LOCAL bool is_recording_; #endif /*! \brief node count used for naming */ std::atomic node_count_{0}; diff --git a/src/ndarray/ndarray.cc b/src/ndarray/ndarray.cc index f2e90dd56f31..48499fa2cafd 100644 --- a/src/ndarray/ndarray.cc +++ b/src/ndarray/ndarray.cc @@ -69,6 +69,7 @@ NDArray NDArray::Slice(index_t begin, index_t end) const { using namespace autograd; NDArray ret = *this; CHECK(!is_none()) << "NDArray is not initialized"; + CHECK_LT(begin, end) << "Invalid slicing range [" << begin << ", " << end << ")"; CHECK_GE(shape_[0], end) << "Slice end index out of range"; size_t length = shape_.ProdShape(1, shape_.ndim()); MSHADOW_TYPE_SWITCH(ret.dtype(), DType, { diff --git a/src/operator/dropout-inl.h b/src/operator/dropout-inl.h index e77d61351be0..57d78146a68d 100644 --- a/src/operator/dropout-inl.h +++ b/src/operator/dropout-inl.h @@ -29,6 +29,7 @@ namespace dropout { enum DropoutOpInputs {kData}; enum DropoutOpOutputs {kOut, kMask}; enum DropoutOpForwardResource {kRandom}; +enum DropoutOpMode {kTraining, kAlways}; } // namespace dropout namespace mxnet { @@ -58,10 +59,16 @@ static void bernoulli_generate(int n, double p, int* r) { struct DropoutParam : public dmlc::Parameter { float p; + int mode; DMLC_DECLARE_PARAMETER(DropoutParam) { DMLC_DECLARE_FIELD(p).set_default(0.5) .set_range(0, 1) .describe("Fraction of the input that gets dropped out during training time."); + DMLC_DECLARE_FIELD(mode) + .add_enum("training", dropout::kTraining) + .add_enum("always", dropout::kAlways) + .set_default(dropout::kTraining) + .describe("Whether to only turn on dropout during training or to also turn on for inference."); } }; // struct DropoutParam @@ -70,6 +77,7 @@ class DropoutOp : public Operator { public: explicit DropoutOp(DropoutParam param) { this->pkeep_ = 1.0f - param.p; + this->mode_ = param.mode; } virtual void Forward(const OpContext &ctx, @@ -86,7 +94,7 @@ class DropoutOp : public Operator { Stream *s = ctx.get_stream(); Tensor data = in_data[dropout::kData].FlatTo2D(s); Tensor out = out_data[dropout::kOut].FlatTo2D(s); - if (ctx.is_train) { + if (ctx.is_train || mode_ == dropout::kAlways) { Tensor mask = out_data[dropout::kMask].FlatTo2D(s); #if !defined(__CUDACC__) && defined(USE_MKL) && defined(_OPENMP) DType* outptr = out.dptr_; @@ -96,7 +104,7 @@ class DropoutOp : public Operator { bernoulli_generate(count, this->pkeep_, maskptr); #pragma omp parallel for for (int i = 0; i < count; ++i) { - outptr[i] = dataptr[i] * maskptr[i]; + outptr[i] = dataptr[i] * maskptr[i] * (1.0f / pkeep_); } #else Random *prnd = ctx.requested[dropout::kRandom].get_random(s); @@ -124,6 +132,7 @@ class DropoutOp : public Operator { Tensor grad = out_grad[dropout::kOut].FlatTo2D(s); Tensor mask = out_data[dropout::kMask].FlatTo2D(s); Tensor gdata = in_grad[dropout::kData].FlatTo2D(s); + if (ctx.is_train || mode_ == dropout::kAlways) { #if !defined(__CUDACC__) && defined(USE_MKL) && defined(_OPENMP) DType* ingradptr = gdata.dptr_; DType* outgradptr = grad.dptr_; @@ -131,17 +140,21 @@ class DropoutOp : public Operator { int count = mask.shape_[0]*mask.shape_[1]; - #pragma omp parallel for + #pragma omp parallel for for (int i = 0; i < count; ++i) { - ingradptr[i] = outgradptr[i] * maskptr[i]; + ingradptr[i] = outgradptr[i] * maskptr[i] * (1.0f / pkeep_); } #else // USE_MKL && _OPENMP Assign(gdata, req[dropout::kData], grad * mask); #endif // USE_MKL && _OPENMP + } else { + Assign(gdata, req[dropout::kData], F(grad)); + } } private: real_t pkeep_; + int mode_; }; // class DropoutOp diff --git a/src/operator/dropout.cc b/src/operator/dropout.cc index 74a50baf80a4..e206214e9b64 100644 --- a/src/operator/dropout.cc +++ b/src/operator/dropout.cc @@ -33,7 +33,8 @@ MXNET_REGISTER_OP_PROPERTY(Dropout, DropoutProp) The whole array is rescaled by :math:`1/(1-p)` to keep the expected sum of the input unchanged. -- During testing, this operator does not change the input. +- During testing, this operator does not change the input if mode is 'training'. + If mode is 'always', the same computaion as during training will be applied. Example:: diff --git a/tests/python/unittest/test_autograd.py b/tests/python/unittest/test_autograd.py index 8dea04da6abc..172075dcfda1 100644 --- a/tests/python/unittest/test_autograd.py +++ b/tests/python/unittest/test_autograd.py @@ -248,6 +248,22 @@ def test_attach_grad(): assert (x.grad.asnumpy() == 2).all() +def test_is_train(): + x = mx.nd.ones((10, 10)) + x.attach_grad() + with record(True): + y = mx.nd.Dropout(x, p=0.5) + assert y.asnumpy().max() == 2 and y.asnumpy().min() == 0 + y.backward() + assert (x.grad.asnumpy() == y.asnumpy()).all() + + with record(False): + y = mx.nd.Dropout(x, p=0.5) + assert (y.asnumpy() == x.asnumpy()).all() + y.backward(is_train=False) + assert (x.grad.asnumpy() == x.asnumpy()).all() + + if __name__ == "__main__": import nose nose.runmodule() diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py index 2f7c3b904e01..51a77e0af221 100644 --- a/tests/python/unittest/test_operator.py +++ b/tests/python/unittest/test_operator.py @@ -3658,6 +3658,42 @@ def test_stack(): check_numeric_gradient(out, inputs) +def test_dropout(): + # test dropout + x = mx.sym.var('data') + y = mx.sym.Dropout(x, p=0.5) + exe = y.simple_bind(ctx=default_context(), data=(10, 10)) + + exe.arg_arrays[0][:] = 1 + exe.forward(is_train=True) + assert exe.outputs[0].asnumpy().max() == 2 + assert exe.outputs[0].asnumpy().min() == 0 + exe.backward([mx.nd.ones((10, 10))]) + assert (exe.grad_arrays[0].asnumpy() == exe.outputs[0].asnumpy()).all() + + exe.forward(is_train=False) + assert (exe.outputs[0].asnumpy() == exe.arg_arrays[0].asnumpy()).all() + exe.backward([mx.nd.ones((10, 10))], is_train=False) + assert (exe.grad_arrays[0].asnumpy() == exe.arg_arrays[0].asnumpy()).all() + + # test permanent dropout + x = mx.sym.var('data') + y = mx.sym.Dropout(x, p=0.5, mode='always') + exe = y.simple_bind(ctx=default_context(), data=(10, 10)) + + exe.arg_arrays[0][:] = 1 + exe.forward(is_train=True) + assert exe.outputs[0].asnumpy().max() == 2 + assert exe.outputs[0].asnumpy().min() == 0 + exe.backward([mx.nd.ones((10, 10))]) + assert (exe.grad_arrays[0].asnumpy() == exe.outputs[0].asnumpy()).all() + + exe.forward(is_train=False) + assert exe.outputs[0].asnumpy().max() == 2 + assert exe.outputs[0].asnumpy().min() == 0 + exe.backward([mx.nd.ones((10, 10))], is_train=False) + assert (exe.grad_arrays[0].asnumpy() == exe.outputs[0].asnumpy()).all() + if __name__ == '__main__': import nose From 5a286b28dd60574ec4d0ede1252eb1ee4dbe3088 Mon Sep 17 00:00:00 2001 From: Madan Jampani Date: Thu, 3 Aug 2017 12:53:06 -0700 Subject: [PATCH 309/834] Fix module tutorial (#7324) --- docs/tutorials/basic/module.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/tutorials/basic/module.md b/docs/tutorials/basic/module.md index 15fdaeef68c4..e0618ca65e4a 100644 --- a/docs/tutorials/basic/module.md +++ b/docs/tutorials/basic/module.md @@ -173,8 +173,8 @@ dataset and evaluates the performance according to the given input metric. It can be used as follows: ```python -score = mod.score(val_iter, ['mse', 'acc']) -print("Accuracy score is %f" % (score)) +score = mod.score(val_iter, ['acc']) +print("Accuracy score is %f" % (score[0][1])) ``` Some of the other metrics which can be used are `top_k_acc`(top-k-accuracy), From 1f0b8130db4d22b5f2b1f3df26e26bc8a65cba2b Mon Sep 17 00:00:00 2001 From: formath Date: Fri, 4 Aug 2017 04:01:44 +0800 Subject: [PATCH 310/834] assert size eq between shared_module.execs and context (#7233) --- python/mxnet/module/module.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/mxnet/module/module.py b/python/mxnet/module/module.py index 2a36c6ad7e7e..b31ea0ffa319 100644 --- a/python/mxnet/module/module.py +++ b/python/mxnet/module/module.py @@ -386,6 +386,7 @@ def bind(self, data_shapes, label_shapes=None, for_training=True, assert isinstance(shared_module, Module) and \ shared_module.binded and shared_module.params_initialized shared_group = shared_module._exec_group + assert len(shared_group.execs) == len(self._context) else: shared_group = None From 4939dc25ff89607b8e0584f9b30a804f7183e07d Mon Sep 17 00:00:00 2001 From: Xu Dong Date: Fri, 4 Aug 2017 04:03:50 +0800 Subject: [PATCH 311/834] Add document for BilinearSampler Op (#7203) * Update document for BilinearSamplerOp * fix lint * remove space --- src/operator/bilinear_sampler.cc | 84 +++++++++++++++++++++++++++----- 1 file changed, 72 insertions(+), 12 deletions(-) diff --git a/src/operator/bilinear_sampler.cc b/src/operator/bilinear_sampler.cc index f76e987440fa..ca83a43343a6 100644 --- a/src/operator/bilinear_sampler.cc +++ b/src/operator/bilinear_sampler.cc @@ -152,17 +152,77 @@ MXNET_REGISTER_OP_PROPERTY(BilinearSampler, BilinearSamplerProp) .add_argument("grid", "NDArray-or-Symbol", "Input grid to the BilinearsamplerOp." "grid has two channels: x_src, y_src") .add_arguments(BilinearSamplerParam::__FIELDS__()) -.describe("Applies bilinear sampling to input feature map," -" which is the key of \"[NIPS2015] Spatial Transformer Networks\"\n " -"output[batch, channel, y_dst, x_dst] = G(data[batch, channel, y_src, x_src)\n " -"x_dst, y_dst enumerate all spatial locations in output\n " -"x_src = grid[batch, 0, y_dst, x_dst]\n " -"y_src = grid[batch, 1, y_dst, x_dst]\n " -"G() denotes the bilinear interpolation kernel\n" -"The out-boundary points will be padded as zeros. (The boundary is defined to be [-1, 1])\n" -"The shape of output will be (data.shape[0], data.shape[1], grid.shape[2], grid.shape[3])\n" -"The operator assumes that grid has been nomalized. " -"If you want to design a CustomOp to manipulate grid, " -"please refer to GridGeneratorOp."); +.describe(R"code(Applies bilinear sampling to input feature map. + +Bilinear Sampling is the key of [NIPS2015] \"Spatial Transformer Networks\". The usage of the operator is very similar to remap function in OpenCV, +except that the operator has the backward pass. + +Given :math:`data` and :math:`grid`, then the output is computed by + +.. math:: + x_{src} = grid[batch, 0, y_{dst}, x_{dst}] \\ + y_{src} = grid[batch, 1, y_{dst}, x_{dst}] \\ + output[batch, channel, y_{dst}, x_{dst}] = G(data[batch, channel, y_{src}, x_{src}) + +:math:`x_{dst}`, :math:`y_{dst}` enumerate all spatial locations in :math:`output`, and :math:`G()` denotes the bilinear interpolation kernel. +The out-boundary points will be padded with zeros.The shape of the output will be (data.shape[0], data.shape[1], grid.shape[2], grid.shape[3]). + +The operator assumes that :math:`data` has 'NCHW' layout and :math:`grid` has been normalized to [-1, 1]. + +BilinearSampler often cooperates with GridGenerator which generates sampling grids for BilinearSampler. +GridGenerator supports two kinds of transformation: ``affine`` and ``warp``. +If users want to design a CustomOp to manipulate :math:`grid`, please firstly refer to the code of GridGenerator. + +Example 1:: + + ## Zoom out data two times + data = array([[[[1, 4, 3, 6], + [1, 8, 8, 9], + [0, 4, 1, 5], + [1, 0, 1, 3]]]]) + + affine_matrix = array([[2, 0, 0], + [0, 2, 0]]) + + affine_matrix = reshape(affine_matrix, shape=(1, 6)) + + grid = GridGenerator(data=affine_matrix, transform_type='affine', target_shape=(4, 4)) + + out = BilinearSampler(data, grid) + + out + [[[[ 0, 0, 0, 0], + [ 0, 3.5, 6.5, 0], + [ 0, 1.25, 2.5, 0], + [ 0, 0, 0, 0]]] + + +Example 2:: + + ## shift data horizontally by -1 pixel + + data = array([[[[1, 4, 3, 6], + [1, 8, 8, 9], + [0, 4, 1, 5], + [1, 0, 1, 3]]]]) + + warp_maxtrix = array([[[[1, 1, 1, 1], + [1, 1, 1, 1], + [1, 1, 1, 1], + [1, 1, 1, 1]], + [[0, 0, 0, 0], + [0, 0, 0, 0], + [0, 0, 0, 0], + [0, 0, 0, 0]]]]) + + grid = GridGenerator(data=warp_matrix, transform_type='warp') + out = BilinearSampler(data, grid) + + out + [[[[ 4, 3, 6, 0], + [ 8, 8, 9, 0], + [ 4, 1, 5, 0], + [ 0, 1, 3, 0]]] +)code" ADD_FILELINE); } // namespace op } // namespace mxnet From 2fe7aa4189941ec5d1673025336417248c22e38a Mon Sep 17 00:00:00 2001 From: Xu Dong Date: Fri, 4 Aug 2017 04:04:52 +0800 Subject: [PATCH 312/834] Fix bug in symbolic RNN (#7282) * Remove forget_bais in ConvLSTM * Remove the hard code about conv_layout * Add interface for initializer * Remove repetitive code in _call_ function --- python/mxnet/rnn/rnn_cell.py | 195 ++++++++++++++---------------- tests/python/unittest/test_rnn.py | 2 +- 2 files changed, 92 insertions(+), 105 deletions(-) diff --git a/python/mxnet/rnn/rnn_cell.py b/python/mxnet/rnn/rnn_cell.py index 99d0e8ad606f..c8213a20f9ef 100644 --- a/python/mxnet/rnn/rnn_cell.py +++ b/python/mxnet/rnn/rnn_cell.py @@ -1072,41 +1072,14 @@ def unroll(self, length, inputs, begin_state=None, layout='NTC', merge_outputs=N class BaseConvRNNCell(BaseRNNCell): - """Abstract base class for Convolutional RNN cells - - Parameters - ---------- - input_shape : tuple of int - Shape of input in single timestep. - num_hidden : int - Number of units in output symbol. - h2h_kernel : tuple of int - Kernel of Convolution operator in state-to-state transitions. - h2h_dilate : tuple of int - Dilation of Convolution operator in state-to-state transitions. - i2h_kernel : tuple of int - Kernel of Convolution operator in input-to-state transitions. - i2h_stride : tuple of int - Stride of Convolution operator in input-to-state transitions. - i2h_pad : tuple of int - Pad of Convolution operator in input-to-state transitions. - i2h_dilate : tuple of int - Dilation of Convolution operator in input-to-state transitions. - activation : str or Symbol, - Type of activation function. - prefix : str, default '' - Prefix for name of layers (and name of weight if params is None). - params : RNNParams, default None - Container for weight sharing between cells. Created if None. - conv_layout : str, , default 'NCHW' - Layout of ConvolutionOp - """ + """Abstract base class for Convolutional RNN cells""" def __init__(self, input_shape, num_hidden, h2h_kernel, h2h_dilate, i2h_kernel, i2h_stride, i2h_pad, i2h_dilate, - activation, - prefix='', params=None, conv_layout='NCHW'): + i2h_weight_initializer, h2h_weight_initializer, + i2h_bias_initializer, h2h_bias_initializer, + activation, prefix='', params=None, conv_layout='NCHW'): super(BaseConvRNNCell, self).__init__(prefix=prefix, params=params) # Convolution setting self._h2h_kernel = h2h_kernel @@ -1137,11 +1110,46 @@ def __init__(self, input_shape, num_hidden, self._state_shape = self._state_shape.infer_shape(data=input_shape)[1][0] self._state_shape = (0, ) + self._state_shape[1:] + # Get params + self._iW = self.params.get('i2h_weight', init=i2h_weight_initializer) + self._hW = self.params.get('h2h_weight', init=h2h_weight_initializer) + self._iB = self.params.get('i2h_bias', init=i2h_bias_initializer) + self._hB = self.params.get('h2h_bias', init=h2h_bias_initializer) + + @property + def _num_gates(self): + return len(self._gate_names) + @property def state_info(self): return [{'shape': self._state_shape, '__layout__': self._conv_layout}, {'shape': self._state_shape, '__layout__': self._conv_layout}] + def _conv_forward(self, inputs, states, name): + + i2h = symbol.Convolution(name='%si2h'%name, + data=inputs, + num_filter=self._num_hidden*self._num_gates, + kernel=self._i2h_kernel, + stride=self._i2h_stride, + pad=self._i2h_pad, + dilate=self._i2h_dilate, + weight=self._iW, + bias=self._iB, + layout=self._conv_layout) + + h2h = symbol.Convolution(name='%sh2h'%name, + data=states[0], + num_filter=self._num_hidden*self._num_gates, + kernel=self._h2h_kernel, + dilate=self._h2h_dilate, + pad=self._h2h_pad, + stride=(1, 1), + weight=self._hW, + bias=self._hB, + layout=self._conv_layout) + return i2h, h2h + def __call__(self, inputs, states): raise NotImplementedError("BaseConvRNNCell is abstract class for convolutional RNN") @@ -1166,6 +1174,16 @@ class ConvRNNCell(BaseConvRNNCell): Pad of Convolution operator in input-to-state transitions. i2h_dilate : tuple of int, default (1, 1) Dilation of Convolution operator in input-to-state transitions. + i2h_weight_initializer : str or Initializer + Initializer for the input weights matrix, used for the convolution + transformation of the inputs. + h2h_weight_initializer : str or Initializer + Initializer for the recurrent weights matrix, used for the convolution + transformation of the recurrent state. + i2h_bias_initializer : str or Initializer, default zeros + Initializer for the bias vector. + h2h_bias_initializer : str or Initializer, default zeros + Initializer for the bias vector. activation : str or Symbol, default functools.partial(symbol.LeakyReLU, act_type='leaky', slope=0.2) Type of activation function. @@ -1180,19 +1198,20 @@ def __init__(self, input_shape, num_hidden, h2h_kernel=(3, 3), h2h_dilate=(1, 1), i2h_kernel=(3, 3), i2h_stride=(1, 1), i2h_pad=(1, 1), i2h_dilate=(1, 1), + i2h_weight_initializer=None, h2h_weight_initializer=None, + i2h_bias_initializer='zeros', h2h_bias_initializer='zeros', activation=functools.partial(symbol.LeakyReLU, act_type='leaky', slope=0.2), prefix='ConvRNN_', params=None, conv_layout='NCHW'): super(ConvRNNCell, self).__init__(input_shape=input_shape, num_hidden=num_hidden, h2h_kernel=h2h_kernel, h2h_dilate=h2h_dilate, i2h_kernel=i2h_kernel, i2h_stride=i2h_stride, i2h_pad=i2h_pad, i2h_dilate=i2h_dilate, + i2h_weight_initializer=i2h_weight_initializer, + h2h_weight_initializer=h2h_weight_initializer, + i2h_bias_initializer=i2h_bias_initializer, + h2h_bias_initializer=h2h_bias_initializer, activation=activation, prefix=prefix, params=params, conv_layout=conv_layout) - # Get params - self._iW = self.params.get('i2h_weight') - self._hW = self.params.get('h2h_weight') - self._iB = self.params.get('i2h_bias') - self._hB = self.params.get('h2h_bias') @property def _gate_names(self): @@ -1201,24 +1220,7 @@ def _gate_names(self): def __call__(self, inputs, states): self._counter += 1 name = '%st%d_'%(self._prefix, self._counter) - i2h = symbol.Convolution(name='%si2h'%name, - data=inputs, - num_filter=self._num_hidden, - kernel=self._i2h_kernel, - stride=self._i2h_stride, - pad=self._i2h_pad, - dilate=self._i2h_dilate, - weight=self._iW, - bias=self._iB,) - h2h = symbol.Convolution(name='%sh2h'%name, - data=states[0], - num_filter=self._num_hidden, - kernel=self._h2h_kernel, - dilate=self._h2h_dilate, - pad=self._h2h_pad, - stride=(1, 1), - weight=self._hW, - bias=self._hB) + i2h, h2h = self._conv_forward(inputs, states, name) output = self._get_activation(i2h + h2h, self._activation, name='%sout'%name) return output, [output] @@ -1248,6 +1250,16 @@ class ConvLSTMCell(BaseConvRNNCell): Pad of Convolution operator in input-to-state transitions. i2h_dilate : tuple of int, default (1, 1) Dilation of Convolution operator in input-to-state transitions. + i2h_weight_initializer : str or Initializer + Initializer for the input weights matrix, used for the convolution + transformation of the inputs. + h2h_weight_initializer : str or Initializer + Initializer for the recurrent weights matrix, used for the convolution + transformation of the recurrent state. + i2h_bias_initializer : str or Initializer, default zeros + Initializer for the bias vector. + h2h_bias_initializer : str or Initializer, default zeros + Initializer for the bias vector. activation : str or Symbol default functools.partial(symbol.LeakyReLU, act_type='leaky', slope=0.2) Type of activation function. @@ -1255,8 +1267,6 @@ class ConvLSTMCell(BaseConvRNNCell): Prefix for name of layers (and name of weight if params is None). params : RNNParams, default None Container for weight sharing between cells. Created if None. - forget_bias : bias added to forget gate, default 1.0. - Jozefowicz et al. 2015 recommends setting this to 1.0 conv_layout : str, , default 'NCHW' Layout of ConvolutionOp """ @@ -1264,23 +1274,22 @@ def __init__(self, input_shape, num_hidden, h2h_kernel=(3, 3), h2h_dilate=(1, 1), i2h_kernel=(3, 3), i2h_stride=(1, 1), i2h_pad=(1, 1), i2h_dilate=(1, 1), + i2h_weight_initializer=None, h2h_weight_initializer=None, + i2h_bias_initializer='zeros', h2h_bias_initializer='zeros', activation=functools.partial(symbol.LeakyReLU, act_type='leaky', slope=0.2), - prefix='ConvLSTM_', params=None, forget_bias=1.0, + prefix='ConvLSTM_', params=None, conv_layout='NCHW'): super(ConvLSTMCell, self).__init__(input_shape=input_shape, num_hidden=num_hidden, h2h_kernel=h2h_kernel, h2h_dilate=h2h_dilate, i2h_kernel=i2h_kernel, i2h_stride=i2h_stride, i2h_pad=i2h_pad, i2h_dilate=i2h_dilate, + i2h_weight_initializer=i2h_weight_initializer, + h2h_weight_initializer=h2h_weight_initializer, + i2h_bias_initializer=i2h_bias_initializer, + h2h_bias_initializer=h2h_bias_initializer, activation=activation, prefix=prefix, params=params, conv_layout=conv_layout) - # Get params - self._iW = self.params.get('i2h_weight') - self._hW = self.params.get('h2h_weight') - # we add the forget_bias to i2h_bias, this adds the bias to the forget gate activation - self._iB = self.params.get('i2h_bias', init=init.LSTMBias(forget_bias=forget_bias)) - self._hB = self.params.get('h2h_bias') - @property def _gate_names(self): return ['_i', '_f', '_c', '_o'] @@ -1288,25 +1297,7 @@ def _gate_names(self): def __call__(self, inputs, states): self._counter += 1 name = '%st%d_'%(self._prefix, self._counter) - i2h = symbol.Convolution(name='%si2h'%name, - data=inputs, - num_filter=self._num_hidden*4, - kernel=self._i2h_kernel, - stride=self._i2h_stride, - pad=self._i2h_pad, - dilate=self._i2h_dilate, - weight=self._iW, - bias=self._iB,) - h2h = symbol.Convolution(name='%sh2h'%name, - data=states[0], - num_filter=self._num_hidden*4, - kernel=self._h2h_kernel, - dilate=self._h2h_dilate, - pad=self._h2h_pad, - stride=(1, 1), - weight=self._hW, - bias=self._hB) - + i2h, h2h = self._conv_forward(inputs, states, name) gates = i2h + h2h slice_gates = symbol.SliceChannel(gates, num_outputs=4, axis=self._conv_layout.find('C'), name="%sslice"%name) @@ -1346,6 +1337,16 @@ class ConvGRUCell(BaseConvRNNCell): Pad of Convolution operator in input-to-state transitions. i2h_dilate : tuple of int, default (1, 1) Dilation of Convolution operator in input-to-state transitions. + i2h_weight_initializer : str or Initializer + Initializer for the input weights matrix, used for the convolution + transformation of the inputs. + h2h_weight_initializer : str or Initializer + Initializer for the recurrent weights matrix, used for the convolution + transformation of the recurrent state. + i2h_bias_initializer : str or Initializer, default zeros + Initializer for the bias vector. + h2h_bias_initializer : str or Initializer, default zeros + Initializer for the bias vector. activation : str or Symbol, default functools.partial(symbol.LeakyReLU, act_type='leaky', slope=0.2) Type of activation function. @@ -1360,19 +1361,20 @@ def __init__(self, input_shape, num_hidden, h2h_kernel=(3, 3), h2h_dilate=(1, 1), i2h_kernel=(3, 3), i2h_stride=(1, 1), i2h_pad=(1, 1), i2h_dilate=(1, 1), + i2h_weight_initializer=None, h2h_weight_initializer=None, + i2h_bias_initializer='zeros', h2h_bias_initializer='zeros', activation=functools.partial(symbol.LeakyReLU, act_type='leaky', slope=0.2), prefix='ConvGRU_', params=None, conv_layout='NCHW'): super(ConvGRUCell, self).__init__(input_shape=input_shape, num_hidden=num_hidden, h2h_kernel=h2h_kernel, h2h_dilate=h2h_dilate, i2h_kernel=i2h_kernel, i2h_stride=i2h_stride, i2h_pad=i2h_pad, i2h_dilate=i2h_dilate, + i2h_weight_initializer=i2h_weight_initializer, + h2h_weight_initializer=h2h_weight_initializer, + i2h_bias_initializer=i2h_bias_initializer, + h2h_bias_initializer=h2h_bias_initializer, activation=activation, prefix=prefix, params=params, conv_layout=conv_layout) - # Get params - self._iW = self.params.get('i2h_weight') - self._hW = self.params.get('h2h_weight') - self._iB = self.params.get('i2h_bias') - self._hB = self.params.get('h2h_bias') @property def _gate_names(self): @@ -1382,22 +1384,7 @@ def __call__(self, inputs, states): self._counter += 1 seq_idx = self._counter name = '%st%d_' % (self._prefix, seq_idx) - i2h = symbol.Convolution(name='%s_i2h'%name, data=inputs, - num_filter=self._num_hidden * 3, - kernel=self._i2h_kernel, - stride=self._i2h_stride, - pad=self._i2h_pad, - dilate=self._i2h_dilate, - weight=self._iW, - bias=self._iB,) - h2h = symbol.Convolution(name='%s_h2h'%name, data=states[0], - num_filter=self._num_hidden * 3, - kernel=self._h2h_kernel, - dilate=self._h2h_dilate, - pad=self._h2h_pad, - stride=(1, 1), - weight=self._hW, - bias=self._hB) + i2h, h2h = self._conv_forward(inputs, states, name) i2h_r, i2h_z, i2h = symbol.SliceChannel(i2h, num_outputs=3, name="%s_i2h_slice" % name) h2h_r, h2h_z, h2h = symbol.SliceChannel(h2h, num_outputs=3, name="%s_h2h_slice" % name) diff --git a/tests/python/unittest/test_rnn.py b/tests/python/unittest/test_rnn.py index 75f41fe13389..e8176bb468c2 100644 --- a/tests/python/unittest/test_rnn.py +++ b/tests/python/unittest/test_rnn.py @@ -254,7 +254,7 @@ def test_convlstm(): h2h_kernel=(3, 3), h2h_dilate=(1, 1), i2h_kernel=(3, 3), i2h_stride=(1, 1), i2h_pad=(1, 1), i2h_dilate=(1, 1), - prefix='rnn_', forget_bias=1.0) + prefix='rnn_') inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)] outputs, _ = cell.unroll(3, inputs) outputs = mx.sym.Group(outputs) From c7d18e077d149784feaf08ad3d41d8302eb8e929 Mon Sep 17 00:00:00 2001 From: MinWoo Byeon Date: Fri, 4 Aug 2017 05:05:42 +0900 Subject: [PATCH 313/834] fix py3 compatibilities (#7305) * fix py3 compatibilities * fix py3 compatibilities --- tools/im2rec.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/tools/im2rec.py b/tools/im2rec.py index a05bfb04621b..30ee3ec92a5e 100644 --- a/tools/im2rec.py +++ b/tools/im2rec.py @@ -11,9 +11,7 @@ import cv2 import time import traceback - -if sys.version_info[0] == 3: - xrange = range +from builtins import range try: import multiprocessing @@ -61,8 +59,8 @@ def make_list(args): random.seed(100) random.shuffle(image_list) N = len(image_list) - chunk_size = (N + args.chunks - 1) / args.chunks - for i in xrange(args.chunks): + chunk_size = (N + args.chunks - 1) // args.chunks + for i in range(args.chunks): chunk = image_list[i * chunk_size:(i + 1) * chunk_size] if args.chunks > 1: str_chunk = '_%d' % i @@ -130,16 +128,16 @@ def image_encode(args, i, item, q_out): return if args.center_crop: if img.shape[0] > img.shape[1]: - margin = (img.shape[0] - img.shape[1]) / 2; + margin = (img.shape[0] - img.shape[1]) // 2; img = img[margin:margin + img.shape[1], :] else: - margin = (img.shape[1] - img.shape[0]) / 2; + margin = (img.shape[1] - img.shape[0]) // 2; img = img[:, margin:margin + img.shape[0]] if args.resize: if img.shape[0] > img.shape[1]: - newsize = (args.resize, img.shape[0] * args.resize / img.shape[1]) + newsize = (args.resize, img.shape[0] * args.resize // img.shape[1]) else: - newsize = (img.shape[1] * args.resize / img.shape[0], args.resize) + newsize = (img.shape[1] * args.resize // img.shape[0], args.resize) img = cv2.resize(img, newsize) try: From 82a3d21104c348610d6f5e224c89a4382302725f Mon Sep 17 00:00:00 2001 From: Guneet Singh Dhillon Date: Thu, 3 Aug 2017 13:08:14 -0700 Subject: [PATCH 314/834] Added sparsity functionality, with tests (#7138) * added pruning for sgd * added pruning for example/image-classification * working example for imagenet to experiment on * added flexibility to start off with pruning * changes to imagenet code * minor changes for testing * changes to imagenet pruning * small changes to parameters for tests * DSD test on mnist added * improved sparsification, added sparse-sparse training, added pruning factor * changed test for more coverage * updated example * updated example to save models * added thresholding by user * made optimizer code cleaner, created tests - mlp and rnn * added thresholding functionality, and related tests * made minor change to tests * updated common file, changed to merger * merging * reverted for mshadow * reverted dmlc-core * back to old examples * removed spaces from code * added comments * another style change * made SparseSGD a subclass * removed dependencies from tests * minor changes * reduced checks - not needed * call sgd from sparsesgd * corrected syntax * corrected syntax * reverted back, handle epoch count myself * added DSD traning to examples * added mask generation logic * added comment on layer-wise vs global pruning * added update message in sparse_sgd * added an example * changes to README --- example/dsd/README.md | 30 +++++++ example/dsd/mlp.py | 125 ++++++++++++++++++++++++++++ example/dsd/sparse_sgd.py | 170 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 325 insertions(+) create mode 100644 example/dsd/README.md create mode 100644 example/dsd/mlp.py create mode 100644 example/dsd/sparse_sgd.py diff --git a/example/dsd/README.md b/example/dsd/README.md new file mode 100644 index 000000000000..0ce5cc5d1f0f --- /dev/null +++ b/example/dsd/README.md @@ -0,0 +1,30 @@ +DSD Training +============ +This folder contains an optimizer class that implements DSD training coupled with SGD. The training +procedure is described in the paper *DSD: Dense-Sparse-Dense Training for Deep Neural Networks*, +available at https://arxiv.org/pdf/1607.04381.pdf + +The optimizer class is flexible in the way it prunes weights. The user can define the following: +- The percentage sparsity they want or the thresholding value for the pruning +- The epochs at which they want a particular level of pruning + +Note that giving the sparsity level induces that level of sparsity in every layer of the neural +network. It layer-wise pruning, and not global pruning (which would require loooking at all the +weights of the neural network at the same time). However, global pruning can be done if the +threshold value is known to the user (by doing some preprocessing), and is passed to the optimizer. + +## Example + +To test out the sparsity feature on a MLP, run the following script: + + python mlp.py --pruning_switch_epoch 4,7,10 --bias_sparsity 0,30,50 --weight_sparsity 0,50,70 + +This will train a MLP with 0% sparsity uptil epoch 4, with 30% bias and 50% weight sparsity uptil +epoch 7, 50% bias and 70% weight sparsity uptil epoch 10. + +To test out the thresholding feature on a MLP, run the following script: + + python mlp.py --pruning_switch_epoch 4,6 --bias_threshold 0,0.01 --weight_threshold 0,0.05 + +This will train a MLP with thresholding at 0 uptil epoch 4, with bias thresholding at 0.01 and +weight thresholding at 0.05 uptil epoch 6. diff --git a/example/dsd/mlp.py b/example/dsd/mlp.py new file mode 100644 index 000000000000..ccb094062f58 --- /dev/null +++ b/example/dsd/mlp.py @@ -0,0 +1,125 @@ +import mxnet as mx +import os +import logging +import argparse +from math import ceil +import sparse_sgd + +# symbol net +def get_symbol(): + data = mx.symbol.Variable('data') + fc1 = mx.symbol.FullyConnected(data, name='fc1', num_hidden=128) + act1 = mx.symbol.Activation(fc1, name='relu1', act_type="relu") + fc2 = mx.symbol.FullyConnected(act1, name='fc2', num_hidden=64) + act2 = mx.symbol.Activation(fc2, name='relu2', act_type="relu") + fc3 = mx.symbol.FullyConnected(act2, name='fc3', num_hidden=10) + softmax = mx.symbol.SoftmaxOutput(fc3, name='sm') + + return softmax + +# download ubyte version of mnist and untar +def download_data(): + if not os.path.isdir("data/"): + os.system("mkdir data/") + if (not os.path.exists('data/train-images-idx3-ubyte')) or \ + (not os.path.exists('data/train-labels-idx1-ubyte')) or \ + (not os.path.exists('data/t10k-images-idx3-ubyte')) or \ + (not os.path.exists('data/t10k-labels-idx1-ubyte')): + os.system("wget -q http://data.mxnet.io/mxnet/data/mnist.zip -P data/") + os.chdir("./data") + os.system("unzip -u mnist.zip") + os.chdir("..") + +# get data iterators +def get_iters(batch_size): + train = mx.io.MNISTIter( + image="data/train-images-idx3-ubyte", + label="data/train-labels-idx1-ubyte", + data_shape=(784,), + label_name='sm_label', + batch_size=batch_size, + shuffle=True, + flat=True, + silent=False, + seed=10) + val = mx.io.MNISTIter( + image="data/t10k-images-idx3-ubyte", + label="data/t10k-labels-idx1-ubyte", + data_shape=(784,), + label_name='sm_label', + batch_size=batch_size, + shuffle=True, + flat=True, + silent=False) + + return (train, val) + +def test_mlp(args): + # get parameters + prefix = './mlp' + batch_size = 100 + pruning_switch_epoch = [int(i) for i in args.pruning_switch_epoch.split(',')] + num_epoch = pruning_switch_epoch[-1] + batches_per_epoch = ceil(60000.0/batch_size) + weight_sparsity = args.weight_sparsity + bias_sparsity = args.bias_sparsity + weight_threshold = args.weight_threshold + bias_threshold = args.bias_threshold + if args.weight_sparsity: + weight_sparsity = [float(i) for i in args.weight_sparsity.split(',')] + bias_sparsity = [float(i) for i in args.bias_sparsity.split(',')] + else: + weight_threshold = [float(i) for i in args.weight_threshold.split(',')] + bias_threshold = [float(i) for i in args.bias_threshold.split(',')] + + # get symbols and iterators + sym = get_symbol() + download_data() + (train, val) = get_iters(batch_size) + + # fit model + model = mx.mod.Module( + sym, + context=[mx.cpu(i) for i in range(2)], + data_names=['data'], + label_names=['sm_label']) + optimizer_params = { + 'learning_rate' : 0.1, + 'wd' : 0.004, + 'momentum' : 0.9, + 'pruning_switch_epoch' : pruning_switch_epoch, + 'batches_per_epoch' : batches_per_epoch, + 'weight_sparsity' : weight_sparsity, + 'bias_sparsity' : bias_sparsity, + 'weight_threshold' : weight_threshold, + 'bias_threshold' : bias_threshold} + logging.info('Start training...') + model.fit(train, + eval_data=val, + eval_metric='acc', + epoch_end_callback=mx.callback.do_checkpoint(prefix), + num_epoch=num_epoch, + optimizer='sparsesgd', + optimizer_params=optimizer_params) + logging.info('Finish traning...') + + # remove files + for i in range(num_epoch): + os.remove('%s-%04d.params' % (prefix, i + 1)) + os.remove('%s-symbol.json' % prefix) + + +if __name__ == "__main__": + + # print logging by default + logging.basicConfig(level=logging.DEBUG) + + parser = argparse.ArgumentParser(description="sparse training") + parser.add_argument('--pruning_switch_epoch', type=str) + parser.add_argument('--weight_sparsity', type=str, default=None) + parser.add_argument('--bias_sparsity', type=str, default=None) + parser.add_argument('--weight_threshold', type=str, default=None) + parser.add_argument('--bias_threshold', type=str, default=None) + args = parser.parse_args() + + test_mlp(args) diff --git a/example/dsd/sparse_sgd.py b/example/dsd/sparse_sgd.py new file mode 100644 index 000000000000..f11a2395c4c0 --- /dev/null +++ b/example/dsd/sparse_sgd.py @@ -0,0 +1,170 @@ +from mxnet.ndarray import NDArray, topk, abs as NDabs +from mxnet.optimizer import SGD, register +import logging + +log = 'Sparsity Update:\t' + +@register +class SparseSGD(SGD): + """The SGD optimizer with weight pruning. + + This class implements the optimizer described in the paper *DSD: Dense-Sparse-Dense Training for + Deep Neural Networks*, available at https://arxiv.org/pdf/1607.04381.pdf + + The optimizer updates the weights the same way as done in SGD, but does the following + preprocessing:: + + if threshold given, all weights below the threshold in absolute value are pruned, + mask = abs(weight) >= threshold + if sparsity level given, the smallest (sparsity)% weights in absolute value are pruned + (or the largest (100-sparsity)% weights in absolute value are used) + mask = topk(abs(weight), ret_typ='mask', k=weight.size*(100-sparsity)/100) + + => mask[i,j] = {0 if weight[i,j] is pruned, 1 otherwise} (for a matrix representation) + + weight = weight * mask + grad = grad * mask + state = state * mask + + This optimizer accepts the following parameters in addition to those accepted + by :class:`.SGD`. + + Parameters + ---------- + pruning_switch_epoch : list of ints, optional + The epochs at which there is a change in sparsity level (should be in ascending order). + + weight_sparsity : list of floats, optional + The sparsity on the weights required on each iteration of sparse training. + + bias_sparsity : list of floats, optional + The sparsity on the biases required on each iteration of sparse training. + + weight_threshold : list of floats, optional + The absolute value threshold on the weights required on each iteration of sparse training. + + bias_threshold : list of floats, optional + The absolute value threshold on the biases required on each iteration of sparse training. + + batches_per_epoch : int, optional + The number of batches in each epoch. + (The ceiling integer value of number_of_examples / batch_size) + """ + def __init__(self, pruning_switch_epoch, batches_per_epoch, + weight_sparsity=None, bias_sparsity=None, + weight_threshold=None, bias_threshold=None, **kwargs): + super(SparseSGD, self).__init__(**kwargs) + + self.masks = [] + self.masks_updated = False + self.epoch = 0 + self.pruning_switch_epoch = pruning_switch_epoch + self.batches_per_epoch = batches_per_epoch + + # get weight and bias sparsity percentages + self.weight_sparsity = weight_sparsity + self.bias_sparsity = bias_sparsity + if weight_sparsity is not None: + assert len(weight_sparsity) == len(bias_sparsity), \ + 'weight_sparsity and bias_sparsity should have same length' + assert len(weight_sparsity) == len(pruning_switch_epoch), \ + 'pruning_switch_epoch and weight_sparsity should have same length' + + # get weight and bias sparsity thresholds + self.weight_threshold = weight_threshold + self.bias_threshold = bias_threshold + if weight_threshold is not None: + assert len(weight_threshold) == len(bias_threshold), \ + 'weight_threshold and bias_threshold should have same length' + assert len(weight_threshold) == len(pruning_switch_epoch), \ + 'pruning_switch_epoch and weight_sparsity_threshold should have same length' + + # either percentages or thresholds must be given + assert weight_sparsity is not None or weight_threshold is not None,\ + 'weight_sparsity or weight_sparsity_threshold should be given' + + def update_masks(self, index, weight): + """Updates the masks for sparse training. + + Parameters + ---------- + index : int + The index for weight. + weight : NDArray + The weight matrix. + + Returns + ------- + boolean + If the masks were changed + """ + # determine number of updates without actually updating the count + if index not in self._index_update_count: + num_update = self.begin_num_update + else: + num_update = self._index_update_count[index] + num_update += 1 + num_update = max(num_update, self.num_update) + + # calculate epoch + epoch = int((num_update - 1) / self.batches_per_epoch) + 1 + + # determine if masks need to be updated, and get corresponding parameters + if index == 0: + self.masks_updated = True + if self.epoch != epoch: + self.epoch = epoch + if epoch == 1: + self.masks_updated = False + if self.weight_sparsity is not None: + logging.info(log + 'bias-sparsity={}, weight-sparsity={}'.format(self.bias_sparsity[0], self.weight_sparsity[0])) + else: + logging.info(log + 'bias-threshold={}, weight-threshold={}'.format(self.bias_threshold[0], self.weight_threshold[0])) + if self.pruning_switch_epoch[0] + 1 == epoch: + self.masks_updated = False + self.pruning_switch_epoch.pop(0) + if self.weight_sparsity is not None: + self.weight_sparsity.pop(0) + self.bias_sparsity.pop(0) + logging.info(log + 'bias-sparsity={}, weight-sparsity={}'.format(self.bias_sparsity[0], self.weight_sparsity[0])) + else: + self.weight_threshold.pop(0) + self.bias_threshold.pop(0) + logging.info(log + 'bias-threshold={}, weight-threshold={}'.format(self.bias_threshold[0], self.weight_threshold[0])) + + # update masks if needed + if not self.masks_updated: + # initialize masks + if epoch == 1: + self.masks.append(None) + # if percentages are given + if self.weight_sparsity is not None: + if len(weight.shape) == 1: + sparsity = self.bias_sparsity[0] + else: + sparsity = self.weight_sparsity[0] + number_unpruned = int((100.0 - sparsity) * weight.size / 100.0) + self.masks[index] = topk(NDabs(weight), axis=None, ret_typ='mask', + k=number_unpruned) + # if thresholds are given + else: + if len(weight.shape) == 1: + threshold = self.bias_threshold[0] + else: + threshold = self.weight_threshold[0] + self.masks[index] = NDabs(weight) >= threshold + + return not self.masks_updated + + def update(self, index, weight, grad, state): + assert(isinstance(weight, NDArray)) + assert(isinstance(grad, NDArray)) + + # preprocessing for pruning + if self.update_masks(index, weight): + weight[:] = weight * self.masks[index] + grad[:] = grad * self.masks[index] + if state is not None: + state[:] = state * self.masks[index] + + super(SparseSGD, self).update(index, weight, grad, state) From 9add5ae417cd6fa5e9153c1f19195f5b88c01305 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=A2=81=E5=BE=B7=E6=BE=8E?= Date: Fri, 4 Aug 2017 04:10:52 +0800 Subject: [PATCH 315/834] improve convert_symbol.py add support to SUM with coeff (#7120) * improve convert_symbol.py add support to SUM with coeff * fix code style * fix code style * fix code style --- tools/caffe_converter/convert_symbol.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tools/caffe_converter/convert_symbol.py b/tools/caffe_converter/convert_symbol.py index fad89c41e83c..c384c7690088 100644 --- a/tools/caffe_converter/convert_symbol.py +++ b/tools/caffe_converter/convert_symbol.py @@ -207,6 +207,7 @@ def _parse_proto(prototxt_fname): need_flatten[name] = need_flatten[mapping[layer.bottom[0]]] if layer.type == 'Eltwise': type_string = 'mx.symbol.broadcast_add' + param = layer.eltwise_param param_string = "" need_flatten[name] = False if layer.type == 'Reshape': @@ -239,8 +240,15 @@ def _parse_proto(prototxt_fname): symbol_string += "%s = %s(name='%s', data=%s %s)\n" % ( name, type_string, name, mapping[bottom[0]], param_string) else: - symbol_string += "%s = %s(name='%s', *[%s] %s)\n" % ( - name, type_string, name, ','.join([mapping[x] for x in bottom]), param_string) + if layer.type == 'Eltwise' and param.operation == 1 and len(param.coeff) > 0: + symbol_string += "%s = " % name + symbol_string += " + ".join(["%s * %s" % ( + mapping[bottom[i]], param.coeff[i]) for i in range(len(param.coeff))]) + symbol_string += "\n" + else: + symbol_string += "%s = %s(name='%s', *[%s] %s)\n" % ( + name, type_string, name, ','.join( + [mapping[x] for x in bottom]), param_string) for j in range(len(layer.top)): mapping[layer.top[j]] = name output_name = name From 8519eafd1d355f66245d9ce074c4f451bceef311 Mon Sep 17 00:00:00 2001 From: qingzhouzhen <576591769@qq.com> Date: Fri, 4 Aug 2017 04:12:28 +0800 Subject: [PATCH 316/834] add mobilenet (#7121) --- .../image-classification/symbols/mobilenet.py | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 example/image-classification/symbols/mobilenet.py diff --git a/example/image-classification/symbols/mobilenet.py b/example/image-classification/symbols/mobilenet.py new file mode 100644 index 000000000000..cf470bace727 --- /dev/null +++ b/example/image-classification/symbols/mobilenet.py @@ -0,0 +1,45 @@ +import mxnet as mx + +def Conv(data, num_filter=1, kernel=(1, 1), stride=(1, 1), pad=(0, 0), num_group=1, name=None, suffix=''): + conv = mx.sym.Convolution(data=data, num_filter=num_filter, kernel=kernel, num_group=num_group, stride=stride, pad=pad, no_bias=True, name='%s%s_conv2d' %(name, suffix)) + bn = mx.sym.BatchNorm(data=conv, name='%s%s_batchnorm' %(name, suffix), fix_gamma=True) + act = mx.sym.Activation(data=bn, act_type='relu', name='%s%s_relu' %(name, suffix)) + return act + +def get_symbol(num_classes, **kwargs): + data = mx.symbol.Variable(name="data") # 224 + conv_1 = Conv(data, num_filter=32, kernel=(3, 3), pad=(1, 1), stride=(2, 2), name="conv_1") # 224/112 + conv_2_dw = Conv(conv_1, num_group=32, num_filter=32, kernel=(3, 3), pad=(1, 1), stride=(1, 1), name="conv_2_dw") # 112/112 + conv_2 = Conv(conv_2_dw, num_filter=64, kernel=(1, 1), pad=(0, 0), stride=(1, 1), name="conv_2") # 112/112 + conv_3_dw = Conv(conv_2, num_group=64, num_filter=64, kernel=(3, 3), pad=(1, 1), stride=(2, 2), name="conv_3_dw") # 112/56 + conv_3 = Conv(conv_3_dw, num_filter=128, kernel=(1, 1), pad=(0, 0), stride=(1, 1), name="conv_3") # 56/56 + conv_4_dw = Conv(conv_3, num_group=128, num_filter=128, kernel=(3, 3), pad=(1, 1), stride=(1, 1), name="conv_4_dw") # 56/56 + conv_4 = Conv(conv_3_dw, num_filter=128, kernel=(1, 1), pad=(0, 0), stride=(1, 1), name="conv_4") # 56/56 + conv_5_dw = Conv(conv_4, num_group=128, num_filter=128, kernel=(3, 3), pad=(1, 1), stride=(2, 2), name="conv_5_dw") # 56/28 + conv_5 = Conv(conv_5_dw, num_filter=256, kernel=(1, 1), pad=(0, 0), stride=(1, 1), name="conv_5") # 28/28 + conv_6_dw = Conv(conv_5, num_group=256, num_filter=256, kernel=(3, 3), pad=(1, 1), stride=(1, 1), name="conv_6_dw") # 28/28 + conv_6 = Conv(conv_6_dw, num_filter=256, kernel=(1, 1), pad=(0, 0), stride=(1, 1), name="conv_6") # 28/28 + conv_7_dw = Conv(conv_6, num_group=256, num_filter=256, kernel=(3, 3), pad=(1, 1), stride=(2, 2), name="conv_7_dw") # 28/14 + conv_7 = Conv(conv_7_dw, num_filter=512, kernel=(1, 1), pad=(0, 0), stride=(1, 1), name="conv_7") # 14/14 + + conv_8_dw = Conv(conv_7, num_group=512, num_filter=512, kernel=(3, 3), pad=(1, 1), stride=(1, 1), name="conv_8_dw") # 14/14 + conv_8 = Conv(conv_8_dw, num_filter=512, kernel=(1, 1), pad=(0, 0), stride=(1, 1), name="conv_8") # 14/14 + conv_9_dw = Conv(conv_8, num_group=512, num_filter=512, kernel=(3, 3), pad=(1, 1), stride=(1, 1), name="conv_9_dw") # 14/14 + conv_9 = Conv(conv_9_dw, num_filter=512, kernel=(1, 1), pad=(0, 0), stride=(1, 1), name="conv_9") # 14/14 + conv_10_dw = Conv(conv_9, num_group=512, num_filter=512, kernel=(3, 3), pad=(1, 1), stride=(1, 1), name="conv_10_dw") # 14/14 + conv_10 = Conv(conv_10_dw, num_filter=512, kernel=(1, 1), pad=(0, 0), stride=(1, 1), name="conv_10") # 14/14 + conv_11_dw = Conv(conv_10, num_group=512, num_filter=512, kernel=(3, 3), pad=(1, 1), stride=(1, 1), name="conv_11_dw") # 14/14 + conv_11 = Conv(conv_11_dw, num_filter=512, kernel=(1, 1), pad=(0, 0), stride=(1, 1), name="conv_11") # 14/14 + conv_12_dw = Conv(conv_11, num_group=512, num_filter=512, kernel=(3, 3), pad=(1, 1), stride=(1, 1), name="conv_12_dw") # 14/14 + conv_12 = Conv(conv_12_dw, num_filter=512, kernel=(1, 1), pad=(0, 0), stride=(1, 1), name="conv_12") # 14/14 + + conv_13_dw = Conv(conv_12, num_group=512, num_filter=512, kernel=(3, 3), pad=(1, 1), stride=(2, 2), name="conv_13_dw") # 14/7 + conv_13 = Conv(conv_13_dw, num_filter=1024, kernel=(1, 1), pad=(0, 0), stride=(1, 1), name="conv_13") # 7/7 + conv_14_dw = Conv(conv_13, num_group=1024, num_filter=1024, kernel=(3, 3), pad=(1, 1), stride=(1, 1), name="conv_14_dw") # 7/7 + conv_14 = Conv(conv_14_dw, num_filter=1024, kernel=(1, 1), pad=(0, 0), stride=(1, 1), name="conv_14") # 7/7 + + pool = mx.sym.Pooling(data=conv_14, kernel=(7, 7), stride=(1, 1), pool_type="avg", name="global_pool") + flatten = mx.sym.Flatten(data=pool, name="flatten") + fc = mx.symbol.FullyConnected(data=flatten, num_hidden=num_classes, name='fc') + softmax = mx.symbol.SoftmaxOutput(data=fc, name='softmax') + return softmax From 76dee53dc494b35ce7dd4ac88cdce817bf9aa1ce Mon Sep 17 00:00:00 2001 From: CNevd Date: Fri, 4 Aug 2017 04:18:07 +0800 Subject: [PATCH 317/834] [cpp-package] add lr scheduler (#6885) * add lr scheduler * Update lr_scheduler.h * Update mlp_gpu.cpp * Update test_score.cpp * update optimizer.hpp --- cpp-package/example/alexnet.cpp | 11 ++- cpp-package/example/charRNN.cpp | 11 ++- cpp-package/example/googlenet.cpp | 19 +++-- cpp-package/example/inception_bn.cpp | 12 ++- cpp-package/example/lenet.cpp | 17 ++-- cpp-package/example/lenet_with_mxdataiter.cpp | 20 +++-- cpp-package/example/mlp_cpu.cpp | 20 +++-- cpp-package/example/mlp_gpu.cpp | 43 ++++++---- cpp-package/example/resnet.cpp | 11 ++- cpp-package/example/test_score.cpp | 22 ++++-- cpp-package/include/mxnet-cpp/executor.h | 12 --- cpp-package/include/mxnet-cpp/executor.hpp | 7 -- cpp-package/include/mxnet-cpp/lr_scheduler.h | 78 +++++++++++++++++++ cpp-package/include/mxnet-cpp/optimizer.h | 22 +++--- cpp-package/include/mxnet-cpp/optimizer.hpp | 42 +++++++--- 15 files changed, 254 insertions(+), 93 deletions(-) create mode 100644 cpp-package/include/mxnet-cpp/lr_scheduler.h diff --git a/cpp-package/example/alexnet.cpp b/cpp-package/example/alexnet.cpp index c0d8273d559b..6a9e01ab01a0 100644 --- a/cpp-package/example/alexnet.cpp +++ b/cpp-package/example/alexnet.cpp @@ -199,6 +199,7 @@ int main(int argc, char const *argv[]) { /*with data and label, executor can be generated automatically*/ auto *exec = Net.SimpleBind(ctx, args_map); + auto arg_names = Net.ListArguments(); aux_map = exec->aux_dict(); args_map = exec->arg_dict(); @@ -240,7 +241,9 @@ int main(int argc, char const *argv[]) { Optimizer* opt = OptimizerRegistry::Find("ccsgd"); opt->SetParam("momentum", 0.9) ->SetParam("rescale_grad", 1.0 / batch_size) - ->SetParam("clip_gradient", 10); + ->SetParam("clip_gradient", 10) + ->SetParam("lr", learning_rate) + ->SetParam("wd", weight_decay); Accuracy acu_train, acu_val; LogLoss logloss_val; @@ -258,7 +261,11 @@ int main(int argc, char const *argv[]) { batch.label.CopyTo(&args_map["label"]); exec->Forward(true); exec->Backward(); - exec->UpdateAll(opt, learning_rate, weight_decay); + for (size_t i = 0; i < arg_names.size(); ++i) { + if (arg_names[i] == "data" || arg_names[i] == "label") continue; + opt->Update(i, exec->arg_arrays[i], exec->grad_arrays[i]); + } + NDArray::WaitAll(); acu_train.Update(batch.label, exec->outputs[0]); } diff --git a/cpp-package/example/charRNN.cpp b/cpp-package/example/charRNN.cpp index 5cb6382137c7..d95c97d8e734 100644 --- a/cpp-package/example/charRNN.cpp +++ b/cpp-package/example/charRNN.cpp @@ -451,6 +451,8 @@ void train(const string file, int batch_size, int max_epoch, int start_epoch) { mx_float learning_rate = 0.0002; mx_float weight_decay = 0.000002; Optimizer* opt = OptimizerRegistry::Find("ccsgd"); + opt->SetParam("lr", learning_rate) + ->SetParam("wd", weight_decay); // opt->SetParam("momentum", 0.9)->SetParam("rescale_grad", 1.0 / batch_size) // ->SetParam("clip_gradient", 10); @@ -470,7 +472,10 @@ void train(const string file, int batch_size, int max_epoch, int start_epoch) { exe->Forward(true); exe->Backward(); - exe->UpdateAll(opt, learning_rate, weight_decay); + for (size_t i = 0; i < exe->arg_arrays.size(); ++i) { + opt->Update(i, exe->arg_arrays[i], exe->grad_arrays[i]); + } + NDArray::WaitAll(); } auto toc = chrono::system_clock::now(); @@ -547,7 +552,9 @@ void trainWithBuiltInRNNOp(const string file, int batch_size, int max_epoch, int exe->Forward(true); exe->Backward(); - exe->UpdateAll(opt, learning_rate, weight_decay); + for (size_t i = 0; i < exe->arg_arrays.size(); ++i) { + opt->Update(i, exe->arg_arrays[i], exe->grad_arrays[i]); + } NDArray::WaitAll(); } auto toc = chrono::system_clock::now(); diff --git a/cpp-package/example/googlenet.cpp b/cpp-package/example/googlenet.cpp index a4dcbbd4a6cf..2e59fbfe45cd 100644 --- a/cpp-package/example/googlenet.cpp +++ b/cpp-package/example/googlenet.cpp @@ -128,7 +128,13 @@ int main(int argc, char const *argv[]) { Optimizer* opt = OptimizerRegistry::Find("ccsgd"); opt->SetParam("momentum", 0.9) ->SetParam("rescale_grad", 1.0 / batch_size) - ->SetParam("clip_gradient", 10); + ->SetParam("clip_gradient", 10) + ->SetParam("lr", learning_rate) + ->SetParam("wd", weight_decay); + + + auto *exec = googlenet.SimpleBind(Context::gpu(), args_map); + auto arg_names = googlenet.ListArguments(); for (int iter = 0; iter < max_epoch; ++iter) { LG << "Epoch: " << iter; @@ -138,11 +144,12 @@ int main(int argc, char const *argv[]) { args_map["data"] = data_batch.data.Copy(Context::gpu()); args_map["data_label"] = data_batch.label.Copy(Context::gpu()); NDArray::WaitAll(); - auto *exec = googlenet.SimpleBind(Context::gpu(), args_map); exec->Forward(true); exec->Backward(); - exec->UpdateAll(opt, learning_rate, weight_decay); - delete exec; + for (size_t i = 0; i < arg_names.size(); ++i) { + if (arg_names[i] == "data" || arg_names[i] == "data_label") continue; + opt->Update(i, exec->arg_arrays[i], exec->grad_arrays[i]); + } } Accuracy acu; @@ -152,14 +159,14 @@ int main(int argc, char const *argv[]) { args_map["data"] = data_batch.data.Copy(Context::gpu()); args_map["data_label"] = data_batch.label.Copy(Context::gpu()); NDArray::WaitAll(); - auto *exec = googlenet.SimpleBind(Context::gpu(), args_map); exec->Forward(false); NDArray::WaitAll(); acu.Update(data_batch.label, exec->outputs[0]); - delete exec; } LG << "Accuracy: " << acu.Get(); } + + delete exec; MXNotifyShutdown(); return 0; } diff --git a/cpp-package/example/inception_bn.cpp b/cpp-package/example/inception_bn.cpp index 5db4f81b0e07..4442e006b5a5 100644 --- a/cpp-package/example/inception_bn.cpp +++ b/cpp-package/example/inception_bn.cpp @@ -156,9 +156,12 @@ int main(int argc, char const *argv[]) { Optimizer* opt = OptimizerRegistry::Find("ccsgd"); opt->SetParam("momentum", 0.9) ->SetParam("rescale_grad", 1.0 / batch_size) - ->SetParam("clip_gradient", 10); + ->SetParam("clip_gradient", 10) + ->SetParam("lr", learning_rate) + ->SetParam("wd", weight_decay); auto *exec = inception_bn_net.SimpleBind(Context::gpu(), args_map); + auto arg_names = inception_bn_net.ListArguments(); for (int iter = 0; iter < max_epoch; ++iter) { LG << "Epoch: " << iter; @@ -171,7 +174,12 @@ int main(int argc, char const *argv[]) { exec->Forward(true); exec->Backward(); - exec->UpdateAll(opt, learning_rate, weight_decay); + // Update parameters + for (size_t i = 0; i < arg_names.size(); ++i) { + if (arg_names[i] == "data" || arg_names[i] == "data_label") continue; + opt->Update(i, exec->arg_arrays[i], exec->grad_arrays[i]); + } + NDArray::WaitAll(); } diff --git a/cpp-package/example/lenet.cpp b/cpp-package/example/lenet.cpp index 91b83a090fa3..56f8d2c8743a 100644 --- a/cpp-package/example/lenet.cpp +++ b/cpp-package/example/lenet.cpp @@ -118,7 +118,12 @@ class Lenet { Optimizer* opt = OptimizerRegistry::Find("ccsgd"); opt->SetParam("momentum", 0.9) ->SetParam("rescale_grad", 1.0) - ->SetParam("clip_gradient", 10); + ->SetParam("clip_gradient", 10) + ->SetParam("lr", learning_rate) + ->SetParam("wd", weight_decay); + + Executor *exe = lenet.SimpleBind(ctx_dev, args_map); + auto arg_names = lenet.ListArguments(); for (int ITER = 0; ITER < max_epoch; ++ITER) { size_t start_index = 0; @@ -135,17 +140,19 @@ class Lenet { start_index += batch_size; NDArray::WaitAll(); - Executor *exe = lenet.SimpleBind(ctx_dev, args_map); exe->Forward(true); exe->Backward(); - exe->UpdateAll(opt, learning_rate, weight_decay); - - delete exe; + // Update parameters + for (size_t i = 0; i < arg_names.size(); ++i) { + if (arg_names[i] == "data" || arg_names[i] == "data_label") continue; + opt->Update(i, exe->arg_arrays[i], exe->grad_arrays[i]); + } } LG << "Iter " << ITER << ", accuracy: " << ValAccuracy(batch_size * 10, lenet); } + delete exe; } private: diff --git a/cpp-package/example/lenet_with_mxdataiter.cpp b/cpp-package/example/lenet_with_mxdataiter.cpp index 85a4b2012eb6..f6301b52a61f 100644 --- a/cpp-package/example/lenet_with_mxdataiter.cpp +++ b/cpp-package/example/lenet_with_mxdataiter.cpp @@ -85,7 +85,13 @@ int main(int argc, char const *argv[]) { Optimizer* opt = OptimizerRegistry::Find("ccsgd"); opt->SetParam("momentum", 0.9) ->SetParam("rescale_grad", 1.0) - ->SetParam("clip_gradient", 10); + ->SetParam("clip_gradient", 10) + ->SetParam("lr", learning_rate) + ->SetParam("wd", weight_decay); + + + auto *exec = lenet.SimpleBind(Context::gpu(), args_map); + auto arg_names = lenet.ListArguments(); for (int iter = 0; iter < max_epoch; ++iter) { LG << "Epoch: " << iter; @@ -95,11 +101,13 @@ int main(int argc, char const *argv[]) { args_map["data"] = data_batch.data.Copy(Context::gpu()); args_map["data_label"] = data_batch.label.Copy(Context::gpu()); NDArray::WaitAll(); - auto *exec = lenet.SimpleBind(Context::gpu(), args_map); exec->Forward(true); exec->Backward(); - exec->UpdateAll(opt, learning_rate, weight_decay); - delete exec; + // Update parameters + for (size_t i = 0; i < arg_names.size(); ++i) { + if (arg_names[i] == "data" || arg_names[i] == "data_label") continue; + opt->Update(i, exec->arg_arrays[i], exec->grad_arrays[i]); + } } Accuracy acu; @@ -109,14 +117,14 @@ int main(int argc, char const *argv[]) { args_map["data"] = data_batch.data.Copy(Context::gpu()); args_map["data_label"] = data_batch.label.Copy(Context::gpu()); NDArray::WaitAll(); - auto *exec = lenet.SimpleBind(Context::gpu(), args_map); exec->Forward(false); NDArray::WaitAll(); acu.Update(data_batch.label, exec->outputs[0]); - delete exec; } LG << "Accuracy: " << acu.Get(); } + + delete exec; MXNotifyShutdown(); return 0; } diff --git a/cpp-package/example/mlp_cpu.cpp b/cpp-package/example/mlp_cpu.cpp index 69486490194c..358e8348ac5e 100644 --- a/cpp-package/example/mlp_cpu.cpp +++ b/cpp-package/example/mlp_cpu.cpp @@ -70,7 +70,13 @@ int main(int argc, char** argv) { // Create sgd optimizer Optimizer* opt = OptimizerRegistry::Find("sgd"); - opt->SetParam("rescale_grad", 1.0/batch_size); + opt->SetParam("rescale_grad", 1.0/batch_size) + ->SetParam("lr", learning_rate) + ->SetParam("wd", weight_decay); + + // Create executor by binding parameters to the model + auto *exec = net.SimpleBind(ctx, args); + auto arg_names = net.ListArguments(); // Start training for (int iter = 0; iter < max_epoch; ++iter) { @@ -85,15 +91,14 @@ int main(int argc, char** argv) { args["X"] = data_batch.data; args["label"] = data_batch.label; - // Create executor by binding parameters to the model - auto *exec = net.SimpleBind(ctx, args); // Compute gradients exec->Forward(true); exec->Backward(); // Update parameters - exec->UpdateAll(opt, learning_rate, weight_decay); - // Remember to free the memory - delete exec; + for (size_t i = 0; i < arg_names.size(); ++i) { + if (arg_names[i] == "X" || arg_names[i] == "label") continue; + opt->Update(i, exec->arg_arrays[i], exec->grad_arrays[i]); + } } auto toc = chrono::system_clock::now(); @@ -103,16 +108,15 @@ int main(int argc, char** argv) { auto data_batch = val_iter.GetDataBatch(); args["X"] = data_batch.data; args["label"] = data_batch.label; - auto *exec = net.SimpleBind(ctx, args); // Forward pass is enough as no gradient is needed when evaluating exec->Forward(false); acc.Update(data_batch.label, exec->outputs[0]); - delete exec; } float duration = chrono::duration_cast(toc - tic).count() / 1000.0; LG << "Epoch: " << iter << " " << samples/duration << " samples/sec Accuracy: " << acc.Get(); } + delete exec; MXNotifyShutdown(); return 0; } diff --git a/cpp-package/example/mlp_gpu.cpp b/cpp-package/example/mlp_gpu.cpp index 23be637437ff..a6281c385dfb 100644 --- a/cpp-package/example/mlp_gpu.cpp +++ b/cpp-package/example/mlp_gpu.cpp @@ -24,7 +24,7 @@ Symbol mlp(const vector &layers) { weights[i], biases[i], layers[i]); - outputs[i] = i == layers.size()-1? fc : Activation(fc, ActivationActType::kRelu); + outputs[i] = i == layers.size()-1 ? fc : Activation(fc, ActivationActType::kRelu); } return SoftmaxOutput(outputs.back(), label); @@ -70,12 +70,24 @@ int main(int argc, char** argv) { // Create sgd optimizer Optimizer* opt = OptimizerRegistry::Find("sgd"); - opt->SetParam("rescale_grad", 1.0/batch_size); + opt->SetParam("rescale_grad", 1.0/batch_size) + ->SetParam("lr", learning_rate) + ->SetParam("wd", weight_decay); + std::unique_ptr lr_sch(new FactorScheduler(5000, 0.1)); + opt->SetLRScheduler(std::move(lr_sch)); + + // Create executor by binding parameters to the model + auto *exec = net.SimpleBind(ctx, args); + auto arg_names = net.ListArguments(); + + // Create metrics + Accuracy train_acc, val_acc; // Start training for (int iter = 0; iter < max_epoch; ++iter) { int samples = 0; train_iter.Reset(); + train_acc.Reset(); auto tic = chrono::system_clock::now(); while (train_iter.Next()) { @@ -87,35 +99,40 @@ int main(int argc, char** argv) { // CopyTo is imperative, need to wait for it to complete. NDArray::WaitAll(); - // Create executor by binding parameters to the model - auto *exec = net.SimpleBind(ctx, args); // Compute gradients exec->Forward(true); exec->Backward(); + // Update parameters - exec->UpdateAll(opt, learning_rate, weight_decay); - // Remember to free the memory - delete exec; + for (size_t i = 0; i < arg_names.size(); ++i) { + if (arg_names[i] == "X" || arg_names[i] == "label") continue; + opt->Update(i, exec->arg_arrays[i], exec->grad_arrays[i]); + } + // Update metric + train_acc.Update(data_batch.label, exec->outputs[0]); } + // one epoch of training is finished auto toc = chrono::system_clock::now(); + float duration = chrono::duration_cast(toc - tic).count() / 1000.0; + LG << "Epoch[" << iter << "] " << samples/duration \ + << " samples/sec " << "Train-Accuracy=" << train_acc.Get();; - Accuracy acc; val_iter.Reset(); + val_acc.Reset(); while (val_iter.Next()) { auto data_batch = val_iter.GetDataBatch(); data_batch.data.CopyTo(&args["X"]); data_batch.label.CopyTo(&args["label"]); NDArray::WaitAll(); - auto *exec = net.SimpleBind(ctx, args); + // Only forward pass is enough as no gradient is needed when evaluating exec->Forward(false); - acc.Update(data_batch.label, exec->outputs[0]); - delete exec; + val_acc.Update(data_batch.label, exec->outputs[0]); } - float duration = chrono::duration_cast(toc - tic).count() / 1000.0; - LG << "Epoch: " << iter << " " << samples/duration << " samples/sec Accuracy: " << acc.Get(); + LG << "Epoch[" << iter << "] Val-Accuracy=" << val_acc.Get(); } + delete exec; MXNotifyShutdown(); return 0; } diff --git a/cpp-package/example/resnet.cpp b/cpp-package/example/resnet.cpp index 5521567e119d..b9766c7a64d0 100644 --- a/cpp-package/example/resnet.cpp +++ b/cpp-package/example/resnet.cpp @@ -165,11 +165,14 @@ int main(int argc, char const *argv[]) { .CreateDataIter(); Optimizer* opt = OptimizerRegistry::Find("ccsgd"); - opt->SetParam("momentum", 0.9) + opt->SetParam("lr", learning_rate) + ->SetParam("wd", weight_decay) + ->SetParam("momentum", 0.9) ->SetParam("rescale_grad", 1.0 / batch_size) ->SetParam("clip_gradient", 10); auto *exec = resnet.SimpleBind(Context::gpu(), args_map); + auto arg_names = resnet.ListArguments(); for (int iter = 0; iter < max_epoch; ++iter) { LG << "Epoch: " << iter; @@ -182,7 +185,11 @@ int main(int argc, char const *argv[]) { exec->Forward(true); exec->Backward(); - exec->UpdateAll(opt, learning_rate, weight_decay); + + for (size_t i = 0; i < arg_names.size(); ++i) { + if (arg_names[i] == "data" || arg_names[i] == "data_label") continue; + opt->Update(i, exec->arg_arrays[i], exec->grad_arrays[i]); + } NDArray::WaitAll(); } diff --git a/cpp-package/example/test_score.cpp b/cpp-package/example/test_score.cpp index 7dccd30b6f94..35342699558f 100644 --- a/cpp-package/example/test_score.cpp +++ b/cpp-package/example/test_score.cpp @@ -72,7 +72,15 @@ int main(int argc, char** argv) { // Create sgd optimizer Optimizer* opt = OptimizerRegistry::Find("sgd"); - opt->SetParam("rescale_grad", 1.0/batch_size); + opt->SetParam("rescale_grad", 1.0/batch_size) + ->SetParam("lr", learning_rate) + ->SetParam("wd", weight_decay); + std::unique_ptr lr_sch(new FactorScheduler(5000, 0.1)); + opt->SetLRScheduler(std::move(lr_sch)); + + // Create executor by binding parameters to the model + auto *exec = net.SimpleBind(ctx, args); + auto arg_names = net.ListArguments(); float score = 0; // Start training @@ -90,15 +98,14 @@ int main(int argc, char** argv) { // CopyTo is imperative, need to wait for it to complete. NDArray::WaitAll(); - // Create executor by binding parameters to the model - auto *exec = net.SimpleBind(ctx, args); // Compute gradients exec->Forward(true); exec->Backward(); // Update parameters - exec->UpdateAll(opt, learning_rate, weight_decay); - // Remember to free the memory - delete exec; + for (size_t i = 0; i < arg_names.size(); ++i) { + if (arg_names[i] == "X" || arg_names[i] == "label") continue; + opt->Update(i, exec->arg_arrays[i], exec->grad_arrays[i]); + } } auto toc = chrono::system_clock::now(); @@ -109,17 +116,16 @@ int main(int argc, char** argv) { data_batch.data.CopyTo(&args["X"]); data_batch.label.CopyTo(&args["label"]); NDArray::WaitAll(); - auto *exec = net.SimpleBind(ctx, args); // Only forward pass is enough as no gradient is needed when evaluating exec->Forward(false); acc.Update(data_batch.label, exec->outputs[0]); - delete exec; } float duration = chrono::duration_cast(toc - tic).count() / 1000.0; LG << "Epoch: " << iter << " " << samples/duration << " samples/sec Accuracy: " << acc.Get(); score = acc.Get(); } + delete exec; MXNotifyShutdown(); return score >= MIN_SCORE ? 0 : 1; } diff --git a/cpp-package/include/mxnet-cpp/executor.h b/cpp-package/include/mxnet-cpp/executor.h index 822344b7efee..67eec0100b65 100644 --- a/cpp-package/include/mxnet-cpp/executor.h +++ b/cpp-package/include/mxnet-cpp/executor.h @@ -79,18 +79,6 @@ class Executor { */ std::string DebugStr(); /*! - * \brief update the arguments with given learning rate and optimizer - * \param opt the pointer to the optimizer - * \param lr learning rate - * \param wd weight decay - * \param arg_update_begin begin index of the arguments to be updated, it - * starts after the input data by default - * \param arg_update_end end index of the arguments to be updated, it ends - * before the label data by default - */ - void UpdateAll(Optimizer *opt, float lr, float wd, int arg_update_begin = 1, - int arg_update_end = -1); - /*! * \brief destructor, free the handle */ ~Executor() { MXExecutorFree(handle_); } diff --git a/cpp-package/include/mxnet-cpp/executor.hpp b/cpp-package/include/mxnet-cpp/executor.hpp index 1a452a1610db..6887956290c2 100644 --- a/cpp-package/include/mxnet-cpp/executor.hpp +++ b/cpp-package/include/mxnet-cpp/executor.hpp @@ -79,13 +79,6 @@ inline std::string Executor::DebugStr() { return std::string(output); } -inline void Executor::UpdateAll(Optimizer *opt, float lr, float wd, - int arg_update_begin, int arg_update_end) { - arg_update_end = arg_update_end < 0 ? arg_arrays.size() - 1 : arg_update_end; - for (int i = arg_update_begin; i < arg_update_end; ++i) { - opt->Update(i, arg_arrays[i], grad_arrays[i], lr, wd); - } -} } // namespace cpp } // namespace mxnet diff --git a/cpp-package/include/mxnet-cpp/lr_scheduler.h b/cpp-package/include/mxnet-cpp/lr_scheduler.h new file mode 100644 index 000000000000..91f9b3c0a952 --- /dev/null +++ b/cpp-package/include/mxnet-cpp/lr_scheduler.h @@ -0,0 +1,78 @@ +/*! +* Copyright (c) 2017 by Contributors +* \file lr_scheduler.h +* \brief Scheduling learning rate +*/ + +#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_LR_SCHEDULER_H_ +#define CPP_PACKAGE_INCLUDE_MXNET_CPP_LR_SCHEDULER_H_ + +#include "dmlc/logging.h" + +namespace mxnet { +namespace cpp { + +/*! +* \brief lr scheduler interface +*/ +class LRScheduler { + public: + /*! + * \brief constructor + * \param base_lr the initial learning rate. + */ + explicit LRScheduler(float base_lr = 0.01) + : base_lr_(base_lr) {} + /*! + * \brief set base lr + * \param lr learning rate from optimizer + */ + void SetLR(const float lr) { base_lr_ = lr; } + /*! + * \brief get a new learning rate + */ + virtual float GetLR(unsigned num_update) = 0; + /*! + * \brief destructor + */ + virtual ~LRScheduler() {} + + protected: + float base_lr_; +}; + +class FactorScheduler : public LRScheduler { + public: + explicit FactorScheduler(int step, float factor = 1, float stop_factor_lr = 1e-8) + : LRScheduler() { + step_ = step; + factor_ = factor; + stop_factor_lr_ = stop_factor_lr; + } + + float GetLR(unsigned num_update) override { + while (num_update > unsigned(count_ + step_)) { + count_ += step_; + base_lr_ *= factor_; + if (base_lr_ < stop_factor_lr_) { + base_lr_ = stop_factor_lr_; + LG << "Update[" << num_update << "]: now learning rate arrived at " \ + << base_lr_ << ", will not change in the future"; + } else { + LG << "Update[" << num_update << "]: Change learning rate to " << base_lr_; + } + } + return base_lr_; + } + + private: + int count_ = 0; + int step_; + float factor_; + float stop_factor_lr_; +}; + +} // namespace cpp +} // namespace mxnet + +#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_LR_SCHEDULER_H_ diff --git a/cpp-package/include/mxnet-cpp/optimizer.h b/cpp-package/include/mxnet-cpp/optimizer.h index 76f8a3564fbe..1bc36d58fd1a 100644 --- a/cpp-package/include/mxnet-cpp/optimizer.h +++ b/cpp-package/include/mxnet-cpp/optimizer.h @@ -17,6 +17,7 @@ #include "dmlc/logging.h" #include "mxnet-cpp/ndarray.h" #include "mxnet-cpp/op_map.h" +#include "mxnet-cpp/lr_scheduler.h" namespace mxnet { namespace cpp { @@ -57,15 +58,16 @@ class Optimizer { return this; } /*! - * \brief Update a weight with gradient. - * \param index the unique index for the weight. - * \param weight the weight to update. - * \param grad gradient for the weight. - * \param lr learning rate. - * \param wd weight decay. + * \bried set the lr scheduler + * \param lrScheduler lr scheduler used for this optimizer + * \return reference if self */ - void Update(int index, NDArray weight, NDArray grad, mx_float lr, - mx_float wd); + Optimizer *SetLRScheduler(std::unique_ptr lrScheduler) { + CHECK(lrScheduler); + lrScheduler_ = std::move(lrScheduler); + lrScheduler_->SetLR(std::stof(params_["lr"])); + return this; + } /*! * \brief Update a weight with gradient. * \param index the unique index for the weight. @@ -92,7 +94,10 @@ class Optimizer { std::map count_; unsigned begin_num_update_, num_update_; unsigned UpdateCount_(int index); + float GetLR_(int index); + float GetWD_(int index); virtual void CreateState_(int index, NDArray weight); + std::unique_ptr lrScheduler_ = nullptr; }; typedef std::function OptimizerCreator; @@ -172,7 +177,6 @@ class AdaDeltaOptimizer : public Optimizer { std::map acc_g_, acc_delta_; }; - } // namespace cpp } // namespace mxnet diff --git a/cpp-package/include/mxnet-cpp/optimizer.hpp b/cpp-package/include/mxnet-cpp/optimizer.hpp index 9dcb158b9e14..0d6a7be9dd6b 100644 --- a/cpp-package/include/mxnet-cpp/optimizer.hpp +++ b/cpp-package/include/mxnet-cpp/optimizer.hpp @@ -42,6 +42,8 @@ namespace cpp { inline Optimizer::Optimizer(unsigned begin_num_update) : begin_num_update_(begin_num_update), num_update_(begin_num_update_) { + params_["lr"] = "0.01f"; + params_["wd"] = "0.f"; } inline std::map& OptimizerRegistry::cmap() { @@ -56,14 +58,6 @@ inline OpMap*& Optimizer::op_map() { inline Optimizer::~Optimizer() {} -inline void Optimizer::Update(int index, NDArray weight, NDArray grad, mx_float lr, - mx_float wd) { - params_["lr"] = std::to_string(lr); - params_["wd"] = std::to_string(wd); - UpdateCount_(index); - Update(index, weight, grad); -} - inline void Optimizer::CreateState_(int index, NDArray weight) { } @@ -100,6 +94,18 @@ inline unsigned Optimizer::UpdateCount_(int index) { return new_count; } +inline float Optimizer::GetLR_(int index) { + if (nullptr != lrScheduler_) { + return lrScheduler_->GetLR(num_update_); + } + return std::stof(params_["lr"]); +} + +inline float Optimizer::GetWD_(int index) { + float wd = std::stof(params_["wd"]); + return wd; +} + inline Optimizer* OptimizerRegistry::Find(const std::string& name) { MXNETCPP_REGISTER_OPTIMIZER(sgd, SGDOptimizer); MXNETCPP_REGISTER_OPTIMIZER(ccsgd, SGDOptimizer); // For backward compatibility @@ -140,6 +146,9 @@ inline void SGDOptimizer::Update(int index, NDArray weight, NDArray grad) { CreateState_(index, weight); } + params_["lr"] = std::to_string(GetLR_(index)); + params_["wd"] = std::to_string(GetWD_(index)); + UpdateCount_(index); auto keys = GetParamKeys_(); auto values = GetParamValues_(); CHECK_EQ(keys.size(), values.size()); @@ -203,6 +212,9 @@ inline void RMSPropOptimizer::Update(int index, NDArray weight, NDArray grad) { CreateState_(index, weight); } + params_["lr"] = std::to_string(GetLR_(index)); + params_["wd"] = std::to_string(GetWD_(index)); + UpdateCount_(index); auto keys = GetParamKeys_(); auto values = GetParamValues_(); CHECK_EQ(keys.size(), values.size()); @@ -257,6 +269,10 @@ inline void AdamOptimizer::Update(int index, NDArray weight, NDArray grad) { if (mean_.count(index) == 0) { CreateState_(index, weight); } + + params_["lr"] = std::to_string(GetLR_(index)); + params_["wd"] = std::to_string(GetWD_(index)); + UpdateCount_(index); auto keys = GetParamKeys_(); auto values = GetParamValues_(); CHECK_EQ(keys.size(), values.size()); @@ -306,9 +322,11 @@ inline void AdaGradOptimizer::Update(int index, NDArray weight, NDArray grad) { if (history_.count(index) == 0) { CreateState_(index, weight); } - float lr = std::stof(params_["lr"]); - float wd = std::stof(params_["wd"]); + float eps = std::stof(params_["eps"]); + float lr = GetLR_(index); + float wd = GetWD_(index); + UpdateCount_(index); if (params_.count("rescale_grad") > 0) { grad *= std::stof(params_["rescale_grad"]); } @@ -345,9 +363,11 @@ inline void AdaDeltaOptimizer::Update(int index, NDArray weight, NDArray grad) { if (acc_g_.count(index) == 0) { CreateState_(index, weight); } - float wd = std::stof(params_["wd"]); + float rho = std::stof(params_["rho"]); float epsilon = std::stof(params_["epsilon"]); + float wd = GetWD_(index); + UpdateCount_(index); if (params_.count("rescale_grad") > 0) { grad *= std::stof(params_["rescale_grad"]); From c9440ba263dac9731975d956c5dabdf70a384809 Mon Sep 17 00:00:00 2001 From: Mengxiao Lin Date: Fri, 4 Aug 2017 04:32:41 +0800 Subject: [PATCH 318/834] update RCNN example for BaseModule::init_params (#6813) * update RCNN example for BaseModule::init_params * Update module.py From d65d363be82fcf8f29994b6506a7047af29b488d Mon Sep 17 00:00:00 2001 From: joey2014 Date: Thu, 3 Aug 2017 15:33:40 -0500 Subject: [PATCH 319/834] [caffe] support convert mtcnn and MobileNet model (#6956) * support convert mtcnn and MobileNet model * pass python lint * put "import re" before "import caffe_parser" as lint required * correct missed checkin and pass pylint --- tools/caffe_converter/convert_model.py | 7 ++++--- tools/caffe_converter/convert_symbol.py | 24 ++++++++++++++++++++++-- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/tools/caffe_converter/convert_model.py b/tools/caffe_converter/convert_model.py index 2d8c9941ddea..d1e4cd07c155 100644 --- a/tools/caffe_converter/convert_model.py +++ b/tools/caffe_converter/convert_model.py @@ -3,6 +3,7 @@ from __future__ import print_function import argparse import sys +import re import caffe_parser import mxnet as mx import numpy as np @@ -53,8 +54,8 @@ def convert_model(prototxt_fname, caffemodel_fname, output_prefix=None): or layer_type == 'Deconvolution' or layer_type == 39: if layer_type == 'PReLU': assert (len(layer_blobs) == 1) - wmat = layer_blobs[0].data weight_name = layer_name + '_gamma' + wmat = np.array(layer_blobs[0].data).reshape(arg_shape_dic[weight_name]) arg_params[weight_name] = mx.nd.zeros(wmat.shape) arg_params[weight_name][:] = wmat continue @@ -148,7 +149,7 @@ def convert_model(prototxt_fname, caffemodel_fname, output_prefix=None): aux_params[var_name] = mx.nd.zeros(var.shape) # Get the original epsilon for idx, layer in enumerate(layers_proto): - if layer.name == bn_name: + if layer.name == bn_name or re.sub('[-/]', '_', layer.name) == bn_name: bn_index = idx eps_caffe = layers_proto[bn_index].batch_norm_param.eps # Compensate for the epsilon shift performed in convert_symbol @@ -180,7 +181,7 @@ def convert_model(prototxt_fname, caffemodel_fname, output_prefix=None): assert len(layer_blobs) == 0 if output_prefix is not None: - model = mx.mod.Module(symbol=sym, label_names=['prob_label', ]) + model = mx.mod.Module(symbol=sym, label_names=[arg_names[-1], ]) model.bind(data_shapes=[('data', tuple(input_dim))]) model.init_params(arg_params=arg_params, aux_params=aux_params) model.save_checkpoint(output_prefix, 0) diff --git a/tools/caffe_converter/convert_symbol.py b/tools/caffe_converter/convert_symbol.py index c384c7690088..100a64fe63c6 100644 --- a/tools/caffe_converter/convert_symbol.py +++ b/tools/caffe_converter/convert_symbol.py @@ -120,6 +120,7 @@ def _parse_proto(prototxt_fname): flatten_count = 0 output_name = "" prev_name = None + _output_name = {} # convert reset layers one by one for i, layer in enumerate(layers): @@ -252,6 +253,22 @@ def _parse_proto(prototxt_fname): for j in range(len(layer.top)): mapping[layer.top[j]] = name output_name = name + for k in range(len(layer.bottom)): + if layer.bottom[k] in _output_name: + _output_name[layer.bottom[k]]['count'] = _output_name[layer.bottom[k]]['count']+1 + else: + _output_name[layer.bottom[k]] = {'count':0} + for k in range(len(layer.top)): + if layer.top[k] in _output_name: + _output_name[layer.top[k]]['count'] = _output_name[layer.top[k]]['count']+1 + else: + _output_name[layer.top[k]] = {'count':0, 'name':name} + + output_name = [] + for i in _output_name: + if 'name' in _output_name[i] and _output_name[i]['count'] == 0: + output_name.append(_output_name[i]['name']) + return symbol_string, output_name, input_dim def convert_symbol(prototxt_fname): @@ -272,8 +289,11 @@ def convert_symbol(prototxt_fname): sym, output_name, input_dim = _parse_proto(prototxt_fname) exec(sym) # pylint: disable=exec-used _locals = locals() - exec("ret = " + output_name, globals(), _locals) # pylint: disable=exec-used - ret = _locals['ret'] + ret = [] + for i in output_name: + exec("ret = " + i, globals(), _locals) # pylint: disable=exec-used + ret.append(_locals['ret']) + ret = mx.sym.Group(ret) return ret, input_dim def main(): From be7e7916f444f67eda5d8acdc8b92d0bf339cb53 Mon Sep 17 00:00:00 2001 From: gurumurthys Date: Thu, 3 Aug 2017 13:34:45 -0700 Subject: [PATCH 320/834] Fixed visualization code error for bi-directional lstms (#6674) --- python/mxnet/visualization.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/mxnet/visualization.py b/python/mxnet/visualization.py index 97b6bfa25b1b..e67fee427be2 100644 --- a/python/mxnet/visualization.py +++ b/python/mxnet/visualization.py @@ -317,7 +317,6 @@ def looks_like_weight(name): params = input_node["attr"] if "num_outputs" in params: key += str(int(params["num_outputs"]) - 1) - params["num_outputs"] = int(params["num_outputs"]) - 1 shape = shape_dict[key][1:] label = "x".join([str(x) for x in shape]) attr["label"] = label From b848c241be41b6933923e9acc7caa40d6c2f76b4 Mon Sep 17 00:00:00 2001 From: Leonard Lausen Date: Fri, 4 Aug 2017 05:35:18 +0900 Subject: [PATCH 321/834] fix example/rnn: Speedometer(..., auto_reset=False) (#6679) If the Speedometer resets the eval_metric and due to an unlucky number of batches the end_of_batch is reached immediately after, the Perplexity will throw an ZeroDivisionError as eval_metric.num_inst == 0. --- example/rnn/cudnn_lstm_bucketing.py | 4 ++-- example/rnn/lstm_bucketing.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/example/rnn/cudnn_lstm_bucketing.py b/example/rnn/cudnn_lstm_bucketing.py index 140f2e697015..fbf32bbacb42 100644 --- a/example/rnn/cudnn_lstm_bucketing.py +++ b/example/rnn/cudnn_lstm_bucketing.py @@ -135,13 +135,13 @@ def sym_gen(seq_len): eval_metric = mx.metric.Perplexity(invalid_label), kvstore = args.kv_store, optimizer = args.optimizer, - optimizer_params = opt_params, + optimizer_params = opt_params, initializer = mx.init.Xavier(factor_type="in", magnitude=2.34), arg_params = arg_params, aux_params = aux_params, begin_epoch = args.load_epoch, num_epoch = args.num_epochs, - batch_end_callback = mx.callback.Speedometer(args.batch_size, args.disp_batches), + batch_end_callback = mx.callback.Speedometer(args.batch_size, args.disp_batches, auto_reset=False), epoch_end_callback = mx.rnn.do_rnn_checkpoint(cell, args.model_prefix, 1) if args.model_prefix else None) diff --git a/example/rnn/lstm_bucketing.py b/example/rnn/lstm_bucketing.py index 6c4371b2fd4e..609276a11f19 100644 --- a/example/rnn/lstm_bucketing.py +++ b/example/rnn/lstm_bucketing.py @@ -107,4 +107,4 @@ def sym_gen(seq_len): 'wd': args.wd }, initializer = mx.init.Xavier(factor_type="in", magnitude=2.34), num_epoch = args.num_epochs, - batch_end_callback = mx.callback.Speedometer(args.batch_size, args.disp_batches)) + batch_end_callback = mx.callback.Speedometer(args.batch_size, args.disp_batches, auto_reset=False)) From edbb4ff26321773789ce3237211587697dbaced0 Mon Sep 17 00:00:00 2001 From: lxn2 Date: Thu, 3 Aug 2017 13:44:54 -0700 Subject: [PATCH 322/834] Test installation of pip --pre (#7314) * Enable test for pip-mkl * Add --pre flag to test pre-releases * Fix conflicts --- tests/jenkins/run_test_pip_installations.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/jenkins/run_test_pip_installations.sh b/tests/jenkins/run_test_pip_installations.sh index 9246708f4329..de235a0a6359 100755 --- a/tests/jenkins/run_test_pip_installations.sh +++ b/tests/jenkins/run_test_pip_installations.sh @@ -42,13 +42,12 @@ for DEV in "${DEVICES[@]}"; do echo "Testing ${PYTHON}" DOCKER_CMD="virtualenv -p \"/usr/bin/${PYTHON}\" ${PYTHON}; source \"${PYTHON}/bin/activate\"; cd ${WORKSPACE};" if [[ "${DEV}" == *"cpu"* ]]; then - DOCKER_CMD="${DOCKER_CMD} pip install mxnet; python tests/python/train/test_conv.py" + DOCKER_CMD="${DOCKER_CMD} pip install mxnet --pre; python tests/python/train/test_conv.py" elif [[ "${DEV}" == *"cu75"* ]]; then - DOCKER_CMD="${DOCKER_CMD} pip install mxnet-cu75; python tests/python/train/test_conv.py --gpu" + DOCKER_CMD="${DOCKER_CMD} pip install mxnet-cu75 --pre; python tests/python/train/test_conv.py --gpu" elif [[ "${DEV}" == *"cu80"* ]]; then - DOCKER_CMD="${DOCKER_CMD} pip install mxnet-cu80; python tests/python/train/test_conv.py --gpu" + DOCKER_CMD="${DOCKER_CMD} pip install mxnet-cu80 --pre; python tests/python/train/test_conv.py --gpu" fi - ${DOCKER_BINARY} run --rm -v ${WORKSPACE}:${WORKSPACE} -w ${WORKSPACE} ${DOCKER_TAG} bash -c "tests/jenkins/run_as_user.sh `id -u` `id -un` `id -g` `id -un` '${DOCKER_CMD}'" done From dd4512f82051711240adc301033e52bec7998abf Mon Sep 17 00:00:00 2001 From: "Qiang Kou (KK)" Date: Thu, 3 Aug 2017 22:22:09 +0000 Subject: [PATCH 323/834] [R] update docs from mx.symbol.MakeLoss. close #2922 (#7325) --- R-package/vignettes/CustomLossFunction.Rmd | 159 +++++++++++++++ docs/tutorials/r/CustomLossFunction.md | 220 +++++++++++++++++---- 2 files changed, 341 insertions(+), 38 deletions(-) create mode 100644 R-package/vignettes/CustomLossFunction.Rmd diff --git a/R-package/vignettes/CustomLossFunction.Rmd b/R-package/vignettes/CustomLossFunction.Rmd new file mode 100644 index 000000000000..1817109e1387 --- /dev/null +++ b/R-package/vignettes/CustomLossFunction.Rmd @@ -0,0 +1,159 @@ +--- +title: "Customized loss function" +output: + md_document: + variant: markdown_github +--- + +```{r setup, include=FALSE} +knitr::opts_chunk$set(echo = TRUE) +``` + +This tutorial provides guidelines for using customized loss function in network construction. + +Model Training Example +---------- + +Let's begin with a small regression example. We can build and train a regression model with the following code: + +```{r} +data(BostonHousing, package = "mlbench") +BostonHousing[, sapply(BostonHousing, is.factor)] <- + as.numeric(as.character(BostonHousing[, sapply(BostonHousing, is.factor)])) +BostonHousing <- data.frame(scale(BostonHousing)) + +test.ind = seq(1, 506, 5) # 1 pt in 5 used for testing +train.x = data.matrix(BostonHousing[-test.ind,-14]) +train.y = BostonHousing[-test.ind, 14] +test.x = data.matrix(BostonHousing[--test.ind,-14]) +test.y = BostonHousing[--test.ind, 14] + +require(mxnet) + +data <- mx.symbol.Variable("data") +label <- mx.symbol.Variable("label") +fc1 <- mx.symbol.FullyConnected(data, num_hidden = 14, name = "fc1") +tanh1 <- mx.symbol.Activation(fc1, act_type = "tanh", name = "tanh1") +fc2 <- mx.symbol.FullyConnected(tanh1, num_hidden = 1, name = "fc2") +lro <- mx.symbol.LinearRegressionOutput(fc2, name = "lro") + +mx.set.seed(0) +model <- mx.model.FeedForward.create(lro, X = train.x, y = train.y, + ctx = mx.cpu(), + num.round = 5, + array.batch.size = 60, + optimizer = "rmsprop", + verbose = TRUE, + array.layout = "rowmajor", + batch.end.callback = NULL, + epoch.end.callback = NULL) + +pred <- predict(model, test.x) +sum((test.y - pred[1,])^2) / length(test.y) +``` + +Besides the `LinearRegressionOutput`, we also provide `LogisticRegressionOutput` and `MAERegressionOutput`. +However, this might not be enough for real-world models. You can provide your own loss function +by using `mx.symbol.MakeLoss` when constructing the network. + +How to Use Your Own Loss Function +--------- + +We still use our previous example, but this time we use `mx.symbol.MakeLoss` to minimize the `(pred-label)^2` + +```{r} +data <- mx.symbol.Variable("data") +label <- mx.symbol.Variable("label") +fc1 <- mx.symbol.FullyConnected(data, num_hidden = 14, name = "fc1") +tanh1 <- mx.symbol.Activation(fc1, act_type = "tanh", name = "tanh1") +fc2 <- mx.symbol.FullyConnected(tanh1, num_hidden = 1, name = "fc2") +lro2 <- mx.symbol.MakeLoss(mx.symbol.square(mx.symbol.Reshape(fc2, shape = 0) - label), name="lro2") +``` + +Then we can train the network just as usual. + +```{r} +mx.set.seed(0) +model2 <- mx.model.FeedForward.create(lro2, X = train.x, y = train.y, + ctx = mx.cpu(), + num.round = 5, + array.batch.size = 60, + optimizer = "rmsprop", + verbose = TRUE, + array.layout = "rowmajor", + batch.end.callback = NULL, + epoch.end.callback = NULL) +``` + +We should get very similar results because we are actually minimizing the same loss function. +However, the result is quite different. + +```{r} +pred2 <- predict(model2, test.x) +sum((test.y - pred2)^2) / length(test.y) +``` + +This is because output of `mx.symbol.MakeLoss` is the gradient of loss with respect to the input data. +We can get the real prediction as below. + +```{r} +internals = internals(model2$symbol) +fc_symbol = internals[[match("fc2_output", outputs(internals))]] + +model3 <- list(symbol = fc_symbol, + arg.params = model2$arg.params, + aux.params = model2$aux.params) + +class(model3) <- "MXFeedForwardModel" + +pred3 <- predict(model3, test.x) +sum((test.y - pred3[1,])^2) / length(test.y) +``` + +We have provided many operations on the symbols. An example of `|pred-label|` can be found below. + +```{r} +lro_abs <- mx.symbol.MakeLoss(mx.symbol.abs(mx.symbol.Reshape(fc2, shape = 0) - label)) +mx.set.seed(0) +model4 <- mx.model.FeedForward.create(lro_abs, X = train.x, y = train.y, + ctx = mx.cpu(), + num.round = 20, + array.batch.size = 60, + optimizer = "sgd", + learning.rate = 0.001, + verbose = TRUE, + array.layout = "rowmajor", + batch.end.callback = NULL, + epoch.end.callback = NULL) + +internals = internals(model4$symbol) +fc_symbol = internals[[match("fc2_output", outputs(internals))]] + +model5 <- list(symbol = fc_symbol, + arg.params = model4$arg.params, + aux.params = model4$aux.params) + +class(model5) <- "MXFeedForwardModel" + +pred5 <- predict(model5, test.x) +sum(abs(test.y - pred5[1,])) / length(test.y) +``` + + +```{r} +lro_mae <- mx.symbol.MAERegressionOutput(fc2, name = "lro") +mx.set.seed(0) +model6 <- mx.model.FeedForward.create(lro_mae, X = train.x, y = train.y, + ctx = mx.cpu(), + num.round = 20, + array.batch.size = 60, + optimizer = "sgd", + learning.rate = 0.001, + verbose = TRUE, + array.layout = "rowmajor", + batch.end.callback = NULL, + epoch.end.callback = NULL) +pred6 <- predict(model6, test.x) +sum(abs(test.y - pred6[1,])) / length(test.y) +``` + diff --git a/docs/tutorials/r/CustomLossFunction.md b/docs/tutorials/r/CustomLossFunction.md index a7104803cacb..afb99518894c 100644 --- a/docs/tutorials/r/CustomLossFunction.md +++ b/docs/tutorials/r/CustomLossFunction.md @@ -3,57 +3,201 @@ Customized loss function This tutorial provides guidelines for using customized loss function in network construction. - Model Training Example ----------- +---------------------- Let's begin with a small regression example. We can build and train a regression model with the following code: +``` r +data(BostonHousing, package = "mlbench") +BostonHousing[, sapply(BostonHousing, is.factor)] <- + as.numeric(as.character(BostonHousing[, sapply(BostonHousing, is.factor)])) +BostonHousing <- data.frame(scale(BostonHousing)) + +test.ind = seq(1, 506, 5) # 1 pt in 5 used for testing +train.x = data.matrix(BostonHousing[-test.ind,-14]) +train.y = BostonHousing[-test.ind, 14] +test.x = data.matrix(BostonHousing[--test.ind,-14]) +test.y = BostonHousing[--test.ind, 14] + +require(mxnet) +``` + + ## Loading required package: mxnet + +``` r +data <- mx.symbol.Variable("data") +label <- mx.symbol.Variable("label") +fc1 <- mx.symbol.FullyConnected(data, num_hidden = 14, name = "fc1") +tanh1 <- mx.symbol.Activation(fc1, act_type = "tanh", name = "tanh1") +fc2 <- mx.symbol.FullyConnected(tanh1, num_hidden = 1, name = "fc2") +lro <- mx.symbol.LinearRegressionOutput(fc2, name = "lro") + +mx.set.seed(0) +model <- mx.model.FeedForward.create(lro, X = train.x, y = train.y, + ctx = mx.cpu(), + num.round = 5, + array.batch.size = 60, + optimizer = "rmsprop", + verbose = TRUE, + array.layout = "rowmajor", + batch.end.callback = NULL, + epoch.end.callback = NULL) +``` + + ## Start training with 1 devices + +``` r +pred <- predict(model, test.x) +``` + + ## Warning in mx.model.select.layout.predict(X, model): Auto detect layout of input matrix, use rowmajor.. + +``` r +sum((test.y - pred[1,])^2) / length(test.y) +``` - ```r - library(mxnet) - data(BostonHousing, package="mlbench") - train.ind = seq(1, 506, 3) - train.x = data.matrix(BostonHousing[train.ind, -14]) - train.y = BostonHousing[train.ind, 14] - test.x = data.matrix(BostonHousing[-train.ind, -14]) - test.y = BostonHousing[-train.ind, 14] - data <- mx.symbol.Variable("data") - fc1 <- mx.symbol.FullyConnected(data, num_hidden=1) - lro <- mx.symbol.LinearRegressionOutput(fc1) - mx.set.seed(0) - model <- mx.model.FeedForward.create( - lro, X=train.x, y=train.y, - eval.data=list(data=test.x, label=test.y), - ctx=mx.cpu(), num.round=10, array.batch.size=20, - learning.rate=2e-6, momentum=0.9, eval.metric=mx.metric.rmse) - ``` - -Besides the `LinearRegressionOutput`, we also provide `LogisticRegressionOutput` and `MAERegressionOutput`. -However, this might not be enough for real-world models. You can provide your own loss function -by using `mx.symbol.MakeLoss` when constructing the network. + ## [1] 0.2485236 +Besides the `LinearRegressionOutput`, we also provide `LogisticRegressionOutput` and `MAERegressionOutput`. However, this might not be enough for real-world models. You can provide your own loss function by using `mx.symbol.MakeLoss` when constructing the network. How to Use Your Own Loss Function ---------- +--------------------------------- + +We still use our previous example, but this time we use `mx.symbol.MakeLoss` to minimize the `(pred-label)^2` + +``` r +data <- mx.symbol.Variable("data") +label <- mx.symbol.Variable("label") +fc1 <- mx.symbol.FullyConnected(data, num_hidden = 14, name = "fc1") +tanh1 <- mx.symbol.Activation(fc1, act_type = "tanh", name = "tanh1") +fc2 <- mx.symbol.FullyConnected(tanh1, num_hidden = 1, name = "fc2") +lro2 <- mx.symbol.MakeLoss(mx.symbol.square(mx.symbol.Reshape(fc2, shape = 0) - label), name="lro2") +``` + +Then we can train the network just as usual. + +``` r +mx.set.seed(0) +model2 <- mx.model.FeedForward.create(lro2, X = train.x, y = train.y, + ctx = mx.cpu(), + num.round = 5, + array.batch.size = 60, + optimizer = "rmsprop", + verbose = TRUE, + array.layout = "rowmajor", + batch.end.callback = NULL, + epoch.end.callback = NULL) +``` + + ## Start training with 1 devices + +We should get very similar results because we are actually minimizing the same loss function. However, the result is quite different. + +``` r +pred2 <- predict(model2, test.x) +``` + + ## Warning in mx.model.select.layout.predict(X, model): Auto detect layout of input matrix, use rowmajor.. + +``` r +sum((test.y - pred2)^2) / length(test.y) +``` + + ## [1] 1.234584 + +This is because output of `mx.symbol.MakeLoss` is the gradient of loss with respect to the input data. We can get the real prediction as below. + +``` r +internals = internals(model2$symbol) +fc_symbol = internals[[match("fc2_output", outputs(internals))]] + +model3 <- list(symbol = fc_symbol, + arg.params = model2$arg.params, + aux.params = model2$aux.params) + +class(model3) <- "MXFeedForwardModel" + +pred3 <- predict(model3, test.x) +``` + + ## Warning in mx.model.select.layout.predict(X, model): Auto detect layout of input matrix, use rowmajor.. + +``` r +sum((test.y - pred3[1,])^2) / length(test.y) +``` + + ## [1] 0.248294 + +We have provided many operations on the symbols. An example of `|pred-label|` can be found below. + +``` r +lro_abs <- mx.symbol.MakeLoss(mx.symbol.abs(mx.symbol.Reshape(fc2, shape = 0) - label)) +mx.set.seed(0) +model4 <- mx.model.FeedForward.create(lro_abs, X = train.x, y = train.y, + ctx = mx.cpu(), + num.round = 20, + array.batch.size = 60, + optimizer = "sgd", + learning.rate = 0.001, + verbose = TRUE, + array.layout = "rowmajor", + batch.end.callback = NULL, + epoch.end.callback = NULL) +``` + + ## Start training with 1 devices + +``` r +internals = internals(model4$symbol) +fc_symbol = internals[[match("fc2_output", outputs(internals))]] + +model5 <- list(symbol = fc_symbol, + arg.params = model4$arg.params, + aux.params = model4$aux.params) + +class(model5) <- "MXFeedForwardModel" + +pred5 <- predict(model5, test.x) +``` + + ## Warning in mx.model.select.layout.predict(X, model): Auto detect layout of input matrix, use rowmajor.. + +``` r +sum(abs(test.y - pred5[1,])) / length(test.y) +``` + + ## [1] 0.7056902 + +``` r +lro_mae <- mx.symbol.MAERegressionOutput(fc2, name = "lro") +mx.set.seed(0) +model6 <- mx.model.FeedForward.create(lro_mae, X = train.x, y = train.y, + ctx = mx.cpu(), + num.round = 20, + array.batch.size = 60, + optimizer = "sgd", + learning.rate = 0.001, + verbose = TRUE, + array.layout = "rowmajor", + batch.end.callback = NULL, + epoch.end.callback = NULL) +``` + + ## Start training with 1 devices -We still use our previous example. +``` r +pred6 <- predict(model6, test.x) +``` - ```r - library(mxnet) - data <- mx.symbol.Variable("data") - fc1 <- mx.symbol.FullyConnected(data, num_hidden=1) - lro <- mx.symbol.MakeLoss(mx.symbol.square(mx.symbol.Reshape(fc1, shape = 0) - label)) - ``` + ## Warning in mx.model.select.layout.predict(X, model): Auto detect layout of input matrix, use rowmajor.. -In the last line of network definition, we do not use the predefined loss function. We define the loss -by ourselves, which is `(pred-label)^2`. +``` r +sum(abs(test.y - pred6[1,])) / length(test.y) +``` -We have provided many operations on the symbols, so you can also define `|pred-label|` using the line below. + ## [1] 0.7056902 - ```r - lro <- mx.symbol.MakeLoss(mx.symbol.abs(mx.symbol.Reshape(fc1, shape = 0) - label)) - ``` ## Next Steps * [Neural Networks with MXNet in Five Minutes](http://mxnet.io/tutorials/r/fiveMinutesNeuralNetwork.html) From feebe774053c3b8361bf6d48dba998186261fe56 Mon Sep 17 00:00:00 2001 From: smarthi Date: Fri, 28 Jul 2017 15:45:56 -0400 Subject: [PATCH 324/834] Add KEYS file --- KEYS | 73 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 KEYS diff --git a/KEYS b/KEYS new file mode 100644 index 000000000000..28c497fceb68 --- /dev/null +++ b/KEYS @@ -0,0 +1,73 @@ +This file contains the PGP keys of various developers. +Please don't use them for email unless you have to. Their main +purpose is code signing. + +Examples of importing this file in your keystore: + gpg --import KEYS.txt + (need pgp and other examples here) + +Examples of adding your key to this file: + pgp -kxa and append it to this file. + (pgpk -ll && pgpk -xa ) >> this file. + (gpg --list-sigs + && gpg --armor --export ) >> this file. + +----------------------------------------------------------------------------------- +pub 4096R/D3541808 2014-01-09 +uid [ultimate] Suneel Marthi (CODE SIGNING KEY) +sig 3 D3541808 2014-01-09 Suneel Marthi (CODE SIGNING KEY) +sub 4096R/AF46E2DE 2014-01-09 +sig D3541808 2014-01-09 Suneel Marthi (CODE SIGNING KEY) + +-----BEGIN PGP PUBLIC KEY BLOCK----- +Comment: GPGTools - https://gpgtools.org + +mQINBFLPJmEBEAC9d/dUZCXeyhB0fVGmJAjdjXfLebav4VqGdNZC+M1T9C3dcVsh +X/JGme5bjJeIgVwiH5UsdNceYn1+hyxs8jXuRAWEWKP76gD+pNrp8Az0ZdBkJoAy +zCywOPtJV2PCOz7+S5ri2nUA2+1Kgcu6IlSLMmYAGO0IAmRrjBEzxy9iGaxiNGTc +LvQt/iVtIXWkKKI8yvpoJ8iFf3TGhpjgaC/h7cJP3zpy0SScmhJJASLXRsfocLv9 +sle6ndN9IPbDtRW8cL7Fk3VQlzp1ToVjmnQTyZZ6S1WafsjzCZ9hLN+k++o8VbvY +v3icY6Sy0BKz0J6KwaxTkuZ6w1K7oUkVOQboKaWFIEdO+jwrEmU+Puyd8Np8jLnF +Q0Y5GPfyMlqM3S/zaDm1t4D1eb5FLciStkxfg5wPVK6TkqB325KVD3aio5C7E7kt +aQechHxaJXCQOtCtVY4X+L4iClnMSuk+hcSc8W8MYRTSVansItK0vI9eQZXMnpan +w9/jk5rS4Gts1rHB7+kdjT3QRJmkyk6fEFT0fz5tfMC7N8waeEUhCaRW6lAoiqDW +NW1h+0UGxJw+9YcGxBC0kkt3iofNOWQWmuf/BS3DHPKT7XV/YtBHe44wW0sF5L5P +nfQUHpnA3pcZ0En6bXAvepKVZTNdOWWJqMyHV+436DA+33h45QL6lWb/GwARAQAB +tDVTdW5lZWwgTWFydGhpIChDT0RFIFNJR05JTkcgS0VZKSA8c21hcnRoaUBhcGFj +aGUub3JnPokCNwQTAQoAIQUCUs8mYQIbAwULCQgHAwUVCgkICwUWAgMBAAIeAQIX +gAAKCRC08czE01QYCOKKEAChRtHBoYNTX+RZbFO0Kl1GlN+i1Ik0shEm5ZJ56XHv +AnFx/gRK7CfZzJswWo7kf2s/dvJiFfs+rrolYVuO6E8gNhAaTEomSuvWQAMHdPcR +9G5APRKCSkbZYugElqplEbSphk78FKoFO+sml52M7Pr9jj88ApBjoFVVY8njdnNq +6DVlaDsg8YninCD78Z7PNFnRGwxyZ8Qd4Dh0rG+MUTfAWopZu6/MxpQxU7QpeVeX +SIMLg7ClFrGfXnZcszYF4dnav1aa0i7W88PAdYNPko7tC5qz5yv2ep7t2gRbcYKf +RXhYC2FHQey3wPhMKjA8V436lAqmfYnY/YdmhEy9Xq/1EdX1nHsQ7OEkfgXK14WM +F+rnqXRAl/0cwiyb41eocdg5kpZFIKgCYT02usLWxwNnd3jOCe109Ze3y3acN/G8 ++xOf9YRfNVAe6pD8H6ieRbv9gRjBmsbz9bXQCmxFnDqxNri5Me6gBAQPNmYTJD0h +jgJTK6o0vJ0pwjBLauasJsLu+1tR3Cb0dxPE+JVaTF26FCd7pM7W6KdVfod9ZfrN +cSyJ/cECc2KvYVGmTjQNVo1dYG0awBachlWnYNt+0Qx4opLsczZOLtPKtFY4BJA7 +aZoXT4Qf9yB8km7x2/cgNExVbFummToJ/IP3M39/EaryspsQQuM5Qu5Q5lZp8Qnn +ybkCDQRSzyZhARAA7bAawFzbJaghYnm6mTZyGG5hQmfAynbF6cPAE+g2SnXcNQjP +6kjYx3tSpb7rEzmjQqs46ztqdec6PIVBMhakON6z27Zz+IviAtO/TcaZHWNuCAjw +FXVQZ+tYsSeiKInttfkrQc8jXAHWwSkSjLqNpvQpBdBEX80MYkFB6ZPOeON2+/Ta +GC1H/HU2YngF0qQSmG33KKG6ezihBJdKxU6t2tsQfTlCmZW6R6MGpS9fVurYMKBk +vR+7RGZ/H6dSjWPcpxhusGg92J9uz7r5SopN1wSdyPMUCMAFGeyoxcAuBDl38quU +H/ENG3x5LDPq2aEH2AJ6yvZfIXbeJ1zmXf2cAHv+HbmvZaTSp0XIjq8Yxh8NkYEC +ZdfRWmsGLIpU16TkBijpK3Dn9MDXjHGT3V8/qfdpURtMvIaL8WFrq9ejcy/vGRFn +mCYqxIIPH+vLiMXKWtuMc61GN3ES21msKQH6IuQxxfQLyhK44L/pv7FpF4E+6LaE +8uRwAex5HIDpR1v4aJq089rRtye9VXTJJLZ7lYs0HctdZ30QbBRWT4jS9d9rj3cr +HgQ7mIGO9TAfK2kWc6AJN/EvxPWNbOwptsTUzAF/adiy9ax8C18iw7nKczC+2eN6 +UcbxXiPdytuKYK7O9A8S9e1w89GwpxYN7Xfn2o6QfpSbL9cLKiinOeV+xikAEQEA +AYkCHwQYAQoACQUCUs8mYQIbDAAKCRC08czE01QYCG7yD/471dmyOD+go8cZkdqR +3CHhjH03odtI0EJNVy4VGEC0r9paz3BWYTy18LqWYkw3ygphOIU1r8/7QK3H5Ke3 +c4yCSUxaMk5SlAJ+iVRek5TABkR8+zI+ZN5pQtqRH+ya5JxV4F/Sx5Q3KWMzpvgY +n6AgSSc3hEfkgdI7SalIeyLaLDWv+RFdGZ5JU5gD28C0G8BeH8L62x6sixZcqoGT +oy9rwkjs45/ZmmvBZhd1wLvC/au8l2Ecou6O8+8m26W8Z7vCuGKxuWn0KV3DLLWe +66uchDVlakGoMJSPIK06JWYUlE+gL0CW+U2ekt/v2qb8hGgMVET3CBAMq+bFWuJ6 +juX7hJd7wHtCFfjnFDDAkdp2IIIZAlBW6FZGv7pJ82xsW6pSAg0A7VrV6nTtMtDv +T8esOfo/t4t0gaL7bivy9DVVdATbUBcJJFpoVoe5MxiyjptveqPzIRwzt04n52Ph +ordVWAnX5AokXWTg+Glem/EWEuf7jUuZArfqCSl/sZoQdXGTjR7G4iFscispji4+ +kNjVQsItqFbgDpuc6n+GcFxlKQ7YMCnu5MVtTV01U4lFs0qy0NTUqsuR35DM4z14 +DkFmj1upWAayCoXTpKzsHBvJZPC+Wqf9Pl3O47apelg7KxU3S011YfXpVPvCTKBv +kD2o/5GKWS5QkSUEUXXY1oDiLg== +=f8kJ +-----END PGP PUBLIC KEY BLOCK----- From 51ec4846409a3ebe09f24c17e2c10b4a255f480c Mon Sep 17 00:00:00 2001 From: qingzhouzhen <576591769@qq.com> Date: Fri, 4 Aug 2017 11:39:53 +0800 Subject: [PATCH 325/834] Mobilenet (#7330) * add mobilenet * modify a mistake in conv_4 --- example/image-classification/symbols/mobilenet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/image-classification/symbols/mobilenet.py b/example/image-classification/symbols/mobilenet.py index cf470bace727..8ad584a50768 100644 --- a/example/image-classification/symbols/mobilenet.py +++ b/example/image-classification/symbols/mobilenet.py @@ -14,7 +14,7 @@ def get_symbol(num_classes, **kwargs): conv_3_dw = Conv(conv_2, num_group=64, num_filter=64, kernel=(3, 3), pad=(1, 1), stride=(2, 2), name="conv_3_dw") # 112/56 conv_3 = Conv(conv_3_dw, num_filter=128, kernel=(1, 1), pad=(0, 0), stride=(1, 1), name="conv_3") # 56/56 conv_4_dw = Conv(conv_3, num_group=128, num_filter=128, kernel=(3, 3), pad=(1, 1), stride=(1, 1), name="conv_4_dw") # 56/56 - conv_4 = Conv(conv_3_dw, num_filter=128, kernel=(1, 1), pad=(0, 0), stride=(1, 1), name="conv_4") # 56/56 + conv_4 = Conv(conv_4_dw, num_filter=128, kernel=(1, 1), pad=(0, 0), stride=(1, 1), name="conv_4") # 56/56 conv_5_dw = Conv(conv_4, num_group=128, num_filter=128, kernel=(3, 3), pad=(1, 1), stride=(2, 2), name="conv_5_dw") # 56/28 conv_5 = Conv(conv_5_dw, num_filter=256, kernel=(1, 1), pad=(0, 0), stride=(1, 1), name="conv_5") # 28/28 conv_6_dw = Conv(conv_5, num_group=256, num_filter=256, kernel=(3, 3), pad=(1, 1), stride=(1, 1), name="conv_6_dw") # 28/28 From 14b83fccef7b96f8d38d780dbce3d0ef47267934 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=A2=81=E5=BE=B7=E6=BE=8E?= Date: Fri, 4 Aug 2017 23:46:51 +0800 Subject: [PATCH 326/834] [Scala] Make Module Api sync with Python interface (#7246) * [Scala] Make Module Api sync with Python interface * fix --- .../ml/dmlc/mxnet/module/BaseModule.scala | 38 +- .../module/DataParallelExecutorGroup.scala | 53 ++- .../scala/ml/dmlc/mxnet/module/Module.scala | 69 +++- .../dmlc/mxnet/module/SequentialModule.scala | 10 +- .../scala/ml/dmlc/mxnet/ModuleSuite.scala | 368 ++++++++++++++++++ .../scala/ml/dmlc/mxnet/OperatorSuite.scala | 2 +- 6 files changed, 514 insertions(+), 26 deletions(-) create mode 100644 scala-package/core/src/test/scala/ml/dmlc/mxnet/ModuleSuite.scala diff --git a/scala-package/core/src/main/scala/ml/dmlc/mxnet/module/BaseModule.scala b/scala-package/core/src/main/scala/ml/dmlc/mxnet/module/BaseModule.scala index c1cb91de56f5..0a73e1afcde1 100644 --- a/scala-package/core/src/main/scala/ml/dmlc/mxnet/module/BaseModule.scala +++ b/scala-package/core/src/main/scala/ml/dmlc/mxnet/module/BaseModule.scala @@ -121,6 +121,7 @@ abstract class BaseModule { private[module] var auxParams: Map[String, NDArray] = null // High Level API + def getSymbol: Symbol = this.symbol // A convenient function that calls both `forward` and `backward`. def forwardBackward(dataBatch: DataBatch): Unit = { @@ -259,7 +260,7 @@ abstract class BaseModule { /** * Get parameters, those are potentially copies of the the actual parameters used * to do computation on the device. - * @return `(arg_params, aux_params)`, a pair of dictionary of name to value mapping. + * @return `(argParams, auxParams)`, a pair of dictionary of name to value mapping. */ def getParams: (Map[String, NDArray], Map[String, NDArray]) @@ -267,41 +268,52 @@ abstract class BaseModule { * Initialize the parameters and auxiliary states. * @param initializer : Initializer * Called to initialize parameters if needed. - * arg_params : dict + * argParams : dict * If not None, should be a dictionary of existing arg_params. Initialization * will be copied from that. - * aux_params : dict + * auxParams : dict * If not None, should be a dictionary of existing aux_params. Initialization * will be copied from that. - * allow_missing : bool + * allowMissing : bool * If true, params could contain missing values, and the initializer will be * called to fill those missing params. - * force_init : bool + * forceInit : bool * If true, will force re-initialize even if already initialized. + * allowExtra : bool + * Whether allow extra parameters that are not needed by symbol. + * If this is True, no error will be thrown when argParams or auxParams + * contain extra parameters that is not needed by the executor. */ def initParams(initializer: Initializer = new Uniform(0.01f), argParams: Map[String, NDArray] = null, auxParams: Map[String, NDArray] = null, - allowMissing: Boolean = false, forceInit: Boolean = false): Unit + allowMissing: Boolean = false, + forceInit: Boolean = false, + allowExtra: Boolean = false): Unit /** * Assign parameter and aux state values. - * arg_params : dict + * argParams : dict * Dictionary of name to value (`NDArray`) mapping. - * aux_params : dict + * auxParams : dict * Dictionary of name to value (`NDArray`) mapping. - * allow_missing : bool + * allowMissing : bool * If true, params could contain missing values, and the initializer will be * called to fill those missing params. - * force_init : bool + * forceInit : bool * If true, will force re-initialize even if already initialized. + * allowExtra : bool + * Whether allow extra parameters that are not needed by symbol. + * If this is True, no error will be thrown when argParams or auxParams + * contain extra parameters that is not needed by the executor. */ def setParams(argParams: Map[String, NDArray], auxParams: Map[String, NDArray], allowMissing: Boolean = false, - forceInit: Boolean = true): Unit = { - initParams(initializer = null, argParams = argParams, auxParams = auxParams, - allowMissing = allowMissing, forceInit = forceInit) + forceInit: Boolean = true, + allowExtra: Boolean = false): Unit = { + initParams(initializer = null, argParams, auxParams, + allowMissing, forceInit, allowExtra) } /** diff --git a/scala-package/core/src/main/scala/ml/dmlc/mxnet/module/DataParallelExecutorGroup.scala b/scala-package/core/src/main/scala/ml/dmlc/mxnet/module/DataParallelExecutorGroup.scala index 2e724c6dc9ce..ea78962d00e8 100644 --- a/scala-package/core/src/main/scala/ml/dmlc/mxnet/module/DataParallelExecutorGroup.scala +++ b/scala-package/core/src/main/scala/ml/dmlc/mxnet/module/DataParallelExecutorGroup.scala @@ -297,6 +297,7 @@ class DataParallelExecutorGroup private[module]( private var batchSize: Int = -1 private var slices: Array[(Int, Int)] = null + private var _defaultExecs: Array[Executor] = null private var execs: Array[Executor] = null private var dataArrays: Seq[Array[((Int, Int), NDArray)]] = null private var labelArrays: Option[Seq[Array[((Int, Int), NDArray)]]] = None @@ -305,8 +306,8 @@ class DataParallelExecutorGroup private[module]( private[module] var auxArrays: IndexedSeq[Array[NDArray]] = null private var inputGradArrays: IndexedSeq[Array[NDArray]] = null - private val dataLayouts = decideSlices(dataShapes) - private val labelLayouts = + private var dataLayouts = decideSlices(dataShapes) + private var labelLayouts = // call it to make sure labels has the same batch size as data if (labelShapes != None) decideSlices(labelShapes.get) else null @@ -349,12 +350,30 @@ class DataParallelExecutorGroup private[module]( * @param dataShapes DataDesc for input data. * @param labelShapes DataDesc for input labels. * @param sharedGroup + * @param reshape */ def bindExec(dataShapes: Seq[DataDesc], labelShapes: Option[Seq[DataDesc]], - sharedGroup: Option[DataParallelExecutorGroup]): Unit = { - execs = (0 until contexts.length).map(i => - bindIthExec(i, dataShapes, labelShapes, sharedGroup) - ).toArray + sharedGroup: Option[DataParallelExecutorGroup], reshape: Boolean = false): Unit = { + this.batchSize = -1 + dataLayouts = decideSlices(dataShapes) + labelLayouts = { + // call it to make sure labels has the same batch size as data + if (labelShapes != None) decideSlices(labelShapes.get) + else null + } + if (reshape) { + (0 until contexts.length).foreach { i => + val dataShapesSliced = slicedShape(dataShapes, i, dataLayouts) + val labelShapesSliced = labelShapes.map(slicedShape(_, i, labelLayouts)) + val inputShapes + = dataShapesSliced.toMap ++ labelShapesSliced.getOrElse(Map.empty[String, Shape]) + execs(i) = _defaultExecs(i).reshape(allowUpSizing = true, kwargs = inputShapes) + } + } else { + execs = (0 until contexts.length).map(i => + bindIthExec(i, dataShapes, labelShapes, sharedGroup) + ).toArray + } // convenient data structures dataArrays = dataShapes.map(dataDesc => @@ -399,13 +418,31 @@ class DataParallelExecutorGroup private[module]( auxArrays = (0 until auxNames.length).map(i => execs.map(_.auxArrays(i))) } + /** + * Reshape executors. + * @param dataShapes + * @param labelShapes + */ + def reshape(dataShapes: Seq[DataDesc], labelShapes: Option[Seq[DataDesc]]): Unit = { + if (!(dataShapes == this.dataShapes && labelShapes == this.labelShapes)) { + if (this._defaultExecs == null) { + this._defaultExecs = this.execs.map(x => x) + } + this.bindExec(dataShapes, labelShapes, None, reshape = true) + } + } + /** * Assign, i.e. copy parameters to all the executors. * @param argParams A dictionary of name to `NDArray` parameter mapping. * @param auxParams A dictionary of name to `NDArray` auxiliary variable mapping. + * @param allowExtra hether allow extra parameters that are not needed by symbol. + * If this is True, no error will be thrown when argParams or auxParams + * contain extra parameters that is not needed by the executor. */ - def setParams(argParams: Map[String, NDArray], auxParams: Map[String, NDArray]): Unit = { - execs.foreach(_.copyParamsFrom(argParams, auxParams)) + def setParams(argParams: Map[String, NDArray], auxParams: Map[String, NDArray], + allowExtra: Boolean = false): Unit = { + execs.foreach(_.copyParamsFrom(argParams, auxParams, allowExtraParams = allowExtra)) } /** diff --git a/scala-package/core/src/main/scala/ml/dmlc/mxnet/module/Module.scala b/scala-package/core/src/main/scala/ml/dmlc/mxnet/module/Module.scala index 2b1d743ea648..b9cc07826504 100644 --- a/scala-package/core/src/main/scala/ml/dmlc/mxnet/module/Module.scala +++ b/scala-package/core/src/main/scala/ml/dmlc/mxnet/module/Module.scala @@ -107,11 +107,16 @@ class Module(symbolVar: Symbol, * @param allowMissing If true, params could contain missing values, * and the initializer will be called to fill those missing params. * @param forceInit If true, will force re-initialize even if already initialized. + * @param allowExtra Whether allow extra parameters that are not needed by symbol. + * If this is True, no error will be thrown when argParams or auxParams + * contain extra parameters that is not needed by the executor. */ override def initParams(initializer: Initializer = new Uniform(0.01f), argParams: Map[String, NDArray] = null, auxParams: Map[String, NDArray] = null, - allowMissing: Boolean = false, forceInit: Boolean = false): Unit = { + allowMissing: Boolean = false, + forceInit: Boolean = false, + allowExtra: Boolean = false): Unit = { if (paramsInitialized && !forceInit) { return } @@ -141,7 +146,7 @@ class Module(symbolVar: Symbol, this.paramsDirty = false // copy the initialized parameters to devices - this.execGroup.setParams(this.argParams, this.auxParams) + this.execGroup.setParams(this.argParams, this.auxParams, allowExtra = allowExtra) } // Internal helper for parameter initialization @@ -261,6 +266,46 @@ class Module(symbolVar: Symbol, } } + /** + * Check that input names matches input data descriptors. + */ + @throws(classOf[IllegalArgumentException]) + private def _checkNamesMatch(dataNames: IndexedSeq[String], dataShapes: IndexedSeq[DataDesc], + name: String, throwEx: Boolean): Unit = { + val actual = dataShapes.map(_.name) + if (dataNames.sorted != actual.sorted) { + val msg = s"Data provided by ${name}_shapes don't match names specified by " + + s"${name}_names (${dataShapes.mkString(", ")} vs. ${dataNames.mkString(", ")})" + if (throwEx) throw new IllegalArgumentException(msg) + else logger.warn(msg) + } + } + + /** + * parse data_attrs into DataDesc format and check that names match + */ + @throws(classOf[IllegalArgumentException]) + private def _parseDataDesc(dataNames: IndexedSeq[String], labelNames: IndexedSeq[String], + dataShapes: IndexedSeq[DataDesc], labelShapes: Option[IndexedSeq[DataDesc]]): + (IndexedSeq[DataDesc], Option[IndexedSeq[DataDesc]]) = { + _checkNamesMatch(dataNames, dataShapes, "data", true) + if (labelShapes != None) _checkNamesMatch(labelNames, labelShapes.get, "label", false) + (dataShapes, labelShapes) + } + + /** + * Reshapes the module for new input shapes. + * @param dataShapes Typically is `dataIter.provideData`. + * @param labelShapes Typically is `dataIter.provideLabel`. + */ + def reshape(dataShapes: IndexedSeq[DataDesc], + labelShapes: Option[IndexedSeq[DataDesc]] = None): Unit = { + require(this.binded) + val (tdataShapes, tlabelShapes) = this._parseDataDesc( + this.dataNames, this.labelNames, dataShapes, labelShapes) + this.execGroup.reshape(tdataShapes, tlabelShapes) + } + /** * Install and initialize optimizers. * @param kvstore @@ -344,6 +389,26 @@ class Module(symbolVar: Symbol, */ def forward(dataBatch: DataBatch, isTrain: Option[Boolean] = None): Unit = { require(binded && paramsInitialized) + val currDataShapes = this.dataShapes.map(_.shape) + val newDataShapes = dataBatch.data.map(_.shape) + if (currDataShapes != newDataShapes) { + val newDShapes: IndexedSeq[DataDesc] = + if (dataBatch.provideData != null) dataBatch.provideData + else { + this.dataShapes.zip(newDataShapes).map { case (i, shape) => + DataDesc(i.name, shape, i.dtype, i.layout) + } + } + val newLShapes: Option[IndexedSeq[DataDesc]] = + if (dataBatch.provideLabel != null) Some(dataBatch.provideLabel) + else if (dataBatch.label != null && dataBatch.label.length > 0 + && this.labelShapes != null) { + Some(this.labelShapes.zip(dataBatch.label).map { case (i, j) => + DataDesc(i.name, j.shape, i.dtype, i.layout) + }) + } else None + this.reshape(newDShapes, newLShapes) + } execGroup.forward(dataBatch, isTrain) } diff --git a/scala-package/core/src/main/scala/ml/dmlc/mxnet/module/SequentialModule.scala b/scala-package/core/src/main/scala/ml/dmlc/mxnet/module/SequentialModule.scala index dfa63ebac629..a77041de5b0a 100644 --- a/scala-package/core/src/main/scala/ml/dmlc/mxnet/module/SequentialModule.scala +++ b/scala-package/core/src/main/scala/ml/dmlc/mxnet/module/SequentialModule.scala @@ -144,11 +144,16 @@ class SequentialModule extends BaseModule { * @param allowMissing If true, params could contain missing values, * and the initializer will be called to fill those missing params. * @param forceInit If true, will force re-initialize even if already initialized. + * @param allowExtra Whether allow extra parameters that are not needed by symbol. + * If this is True, no error will be thrown when argParams or auxParams + * contain extra parameters that is not needed by the executor. */ override def initParams(initializer: Initializer = new Uniform(0.01f), argParams: Map[String, NDArray] = null, auxParams: Map[String, NDArray] = null, - allowMissing: Boolean = false, forceInit: Boolean = false): Unit = { + allowMissing: Boolean = false, + forceInit: Boolean = false, + allowExtra: Boolean = false): Unit = { if (this.paramsInitialized && !forceInit) { return } @@ -156,7 +161,8 @@ class SequentialModule extends BaseModule { for (module <- this.modules) { module.initParams(initializer = initializer, argParams = argParams, - auxParams = auxParams, allowMissing = allowMissing, forceInit = forceInit) + auxParams = auxParams, allowMissing = allowMissing, + forceInit = forceInit, allowExtra = allowExtra) } // Internal function to help checking duplicated names, diff --git a/scala-package/core/src/test/scala/ml/dmlc/mxnet/ModuleSuite.scala b/scala-package/core/src/test/scala/ml/dmlc/mxnet/ModuleSuite.scala new file mode 100644 index 000000000000..ab48ef7d1928 --- /dev/null +++ b/scala-package/core/src/test/scala/ml/dmlc/mxnet/ModuleSuite.scala @@ -0,0 +1,368 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ml.dmlc.mxnet + +import org.scalatest.{BeforeAndAfterAll, FunSuite} +import ml.dmlc.mxnet.CheckUtils._ +import ml.dmlc.mxnet.module._ +import ml.dmlc.mxnet.optimizer._ +import ml.dmlc.mxnet.io._ + +class ModuleSuite extends FunSuite with BeforeAndAfterAll { + test ("model dtype") { + val dType = DType.Float16 + val dShape = Shape(3, 8, 7) + + var sym = Symbol.Variable("data") + sym = Symbol.Activation(attr = Map("__layout__" -> "TNC"))()( + Map("data" -> sym, "act_type" -> "relu")) + + val mod = new Module(sym, IndexedSeq("data"), null, + contexts = Array(Context.cpu(0), Context.cpu(1))) + mod.bind(dataShapes = IndexedSeq(DataDesc("data", dShape, dType, "TNC"))) + mod.initParams() + mod.forward(new DataBatch( + data = IndexedSeq(NDArray.ones(dShape, dtype = dType)), + label = null, index = null, pad = 0)) + mod.backward(Array(NDArray.ones(dShape, dtype = dType))) + + assert(mod.getOutputs.flatten.forall(_.dtype == dType)) + } + + test ("module input_grads") { + val a = Symbol.Variable("a", kwargs = Map("__layout__" -> "NC")) + val b = Symbol.Variable("b", kwargs = Map("__layout__" -> "NC")) + var c = Symbol.Variable("c", kwargs = Map("__layout__" -> "NC")) + + import SymbolConversions._ + c = a + 2 * b + 3 * c + + val mod = new Module(c, IndexedSeq("b", "c", "a"), null, + contexts = Array(Context.cpu(0), Context.cpu(1))) + mod.bind(dataShapes = IndexedSeq( + DataDesc("b", Shape(5, 5)), + DataDesc("c", Shape(5, 5)), + DataDesc("a", Shape(5, 5))), + inputsNeedGrad = true + ) + mod.initParams() + mod.forward(new DataBatch( + data = IndexedSeq( + NDArray.ones(5, 5), NDArray.ones(5, 5), NDArray.ones(5, 5)), + label = null, index = null, pad = 0)) + mod.backward(Array(NDArray.ones(5, 5))) + + val inputGrads = mod.getInputGradsMerged() + val aGrad = inputGrads(0).toArray + val bGrad = inputGrads(1).toArray + val cGrad = inputGrads(2).toArray + + assert(aGrad.forall(_ == 1f)) + assert(bGrad.forall(_ == 2f)) + assert(cGrad.forall(_ == 3f)) + } + + test ("module layout") { + var sym = Symbol.Variable("data") + sym = Symbol.Activation(attr = Map("__layout__" -> "TNC"))()( + Map("data" -> sym, "act_type" -> "relu")) + + val dShape = Shape(3, 8, 7) + val mod = new Module(sym, IndexedSeq("data"), null, + contexts = Array(Context.cpu(0), Context.cpu(1))) + mod.bind(dataShapes = IndexedSeq(DataDesc("data", dShape, layout = "TNC"))) + mod.initParams() + mod.forward(new DataBatch( + data = IndexedSeq(NDArray.ones(dShape)), + label = null, index = null, pad = 0)) + mod.backward(Array(NDArray.ones(dShape))) + assert(mod.getOutputsMerged()(0).shape == dShape) + + val hdShape = Shape(3, 4, 7) + for (x <- mod.getOutputs) assert(x(0).shape == hdShape) + } + + test ("save load") { + def mapEqu(a: Map[String, NDArray], b: Map[String, NDArray]): Unit = { + assert(a.toSet == b.toSet) + for (k <- a.keys) assert(a(k) == b(k)) + } + + var sym = Symbol.Variable("data") + sym = Symbol.FullyConnected()()(Map("data" -> sym, "num_hidden" -> 100)) + + // single device + var mod = new Module(sym, IndexedSeq("data"), null) + mod.bind(dataShapes = IndexedSeq(DataDesc("data", Shape(10, 10)))) + mod.initParams() + mod.initOptimizer(optimizer = new SGD(learningRate = 0.1f, momentum = 0.9f)) + mod.update() + mod.saveCheckpoint("test", 0, saveOptStates = true) + + var mod2 = Module.loadCheckpoint("test", 0, loadOptimizerStates = true) + mod2.bind(dataShapes = IndexedSeq(DataDesc("data", Shape(10, 10)))) + mod2.initOptimizer(optimizer = new SGD(learningRate = 0.1f, momentum = 0.9f)) + assert(mod.getSymbol.toJson == mod2.getSymbol.toJson) + mapEqu(mod.getParams._1, mod2.getParams._1) + + // multi device + mod = new Module(sym, IndexedSeq("data"), null, + contexts = Array(Context.cpu(0), Context.cpu(1))) + mod.bind(dataShapes = IndexedSeq(DataDesc("data", Shape(10, 10)))) + mod.initParams() + mod.initOptimizer(optimizer = new SGD(learningRate = 0.1f, momentum = 0.9f)) + mod.update() + mod.saveCheckpoint("test", 0, saveOptStates = true) + + mod2 = Module.loadCheckpoint("test", 0, loadOptimizerStates = true) + mod2.bind(dataShapes = IndexedSeq(DataDesc("data", Shape(10, 10)))) + mod2.initOptimizer(optimizer = new SGD(learningRate = 0.1f, momentum = 0.9f)) + assert(mod.getSymbol.toJson == mod2.getSymbol.toJson) + mapEqu(mod.getParams._1, mod2.getParams._1) + } + + test ("module reshape") { + var sym = Symbol.Variable("data") + sym = Symbol.FullyConnected("fc")()(Map("data" -> sym, "num_hidden" -> 20)) + + var dShape = Shape(7, 20) + val mod = new Module(sym, IndexedSeq("data"), null, + contexts = Array(Context.cpu(0), Context.cpu(1))) + mod.bind(dataShapes = IndexedSeq(DataDesc("data", dShape))) + mod.initParams() + mod.initOptimizer(optimizer = new SGD(learningRate = 1f)) + + mod.forward(new DataBatch( + data = IndexedSeq(NDArray.ones(dShape)), + label = null, index = null, pad = 0)) + mod.backward(Array(NDArray.ones(dShape))) + mod.update() + assert(mod.getOutputsMerged()(0).shape == dShape) + assert(mod.getParams._1("fc_bias").toArray.forall(_ == -1f)) + + dShape = Shape(14, 20) + mod.reshape(IndexedSeq(DataDesc("data", dShape))) + mod.forward(new DataBatch( + data = IndexedSeq(NDArray.ones(dShape)), + label = null, index = null, pad = 0)) + mod.backward(Array(NDArray.ones(dShape))) + mod.update() + assert(mod.getOutputsMerged()(0).shape == dShape) + assert(mod.getParams._1("fc_bias").toArray.forall(x => (x - -3f) < 1e-3)) + } + + test ("module setParams") { + val data = NDArray.array(Array(0.05f, 0.1f), Shape(1, 2)) + val label = NDArray.array(Array(0.01f, 0.99f), Shape(1, 2)) + val trainData = new NDArrayIter( + IndexedSeq(data), IndexedSeq(label), labelName = "softmax_label") + + // symbols + var x = Symbol.Variable("data") + x = Symbol.FullyConnected(name = "fc_0")()(Map("data" -> x, "num_hidden" -> 2)) + x = Symbol.Activation(name = "act_0")()(Map("data" -> x, "act_type" -> "sigmoid")) + x = Symbol.FullyConnected(name = "fc_1")()(Map("data" -> x, "num_hidden" -> 2)) + x = Symbol.Activation(name = "act_1")()(Map("data" -> x, "act_type" -> "sigmoid")) + x = Symbol.LinearRegressionOutput(name = "softmax")()(Map("data" -> x, "grad_scale" -> 2)) + + // create module + val mod = new Module(x, contexts = Array(Context.cpu())) + mod.bind(dataShapes = trainData.provideData, + Option(trainData.provideLabel)) + val argParamsCorrect = Map( + "fc_0_weight" -> NDArray.array(Array(0.15f, 0.2f, 0.25f, 0.3f), Shape(2, 2)), + "fc_0_bias" -> NDArray.array(Array(0.35f, 0.35f), Shape(2)), + "fc_1_weight" -> NDArray.array(Array(0.4f, 0.45f, 0.5f, 0.55f), Shape(2, 2)), + "fc_1_bias" -> NDArray.array(Array(0.6f, 0.6f), Shape(2)) + ) + val argParamsMissing = Map( + "fc_0_weight" -> NDArray.array(Array(0.15f, 0.2f, 0.25f, 0.3f), Shape(2, 2)), + "fc_0_bias" -> NDArray.array(Array(0.35f, 0.35f), Shape(2)), + "fc_1_weight" -> NDArray.array(Array(0.4f, 0.45f, 0.5f, 0.55f), Shape(2, 2)) + ) + val argParamsExtra = Map( + "fc_0_weight" -> NDArray.array(Array(0.15f, 0.2f, 0.25f, 0.3f), Shape(2, 2)), + "fc_0_bias" -> NDArray.array(Array(0.35f, 0.35f), Shape(2)), + "fc_1_weight" -> NDArray.array(Array(0.4f, 0.45f, 0.5f, 0.55f), Shape(2, 2)), + "fc_1_bias" -> NDArray.array(Array(0.6f, 0.6f), Shape(2)), + "fc_2_weight" -> NDArray.array(Array(0.6f, 0.6f), Shape(2)) + ) + + mod.setParams(forceInit = true, argParams = argParamsCorrect, + auxParams = null) + + // test allow missing + mod.setParams(forceInit = true, argParams = argParamsMissing, + auxParams = null, allowMissing = true) + + // test allow extra + mod.setParams(forceInit = true, argParams = argParamsExtra, auxParams = null, + allowMissing = true, allowExtra = true) + } + + test ("monitor") { + // data iter + val data = NDArray.array(Array(0.05f, 0.1f), Shape(1, 2)) + val label = NDArray.array(Array(0.01f, 0.99f), Shape(1, 2)) + val trainData = new NDArrayIter( + IndexedSeq(data), IndexedSeq(label), labelName = "softmax_label") + + // symbols + var x = Symbol.Variable("data") + x = Symbol.FullyConnected(name = "fc_0")()(Map("data" -> x, "num_hidden" -> 2)) + x = Symbol.Activation(name = "act_0")()(Map("data" -> x, "act_type" -> "sigmoid")) + x = Symbol.FullyConnected(name = "fc_1")()(Map("data" -> x, "num_hidden" -> 2)) + x = Symbol.Activation(name = "act_1")()(Map("data" -> x, "act_type" -> "sigmoid")) + x = Symbol.LinearRegressionOutput(name = "softmax")()(Map("data" -> x, "grad_scale" -> 2)) + + // create monitor + def meanAbs(x: NDArray): NDArray = { + val sumAbs = NDArray.sum(NDArray.abs(x)) + sumAbs / x.shape.product + } + val mon = new Monitor(1, statFunc = meanAbs) + + // create module + val mod = new Module(x, contexts = Array(Context.cpu())) + mod.bind(dataShapes = trainData.provideData, + Option(trainData.provideLabel)) + mod.installMonitor(mon) + val argParams = Map( + "fc_0_weight" -> NDArray.array(Array(0.15f, 0.2f, 0.25f, 0.3f), Shape(2, 2)), + "fc_0_bias" -> NDArray.array(Array(0.35f, 0.35f), Shape(2)), + "fc_1_weight" -> NDArray.array(Array(0.4f, 0.45f, 0.5f, 0.55f), Shape(2, 2)), + "fc_1_bias" -> NDArray.array(Array(0.6f, 0.6f), Shape(2)) + ) + mod.initParams(argParams = argParams) + + val dataBatch = trainData.next() + mon.tic() + mod.forwardBackward(dataBatch) + val res = mon.toc() + val keys = Array("act_0", "act_1", "data", "fc_0", "fc_1", "softmax") + val monResultCounts = Array(0, 0, 0, 0, 0, 0) + assert(res.length == 21) + for ((n, k, v) <- res) { + var break = false + for ((key, idx) <- keys.zipWithIndex) { + if (!break && k.startsWith(key)) { + monResultCounts(idx) += 1 + break = true + } + } + } + assert(monResultCounts.zip(Array(2, 2, 1, 6, 6, 4)).forall(x => x._1 == x._2)) + } + + test ("forward reshape") { + val numClass = 10 + val data1 = Symbol.Variable("data1") + val data2 = Symbol.Variable("data2") + val conv1 = Symbol.Convolution()()(Map("data" -> data1, + "kernel" -> "(2, 2)", "num_filter" -> 2, "stride" -> "(2, 2)")) + val conv2 = Symbol.Convolution()()(Map("data" -> data2, + "kernel" -> "(3, 3)", "num_filter" -> 3, "stride" -> "(1, 1)")) + val pooling1 = Symbol.Pooling()()(Map("data" -> conv1, + "kernel" -> "(2, 2)", "pool_type" -> "avg", "stride" -> "(1, 1)")) + val pooling2 = Symbol.Pooling()()(Map("data" -> conv2, + "kernel" -> "(2, 2)", "pool_type" -> "max", "stride" -> "(1, 1)")) + val flatten1 = Symbol.flatten()()(Map("data" -> pooling1)) + val flatten2 = Symbol.flatten()()(Map("data" -> pooling2)) + val sum = Symbol.sum()()(Map("data" -> flatten1, "axis" -> 1)) + + Symbol.sum()()(Map("data" -> flatten2, "axis" -> 1)) + val fc = Symbol.FullyConnected()()( + Map("data" -> sum, "num_hidden" -> numClass)) + val sym = Symbol.SoftmaxOutput(name = "softmax")()(Map("data" -> fc)) + + var dShape1 = Shape(10, 3, 64, 64) + var dShape2 = Shape(10, 3, 32, 32) + var lShape = Shape(10) + + val mod = new Module(sym, IndexedSeq("data1", "data2")) + mod.bind(dataShapes = IndexedSeq( + DataDesc("data1", dShape1), DataDesc("data2", dShape2)), + labelShapes = Option(IndexedSeq(DataDesc("softmax_label", lShape))) + ) + mod.initParams() + mod.initOptimizer(optimizer = new SGD(learningRate = 0.01f)) + + // Train with original data shapes + var dataBatch = new DataBatch( + data = IndexedSeq( + NDArray.random_uniform(Map("low" -> 0, "high" -> 9, "shape" -> dShape1.toString()))(), + NDArray.random_uniform(Map("low" -> 5, "high" -> 15, "shape" -> dShape2.toString()))()), + label = IndexedSeq(NDArray.ones(lShape)), index = null, pad = 0) + mod.forward(dataBatch) + assert(mod.getOutputsMerged()(0).shape == Shape(lShape(0), numClass)) + mod.backward() + mod.update() + + dShape1 = Shape(3, 3, 64, 64) + dShape2 = Shape(3, 3, 32, 32) + lShape = Shape(3) + dataBatch = new DataBatch( + data = IndexedSeq( + NDArray.random_uniform(Map("low" -> 0, "high" -> 9, "shape" -> dShape1.toString()))(), + NDArray.random_uniform(Map("low" -> 5, "high" -> 15, "shape" -> dShape2.toString()))()), + label = IndexedSeq(NDArray.ones(lShape)), index = null, pad = 0) + mod.forward(dataBatch) + assert(mod.getOutputsMerged()(0).shape == Shape(lShape(0), numClass)) + mod.backward() + mod.update() + + dShape1 = Shape(20, 3, 64, 64) + dShape2 = Shape(20, 3, 32, 32) + lShape = Shape(20) + dataBatch = new DataBatch( + data = IndexedSeq( + NDArray.random_uniform(Map("low" -> 3, "high" -> 5, "shape" -> dShape1.toString()))(), + NDArray.random_uniform(Map("low" -> 10, "high" -> 25, "shape" -> dShape2.toString()))()), + label = IndexedSeq(NDArray.ones(lShape)), index = null, pad = 0) + mod.forward(dataBatch) + assert(mod.getOutputsMerged()(0).shape == Shape(lShape(0), numClass)) + mod.backward() + mod.update() + + // Train with both different batch size and data shapes + dShape1 = Shape(20, 3, 120, 120) + dShape2 = Shape(20, 3, 32, 64) + lShape = Shape(20) + dataBatch = new DataBatch( + data = IndexedSeq( + NDArray.random_uniform(Map("low" -> 0, "high" -> 9, "shape" -> dShape1.toString()))(), + NDArray.random_uniform(Map("low" -> 5, "high" -> 15, "shape" -> dShape2.toString()))()), + label = IndexedSeq(NDArray.ones(lShape)), index = null, pad = 0) + mod.forward(dataBatch) + assert(mod.getOutputsMerged()(0).shape == Shape(lShape(0), numClass)) + mod.backward() + mod.update() + + dShape1 = Shape(5, 3, 28, 40) + dShape2 = Shape(5, 3, 24, 16) + lShape = Shape(5) + dataBatch = new DataBatch( + data = IndexedSeq( + NDArray.random_uniform(Map("low" -> 0, "high" -> 9, "shape" -> dShape1.toString()))(), + NDArray.random_uniform(Map("low" -> 15, "high" -> 25, "shape" -> dShape2.toString()))()), + label = IndexedSeq(NDArray.ones(lShape)), index = null, pad = 0) + mod.forward(dataBatch) + assert(mod.getOutputsMerged()(0).shape == Shape(lShape(0), numClass)) + mod.backward() + mod.update() + } +} diff --git a/scala-package/core/src/test/scala/ml/dmlc/mxnet/OperatorSuite.scala b/scala-package/core/src/test/scala/ml/dmlc/mxnet/OperatorSuite.scala index 187869c3af21..ac1cee202e5b 100644 --- a/scala-package/core/src/test/scala/ml/dmlc/mxnet/OperatorSuite.scala +++ b/scala-package/core/src/test/scala/ml/dmlc/mxnet/OperatorSuite.scala @@ -239,7 +239,7 @@ class OperatorSuite extends FunSuite with BeforeAndAfterAll var exe = x.simpleBind(ctx = Context.cpu(), gradReq = "write", shapeDict = Map()) exe.forward(isTrain = false) assert(exe.gradArrays.length == 0) - assert(CheckUtils.reldiff(result.toArray, exe.outputs.head.toArray) <= 1e-5f) + assert(CheckUtils.reldiff(result.toArray, exe.outputs.head.toArray) <= 1e-4f) } } From c9aacaa7bfa9de0036837a504b67229be8404c67 Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Fri, 4 Aug 2017 10:12:31 -0700 Subject: [PATCH 327/834] fix cpplint (#7332) --- cpp-package/include/mxnet-cpp/lr_scheduler.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cpp-package/include/mxnet-cpp/lr_scheduler.h b/cpp-package/include/mxnet-cpp/lr_scheduler.h index 91f9b3c0a952..4c56b7ab3f0b 100644 --- a/cpp-package/include/mxnet-cpp/lr_scheduler.h +++ b/cpp-package/include/mxnet-cpp/lr_scheduler.h @@ -4,8 +4,8 @@ * \brief Scheduling learning rate */ -#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_LR_SCHEDULER_H_ -#define CPP_PACKAGE_INCLUDE_MXNET_CPP_LR_SCHEDULER_H_ +#ifndef MXNET_CPP_LR_SCHEDULER_H_ +#define MXNET_CPP_LR_SCHEDULER_H_ #include "dmlc/logging.h" @@ -75,4 +75,4 @@ class FactorScheduler : public LRScheduler { } // namespace cpp } // namespace mxnet -#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_LR_SCHEDULER_H_ +#endif // MXNET_CPP_LR_SCHEDULER_H_ From c8db271dcea47ca60222d30aeeee8ff2c5336af9 Mon Sep 17 00:00:00 2001 From: Leonard Lausen Date: Sat, 5 Aug 2017 02:14:43 +0900 Subject: [PATCH 328/834] Add Autograd doc to gluon page (#7327) * Add Autograd doc to gluon page * Make autograd doc toplevel --- docs/api/python/autograd.md | 32 ++++++++++++++++++++++++++++++++ docs/api/python/index.md | 1 + 2 files changed, 33 insertions(+) create mode 100644 docs/api/python/autograd.md diff --git a/docs/api/python/autograd.md b/docs/api/python/autograd.md new file mode 100644 index 000000000000..440a1e4de289 --- /dev/null +++ b/docs/api/python/autograd.md @@ -0,0 +1,32 @@ +# Autograd Package + + +```eval_rst +.. currentmodule:: mxnet.autograd +``` + +```eval_rst +.. warning:: This package is currently experimental and may change in the near future. +``` + + + +## Autograd + +```eval_rst +.. currentmodule:: mxnet.autograd +``` + + +```eval_rst +.. autosummary:: + :nosignatures: + + record + pause + mark_variables + backward + set_training + set_recording +``` + diff --git a/docs/api/python/index.md b/docs/api/python/index.md index 4014a886a0d4..964ccde0145a 100644 --- a/docs/api/python/index.md +++ b/docs/api/python/index.md @@ -28,6 +28,7 @@ imported by running: ndarray symbol module + autograd gluon rnn kvstore From 79231e22fcc00dc47ba3ae34944891fcc111799d Mon Sep 17 00:00:00 2001 From: Dick Carter Date: Fri, 4 Aug 2017 10:22:40 -0700 Subject: [PATCH 329/834] Fixes improper deconv workspace alloc (#7326) --- src/operator/cudnn_convolution-inl.h | 40 +++++++++++------- src/operator/cudnn_deconvolution-inl.h | 57 +++++++++++++++++--------- tests/python/unittest/test_operator.py | 4 +- 3 files changed, 64 insertions(+), 37 deletions(-) diff --git a/src/operator/cudnn_convolution-inl.h b/src/operator/cudnn_convolution-inl.h index 508b1f8be84d..06887a94aa70 100644 --- a/src/operator/cudnn_convolution-inl.h +++ b/src/operator/cudnn_convolution-inl.h @@ -95,9 +95,8 @@ class CuDNNConvolutionOp : public Operator { CHECK_EQ(out_data.size(), 1U); Stream *s = ctx.get_stream(); GetTempSize(ctx); - Tensor workspace = - ctx.requested[conv::kTempSpace].get_space_typed( - mshadow::Shape1(forward_workspace_), s); + Tensor workspace = AllocateTempWorkspace(ctx, forward_workspace_byte_); + size_t workspace_size = TensorSizeBytes(workspace); if (param_.kernel.ndim() == 2) { Tensor data = in_data[conv::kData].get(s); @@ -133,7 +132,7 @@ class CuDNNConvolutionOp : public Operator { forward_conv_desc_, algo_, workspace.dptr_, - forward_workspace_byte_, + workspace_size, req[conv::kOut] == kAddTo? &beta_add : &beta, out_desc_, out_ptr + out_offset_ * g)); @@ -203,9 +202,8 @@ class CuDNNConvolutionOp : public Operator { data_ptr = data.dptr_; gdata_ptr = gdata.dptr_; } - Tensor workspace = - ctx.requested[conv::kTempSpace].get_space_typed( - mshadow::Shape1(backward_workspace_), s); + Tensor workspace = AllocateTempWorkspace(ctx, backward_workspace_byte_); + size_t workspace_size = TensorSizeBytes(workspace); for (uint32_t g = 0; g < param_.num_group; ++g) { typename DataType::ScaleType alpha = 1.0f; typename DataType::ScaleType beta = 0.0f; @@ -231,7 +229,7 @@ class CuDNNConvolutionOp : public Operator { backward_conv_desc_, back_algo_w_, workspace.dptr_, - backward_workspace_byte_, + workspace_size, req[conv::kWeight] == kAddTo? &beta_add : &beta, filter_desc_, gwmat_ptr + weight_offset_ * g)); @@ -245,7 +243,7 @@ class CuDNNConvolutionOp : public Operator { backward_conv_desc_, back_algo_w_, workspace.dptr_, - backward_workspace_byte_, + workspace_size, req[conv::kWeight] == kAddTo? &beta_add : &beta, filter_desc_, gwmat_ptr + weight_offset_ * g)); @@ -262,7 +260,7 @@ class CuDNNConvolutionOp : public Operator { backward_conv_desc_, back_algo_, workspace.dptr_, - backward_workspace_byte_, + workspace_size, req[conv::kData] == kAddTo? &beta_add : &beta, in_desc_, gdata_ptr + data_offset_ * g)); @@ -276,7 +274,7 @@ class CuDNNConvolutionOp : public Operator { backward_conv_desc_, back_algo_, workspace.dptr_, - backward_workspace_byte_, + workspace_size, req[conv::kData] == kAddTo? &beta_add : &beta, in_desc_, gdata_ptr + data_offset_ * g)); @@ -667,8 +665,6 @@ class CuDNNConvolutionOp : public Operator { algo_, &forward_workspace_byte_)); - forward_workspace_ = forward_workspace_byte_ / sizeof(DType) + 1; - backward_workspace_ = backward_workspace_byte_ / sizeof(DType) + 1; init_temp_size_ = true; } @@ -684,15 +680,29 @@ class CuDNNConvolutionOp : public Operator { CastTShapeToIntPtr(param_.pad, ¶m_pad_); } + // Allocates a 1D Tensor of words with size in bytes >= `size_bytes`. + // Always allocates at least one word. + mshadow::Tensor AllocateTempWorkspace(const OpContext &ctx, size_t size_bytes) { + mshadow::Stream *s = ctx.get_stream(); + size_t size_words = size_bytes / sizeof(DType) + 1; + return ctx.requested[conv::kTempSpace].get_space_typed( + mshadow::Shape1(size_words), s); + } + + // Returns the size in bytes of the 1D Tensor of words. + size_t TensorSizeBytes(const mshadow::Tensor &tensor) { + return tensor.MSize() * sizeof(DType); + } + std::vector param_stride_; std::vector param_dilate_; std::vector param_pad_; bool init_cudnn_; bool init_temp_size_; - size_t forward_workspace_; - size_t backward_workspace_; + // Temp workspace size in bytes needed for Forward() operation. size_t forward_workspace_byte_; + // Temp workspace size in bytes needed for Backward() operation. size_t backward_workspace_byte_; size_t data_offset_; size_t out_offset_; diff --git a/src/operator/cudnn_deconvolution-inl.h b/src/operator/cudnn_deconvolution-inl.h index 5bba1e5278fa..2e2ae3a8cb8f 100644 --- a/src/operator/cudnn_deconvolution-inl.h +++ b/src/operator/cudnn_deconvolution-inl.h @@ -92,9 +92,8 @@ class CuDNNDeconvolutionOp : public Operator { CHECK_EQ(out_data.size(), 1U); Stream *s = ctx.get_stream(); GetTempSize(ctx); - Tensor workspace = - ctx.requested[deconv::kTempSpace].get_space_typed( - mshadow::Shape1(forward_workspace_), s); + Tensor workspace = AllocateTempWorkspace(ctx, forward_workspace_byte_); + size_t workspace_size = TensorSizeBytes(workspace); if (param_.kernel.ndim() == 2) { Tensor data = in_data[deconv::kData].get(s); @@ -131,7 +130,7 @@ class CuDNNDeconvolutionOp : public Operator { forward_conv_desc_, // this backward algorithm used for inference back_algo_, workspace.dptr_, - backward_workspace_byte_, + workspace_size, &beta, out_desc_, out.dptr_ + out_offset_ * g)); @@ -145,7 +144,7 @@ class CuDNNDeconvolutionOp : public Operator { forward_conv_desc_, // this backward algorithm used for inference back_algo_, workspace.dptr_, - backward_workspace_byte_, + workspace_size, &beta, out_desc_, out_ptr + out_offset_ * g)); @@ -222,9 +221,8 @@ class CuDNNDeconvolutionOp : public Operator { CHECK_NE(req[deconv::kBias], kWriteInplace); } CHECK_NE(req[deconv::kData], kWriteInplace); - Tensor workspace = - ctx.requested[deconv::kTempSpace].get_space_typed( - mshadow::Shape1(backward_workspace_), s); + Tensor workspace = AllocateTempWorkspace(ctx, backward_workspace_byte_); + size_t workspace_size = TensorSizeBytes(workspace); for (uint32_t g = 0; g < param_.num_group; ++g) { typename DataType::ScaleType alpha = 1.0f; typename DataType::ScaleType bias_beta = 0.0f; @@ -257,7 +255,7 @@ class CuDNNDeconvolutionOp : public Operator { backward_conv_desc_, back_algo_w_, workspace.dptr_, - backward_workspace_byte_, + workspace_size, &weight_beta, filter_desc_, gwmat.dptr_ + weight_offset_ * g)); @@ -272,7 +270,7 @@ class CuDNNDeconvolutionOp : public Operator { backward_conv_desc_, back_algo_w_, workspace.dptr_, - backward_workspace_byte_, + workspace_size, &weight_beta, filter_desc_, gwmat_ptr + weight_offset_ * g)); @@ -288,7 +286,7 @@ class CuDNNDeconvolutionOp : public Operator { backward_conv_desc_, algo_, workspace.dptr_, - forward_workspace_byte_, + workspace_size, &data_beta, in_desc_, gdata_ptr + data_offset_ * g)); @@ -664,32 +662,34 @@ class CuDNNDeconvolutionOp : public Operator { void GetTempSize(const OpContext& ctx) { if (init_temp_size_) return; mshadow::Stream *s = ctx.get_stream(); - size_t back_size = 0, back_size_w = 0; + size_t back_data_algo_workspace_size = 0; + size_t back_filter_algo_workspace_size = 0; + size_t forward_algo_workspace_size = 0; CUDNN_CALL(cudnnGetConvolutionBackwardDataWorkspaceSize(s->dnn_handle_, filter_desc_, in_desc_, forward_conv_desc_, out_desc_, back_algo_, - &back_size)); + &back_data_algo_workspace_size)); CUDNN_CALL(cudnnGetConvolutionBackwardFilterWorkspaceSize(s->dnn_handle_, out_desc_, in_desc_, backward_conv_desc_, filter_desc_, back_algo_w_, - &back_size_w)); - backward_workspace_byte_ = std::max(back_size, back_size_w); + &back_filter_algo_workspace_size)); CUDNN_CALL(cudnnGetConvolutionForwardWorkspaceSize(s->dnn_handle_, out_desc_, filter_desc_, backward_conv_desc_, in_desc_, algo_, - &forward_workspace_byte_)); + &forward_algo_workspace_size)); - forward_workspace_ = forward_workspace_byte_ / sizeof(DType) + 1; - backward_workspace_ = backward_workspace_byte_ / sizeof(DType) + 1; + forward_workspace_byte_ = back_data_algo_workspace_size; + backward_workspace_byte_ = std::max(forward_algo_workspace_size, + back_filter_algo_workspace_size); init_temp_size_ = true; } @@ -704,14 +704,31 @@ class CuDNNDeconvolutionOp : public Operator { CastTShapeToIntPtr(param_.dilate, ¶m_dilate_); } + // Allocates a 1D Tensor of words with size in bytes >= `size_bytes`. + // Always allocates at least one word. + mshadow::Tensor AllocateTempWorkspace(const OpContext &ctx, size_t size_bytes) { + mshadow::Stream *s = ctx.get_stream(); + size_t size_words = size_bytes / sizeof(DType) + 1; + return ctx.requested[deconv::kTempSpace].get_space_typed( + mshadow::Shape1(size_words), s); + } + + // Returns the size in bytes of the 1D Tensor of words. + size_t TensorSizeBytes(const mshadow::Tensor &tensor) { + return tensor.MSize() * sizeof(DType); + } + std::vector param_stride_; std::vector param_dilate_; bool init_cudnn_; bool init_temp_size_; - size_t forward_workspace_; - size_t backward_workspace_; + // Temp workspace size in bytes needed for Forward() operation. Note that + // in deconvolution, this is handled by the cuDNN backprop-to-data kernel. size_t forward_workspace_byte_; + // Temp workspace size in bytes needed for Backward() operation. Note that + // in deconvolution, this is handled by the cuDNN forward kernel and the + // the cuDNN backprop-to-filter kernel. size_t backward_workspace_byte_; size_t data_offset_; size_t out_offset_; diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py index 51a77e0af221..7007da6a2910 100644 --- a/tests/python/unittest/test_operator.py +++ b/tests/python/unittest/test_operator.py @@ -677,7 +677,7 @@ def check_deconvolution_forward_backward(input_shape, num_filter, kernel, stride exe.forward(is_train=True) out = exe.outputs[0].asnumpy() exe.backward(out_grad) - assert_almost_equal(out, args_grad[0].asnumpy(), rtol=1E-3, atol=1e-4) + assert_almost_equal(out, args_grad[0].asnumpy(), rtol=1E-3, atol=1e-3) args_grad_addto_npy = [np.random.normal(size=s) for s in arg_shapes] args_grad_addto = [mx.nd.array(ele) for ele in args_grad_addto_npy] @@ -685,7 +685,7 @@ def check_deconvolution_forward_backward(input_shape, num_filter, kernel, stride exe.forward(is_train=True) out = exe.outputs[0].asnumpy() exe.backward(out_grad) - assert_almost_equal(out + args_grad_addto_npy[0], args_grad_addto[0].asnumpy(), rtol=1e-4, atol=1e-4) + assert_almost_equal(out + args_grad_addto_npy[0], args_grad_addto[0].asnumpy(), rtol=1e-4, atol=1e-3) def check_deconvolution_gradient(input_shape, num_filter, pad): From b230d1745fd8e5b01c9c741d9567e5561ea538cc Mon Sep 17 00:00:00 2001 From: Sergey Kolychev Date: Fri, 4 Aug 2017 11:57:31 -0700 Subject: [PATCH 330/834] Attempting to add Perl interface to Apache CI. (#7170) * attempting to add Perl interface to Apache CI. * second attempt. * forgot the test file. * changed the working dir for tests, removed test that started to fail because upstream bug. --- Jenkinsfile | 22 +++++++ perl-package/AI-MXNet/t/test_model_parallel.t | 62 ------------------- perl-package/test.sh | 16 +++++ tests/ci_build/Dockerfile.cpu | 2 + tests/ci_build/Dockerfile.gpu | 2 + tests/ci_build/install/ubuntu_install_perl.sh | 4 ++ 6 files changed, 46 insertions(+), 62 deletions(-) delete mode 100644 perl-package/AI-MXNet/t/test_model_parallel.t create mode 100755 perl-package/test.sh create mode 100755 tests/ci_build/install/ubuntu_install_perl.sh diff --git a/Jenkinsfile b/Jenkinsfile index e48ecf207955..b3bf82689e56 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -281,6 +281,28 @@ try { } } }, + 'Perl: CPU': { + node('mxnetlinux') { + ws('workspace/ut-perl-cpu') { + init_git() + unpack_lib('cpu') + timeout(time: max_time, unit: 'MINUTES') { + sh "${docker_run} cpu ./perl-package/test.sh /workspace/ut-perl-cpu /workspace/ut-perl-cpu" + } + } + } + }, + 'Perl: GPU': { + node('mxnetlinux') { + ws('workspace/ut-perl-gpu') { + init_git() + unpack_lib('gpu') + timeout(time: max_time, unit: 'MINUTES') { + sh "${docker_run} gpu ./perl-package/test.sh /workspace/ut-perl-gpu /workspace/ut-perl-gpu" + } + } + } + }, 'R: CPU': { node('mxnetlinux') { ws('workspace/ut-r-cpu') { diff --git a/perl-package/AI-MXNet/t/test_model_parallel.t b/perl-package/AI-MXNet/t/test_model_parallel.t deleted file mode 100644 index e20b208029b5..000000000000 --- a/perl-package/AI-MXNet/t/test_model_parallel.t +++ /dev/null @@ -1,62 +0,0 @@ -use strict; -use warnings; -use Test::More tests => 3; -use AI::MXNet qw(mx); -use AI::MXNet::TestUtils qw(reldiff); -use AI::MXNet::Base; - -sub test_chain -{ - my $n = 2; - my $data1 = mx->sym->Variable('data1'); - my $data2 = mx->sym->Variable('data2'); - my $net; - { - local($mx::AttrScope) = mx->AttrScope(ctx_group=>'dev1'); - $net = $data1 + $data2; - $net = $net * 3; - } - - { - local($mx::AttrScope) = mx->AttrScope(ctx_group=>'dev2'); - $net = $net + $data1; - } - my $arr; - my $arr_grad; - my $shape = [4, 5]; - { - local($mx::Context) = mx->Context(mx->cpu(0)); - $arr = [map { mx->nd->empty($shape) } 0..$n-1]; - $arr_grad = [map { mx->nd->empty($shape) } 0..$n-1]; - } - - my $exec1 = $net->bind( - ctx => mx->cpu(), - args => $arr, - args_grad => $arr_grad, - group2ctx => { dev1 => mx->cpu(0), dev2 => mx->cpu(1) } - ); - $arr->[0] .= 1; - $arr->[1] .= 2; - my $arr2 = [map { $_->copyto(mx->cpu()) } @$arr]; - my $arr_grad2 = [map { $_->copyto(mx->cpu()) } @$arr_grad]; - my $exec2 = $net->bind( - ctx => mx->cpu(), - args => $arr2, - args_grad => $arr_grad2 - ); - - $exec1->forward(1); - $exec2->forward(1); - ok(reldiff($exec1->outputs->[0]->aspdl, $exec2->outputs->[0]->aspdl) < 1e-6); - my $out_grad = mx->nd->empty($shape, ctx => mx->cpu(1)); - $out_grad .= 1; - $exec1->backward([$out_grad]); - $exec2->backward([$out_grad->copyto(mx->cpu())]); - zip(sub { - my ($a, $b) = @_; - ok(reldiff($a->aspdl, $b->aspdl) < 1e-6); - }, $arr_grad, $arr_grad2); -} - -test_chain(); diff --git a/perl-package/test.sh b/perl-package/test.sh new file mode 100755 index 000000000000..c83120f31546 --- /dev/null +++ b/perl-package/test.sh @@ -0,0 +1,16 @@ +MXNET_HOME=$1 +HOME=$2 +export LD_LIBRARY_PATH=${MXNET_HOME}/lib +export PERL5LIB=${HOME}/perl5/lib/perl5 + +cd ${MXNET_HOME}/perl-package/AI-MXNetCAPI/ +perl Makefile.PL INSTALL_BASE=${HOME}/perl5 +make install || exit -1 + +cd ${MXNET_HOME}/perl-package/AI-NNVMCAPI/ +perl Makefile.PL INSTALL_BASE=${HOME}/perl5 +make install || exit -1 + +cd ${MXNET_HOME}/perl-package/AI-MXNet/ +perl Makefile.PL INSTALL_BASE=${HOME}/perl5 +make test || exit -1 diff --git a/tests/ci_build/Dockerfile.cpu b/tests/ci_build/Dockerfile.cpu index c9ba57c6ad46..c7bb0af0f79c 100644 --- a/tests/ci_build/Dockerfile.cpu +++ b/tests/ci_build/Dockerfile.cpu @@ -8,3 +8,5 @@ COPY install/ubuntu_install_scala.sh /install/ RUN /install/ubuntu_install_scala.sh COPY install/ubuntu_install_r.sh /install/ RUN /install/ubuntu_install_r.sh +COPY install/ubuntu_install_perl.sh /install/ +RUN /install/ubuntu_install_perl.sh diff --git a/tests/ci_build/Dockerfile.gpu b/tests/ci_build/Dockerfile.gpu index cd9986ec01a2..a2893a9fb44f 100644 --- a/tests/ci_build/Dockerfile.gpu +++ b/tests/ci_build/Dockerfile.gpu @@ -8,3 +8,5 @@ COPY install/ubuntu_install_scala.sh /install/ RUN /install/ubuntu_install_scala.sh COPY install/ubuntu_install_r.sh /install/ RUN /install/ubuntu_install_r.sh +COPY install/ubuntu_install_perl.sh /install/ +RUN /install/ubuntu_install_perl.sh diff --git a/tests/ci_build/install/ubuntu_install_perl.sh b/tests/ci_build/install/ubuntu_install_perl.sh new file mode 100755 index 000000000000..da4df67a464a --- /dev/null +++ b/tests/ci_build/install/ubuntu_install_perl.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# install libraries for mxnet's perl package on ubuntu +apt-get update && apt-get install -y libmouse-perl pdl cpanminus swig libgraphviz-perl +cpanm -q Function::Parameters From 43d1d2c58fca2654818d5f16f664859234dc8dac Mon Sep 17 00:00:00 2001 From: Yao Wang Date: Fri, 4 Aug 2017 15:41:28 -0700 Subject: [PATCH 331/834] Build versioning website (#7340) * Build versioning website Fix build versioned doc More fix More fix More fix Fix Fix More fix Change url Fix more Fix more Fix Fix Fix Rollback Rollback Fix Fix Fix Fix Fix Fix Fix Fix Fix Fix Add url as command line arguments More fix Change addversion path Fix Fix Fix Fix Fix * Small changes --- docs/_static/js/navbar.js | 6 ++- docs/_static/mxnet-theme/navbar.html | 4 +- docs/_static/mxnet.css | 5 +- docs/build_version_doc/AddVersion.py | 58 ++++++++++++++++++++++ docs/build_version_doc/build_doc.sh | 74 ++++++++++++++++++++++++++++ tests/ci_build/Dockerfile.doc | 2 +- 6 files changed, 143 insertions(+), 6 deletions(-) create mode 100644 docs/build_version_doc/AddVersion.py create mode 100755 docs/build_version_doc/build_doc.sh diff --git a/docs/_static/js/navbar.js b/docs/_static/js/navbar.js index 9c3164ee18ea..91e0356d9263 100644 --- a/docs/_static/js/navbar.js +++ b/docs/_static/js/navbar.js @@ -3,6 +3,7 @@ var TITLE = ['/get_started/', '/tutorials/', '/how_to/', '/api/', '/architecture var APIsubMenu; $("#burgerMenu").children().each(function () { if($(this).children().first().html() == 'API') APIsubMenu = $(this).clone() + if($(this).children().first().html().startsWith('Versions')) VersionsubMenu = $(this).clone() }); function navbar() { @@ -38,9 +39,12 @@ function navbar() { } $("#plusMenu").empty(); for (var i = 0; i < plusMenuList.length; ++i) { - if(plusMenuList[i].html().length > 20) { + if(plusMenuList[i].attr('id') == 'dropdown-menu-position-anchor') { $("#plusMenu").append(APIsubMenu); } + else if(plusMenuList[i].attr('id') == 'dropdown-menu-position-anchor-version') { + $("#plusMenu").append(VersionsubMenu); + } else { $("#plusMenu").append("
  • "); plusMenuList[i].removeClass("main-nav-link"); diff --git a/docs/_static/mxnet-theme/navbar.html b/docs/_static/mxnet-theme/navbar.html index 1887f8cf520d..c88fb58bb5c2 100644 --- a/docs/_static/mxnet-theme/navbar.html +++ b/docs/_static/mxnet-theme/navbar.html @@ -74,7 +74,7 @@

    ☰ -
    - +
    {{searchform('', False)}} diff --git a/docs/_static/mxnet.css b/docs/_static/mxnet.css index c71d9ef6880a..6f6d8cda1351 100644 --- a/docs/_static/mxnet.css +++ b/docs/_static/mxnet.css @@ -189,7 +189,9 @@ img { text-decoration: none; } -#dropdown-menu-position-anchor { +#dropdown-menu-position-anchor, +#dropdown-menu-position-anchor-version, +#dropdown-menu-position-anchor-version-mobile { position: relative; } @@ -358,7 +360,6 @@ div .burgerIcon a { li.dropdown-submenu ul.dropdown-menu { min-width: 75px; - width: 75px } li.dropdown-submenu ul.dropdown-menu li { diff --git a/docs/build_version_doc/AddVersion.py b/docs/build_version_doc/AddVersion.py new file mode 100644 index 000000000000..ee46ef5ffd12 --- /dev/null +++ b/docs/build_version_doc/AddVersion.py @@ -0,0 +1,58 @@ +import os +import argparse +from bs4 import BeautifulSoup as bs + +parser = argparse.ArgumentParser(description="Manipulate index page", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument('--file_path', type=str, default='mxnet/docs/_build/html/', + help='file to be modified') +parser.add_argument('--current_version', type=str, default='master', + help='Current version') +parser.add_argument('--root_url', type=str, default='https://mxnet.io', + help='Root URL') + +if __name__ == '__main__': + args = parser.parse_args() + + root_url = args.root_url + tag_list = list() + with open('tag_list.txt', 'r') as tag_file: + for line in tag_file: + tag_list.append(line.lstrip().rstrip()) + tag_list.append('master') + + version_str = '' \ + 'Versions(%s)' \ + '' + + for path, subdirs, files in os.walk(args.file_path): + for name in files: + if not name.endswith('.html'): + continue + with open(os.path.join(path, name), 'r') as html_file: + content = bs(html_file, 'html.parser') + navbar = content.find(id="main-nav") + navbar_mobile = content.find(id="burgerMenu") + if navbar and navbar_mobile: + version_tag = content.find(id="dropdown-menu-position-anchor-version") + version_tag_mobile = content.find(id="dropdown-menu-position-anchor-version-mobile") + if version_tag: + version_tag.extract() + if version_tag_mobile: + version_tag_mobile.extract() + navbar.append(version_str) + navbar_mobile.append(version_str_mobile) + outstr = str(content).replace('<', '<').replace('>', '>') + with open(os.path.join(path, name), "w") as outf: + outf.write(outstr) \ No newline at end of file diff --git a/docs/build_version_doc/build_doc.sh b/docs/build_version_doc/build_doc.sh new file mode 100755 index 000000000000..046dae2d02d0 --- /dev/null +++ b/docs/build_version_doc/build_doc.sh @@ -0,0 +1,74 @@ +#!/bin/bash + +web_url="$1" +web_folder="VersionedWeb" +local_build="latest" +web_branch="$2" +git clone $web_url $web_folder +cd $web_folder +git checkout -b $web_branch "origin/$web_branch" +cd .. +mkdir "$local_build" + +# Fetch tag information +tag_list_file="tag_list.txt" +cp "$web_folder/tag.txt" "$tag_list_file" +tag_list=() +while read -r line +do + tag_list+=("$line") +done < "$tag_list_file" +latest_tag=${tag_list[0]} +echo "latest_tag is: $latest_tag" +commit_id=$(git rev-parse HEAD) +curr_tag=${TAG} +curr_tag=${curr_tag:5} +echo "Current tag is $curr_tag" +if [[ "$curr_tag" != 'master' ]] && [ $curr_tag != $latest_tag ] +then + latest_tag=$curr_tag +fi + +# Build new released tag +if [ $latest_tag != ${tag_list[0]} ] +then + echo "Building new tag" + git submodule update + make docs || exit 1 + echo -e "$latest_tag\n$(cat $tag_list_file)" > "$tag_list_file" + cat $tag_list_file + tests/ci_build/ci_build.sh doc python docs/build_version_doc/AddVersion.py --file_path "docs/_build/html/" \ + --current_version "$latest_tag" --root_url "http://mxnet.incubator.apache.org/" + cp -a "docs/_build/html/." "$local_build" + cp $tag_list_file "$local_build/tag.txt" + rm -rf "$web_folder/.git" + cp -a "$web_folder/versions/." "$local_build/versions" + mkdir "$local_build/versions/${tag_list[0]}" + cp -a "$web_folder/." "$local_build/versions/${tag_list[0]}" || exit 1 + rm -rf "$local_build/versions/${tag_list[0]}/versions" + rm -rf "$web_folder/*" + cp -a "$local_build/." "$web_folder" +fi + +# Build latest master +git checkout VersionedDoc +git checkout -- . +git submodule update +echo "Building master" +make docs || exit 1 + +rm -rfv "$web_folder/versions/master/*" +cp -a "docs/_build/html/." "$web_folder/versions/master" +tests/ci_build/ci_build.sh doc python docs/build_version_doc/AddVersion.py --file_path "$web_folder/versions/master" \ + --root_url "http://mxnet.incubator.apache.org/" + +# Update version list for all previous version website +if [ $latest_tag != ${tag_list[0]} ] +then + total=${#tag_list[*]} + for (( i=0; i<=$(( $total -1 )); i++ )) + do + tests/ci_build/ci_build.sh doc python docs/build_version_doc/AddVersion.py --file_path "$web_folder/versions/${tag_list[$i]}" \ + --current_version "${tag_list[$i]}" --root_url "http://mxnet.incubator.apache.org/" + done +fi diff --git a/tests/ci_build/Dockerfile.doc b/tests/ci_build/Dockerfile.doc index 622d946665cc..43d1fa97ac37 100644 --- a/tests/ci_build/Dockerfile.doc +++ b/tests/ci_build/Dockerfile.doc @@ -12,4 +12,4 @@ RUN wget http://downloads.lightbend.com/scala/2.11.8/scala-2.11.8.deb && \ dpkg -i scala-2.11.8.deb && rm scala-2.11.8.deb RUN apt-get install -y doxygen libatlas-base-dev graphviz pandoc -RUN pip install sphinx==1.3.5 CommonMark==0.5.4 breathe mock recommonmark pypandoc +RUN pip install sphinx==1.3.5 CommonMark==0.5.4 breathe mock recommonmark pypandoc beautifulsoup4 From 15ffc8231f3f3c870770055294c50bbeb13ac39a Mon Sep 17 00:00:00 2001 From: lxn2 Date: Fri, 4 Aug 2017 17:32:06 -0700 Subject: [PATCH 332/834] Add DISCLAIMER and lxn2 GPG keys (#7344) --- DISCLAIMER | 12 +++++++++++ KEYS | 59 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+) create mode 100644 DISCLAIMER diff --git a/DISCLAIMER b/DISCLAIMER new file mode 100644 index 000000000000..8adc57f6e6b0 --- /dev/null +++ b/DISCLAIMER @@ -0,0 +1,12 @@ +Apache MXNet (incubating) is an effort undergoing incubation at The +Apache Software Foundation (ASF), sponsored by the Apache Incubator PMC. + +Incubation is required of all newly accepted +projects until a further review indicates that the +infrastructure, communications, and decision making process have +stabilized in a manner consistent with other successful ASF +projects. + +While incubation status is not necessarily a reflection +of the completeness or stability of the code, it does indicate +that the project has yet to be fully endorsed by the ASF. diff --git a/KEYS b/KEYS index 28c497fceb68..19ec1a3e5f15 100644 --- a/KEYS +++ b/KEYS @@ -71,3 +71,62 @@ DkFmj1upWAayCoXTpKzsHBvJZPC+Wqf9Pl3O47apelg7KxU3S011YfXpVPvCTKBv kD2o/5GKWS5QkSUEUXXY1oDiLg== =f8kJ -----END PGP PUBLIC KEY BLOCK----- +pub rsa4096 2017-07-12 [SC] + 406DCA257CD2BE237B79AE6BC9D353CA4AFF2E24 +uid [ultimate] Ly Nguyen (CODE SIGNING KEY) +sig 3 C9D353CA4AFF2E24 2017-07-12 Ly Nguyen (CODE SIGNING KEY) +sub rsa4096 2017-07-12 [E] +sig C9D353CA4AFF2E24 2017-07-12 Ly Nguyen (CODE SIGNING KEY) + +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBFlmSIMBEADIr6FzNJ6o/owjqgqWdOtreIRuU47/uzNRZw8c2lEys2Fw+3CI +iUitkWpb7jR0BGLk+8yUk+1VGdXPuJ+zj8XWcCnCJ7TUy3Hudp/BrX7y388m9hP9 +3LP5yx+AUKbXRZiEr5EG2lyTmJBB5lmreVlRMs74Ie3uFtH6US/DVZMqULEtumcH +yCL30kKugUjfftO1mbx901kB0WpB705od3Wrde0Jd9sniMz4HkXMsd93gExh/s1H +3XApXes+yDIEILiUJRawgzgcPIuTyOq4bbafoiFd8ipZU0G7AQPtNUAnpTUtrUaJ +5CDGzOiqGUgwi+M3zwsRcW2MjDi9MyNTmlW2P6Gifzn3EaJ0EVdz4fPmIokC5h+H +6nMHqSPUEu0WA/qpirVrOiUku34lpkP0vZwb8UOyjgBCFTxDMPX70DuUmCbij1rr +vGM0rKLV+LFclEQFpnXckUnza8f/Zbk9T3yWcPQykXyi7+1Z1WJSPVkF4l8ynpDy +4DdUnLGdF8HZAGHdroi/jGVrH2NYy42XQqOZoLfk2BTGiFYpQem/Bfzo3OdEPBT7 +zpZUVqixtXbnGseL1sdHao1BdinIbvSpPOPEbObINenk65NtXWc+9YbauGkJ5kwd +opAkBmZC4IycFWkpmHecbGXJN61eYvARuXKAev7DeYH7g6Zuzp4n07rtIwARAQAB +tC5MeSBOZ3V5ZW4gKENPREUgU0lHTklORyBLRVkpIDxseG4yQGFwYWNoZS5vcmc+ +iQJOBBMBCgA4FiEEQG3KJXzSviN7ea5rydNTykr/LiQFAllmSIMCGwMFCwkIBwMF +FQoJCAsFFgIDAQACHgECF4AACgkQydNTykr/LiT2/Q//aW1qOLX7msuJDqhlHFIM +hCUZzWClljfCHMHZJooJY5YOcvzE5mVgwVdWjgAgZfgk/bFsNhuOb+jIqlatsNfI +Eg7sm6VjfHRo3pP1W7NN+CQNu5JnEEZAIVLy2gn+Eq1rQc7g2pfylVh/HV14TGon +OWbk7BfaZubGLtLJTIimHAPd+TrRsGsLnd9JiDZj0gsPPKV6HHXHgZoAeStIUPNX +13mN/WMDAAqroPPUfMEMXPbmJgNf/ukIFxsS/y8MwU32BjVCBvvh8ojN3RIgUJnX +chdjT9i/QVKi9TyoF20R7mR80x/P9CBwqKoN9+QuHjTPDuZkol4xD3jyzOsKHPwZ +CpltwdhI2JCYJzEIFtrZ0R59fXJ+8NNXZzIOqnx83qarC+eSf8cunqPS/ZBIvEJ0 +qM1adZlJiY96La10wXSjYnEc+XEw+dad3D3ChVsvDceJirelaAVrRS2Dz4ugNShy +W0cZFFUL0aCTNNJnF9sHAfexbbg06BTzSSAeYrEWLmmpjEYHXAtFyToHzk0jTUr4 +66SeIUVHIqBLk8yx1L9zQK38JS9usYj1PFJri9J6iYyqiIS7zRinoO8MIySZOOGp +Z3Q5xJbnwzjwl4frGaXg2/zyD7rfQGG3P23WOselgNWMKuYtVAA+AHo/CxLIinKk +JAMljesV3vfeawK5HHnfcgK5Ag0EWWZIgwEQAMsmr5lOFe4n9iGdTciYFXxZYSEX +ZqmtWyxNsXkih2icfohygx/YLFBSkdXSfIywS7w7+Na4OYdhp3uaRdU+yA4ianY7 +qH5guni98KtyZmsRnnjT1DgyR0pNNqAdAyfWeCglMx5SWLLtzKxHazqF0t6Jb6M/ +sAew+KdoTXsYzKb9d/R81spvefJoBopaxKLF1tijaX98RiquKLlFBD+88XP6pxSB +nwNxNybgJVlGT/RdxPiRiRj0CySuvx27i8w8Rc2HaT9CFumzdy6moz+RJbuuIjDN +QzIOpNy4+LJKSysPGh8AwRu6xCl9gnfbJ9thiFwYGZ7S3lVvS23/poI1YzLZZY+5 +XvpiiogF7j5Aj/zTTli8BI/CiNVrGKJuzeJJyLFfBMmrbysi9mV/fR8wC7xd5P9g +LjElkA4j1Xv5I47AVsILAbHLhphpxNDoKBmr1EbP/CJitEYjRmdjn4Mo6sYwMlVN +CA+rl/VMS3Nc0Iixu/Y070H3kE9IfitksiuXIJfeX5RW/uWegEO1e1dSpi+rreb8 +lvVtQk4tMUHyM16qPqO08tPGSunt6J0HiPi7J+xDwbJjJS7gNDW4AYHG5q4/dZsx +PtpcZC7zFOlFV0BwFftYnluccDhsWPc48mDmmhOe9p42irMAx6ms/Y42jgh4OmgD +bjMzKIyYFI40URGnABEBAAGJAjYEGAEKACAWIQRAbcolfNK+I3t5rmvJ01PKSv8u +JAUCWWZIgwIbDAAKCRDJ01PKSv8uJCAtD/97SuVGnCP3kbWfI/qfTTVKwuWTdbIg +rPvOjGo5F57l1PAgARt8N1ccqREbR3JwhRdsU3ewz5eDQEyEZVffPgufhqZr8liI +EP783m83VgRSMKYt6HzORX0os2BapsHHuejvlME9XpN0UG5AnvbzXDxP3wJufB1K +GkmC+rlpqfyMu60xFXzym9QuePksbdf/xXZduvLGaB1u+AYtvHp3+NGV382vat7C +xwRShVJTb8Zr9y5tA+JDqfhDDb5CepcPH6Uk2frU8aV7vZ3hmVmGcDcUddu3U9hg +L7Lcpr1E0D7xOuQ4QMAFhcDO+aB8aPv+JRkH4Y6wDFPrEgcEJ1YK6hhW5KSdslyK +QrKHKMSl+hwPmh9fKX4wC+FjMMXJ/PHtEG3N3f7/TyyO4iza5xDIJkYcyKkDXc0l +VcHLJvtjsJziMJNV3lKAeTp/uzbaJHRhLmpPHukQPnlpjfhnmsYh3wydnd03pfzQ +k6XJ4iGeSSQqtW6T14yqkCl5HDH2ms1ufhe4Os217CMXnaRbM/K6Zl4iGGozzXgd +no02+jTN3NqmUw0hUBR/9ZEn+IKmZ6f0Azsgio0M9ez1T0CCDZvo19kJw9b3VdOF +TZQhIRekaaV+bCQQxnwDOJ31bIUUpxaMdvygjq55Gri/5C75TsMNcgbhqYWLGKe2 +kRsGTxyO+fQ6/Q== +=FuXU +-----END PGP PUBLIC KEY BLOCK----- From 0d8d27ece04613197711be0843af0de79822aa3b Mon Sep 17 00:00:00 2001 From: Yao Wang Date: Fri, 4 Aug 2017 22:03:58 -0700 Subject: [PATCH 333/834] Fix gluon bottleneck v2 (#7339) * Fix Gluon Resnet BottleneckV2 * Fix --- python/mxnet/gluon/model_zoo/vision/resnet.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/mxnet/gluon/model_zoo/vision/resnet.py b/python/mxnet/gluon/model_zoo/vision/resnet.py index 5e2adad52781..48ba07941acb 100644 --- a/python/mxnet/gluon/model_zoo/vision/resnet.py +++ b/python/mxnet/gluon/model_zoo/vision/resnet.py @@ -177,11 +177,11 @@ class BottleneckV2(HybridBlock): def __init__(self, channels, stride, downsample=False, in_channels=0, **kwargs): super(BottleneckV2, self).__init__(**kwargs) self.bn1 = nn.BatchNorm() - self.conv1 = _conv3x3(channels//4, 1, in_channels) + self.conv1 = nn.Conv2D(channels//4, kernel_size=1, strides=1, use_bias=False) self.bn2 = nn.BatchNorm() self.conv2 = _conv3x3(channels//4, stride, channels//4) self.bn3 = nn.BatchNorm() - self.conv3 = _conv3x3(channels, 1, channels//4) + self.conv3 = nn.Conv2D(channels, kernel_size=1, strides=1, use_bias=False) if downsample: self.downsample = nn.Conv2D(channels, 1, stride, use_bias=False, in_channels=in_channels) From b2360f4ff5cccbcf09a780d8065ecc4a64ce470f Mon Sep 17 00:00:00 2001 From: "Qiang Kou (KK)" Date: Sat, 5 Aug 2017 06:34:03 +0000 Subject: [PATCH 334/834] [R] fix mx.symbol.min. close #7219 (#7342) --- R-package/R/symbol.R | 11 +++++++++++ R-package/src/export.cc | 5 ++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/R-package/R/symbol.R b/R-package/R/symbol.R index 541cce456f9f..b97b19394209 100644 --- a/R-package/R/symbol.R +++ b/R-package/R/symbol.R @@ -60,6 +60,17 @@ mx.symbol.Concat <- function(data, num.args, dim = NULL, name = NULL) { mx.symbol.concat(data, num.args, dim, name) } +#' @export +mx.symbol.min <- function(e1, e2) { + if (is.mx.symbol(e1) && is.mx.symbol(e2)) { + mx.varg.symbol.internal.minimum(list(e1, e2)) + } else if (is.mx.symbol(e1)) { + mx.varg.symbol.internal.minimum_scalar(list(e1, scalar = e2)) + } else if (is.mx.symbol(e2)) { + mx.varg.symbol.internal.minimum_scalar(list(e2, scalar = e1)) + } +} + #' Save an mx.symbol object #' #' @param symbol the \code{mx.symbol} object diff --git a/R-package/src/export.cc b/R-package/src/export.cc index 0e77c1c3b58e..2377a02fbc86 100644 --- a/R-package/src/export.cc +++ b/R-package/src/export.cc @@ -93,7 +93,10 @@ void Exporter::Export(const std::string& path) { std::string fname = Rcpp::as(func_names[i]); // skip internal functions if (fname.find("internal.") != std::string::npos) continue; - if (fname == "mx.varg.symbol.Concat" || fname == "mx.varg.symbol.concat") continue; + if (fname == "mx.varg.symbol.Concat" + || fname == "mx.varg.symbol.concat" + || fname == "mx.varg.symbol.min_axis" + || fname == "mx.varg.symbol.min") continue; Rcpp::List func_info(scope->get_function(fname)); std::string docstr = Rcpp::as(func_info[2]); if (docstr.find("@export") == std::string::npos) continue; From be0579ce7519cd910dab8c0261df7212d155d0b1 Mon Sep 17 00:00:00 2001 From: Eric Junyuan Xie Date: Sat, 5 Aug 2017 21:28:24 -0700 Subject: [PATCH 335/834] refactor gluon trainer (#7338) * fix optimizer * Update trainer.py --- python/mxnet/gluon/parameter.py | 2 +- python/mxnet/gluon/trainer.py | 59 +++++++++++-------- python/mxnet/optimizer.py | 12 +++- .../unittest/{test_nn.py => test_gluon.py} | 23 ++++++++ 4 files changed, 69 insertions(+), 27 deletions(-) rename tests/python/unittest/{test_nn.py => test_gluon.py} (93%) diff --git a/python/mxnet/gluon/parameter.py b/python/mxnet/gluon/parameter.py index 0ae829ab79ec..bdc967490dc5 100644 --- a/python/mxnet/gluon/parameter.py +++ b/python/mxnet/gluon/parameter.py @@ -361,7 +361,7 @@ class ParameterDict(object): """ def __init__(self, prefix='', shared=None): self._prefix = prefix - self._params = {} + self._params = OrderedDict() self._shared = shared def __getitem__(self, key): diff --git a/python/mxnet/gluon/trainer.py b/python/mxnet/gluon/trainer.py index 5483f6bc7d9c..e8aae71cfd99 100644 --- a/python/mxnet/gluon/trainer.py +++ b/python/mxnet/gluon/trainer.py @@ -15,14 +15,19 @@ class Trainer(object): params : ParameterDict The set of parameters to optimize. optimizer : str or Optimizer - The optimizer to use. + The optimizer to use. See + `help `_ + on Optimizer for a list of available optimizers. optimizer_params : dict Key-word arguments to be passed to optimizer constructor. For example, - `{'learning_rate': 0.1}` + `{'learning_rate': 0.1}`. All optimizers accept learning_rate, wd (weight decay), + clip_gradient, and lr_scheduler. See each optimizer's + constructor for a list of additional supported arguments. kvstore : str or KVStore - kvstore type for multi-gpu and distributed training. + kvstore type for multi-gpu and distributed training. See help on + :any:`mxnet.kvstore.create` for more information. """ - def __init__(self, params, optimizer, optimizer_params, kvstore='device'): + def __init__(self, params, optimizer, optimizer_params=None, kvstore='device'): if isinstance(params, (dict, ParameterDict)): params = list(params.values()) if not isinstance(params, (list, tuple)): @@ -35,9 +40,9 @@ def __init__(self, params, optimizer, optimizer_params, kvstore='device'): raise ValueError( "First argument must be a list or dict of Parameters, " \ "got list of %s."%(type(param))) - if param.grad_req != 'null': - self._params.append(param) + self._params.append(param) + optimizer_params = optimizer_params if optimizer_params else {} self._scale = optimizer_params.get('rescale_grad', 1.0) self._contexts = self._check_contexts() self._init_optimizer(optimizer, optimizer_params) @@ -56,32 +61,39 @@ def _check_contexts(self): return contexts def _init_optimizer(self, optimizer, optimizer_params): - self._optimizer = opt.create(optimizer, **optimizer_params) - - lr_mult = {} - wd_mult = {} - for i, param in enumerate(self._params): - lr_mult[i] = param.lr_mult - wd_mult[i] = param.wd_mult - self._optimizer.set_lr_mult(lr_mult) - self._optimizer.set_wd_mult(wd_mult) + param_dict = {i: param for i, param in enumerate(self._params)} + if isinstance(optimizer, opt.Optimizer): + assert not optimizer_params, \ + "optimizer_params must be None if optimizer is an instance of " \ + "Optimizer instead of str" + self._optimizer = optimizer + self._optimizer.param_dict = param_dict + else: + self._optimizer = opt.create(optimizer, param_dict=param_dict, + **optimizer_params) self._updaters = [opt.get_updater(self._optimizer) \ for _ in self._contexts] def _init_kvstore(self): arg_arrays = {param.name: param.data(self._contexts[0]) for param in self._params} - kvstore, update_on_kvstore = _create_kvstore(self._kvstore, len(self._contexts), arg_arrays) - self._kvstore = kvstore - self._update_on_kvstore = update_on_kvstore + kvstore, update_on_kvstore = _create_kvstore(self._kvstore, len(self._contexts), + arg_arrays) if kvstore: - assert 'dist' not in self._kvstore.type, "distributed training not supported yet" + if 'dist' in kvstore.type: + update_on_kvstore = False for i, param in enumerate(self._params): param_arrays = param.list_data() kvstore.init(i, param_arrays[0]) kvstore.pull(i, param_arrays, priority=-i) if update_on_kvstore: kvstore.set_optimizer(self._optimizer) + self._kvstore = kvstore + self._update_on_kvstore = update_on_kvstore + else: + self._kvstore = None + self._update_on_kvstore = None + self._kv_initialized = True def step(self, batch_size, ignore_stale_grad=False): @@ -103,9 +115,8 @@ def step(self, batch_size, ignore_stale_grad=False): self._optimizer.rescale_grad = self._scale / batch_size for i, param in enumerate(self._params): - assert param.list_ctx() == self._contexts, \ - "Parameter %s's contexts changed after Optim initialization: " \ - "was %s, now %s"%(param.name, self._contexts, param.list_ctx()) + if param.grad_req == 'null': + continue if not ignore_stale_grad: for data in param.list_data(): if not data._fresh_grad: @@ -117,6 +128,7 @@ def step(self, batch_size, ignore_stale_grad=False): "call step with ignore_stale_grad=True to suppress this " "warning and skip updating of Parameters with stale gradient" \ %(param.name, str(data.context))) + if self._kvstore: self._kvstore.push(i, param.list_grad(), priority=-i) if self._update_on_kvstore: @@ -124,7 +136,8 @@ def step(self, batch_size, ignore_stale_grad=False): continue else: self._kvstore.pull(i, param.list_grad(), priority=-i) + for upd, arr, grad in zip(self._updaters, param.list_data(), param.list_grad()): - if arr._fresh_grad: + if not ignore_stale_grad or arr._fresh_grad: upd(i, grad, arr) arr._fresh_grad = False diff --git a/python/mxnet/optimizer.py b/python/mxnet/optimizer.py index 57fadf44335e..934566ec9d17 100644 --- a/python/mxnet/optimizer.py +++ b/python/mxnet/optimizer.py @@ -43,7 +43,8 @@ class Optimizer(object): """ def __init__(self, rescale_grad=1., param_idx2name=None, wd=0., clip_gradient=None, learning_rate=0.01, - lr_scheduler=None, sym=None, begin_num_update=0): + lr_scheduler=None, sym=None, begin_num_update=0, + param_dict=None): self.rescale_grad = rescale_grad self.lr = learning_rate self.lr_scheduler = lr_scheduler @@ -64,6 +65,7 @@ def __init__(self, rescale_grad=1., param_idx2name=None, wd=0., 'param_idx2name should be a dict of param indexes to names.' self.idx2name = param_idx2name.copy() self.sym = sym + self.param_dict = param_dict if param_dict else {} self.set_lr_mult({}) self.set_wd_mult({}) @@ -277,7 +279,9 @@ def _get_lr(self, index): else: lr = self.lr - if index in self.lr_mult: + if index in self.param_dict: + lr *= self.param_dict[index].lr_mult + elif index in self.lr_mult: lr *= self.lr_mult[index] elif index in self.idx2name: lr *= self.lr_mult.get(self.idx2name[index], 1.0) @@ -298,7 +302,9 @@ def _get_wd(self, index): Weight decay for this index. """ wd = self.wd - if index in self.wd_mult: + if index in self.param_dict: + wd *= self.param_dict[index].wd_mult + elif index in self.wd_mult: wd *= self.wd_mult[index] elif index in self.idx2name: wd *= self.wd_mult.get(self.idx2name[index], 1.0) diff --git a/tests/python/unittest/test_nn.py b/tests/python/unittest/test_gluon.py similarity index 93% rename from tests/python/unittest/test_nn.py rename to tests/python/unittest/test_gluon.py index e29306326e0c..8256c719c386 100644 --- a/tests/python/unittest/test_nn.py +++ b/tests/python/unittest/test_gluon.py @@ -302,6 +302,29 @@ def test_flatten(): assert flatten(x).shape == (3, 1) +def test_trainer(): + x = gluon.Parameter('x', shape=(10,)) + x.initialize(ctx=[mx.cpu(0), mx.cpu(1)], init='zeros') + trainer = gluon.Trainer([x], 'sgd', {'learning_rate': 1.0}) + with mx.autograd.record(): + for w in x.list_data(): + y = w + 1 + y.backward() + trainer.step(1) + + assert (x.data(mx.cpu(1)).asnumpy() == -2).all() + + x.lr_mult = 0.5 + + with mx.autograd.record(): + for w in x.list_data(): + y = w + 1 + y.backward() + trainer.step(1) + + assert (x.data(mx.cpu(1)).asnumpy() == -3).all() + + if __name__ == '__main__': import nose nose.runmodule() From b0c8c6c6e316d5d9b1a54146ba88caa63bfe4dc0 Mon Sep 17 00:00:00 2001 From: Sergey Kolychev Date: Sun, 6 Aug 2017 13:28:35 -0700 Subject: [PATCH 336/834] [Perl] Fix for CI (#7343) * auto determiming the mxnet home. * autograd needs rework. --- Jenkinsfile | 4 +- perl-package/AI-MXNet/t/test_autograd.t | 96 ------------------------- perl-package/test.sh | 11 ++- 3 files changed, 7 insertions(+), 104 deletions(-) delete mode 100644 perl-package/AI-MXNet/t/test_autograd.t diff --git a/Jenkinsfile b/Jenkinsfile index b3bf82689e56..49633d484981 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -287,7 +287,7 @@ try { init_git() unpack_lib('cpu') timeout(time: max_time, unit: 'MINUTES') { - sh "${docker_run} cpu ./perl-package/test.sh /workspace/ut-perl-cpu /workspace/ut-perl-cpu" + sh "${docker_run} cpu ./perl-package/test.sh" } } } @@ -298,7 +298,7 @@ try { init_git() unpack_lib('gpu') timeout(time: max_time, unit: 'MINUTES') { - sh "${docker_run} gpu ./perl-package/test.sh /workspace/ut-perl-gpu /workspace/ut-perl-gpu" + sh "${docker_run} gpu ./perl-package/test.sh" } } } diff --git a/perl-package/AI-MXNet/t/test_autograd.t b/perl-package/AI-MXNet/t/test_autograd.t deleted file mode 100644 index 60cfd3b8bf98..000000000000 --- a/perl-package/AI-MXNet/t/test_autograd.t +++ /dev/null @@ -1,96 +0,0 @@ -use strict; -use warnings; -use AI::MXNet qw(mx); -use AI::MXNet::TestUtils qw(same zip); -use Test::More tests => 31; - -sub autograd_assert -{ - my ($args, $kwargs) = @_; - my $func = $kwargs->{func}; - my $grad_f = $kwargs->{grad_func}; - my $argnum = $kwargs->{argnum}; - - my $grad_func = mx->contrib->autograd->grad_and_loss($func, $argnum); - my ($grad_vals, $output) = $grad_func->(@$args); - my $res = $func->(@$args); - ok(same($output->aspdl, $res->aspdl)); - my $grad_res = &{$grad_f}(@$args); - is(scalar(@$grad_vals), scalar(@$grad_res)); - zip(sub { - ok(same($_[0]->aspdl, $_[1]->aspdl)); - }, $grad_vals, $grad_res); -} - -sub test_unary_func -{ - my $x = mx->nd->uniform({ shape=>[4, 5] }); - my $f_exp = sub { $_[0]->exp }; - my $f_exp_grad = sub { [$_[0]->exp] }; - autograd_assert([$x], { func=>$f_exp, grad_func=>$f_exp_grad }); - my $f_half = sub { $_[0]/2 }; - my $f_half_grad = sub { [mx->nd->ones($_[0]->shape) * 0.5] }; - autograd_assert([$x], { func=>$f_half, grad_func=>$f_half_grad }); - my $f_square = sub { $_[0]**2 }; - my $f_square_grad = sub { [2*$_[0]] }; - autograd_assert([$x],{ func=>$f_square, grad_func=>$f_square_grad }); -} - -test_unary_func(); - -sub test_binary_func -{ - my $x = mx->nd->uniform({ shape=>[4, 5] }); - my $y = mx->nd->uniform({ shape=>[4, 5] }); - my $f_add = sub { $_[0] + $_[1] }; - my $f_add_grad = sub { [mx->nd->ones($_[0]->shape), mx->nd->ones($_[1]->shape)] }; - autograd_assert([$x, $y], { func=>$f_add, grad_func=>$f_add_grad }); - my $f_mul = sub { $_[0] * $_[1] }; - my $f_mul_grad = sub { [$_[1], $_[0]] }; - autograd_assert([$x, $y], { func=>$f_mul, grad_func=>$f_mul_grad }); - my $f_compose = sub { $_[0] + $_[0]*$_[1] }; - my $f_compose_grad = sub { [mx->nd->ones($_[0]->shape) + $_[1], $_[0]] }; - autograd_assert([$x, $y], { func=>$f_compose, grad_func=>$f_compose_grad }); -} - -test_binary_func(); - -sub test_argnum -{ - - my $f_with_mode = sub { my ($a, $b, $mode) = @_; - if($mode) - { - return $a+$b; - } - else - { - return $a*$b - } - }; - - my $a = mx->nd->uniform({ shape=>[3, 2] }); - my $b = mx->nd->uniform({ shape=>[3, 2] }); - my $f_add_grad = sub { [mx->nd->ones($_[0]->shape), mx->nd->ones($_[1]->shape)] }; - my $f_mul_grad = sub { [$_[1], $_[0]] }; - autograd_assert([$a, $b, 1], - { argnum=>[0, 1], func=>$f_with_mode, grad_func=>$f_add_grad }); - autograd_assert([$a, $b, 0], - { argnum=>[0, 1], func=>$f_with_mode, grad_func=>$f_mul_grad }); -} - -test_argnum(); - -sub test_training -{ - my $x = mx->nd->ones([10, 10]); - mx->contrib->autograd->set_is_training(1); - my $y = mx->nd->Dropout($x, { p=>0.5 }); - ok(not ($y->aspdl== $x->aspdl)->all); - mx->contrib->autograd->set_is_training(0); - $y = mx->nd->Dropout($x, { p=>0.5 }); - ok(($y->aspdl== $x->aspdl)->all); -} - -test_training(); - diff --git a/perl-package/test.sh b/perl-package/test.sh index c83120f31546..5aef8e6b82b1 100755 --- a/perl-package/test.sh +++ b/perl-package/test.sh @@ -1,16 +1,15 @@ -MXNET_HOME=$1 -HOME=$2 +MXNET_HOME=${PWD} export LD_LIBRARY_PATH=${MXNET_HOME}/lib -export PERL5LIB=${HOME}/perl5/lib/perl5 +export PERL5LIB=${MXNET_HOME}/perl5/lib/perl5 cd ${MXNET_HOME}/perl-package/AI-MXNetCAPI/ -perl Makefile.PL INSTALL_BASE=${HOME}/perl5 +perl Makefile.PL INSTALL_BASE=${MXNET_HOME}/perl5 make install || exit -1 cd ${MXNET_HOME}/perl-package/AI-NNVMCAPI/ -perl Makefile.PL INSTALL_BASE=${HOME}/perl5 +perl Makefile.PL INSTALL_BASE=${MXNET_HOME}/perl5 make install || exit -1 cd ${MXNET_HOME}/perl-package/AI-MXNet/ -perl Makefile.PL INSTALL_BASE=${HOME}/perl5 +perl Makefile.PL INSTALL_BASE=${MXNET_HOME}/perl5 make test || exit -1 From 2f257e8197ba23bf5e25db0feb050b1c8bb79072 Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Sun, 6 Aug 2017 23:20:59 -0700 Subject: [PATCH 337/834] fix build (#7358) --- .../core/src/test/scala/ml/dmlc/mxnet/OperatorSuite.scala | 2 +- tests/python/gpu/test_operator_gpu.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scala-package/core/src/test/scala/ml/dmlc/mxnet/OperatorSuite.scala b/scala-package/core/src/test/scala/ml/dmlc/mxnet/OperatorSuite.scala index ac1cee202e5b..86f04366a938 100644 --- a/scala-package/core/src/test/scala/ml/dmlc/mxnet/OperatorSuite.scala +++ b/scala-package/core/src/test/scala/ml/dmlc/mxnet/OperatorSuite.scala @@ -37,7 +37,7 @@ class OperatorSuite extends FunSuite with BeforeAndAfterAll exec.forward() val forwardOutput = exec.outputs(0) val forwardOutputExpected = arr.reduce(_ + _) - assert(reldiff(forwardOutput, forwardOutputExpected) < 1e-6) + assert(reldiff(forwardOutput, forwardOutputExpected) < 2e-6) // backward val outGrad = Random.uniform(-10, 10, shape) diff --git a/tests/python/gpu/test_operator_gpu.py b/tests/python/gpu/test_operator_gpu.py index 3c319f84bf29..6fef4e24a24c 100644 --- a/tests/python/gpu/test_operator_gpu.py +++ b/tests/python/gpu/test_operator_gpu.py @@ -11,7 +11,7 @@ from test_operator import * from test_optimizer import * from test_random import * -from test_nn import * +from test_gluon import * #from test_rnn import * from test_gluon_rnn import * From c1985725c4a877a3658cafd4e791aafb4c063e55 Mon Sep 17 00:00:00 2001 From: Yao Wang Date: Mon, 7 Aug 2017 10:08:11 -0700 Subject: [PATCH 338/834] Fix data tutorial (#7329) --- docs/tutorials/basic/data.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/tutorials/basic/data.md b/docs/tutorials/basic/data.md index dba13918aa0e..93a1db066a8c 100644 --- a/docs/tutorials/basic/data.md +++ b/docs/tutorials/basic/data.md @@ -19,7 +19,7 @@ $ pip install opencv-python requests matplotlib jupyter ``` $ git clone https://github.com/dmlc/mxnet ~/mxnet -$ MXNET_HOME = '~/mxnet' +$ export MXNET_HOME='~/mxnet' ``` ## MXNet Data Iterator @@ -366,7 +366,7 @@ Now let's convert them into record io format using the `im2rec.py` utility scrip First, we need to make a list that contains all the image files and their categories: ```python -os.system('python %s/tools/im2rec.py --list=1 --recursive=1 --shuffle=1 --test-ratio=0.2 data/caltech data/101_ObjectCategories'%MXNET_HOME) +os.system('python %s/tools/im2rec.py --list=1 --recursive=1 --shuffle=1 --test-ratio=0.2 data/caltech data/101_ObjectCategories'%os.environ['MXNET_HOME']) ``` The resulting list file (./data/caltech_train.lst) is in the format `index\t(one or more label)\tpath`. In this case, there is only one label for each image but you can modify the list to add in more for multi-label training. @@ -375,7 +375,7 @@ Then we can use this list to create our record io file: ```python -os.system("python %s/tools/im2rec.py --num-thread=4 --pass-through=1 data/caltech data/101_ObjectCategories"%MXNET_HOME) +os.system("python %s/tools/im2rec.py --num-thread=4 --pass-through=1 data/caltech data/101_ObjectCategories"%os.environ['MXNET_HOME']) ``` The record io files are now saved at here (./data) From ef383c28bd4fdd69b2a489bb51c86dcc73b7b104 Mon Sep 17 00:00:00 2001 From: "Joshua Z. Zhang" Date: Mon, 7 Aug 2017 13:43:20 -0700 Subject: [PATCH 339/834] disable vgg-16 resnet converter check (#7369) --- tools/caffe_converter/test_converter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/caffe_converter/test_converter.py b/tools/caffe_converter/test_converter.py index c7eb86b2e437..7572d2937186 100644 --- a/tools/caffe_converter/test_converter.py +++ b/tools/caffe_converter/test_converter.py @@ -78,7 +78,7 @@ def main(): assert gpus, 'At least one GPU is needed to run test_converter in GPU mode' batch_size = 32 * len(gpus) - models = ['bvlc_googlenet', 'vgg-16', 'resnet-50'] + models = ['bvlc_googlenet'] val = download_data() for m in models: From 63ae4c9865d19bff68ddf0f00de5f5ebe88466eb Mon Sep 17 00:00:00 2001 From: Yao Wang Date: Mon, 7 Aug 2017 14:15:33 -0700 Subject: [PATCH 340/834] Small fix for versioning doc build (#7371) --- docs/build_version_doc/build_doc.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/build_version_doc/build_doc.sh b/docs/build_version_doc/build_doc.sh index 046dae2d02d0..99b6bd81b517 100755 --- a/docs/build_version_doc/build_doc.sh +++ b/docs/build_version_doc/build_doc.sh @@ -6,7 +6,7 @@ local_build="latest" web_branch="$2" git clone $web_url $web_folder cd $web_folder -git checkout -b $web_branch "origin/$web_branch" +git checkout $web_branch cd .. mkdir "$local_build" @@ -51,7 +51,7 @@ then fi # Build latest master -git checkout VersionedDoc +git checkout master git checkout -- . git submodule update echo "Building master" From 7e24097be4505dda3abad7f8939cf60cf41fd34d Mon Sep 17 00:00:00 2001 From: lxn2 Date: Mon, 7 Aug 2017 17:07:45 -0700 Subject: [PATCH 341/834] Seed numpy random (#7372) --- tests/python/unittest/test_operator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py index 7007da6a2910..62a064a49e0d 100644 --- a/tests/python/unittest/test_operator.py +++ b/tests/python/unittest/test_operator.py @@ -1986,7 +1986,8 @@ def test_instance_normalization(): def check_l2_normalization(in_shape, mode, ctx=default_context(), norm_eps=1e-10): data = mx.symbol.Variable('data') out = mx.symbol.L2Normalization(data=data, mode=mode, eps=norm_eps) - np.random.seed() + # TODO(szha): Seeding this masks failures. We need to do a deep dive for failures without this seed. + np.random.seed(1234) in_data = np.random.uniform(-1, 1, in_shape) # calculate numpy results if mode == 'channel': From ada6d4e0bbfb6a244a868c8ef6edf40529dd996d Mon Sep 17 00:00:00 2001 From: Eric Junyuan Xie Date: Mon, 7 Aug 2017 19:41:25 -0700 Subject: [PATCH 342/834] Update custom.cc (#7373) --- src/operator/custom/custom.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/operator/custom/custom.cc b/src/operator/custom/custom.cc index ee420635f824..5a40be92b68e 100644 --- a/src/operator/custom/custom.cc +++ b/src/operator/custom/custom.cc @@ -268,13 +268,13 @@ void Forward(const OpStatePtr& state, tags.push_back(4); } - bool old = autograd::AutogradRuntime::Get()->SetIsTraining(false); + bool old = autograd::AutogradRuntime::Get()->SetIsRecording(false); CHECK(reinterpret_cast(params.info->callbacks[kCustomOpForward])( ptrs.size(), ptrs.data(), tags.data(), reinterpret_cast(req.data()), static_cast(ctx.is_train), params.info->contexts[kCustomOpForward])); - autograd::AutogradRuntime::Get()->SetIsTraining(old); + autograd::AutogradRuntime::Get()->SetIsRecording(old); } @@ -312,13 +312,13 @@ void Backward(const OpStatePtr& state, tags.push_back(4); } - bool old = autograd::AutogradRuntime::Get()->SetIsTraining(false); + bool old = autograd::AutogradRuntime::Get()->SetIsRecording(false); CHECK(reinterpret_cast(params.info->callbacks[kCustomOpBackward])( - ptrs.size(), ptrs.data(), tags.data(), reinterpret_cast(req.data()), 1, - params.info->contexts[kCustomOpBackward])); + ptrs.size(), ptrs.data(), tags.data(), reinterpret_cast(req.data()), + static_cast(ctx.is_train), params.info->contexts[kCustomOpBackward])); - autograd::AutogradRuntime::Get()->SetIsTraining(old); + autograd::AutogradRuntime::Get()->SetIsRecording(old); } From d08abf54e90382888984ddae70b0da97b4ce9876 Mon Sep 17 00:00:00 2001 From: Yash Date: Tue, 8 Aug 2017 10:27:22 -0700 Subject: [PATCH 343/834] Docs for GAN (#7378) * gan.md * fix caps * fix formatting * title change --- docs/tutorials/unsupervised_learning/gan.md | 386 +++++++++++++++++++- 1 file changed, 382 insertions(+), 4 deletions(-) diff --git a/docs/tutorials/unsupervised_learning/gan.md b/docs/tutorials/unsupervised_learning/gan.md index 6491806c0acc..709e1323c6f6 100644 --- a/docs/tutorials/unsupervised_learning/gan.md +++ b/docs/tutorials/unsupervised_learning/gan.md @@ -1,5 +1,383 @@ -# Generative Adversarial Network -Get the source code for an example of a generative adversarial network (GAN) running on MXNet on GitHub in the [gan](https://github.com/dmlc/mxnet/tree/master/example/gan) folder. +# Generative Adversarial Networks -## Next Steps -* [MXNet tutorials index](http://mxnet.io/tutorials/index.html) \ No newline at end of file +GANs are an application of unsupervised learning - you don't need labels for your dataset in order to train a GAN. + +The GAN framework composes of two neural networks: a generator network and a discriminator network. + +The generator's job is to take a set of random numbers and produce data (such as images or text). + +The discriminator then takes in that data as well as samples of that data from a dataset and tries to determine if is "fake" (created by the generator network) or "real" (from the original dataset). + +During training, the two networks play a game against each other. The generator tries to create realistic data, so that it can fool the discriminator into thinking that the data it generated is from the original dataset. At the same time, the discriminator tries to not be fooled - it learns to become better at determining if data is real or fake. + +Since the two networks are fighting in this game, they can be seen as as adversaries, which is where the term "Generative Adverserial Network" comes from. + +## Deep Convolutional Generative Adversarial Networks + +This tutorial takes a look at Deep Convolutional Generative Adversarial Networks (DCGAN), which combines Convolutional Neural Networks (CNNs) and GANs. + +We will create a DCGAN that is able to create images of handwritten digits from random numbers.The tutorial uses the neural net architecture and guidelines outlined in [this paper](https://arxiv.org/abs/1511.06434), and the MNIST dataset. + +##How to Use This Tutorial +You can use this tutorial by executing each snippet of python code in order as it appears in the tutorial. + + +1. The first net is the "generator" and creates images of handwritten digits from random numbers. +2. The second net is the "discriminator" and determines if the image created by the generator is real (a realistic looking image of handwritten digits) or fake (an image that doesn't look like it came from the original dataset). + +Apart from creating a DCGAN, you'll also learn: + +- How to manipulate and iterate through batches images that you can feed into your neural network. + +- How to create a custom MXNet data iterator that generates random numbers from a normal distribution. + +- How to create a custom training process in MXNet, using lower level functions from the MXNet Module API such as .bind() .forward() and .backward(). The training process for a DCGAN is more complex than many other neural net's, so we need to use these functions instead of using the higher level .fit() function. + +- How to visualize images as they are going through the training process + +## Prerequisites + +This tutorial assumes you're familiar with the concept of CNN's and have implemented one in MXNet. You should also be familiar with the concept of logistic regression. Having a basic understanding for MXNet data iterators helps, since we'll create a custom Data Iterator to iterate though random numbers as inputs to our generator network. + +This example is designed to be trained on a single GPU. Training this network on CPU can be slow, so it's recommended that you use a GPU for training. + +To complete this tutorial, you need: + +- MXNet +- Python 2.7, and the following libraries for Python: + - Numpy - for matrix math + - OpenCV - for image manipulation + - Scikit-learn - to easily get our dataset + - Matplotlib - to visualize our output + +## The Data +We need two pieces of data to train our DCGAN: + 1. Images of handwritten digits from the MNIST dataset + 2. Random numbers from a normal distribution + +Our generator network will use the random numbers as the input to produce images of handwritten digits, and out discriminator network will use images of handwritten digits from the MNIST dataset to determine if images produced by our generator are realistic. + +We are going to use the python library, scikit-learn, to get the MNIST dataset. Scikit-learn comes with a function that gets the dataset for us, which we will then manipulate to create our training and testing inputs. + +The MNIST dataset contains 70,000 images of handwritten digits. Each image is 28x28 pixels in size. To create random numbers, we're going to create a custom MXNet data iterator, which will returns random numbers from a normal distribution as we need then. + +## Prepare the Data + +### 1. Preparing the MNSIT dataset + +Let's start by preparing our handwritten digits from the MNIST dataset. We import the fetch_mldata function from scikit-learn, and use it to get the MNSIT dataset. Notice that it's shape is 70000x784. This contains the 70000 images on every row and 784 pixels of each image in the columns of each row. Each image is 28x28 pixels, but has been flattened so that all 784 images are represented in a single list. +```python +from sklearn.datasets import fetch_mldata +mnist = fetch_mldata('MNIST original') +``` + +Next, we'll randomize the handwritten digits by using numpy to create random permutations on the dataset on our rows (images). We'll then reshape our dataset from 70000x786 to 70000x28x28, so that every image in our dataset is arranged into a 28x28 grid, where each cell in the grid represents 1 pixel of the image. + +```python +import numpy as np +#Use a seed so that we get the same random permutation each time +np.random.seed(1) +p = np.random.permutation(mnist.data.shape[0]) +X = mnist.data[p] +X = X.reshape((70000, 28, 28)) +``` +Since the DCGAN that we're creating takes in a 64x64 image as the input, we'll use OpenCV to resize the each 28x28 image to 64x64 images: +```python +import cv2 +X = np.asarray([cv2.resize(x, (64,64)) for x in X]) +``` +Each pixel in our 64x64 image is represented by a number between 0-255, that represents the intensity of the pixel. However, we want to input numbers between -1 and 1 into our DCGAN, as suggested by the research paper. To rescale our pixels to be in the range of -1 to 1, we'll divide each pixel by (255/2). This put our images on a scale of 0-2. We can then subtract by 1, to get them in the range of -1 to 1. +```python +X = X.astype(np.float32)/(255.0/2) - 1.0 +``` +Ultimately, images are inputted into our neural net from a 70000x3x64x64 array, and they are currently in a 70000x64x64 array. We need to add 3 channels to our images. Typically when we are working with images, the 3 channels represent the red, green, and blue components of each image. Since the MNIST dataset is grayscale, we only need 1 channel to represent our dataset. We will pad the other channels with 0's: + +```python +X = X.reshape((70000, 1, 64, 64)) +X = np.tile(X, (1, 3, 1, 1)) +``` +Finally, we'll put our images into MXNet's NDArrayIter, which will allow MXNet to easily iterate through our images during training. We'll also split up them images into a batches, with 64 images in each batch. Every time we iterate, we'll get a 4 dimensional array with size (64, 3, 64, 64), representing a batch of 64 images. +```python +import mxnet as mx +batch_size = 64 +image_iter = mx.io.NDArrayIter(X, batch_size=batch_size) +``` +### 2. Preparing Random Numbers + +We need to input random numbers from a normal distribution to our generator network, so we'll create an MXNet DataIter that produces random numbers for each training batch. The DataIter is the base class of MXNet's Data Loading API. Below, we create a class called RandIter which is a subclass of DataIter. We use MXNet's built in mx.random.normal function in order to return the normally distributed random numbers every time we iterate. +```python +class RandIter(mx.io.DataIter): + def __init__(self, batch_size, ndim): + self.batch_size = batch_size + self.ndim = ndim + self.provide_data = [('rand', (batch_size, ndim, 1, 1))] + self.provide_label = [] + + def iter_next(self): + return True + + def getdata(self): + #Returns random numbers from a gaussian (normal) distribution + #with mean=0 and standard deviation = 1 + return [mx.random.normal(0, 1.0, shape=(self.batch_size, self.ndim, 1, 1))] +``` +When we initalize our RandIter, we need to provide two numbers: the batch size and how many random numbers we want to produce a single image from. This number is referred to as Z, and we'll set this to 100. This value comes from the research paper on the topic. Every time we iterate and get a batch of random numbers, we will get a 4 dimensional array with shape: (batch_size, Z, 1, 1), which in our example is (64, 100, 1, 1). +```python +Z = 100 +rand_iter = RandIter(batch_size, Z) +``` +## Create the Model + +Our model has two networks that we will train together - the generator network and the disciminator network. + +### The Generator + +Let's start off by defining the generator network, which uses deconvolutional layers (also callled fractionally strided layers) to generate an image form random numbers : +```python +no_bias = True +fix_gamma = True +epsilon = 1e-5 + 1e-12 + +rand = mx.sym.Variable('rand') + +g1 = mx.sym.Deconvolution(rand, name='g1', kernel=(4,4), num_filter=1024, no_bias=no_bias) +gbn1 = mx.sym.BatchNorm(g1, name='gbn1', fix_gamma=fix_gamma, eps=epsilon) +gact1 = mx.sym.Activation(gbn1, name='gact1', act_type='relu') + +g2 = mx.sym.Deconvolution(gact1, name='g2', kernel=(4,4), stride=(2,2), pad=(1,1), num_filter=512, no_bias=no_bias) +gbn2 = mx.sym.BatchNorm(g2, name='gbn2', fix_gamma=fix_gamma, eps=epsilon) +gact2 = mx.sym.Activation(gbn2, name='gact2', act_type='relu') + +g3 = mx.sym.Deconvolution(gact2, name='g3', kernel=(4,4), stride=(2,2), pad=(1,1), num_filter=256, no_bias=no_bias) +gbn3 = mx.sym.BatchNorm(g3, name='gbn3', fix_gamma=fix_gamma, eps=epsilon) +gact3 = mx.sym.Activation(gbn3, name='gact3', act_type='relu') + +g4 = mx.sym.Deconvolution(gact3, name='g4', kernel=(4,4), stride=(2,2), pad=(1,1), num_filter=128, no_bias=no_bias) +gbn4 = mx.sym.BatchNorm(g4, name='gbn4', fix_gamma=fix_gamma, eps=epsilon) +gact4 = mx.sym.Activation(gbn4, name='gact4', act_type='relu') + +g5 = mx.sym.Deconvolution(gact4, name='g5', kernel=(4,4), stride=(2,2), pad=(1,1), num_filter=3, no_bias=no_bias) +generatorSymbol = mx.sym.Activation(g5, name='gact5', act_type='tanh') +``` + +Our generator image starts with random numbers that will be obtained from the RandIter we created earlier, so we created the rand variable for this input. +We then start creating the model starting with a Deconvolution layer (sometimes called 'fractionally strided layer'). We apply batch normalization and ReLU activation after the Deconvolution layer. + +We repeat this process 4 times, applying a (2,2) stride and (1,1) pad at each Deconvolutional layer, which doubles the size of our image at each layer. By creating these layers, our generator network will have to learn to upsample our input vector of random numbers, Z at each layer, so that network output a final image. We also reduce half the number of filters at each layer, reducing dimensionality at each layer. Ultimatley, our output layer is a 64x64x3 layer, representing the size and channels of our image. We use tanh activation instead of relu on the last layer, as recommended by the research on DCGANs. The output of neurons in the final gout layer represent the pixels of generated image. + +Notice we used 3 parameters to help us create our model: no_bias, fixed_gamma, and epsilon. Neurons in our network won't have a bias added to them, this seems to work better in practice for the DCGAN. In our batch norm layer, we set fixed_gamma=True, which means gamma=1 for all of our batch norm layers. epsilon is a small number that gets added to our batch norm so that we don't end up dividing by zero. By default, CuDNN requires that this number is greater than 1e-5, so we add a small number to this value, ensuring this values stays small. + +### The Discriminator + +Let's now create our discriminator network, which will take in images of handwritten digits from the MNIST dataset and images created by the generator network: +```python +data = mx.sym.Variable('data') + +d1 = mx.sym.Convolution(data, name='d1', kernel=(4,4), stride=(2,2), pad=(1,1), num_filter=128, no_bias=no_bias) +dact1 = mx.sym.LeakyReLU(d1, name='dact1', act_type='leaky', slope=0.2) + +d2 = mx.sym.Convolution(dact1, name='d2', kernel=(4,4), stride=(2,2), pad=(1,1), num_filter=256, no_bias=no_bias) +dbn2 = mx.sym.BatchNorm(d2, name='dbn2', fix_gamma=fix_gamma, eps=epsilon) +dact2 = mx.sym.LeakyReLU(dbn2, name='dact2', act_type='leaky', slope=0.2) + +d3 = mx.sym.Convolution(dact2, name='d3', kernel=(4,4), stride=(2,2), pad=(1,1), num_filter=512, no_bias=no_bias) +dbn3 = mx.sym.BatchNorm(d3, name='dbn3', fix_gamma=fix_gamma, eps=epsilon) +dact3 = mx.sym.LeakyReLU(dbn3, name='dact3', act_type='leaky', slope=0.2) + +d4 = mx.sym.Convolution(dact3, name='d4', kernel=(4,4), stride=(2,2), pad=(1,1), num_filter=1024, no_bias=no_bias) +dbn4 = mx.sym.BatchNorm(d4, name='dbn4', fix_gamma=fix_gamma, eps=epsilon) +dact4 = mx.sym.LeakyReLU(dbn4, name='dact4', act_type='leaky', slope=0.2) + +d5 = mx.sym.Convolution(dact4, name='d5', kernel=(4,4), num_filter=1, no_bias=no_bias) +d5 = mx.sym.Flatten(d5) + +label = mx.sym.Variable('label') +discriminatorSymbol = mx.sym.LogisticRegressionOutput(data=d5, label=label, name='dloss') +``` + +We start off by creating the data variable, which is used to hold our input images to the discriminator. + +The discriminator then goes through a series of 5 convolutional layers, each with a 4x4 kernel, 2x2 stride, and 1x1 pad. These layers half the size of the image (which starts at 64x64) at each convolutional layer. Our model also increases dimensionality at each layer by doubling the number of filters per convolutional layer, starting at 128 filters and ending at 1024 filters before we flatten the output. + +At the final convolution, we flatten the neural net to get one number as the final output of discriminator network. This number is the probability the image is real, as determined by our discriminator. We use logistic regression to determine this probability. When we pass in "real" images from the MNIST dataset, we can label these as 1 and we can label the "fake" images from the generator net as 0 to perform logistic regression on the discriminator network. +Prepare the models using the Module API + +So far we have defined a MXNet Symbol for both the generator and the discriminator network. Before we can train our model, we need to bind these symbols using the Module API, which creates the computation graph for our models. It also allows us to decide how we want to initialize our model and what type of optimizer we want to use. Let's set up Module for both of our networks: +```python +#Hyperperameters +sigma = 0.02 +lr = 0.0002 +beta1 = 0.5 +ctx = mx.gpu(0) + +#=============Generator Module============= +generator = mx.mod.Module(symbol=generatorSymbol, data_names=('rand',), label_names=None, context=ctx) +generator.bind(data_shapes=rand_iter.provide_data) +generator.init_params(initializer=mx.init.Normal(sigma)) +generator.init_optimizer( + optimizer='adam', + optimizer_params={ + 'learning_rate': lr, + 'beta1': beta1, + }) +mods = [generator] + +# =============Discriminator Module============= +discriminator = mx.mod.Module(symbol=discriminatorSymbol, data_names=('data',), label_names=('label',), context=ctx) +discriminator.bind(data_shapes=image_iter.provide_data, + label_shapes=[('label', (batch_size,))], + inputs_need_grad=True) +discriminator.init_params(initializer=mx.init.Normal(sigma)) +discriminator.init_optimizer( + optimizer='adam', + optimizer_params={ + 'learning_rate': lr, + 'beta1': beta1, + }) +mods.append(discriminator) +``` +First, we create Modules for our networks and then bind the symbols that we've created in the previous steps to our modules. +We use rand_iter.provide_data as the data_shape to bind our generator network. This means that as we iterate though batches of data on the generator Module, our RandIter will provide us with random numbers to feed our Module using it's provide_data function. + +Similarly, we bind the discriminator Module to image_iter.provide_data, which gives us images from MNIST from the NDArrayIter we had set up earlier, called image_iter. + +Notice that we're using the Normal initialization, with the hyperparameter sigma=0.02. This means our weight initializations for the neurons in our networks will random numbers from a Gaussian (normal) distribution with a mean of 0 and a standard deviation of 0.02. + +We also use the adam optimizer for gradient decent. We've set up two hyperparameters, lr and beta1 based on the values used in the DCGAN paper. We're using a single gpu, gpu(0) for training. + +### Visualizing Our Training +Before we train the model, let's set up some helper functions that will help visualize what our generator is producing, compared to what the real image is: +```python +from matplotlib import pyplot as plt + +#Takes the images in our batch and arranges them in an array so that they can be +#Plotted using matplotlib +def fill_buf(buf, num_images, img, shape): + width = buf.shape[0]/shape[1] + height = buf.shape[1]/shape[0] + img_width = (num_images%width)*shape[0] + img_hight = (num_images/height)*shape[1] + buf[img_hight:img_hight+shape[1], img_width:img_width+shape[0], :] = img + +#Plots two images side by side using matplotlib +def visualize(fake, real): + #64x3x64x64 to 64x64x64x3 + fake = fake.transpose((0, 2, 3, 1)) + #Pixel values from 0-255 + fake = np.clip((fake+1.0)*(255.0/2.0), 0, 255).astype(np.uint8) + #Repeat for real image + real = real.transpose((0, 2, 3, 1)) + real = np.clip((real+1.0)*(255.0/2.0), 0, 255).astype(np.uint8) + + #Create buffer array that will hold all the images in our batch + #Fill the buffer so to arrange all images in the batch onto the buffer array + n = np.ceil(np.sqrt(fake.shape[0])) + fbuff = np.zeros((int(n*fake.shape[1]), int(n*fake.shape[2]), int(fake.shape[3])), dtype=np.uint8) + for i, img in enumerate(fake): + fill_buf(fbuff, i, img, fake.shape[1:3]) + rbuff = np.zeros((int(n*real.shape[1]), int(n*real.shape[2]), int(real.shape[3])), dtype=np.uint8) + for i, img in enumerate(real): + fill_buf(rbuff, i, img, real.shape[1:3]) + + #Create a matplotlib figure with two subplots: one for the real and the other for the fake + #fill each plot with our buffer array, which creates the image + fig = plt.figure() + ax1 = fig.add_subplot(2,2,1) + ax1.imshow(fbuff) + ax2 = fig.add_subplot(2,2,2) + ax2.imshow(rbuff) + plt.show() +``` + +## Fit the Model +Training the DCGAN is a complex process that requires multiple steps. +To fit the model, for every batch of data in our dataset: + +1. Use the Z vector, which contains our random numbers to do a forward pass through our generator. This outputs the "fake" image, since it's created from our generator. + +2. Use the fake image as the input to do a forward and backwards pass through the discriminator network. We set our labels for our logistic regression to 0 to represent that this is a fake image. This trains the discriminator to learn what a fake image looks like. We save the gradient produced in backpropogation for the next step. + +3. Do a forwards and backwards pass through the discriminator using a real image from our dataset. Our label for logistic regression will now be 1 to represent real images, so our discriminator can learn to recognize a real image. + +4. Update the discriminator by adding the result of the gradient generated during backpropogation on the fake image with the gradient from backpropogation on the real image. + +5. Now that the discriminator has been updated for the this batch, we still need to update the generator. First, do a forward and backwards pass with the same batch on the updated discriminator, to produce a new gradient. Use the new gradient to do a backwards pass + +Here's the main training loop for our DCGAN: + +```python +# =============train=============== +print('Training...') +for epoch in range(1): + image_iter.reset() + for i, batch in enumerate(image_iter): + #Get a batch of random numbers to generate an image from the generator + rbatch = rand_iter.next() + #Forward pass on training batch + generator.forward(rbatch, is_train=True) + #Output of training batch is the 64x64x3 image + outG = generator.get_outputs() + + #Pass the generated (fake) image through the discriminator, and save the gradient + #Label (for logistic regression) is an array of 0's since this image is fake + label = mx.nd.zeros((batch_size,), ctx=ctx) + #Forward pass on the output of the discriminator network + discriminator.forward(mx.io.DataBatch(outG, [label]), is_train=True) + #Do the backwards pass and save the gradient + discriminator.backward() + gradD = [[grad.copyto(grad.context) for grad in grads] for grads in discriminator._exec_group.grad_arrays] + + #Pass a batch of real images from MNIST through the discriminator + #Set the label to be an array of 1's because these are the real images + label[:] = 1 + batch.label = [label] + #Forward pass on a batch of MNIST images + discriminator.forward(batch, is_train=True) + #Do the backwards pass and add the saved gradient from the fake images to the gradient + #generated by this backwards pass on the real images + discriminator.backward() + for gradsr, gradsf in zip(discriminator._exec_group.grad_arrays, gradD): + for gradr, gradf in zip(gradsr, gradsf): + gradr += gradf + #Update gradient on the discriminator + discriminator.update() + + #Now that we've updated the discriminator, let's update the generator + #First do a forward pass and backwards pass on the newly updated discriminator + #With the current batch + discriminator.forward(mx.io.DataBatch(outG, [label]), is_train=True) + discriminator.backward() + #Get the input gradient from the backwards pass on the discriminator, + #and use it to do the backwards pass on the generator + diffD = discriminator.get_input_grads() + generator.backward(diffD) + #Update the gradients on the generator + generator.update() + + #Increment to the next batch, printing every 50 batches + i += 1 + if i % 50 == 0: + print('epoch:', epoch, 'iter:', i) + print + print(" From generator: From MNIST:") + + visualize(outG[0].asnumpy(), batch.data[0].asnumpy()) +``` + +This causes our GAN to train and we can visualize the progress that we're making as our networks train. After every 25 iterations, we're calling the visualize function that we created earlier, which creates the visual plots during training. + +The plot on our left will represent what our generator created (the fake image) in the most recent iteration. The plot on the right will represent the original (real) image from the MNIST dataset that was inputted to the discriminator on the same iteration. + +As training goes on the generator becomes better at generating realistic images. You can see this happening since images on the left become closer to the original dataset with each iteration. + +## Summary + +We've now sucessfully used Apache MXNet to train a Deep Convolutional GAN using the MNIST dataset. + +As a result, we've created two neural nets: a generator, which is able to create images of handwritten digits from random numbers, and a discriminator, which is able to take an image and determine if it is an image of handwritten digits. + +Along the way, we've learned how to do the image manipulation and visualization that's associted with training deep neural nets. We've also learned how to some of MXNet's advanced training functionality to fit our model. + +## Acknowledgements +This tutorial is based on [MXNet DCGAN codebase](https://github.com/apache/incubator-mxnet/blob/master/example/gan/dcgan.py), +[The original paper on GANs](https://arxiv.org/abs/1406.2661), as well as [this paper on deep convolutional GANs](https://arxiv.org/abs/1511.06434). \ No newline at end of file From 1a617fadc3cf612f964722e09571e1b80ad16f68 Mon Sep 17 00:00:00 2001 From: Peter Schneider Date: Tue, 8 Aug 2017 19:30:03 +0200 Subject: [PATCH 344/834] fixes the incorrect retrieval of the batch size in the RNN forward pass. (#7385) --- python/mxnet/gluon/rnn/rnn_layer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/mxnet/gluon/rnn/rnn_layer.py b/python/mxnet/gluon/rnn/rnn_layer.py index d5673690e587..deb6898be55c 100644 --- a/python/mxnet/gluon/rnn/rnn_layer.py +++ b/python/mxnet/gluon/rnn/rnn_layer.py @@ -154,7 +154,7 @@ def begin_state(self, batch_size=0, func=ndarray.zeros, **kwargs): def forward(self, inputs, states): if isinstance(states, ndarray.NDArray): states = [states] - batch_size = states[0].shape[self._layout.find('N')] + batch_size = inputs.shape[self._layout.find('N')] for state, info in zip(states, self.state_info(batch_size)): if state.shape != info['shape']: raise ValueError( From 07661ae9a627d2a90b15c04b665fdb0773920285 Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Tue, 8 Aug 2017 15:13:29 -0700 Subject: [PATCH 345/834] decouple record/train and add state readers (#7356) * decouple record/train and add state readers * update per comments * update per concensus * add API doc * fix --- docs/api/python/autograd.md | 21 ++-- include/mxnet/c_api.h | 12 +++ python/mxnet/autograd.py | 136 ++++++++++++++++++------- python/mxnet/ndarray.py | 7 +- src/c_api/c_api_ndarray.cc | 12 +++ tests/python/unittest/test_autograd.py | 37 ++++++- 6 files changed, 174 insertions(+), 51 deletions(-) diff --git a/docs/api/python/autograd.md b/docs/api/python/autograd.md index 440a1e4de289..d204a2ce4464 100644 --- a/docs/api/python/autograd.md +++ b/docs/api/python/autograd.md @@ -13,20 +13,29 @@ ## Autograd -```eval_rst -.. currentmodule:: mxnet.autograd -``` - - ```eval_rst .. autosummary:: :nosignatures: record pause - mark_variables + train_mode + predict_mode backward set_training + is_training set_recording + is_recording + mark_variables +``` + +## API Reference + + + +```eval_rst +.. automodule:: mxnet.autograd + :members: ``` + diff --git a/include/mxnet/c_api.h b/include/mxnet/c_api.h index d9a5315c9167..3b8d54ce9725 100644 --- a/include/mxnet/c_api.h +++ b/include/mxnet/c_api.h @@ -565,6 +565,18 @@ MXNET_DLL int MXAutogradSetIsRecording(int is_recording, int* prev); * \return 0 when success, -1 when failure happens */ MXNET_DLL int MXAutogradSetIsTraining(int is_training, int* prev); +/*! + * \brief get whether autograd recording is on + * \param curr returns the current status. + * \return 0 when success, -1 when failure happens + */ +MXNET_DLL int MXAutogradIsRecording(bool* curr); +/*! + * \brief get whether training mode is on + * \param curr returns the current status. + * \return 0 when success, -1 when failure happens + */ +MXNET_DLL int MXAutogradIsTraining(bool* curr); /*! * \brief mark NDArrays as variables to compute gradient for autograd * \param num_var number of variable NDArrays diff --git a/python/mxnet/autograd.py b/python/mxnet/autograd.py index 2f33052e663e..2c3feabd525c 100644 --- a/python/mxnet/autograd.py +++ b/python/mxnet/autograd.py @@ -10,7 +10,7 @@ from .symbol import _GRAD_REQ_MAP -def set_recording(is_recording): +def set_recording(is_recording): #pylint: disable=redefined-outer-name """Set status to recording/not recording. When recording, graph will be constructed for gradient computation. @@ -27,14 +27,14 @@ def set_recording(is_recording): ctypes.c_int(is_recording), ctypes.byref(prev))) return bool(prev.value) -def set_training(is_train): - """Set status to training/not training. This affects ctx.is_train in operator +def set_training(train_mode): #pylint: disable=redefined-outer-name + """Set status to training/predicting. This affects ctx.is_train in operator running context. For example, Dropout will drop inputs randomly when - is_train=True while simply passing through if is_train=False. + train_mode=True while simply passing through if train_mode=False. Parameters ---------- - is_train: bool + train_mode: bool Returns ------- @@ -42,43 +42,70 @@ def set_training(is_train): """ prev = ctypes.c_int() check_call(_LIB.MXAutogradSetIsTraining( - ctypes.c_int(is_train), ctypes.byref(prev))) + ctypes.c_int(train_mode), ctypes.byref(prev))) return bool(prev.value) +def is_recording(): + """Get status on recording/not recording. -class RecordingStateScope(object): + Returns + ------- + Current state of recording. + """ + curr = ctypes.c_bool() + check_call(_LIB.MXAutogradIsRecording(ctypes.byref(curr))) + return curr.value + +def is_training(): + """Get status on training/predicting. + + Returns + ------- + Current state of training/predicting. + """ + curr = ctypes.c_bool() + check_call(_LIB.MXAutogradIsTraining(ctypes.byref(curr))) + return curr.value + + +class _RecordingStateScope(object): """Scope for managing training state. Example:: - with RecordingStateScope(True, True): + + with _RecordingStateScope(True, True): y = model(x) backward([y]) + """ - def __init__(self, enter_state, is_train): - self._enter_state = enter_state - self._enter_is_train = is_train - self._prev = None - self._prev_is_train = None + def __init__(self, is_record, train_mode): #pylint: disable=redefined-outer-name + self._enter_is_record = is_record + self._enter_train_mode = train_mode + self._prev_is_record = None + self._prev_train_mode = None def __enter__(self): - self._prev = set_recording(self._enter_state) - self._prev_is_train = set_training(self._enter_is_train) + if self._enter_is_record is not None: + self._prev_is_record = set_recording(self._enter_is_record) + if self._enter_train_mode is not None: + self._prev_train_mode = set_training(self._enter_train_mode) def __exit__(self, ptype, value, trace): - if self._prev != self._enter_state: - set_recording(self._prev) - if self._prev_is_train != self._enter_is_train: - set_training(self._prev_is_train) + if self._enter_is_record is not None and self._prev_is_record != self._enter_is_record: + set_recording(self._prev_is_record) + if self._enter_train_mode is not None and self._prev_train_mode != self._enter_train_mode: + set_training(self._prev_train_mode) -def record(is_train=True): - """Returns a training scope context to be used in 'with' statement - and captures training code. +def record(train_mode=True): #pylint: disable=redefined-outer-name + """Returns an autograd recording scope context to be used in 'with' statement + and captures code that needs gradients to be calculated. - .. note:: When forwarding with is_train=False, the corresponding backward - should also use is_train=False, otherwise gradient is undefined. + .. note:: When forwarding with train_mode=False, the corresponding backward + should also use train_mode=False, otherwise gradient is undefined. Example:: + with autograd.record(): y = model(x) backward([y]) @@ -87,17 +114,19 @@ def record(is_train=True): Parameters ---------- - is_train: bool, default True - Whether to do forward for training or inference. + train_mode: bool, default True + Whether the forward pass is in training or predicting mode. This controls the behavior + of some layers such as Dropout, BatchNorm. """ - return RecordingStateScope(True, is_train) + return _RecordingStateScope(True, train_mode) -def pause(is_train=False): - """Returns a testing scope context to be used in 'with' statement - and captures testing code. +def pause(train_mode=False): #pylint: disable=redefined-outer-name + """Returns a scope context to be used in 'with' statement for codes that do not need + gradients to be calculated. Example:: + with autograd.record(): y = model(x) backward([y]) @@ -106,10 +135,41 @@ def pause(is_train=False): Parameters ---------- - is_train: bool, default False - Whether to do forward for training or inference. + train_mode: bool, default False + Whether to do forward for training or predicting. + """ + return _RecordingStateScope(False, train_mode) + + +def train_mode(): + """Returns a scope context to be used in 'with' statement + in which forward pass behavior is set to training mode, + without changing the recording states. + + Example:: + + y = model(x) + with autograd.train_mode(): + y = dropout(y) + + """ + return _RecordingStateScope(None, True) + + +def predict_mode(): + """Returns a scope context to be used in 'with' statement + in which forward pass behavior is set to inference mode, + without changing the recording states. + + Example:: + + with autograd.record(): + y = model(x) + with autograd.predict_mode(): + y = sampling(y) + backward([y]) """ - return RecordingStateScope(False, is_train) + return _RecordingStateScope(None, False) def mark_variables(variables, gradients, grad_reqs='write'): @@ -143,7 +203,7 @@ def mark_variables(variables, gradients, grad_reqs='write'): c_array(NDArrayHandle, gradient_handles))) -def backward(heads, head_grads=None, retain_graph=False, is_train=True): +def backward(heads, head_grads=None, retain_graph=False, train_mode=True): #pylint: disable=redefined-outer-name """Compute the gradients of heads w.r.t previously marked variables. Parameters @@ -152,8 +212,8 @@ def backward(heads, head_grads=None, retain_graph=False, is_train=True): Output NDArray(s) head_grads: NDArray or list of NDArray or None Gradients with respect to heads. - is_train: bool, optional - Whether to do backward for training or inference. + train_mode: bool, optional + Whether to do backward for training or predicting. """ if isinstance(heads, NDArray): assert head_grads is None or isinstance(head_grads, NDArray) @@ -170,7 +230,7 @@ def backward(heads, head_grads=None, retain_graph=False, is_train=True): c_array(NDArrayHandle, output_handles), ctypes.c_void_p(0), ctypes.c_int(retain_graph), - ctypes.c_int(is_train))) + ctypes.c_int(train_mode))) return ograd_handles = [] @@ -187,4 +247,4 @@ def backward(heads, head_grads=None, retain_graph=False, is_train=True): c_array(NDArrayHandle, output_handles), c_array(NDArrayHandle, ograd_handles), ctypes.c_int(retain_graph), - ctypes.c_int(is_train))) + ctypes.c_int(train_mode))) diff --git a/python/mxnet/ndarray.py b/python/mxnet/ndarray.py index b2178a98a84e..d4a0cdbd8d51 100644 --- a/python/mxnet/ndarray.py +++ b/python/mxnet/ndarray.py @@ -1059,7 +1059,7 @@ def detach(self): check_call(_LIB.MXNDArrayDetach(self.handle, ctypes.byref(hdl))) return NDArray(hdl) - def backward(self, out_grad=None, retain_graph=False, is_train=True): + def backward(self, out_grad=None, retain_graph=False, train_mode=True): """Compute the gradients of this NDArray w.r.t variables. Parameters @@ -1070,7 +1070,7 @@ def backward(self, out_grad=None, retain_graph=False, is_train=True): Whether to retain the computaion graph for another backward pass on the same graph. By default the computaion history is cleared. - is_train : bool, optional + train_mode : bool, optional Whether to compute gradient for training or inference. """ if out_grad is None: @@ -1082,7 +1082,7 @@ def backward(self, out_grad=None, retain_graph=False, is_train=True): 1, c_array(NDArrayHandle, [self.handle]), c_array(NDArrayHandle, ograd_handles), ctypes.c_int(retain_graph), - ctypes.c_int(is_train))) + ctypes.c_int(train_mode))) def onehot_encode(indices, out): @@ -2538,7 +2538,6 @@ def _make_ndarray_function(handle, name): else: signature.append('%s=_Null'%name) kwarg_names.append(name) - #signature.append('is_train=False') signature.append('out=None') signature.append('name=None') signature.append('**kwargs') diff --git a/src/c_api/c_api_ndarray.cc b/src/c_api/c_api_ndarray.cc index f40139424b31..a37e3144c303 100644 --- a/src/c_api/c_api_ndarray.cc +++ b/src/c_api/c_api_ndarray.cc @@ -522,12 +522,24 @@ int MXInvokeCachedOp(CachedOpHandle handle, API_END(); } +int MXAutogradIsTraining(bool* curr) { + API_BEGIN(); + *curr = AutogradRuntime::Get()->IsTraining(); + API_END(); +} + int MXAutogradSetIsTraining(int is_training, int* prev) { API_BEGIN(); *prev = AutogradRuntime::Get()->SetIsTraining(static_cast(is_training)); API_END(); } +int MXAutogradIsRecording(bool* curr) { + API_BEGIN(); + *curr = AutogradRuntime::Get()->IsRecording(); + API_END(); +} + int MXAutogradSetIsRecording(int is_recording, int* prev) { API_BEGIN(); *prev = AutogradRuntime::Get()->SetIsRecording(static_cast(is_recording)); diff --git a/tests/python/unittest/test_autograd.py b/tests/python/unittest/test_autograd.py index 172075dcfda1..7ee35009a164 100644 --- a/tests/python/unittest/test_autograd.py +++ b/tests/python/unittest/test_autograd.py @@ -251,18 +251,49 @@ def test_attach_grad(): def test_is_train(): x = mx.nd.ones((10, 10)) x.attach_grad() - with record(True): + with record(train_mode=True): + assert is_recording() + assert is_training() y = mx.nd.Dropout(x, p=0.5) assert y.asnumpy().max() == 2 and y.asnumpy().min() == 0 y.backward() assert (x.grad.asnumpy() == y.asnumpy()).all() - with record(False): + with predict_mode(): + assert is_recording() + assert not is_training() + y = mx.nd.Dropout(x, p=0.5) + assert (y.asnumpy() == x.asnumpy()).all() + y.backward(train_mode=False) + assert (x.grad.asnumpy() == x.asnumpy()).all() + + with record(train_mode=False): + assert is_recording() + assert not is_training() y = mx.nd.Dropout(x, p=0.5) assert (y.asnumpy() == x.asnumpy()).all() - y.backward(is_train=False) + y.backward(train_mode=False) assert (x.grad.asnumpy() == x.asnumpy()).all() + with train_mode(): + assert is_recording() + assert is_training() + y = mx.nd.Dropout(x, p=0.5) + assert y.asnumpy().max() == 2 and y.asnumpy().min() == 0 + y.backward() + assert (x.grad.asnumpy() == y.asnumpy()).all() + + assert not is_recording() + assert not is_training() + y = mx.nd.Dropout(x, p=0.5) + assert (y.asnumpy() == x.asnumpy()).all() + + with train_mode(): + assert not is_recording() + assert is_training() + y = mx.nd.Dropout(x, p=0.5) + assert y.asnumpy().max() == 2 and y.asnumpy().min() == 0 + if __name__ == "__main__": import nose From 4c13b96b9ae73ac2809339798a923d2270c54f16 Mon Sep 17 00:00:00 2001 From: Eric Junyuan Xie Date: Tue, 8 Aug 2017 15:20:20 -0700 Subject: [PATCH 346/834] Update conv_layers.py (#7388) * Update conv_layers.py * Update conv_layers.py --- python/mxnet/gluon/nn/conv_layers.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python/mxnet/gluon/nn/conv_layers.py b/python/mxnet/gluon/nn/conv_layers.py index d9608a151220..9094eddee3f3 100644 --- a/python/mxnet/gluon/nn/conv_layers.py +++ b/python/mxnet/gluon/nn/conv_layers.py @@ -65,7 +65,7 @@ class _Conv(HybridBlock): def __init__(self, channels, kernel_size, strides, padding, dilation, groups, layout, in_channels=0, activation=None, use_bias=True, weight_initializer=None, bias_initializer='zeros', - op_name='Convolution', prefix=None, params=None, **kwargs): + op_name='Convolution', adj=None, prefix=None, params=None): super(_Conv, self).__init__(prefix=prefix, params=params) with self.name_scope(): self._channels = channels @@ -81,7 +81,8 @@ def __init__(self, channels, kernel_size, strides, padding, dilation, 'kernel': kernel_size, 'stride': strides, 'dilate': dilation, 'pad': padding, 'num_filter': channels, 'num_group': groups, 'no_bias': not use_bias, 'layout': layout} - self._kwargs.update(kwargs) + if adj is not None: + self._kwargs['adj'] = adj dshape = [0]*(len(kernel_size) + 2) dshape[layout.find('N')] = 1 From 251ae71a20d8318ab20f4c19520f74b881fdf3ff Mon Sep 17 00:00:00 2001 From: Mu Li Date: Tue, 8 Aug 2017 16:36:23 -0700 Subject: [PATCH 347/834] Add license header (#7379) * add * add .py and ci * fix pylint * update --- Jenkinsfile | 1 + amalgamation/amalgamation.py | 19 +- amalgamation/dmlc-minimum0.cc | 20 +- amalgamation/jni/org_dmlc_mxnet_Predictor.h | 19 + amalgamation/jni/predictor.cc | 19 + amalgamation/mxnet_predict0.cc | 19 + amalgamation/python/mxnet_predict.py | 17 + cmake/Modules/FindAccelerate.cmake | 17 + cmake/Modules/FindAtlas.cmake | 17 + cmake/Modules/FindJeMalloc.cmake | 17 + cmake/Modules/FindMKL.cmake | 17 + cmake/Modules/FindOpenBLAS.cmake | 17 + cmake/Utils.cmake | 17 + cpp-package/example/feature_extract/run.sh | 17 + cpp-package/example/get_mnist.sh | 17 + .../example/run_lenet_with_mxdataiter.sh | 17 + cpp-package/include/mxnet-cpp/MxNetCpp.h | 20 +- cpp-package/include/mxnet-cpp/base.h | 20 +- cpp-package/include/mxnet-cpp/executor.h | 20 +- cpp-package/include/mxnet-cpp/initializer.h | 20 +- cpp-package/include/mxnet-cpp/io.h | 20 +- cpp-package/include/mxnet-cpp/kvstore.h | 20 +- cpp-package/include/mxnet-cpp/lr_scheduler.h | 20 +- cpp-package/include/mxnet-cpp/metric.h | 20 +- cpp-package/include/mxnet-cpp/model.h | 20 +- cpp-package/include/mxnet-cpp/monitor.h | 20 +- cpp-package/include/mxnet-cpp/ndarray.h | 20 +- cpp-package/include/mxnet-cpp/op_map.h | 20 +- cpp-package/include/mxnet-cpp/op_suppl.h | 20 +- cpp-package/include/mxnet-cpp/op_util.h | 20 +- cpp-package/include/mxnet-cpp/operator.h | 20 +- cpp-package/include/mxnet-cpp/optimizer.h | 20 +- cpp-package/include/mxnet-cpp/shape.h | 20 +- cpp-package/include/mxnet-cpp/symbol.h | 20 +- cpp-package/scripts/OpWrapperGenerator.py | 19 +- cpp-package/scripts/lint.py | 18 + cpp-package/tests/ci_test.sh | 17 + cpp-package/tests/travis/run_test.sh | 18 + cpp-package/tests/travis/setup.sh | 18 + docker/Dockerfiles/Dockerfile.in.scala | 19 + docker/install/cpp.sh | 18 + docker/install/julia.sh | 18 + docker/install/perl.sh | 18 + docker/install/python.sh | 18 + docker/install/r.sh | 18 + docker/install/scala.sh | 18 + docker/run.sh | 18 + docker/tool.sh | 18 + docs/build_version_doc/AddVersion.py | 19 +- docs/build_version_doc/build_doc.sh | 22 +- docs/conf.py | 17 + docs/mxdoc.py | 17 + example/adversary/data.py | 17 + example/autoencoder/autoencoder.py | 17 + example/autoencoder/data.py | 17 + example/autoencoder/mnist_sae.py | 71 +- example/autoencoder/model.py | 17 + example/autoencoder/solver.py | 17 + example/bayesian-methods/algos.py | 17 + example/bayesian-methods/bdk_demo.py | 17 + example/bayesian-methods/data_loader.py | 17 + example/bayesian-methods/utils.py | 19 +- example/bi-lstm-sort/infer_sort.py | 25 +- example/bi-lstm-sort/lstm.py | 25 +- example/bi-lstm-sort/lstm_sort.py | 17 + example/bi-lstm-sort/rnn_model.py | 19 +- example/bi-lstm-sort/sort_io.py | 17 + example/caffe/caffe_net.py | 19 +- example/caffe/data.py | 17 + example/caffe/train_model.py | 17 + .../cnn_text_classification/data_helpers.py | 17 + .../cnn_text_classification/old/text_cnn.py | 20 +- example/cnn_text_classification/text_cnn.py | 22 +- example/ctc/lstm.py | 17 + example/ctc/lstm_ocr.py | 17 + example/ctc/ocr_predict.py | 18 + example/dec/dec.py | 19 +- example/dsd/mlp.py | 17 + example/dsd/sparse_sgd.py | 17 + example/fcn-xs/data.py | 17 + example/fcn-xs/fcn_xs.py | 17 + example/fcn-xs/image_segmentaion.py | 133 +- example/fcn-xs/init_fcnxs.py | 195 +-- example/fcn-xs/run_fcnxs.sh | 17 + example/fcn-xs/solver.py | 269 ++-- example/fcn-xs/symbol_fcnxs.py | 395 +++--- example/gan/dcgan.py | 17 + example/gluon/actor_critic.py | 17 + example/gluon/data.py | 17 + example/gluon/dcgan.py | 17 + example/gluon/image_classification.py | 17 + example/gluon/lstm_crf.py | 17 + example/gluon/mnist.py | 17 + example/gluon/super_resolution.py | 17 + example/gluon/tree_lstm/dataset.py | 17 + .../gluon/tree_lstm/fetch_and_preprocess.sh | 20 +- example/gluon/tree_lstm/main.py | 17 + example/gluon/tree_lstm/scripts/download.py | 17 + .../tree_lstm/scripts/preprocess-sick.py | 17 + example/gluon/tree_lstm/tree_lstm.py | 17 + example/gluon/word_language_model/data.py | 17 + .../gluon/word_language_model/get_ptb_data.sh | 18 + example/gluon/word_language_model/model.py | 17 + example/gluon/word_language_model/train.py | 17 + example/image-classification/benchmark.py | 17 + .../image-classification/benchmark_score.py | 17 + example/image-classification/common/data.py | 17 + .../image-classification/common/find_mxnet.py | 17 + example/image-classification/common/fit.py | 17 + .../image-classification/common/modelzoo.py | 17 + example/image-classification/common/util.py | 17 + .../image-classification/data/caltech256.sh | 18 + .../data/imagenet1k-val.sh | 18 + example/image-classification/fine-tune.py | 17 + .../image-classification-predict.cc | 21 +- example/image-classification/score.py | 17 + .../image-classification/symbols/alexnet.py | 17 + .../image-classification/symbols/googlenet.py | 17 + .../symbols/inception-bn.py | 17 + .../symbols/inception-resnet-v2.py | 27 +- .../symbols/inception-v3.py | 17 + .../symbols/inception-v4.py | 17 + example/image-classification/symbols/lenet.py | 17 + example/image-classification/symbols/mlp.py | 17 + .../image-classification/symbols/mobilenet.py | 17 + .../image-classification/symbols/resnet-v1.py | 17 + .../image-classification/symbols/resnet.py | 17 + .../image-classification/symbols/resnext.py | 33 +- example/image-classification/symbols/vgg.py | 17 + example/image-classification/test_score.py | 17 + example/image-classification/train_cifar10.py | 17 + .../image-classification/train_imagenet.py | 17 + example/image-classification/train_mnist.py | 21 +- example/kaggle-ndsb1/gen_img_list.py | 21 +- example/kaggle-ndsb1/predict_dsb.py | 17 + example/kaggle-ndsb1/submission_dsb.py | 29 +- example/kaggle-ndsb1/symbol_dsb.py | 17 + example/kaggle-ndsb1/train_dsb.py | 23 +- example/kaggle-ndsb1/training_curves.py | 17 + example/kaggle-ndsb2/Preprocessing.py | 17 + example/kaggle-ndsb2/Train.py | 17 + example/memcost/inception_memcost.py | 17 + example/model-parallel-lstm/get_ptb_data.sh | 18 + example/model-parallel-lstm/lstm.py | 37 +- example/model-parallel-lstm/lstm_ptb.py | 19 +- example/module/lstm_bucketing.py | 17 + example/module/mnist_mlp.py | 17 + example/module/python_loss.py | 17 + example/module/sequential_module.py | 17 + example/module/train_cifar10.py | 19 +- example/multi-task/data.py | 17 + example/multi-task/example_multi_task.py | 17 + example/nce-loss/get_text8.sh | 17 + example/nce-loss/lstm_word.py | 29 +- example/nce-loss/nce.py | 17 + example/nce-loss/toy_nce.py | 25 +- example/nce-loss/toy_softmax.py | 21 +- example/nce-loss/wordvec.py | 29 +- example/nce-loss/wordvec_subwords.py | 17 + example/neural-style/download.sh | 18 + example/neural-style/end_to_end/basic.py | 17 + .../end_to_end/boost_inference.py | 17 + .../neural-style/end_to_end/boost_train.py | 17 + .../end_to_end/data_processing.py | 17 + example/neural-style/end_to_end/gen_v3.py | 17 + example/neural-style/end_to_end/gen_v4.py | 17 + .../neural-style/end_to_end/model_vgg19.py | 17 + example/neural-style/find_mxnet.py | 17 + example/neural-style/model_vgg19.py | 17 + example/neural-style/nstyle.py | 19 +- example/numpy-ops/custom_softmax.py | 19 +- example/numpy-ops/data.py | 17 + example/numpy-ops/ndarray_softmax.py | 19 +- example/numpy-ops/numpy_softmax.py | 19 +- .../numpy-ops/weighted_logistic_regression.py | 17 + example/profiler/profiler_executor.py | 17 + example/profiler/profiler_imageiter.py | 21 +- example/profiler/profiler_matmul.py | 17 + example/profiler/profiler_ndarray.py | 17 + example/python-howto/data.py | 17 + example/python-howto/data_iter.py | 17 + example/python-howto/debug_conv.py | 19 +- example/python-howto/monitor_weights.py | 19 +- example/python-howto/multiple_outputs.py | 17 + example/rcnn/demo.py | 17 + example/rcnn/rcnn/config.py | 17 + example/rcnn/rcnn/core/callback.py | 17 + example/rcnn/rcnn/core/loader.py | 17 + example/rcnn/rcnn/core/metric.py | 17 + example/rcnn/rcnn/core/module.py | 17 + example/rcnn/rcnn/core/tester.py | 17 + example/rcnn/rcnn/cython/nms_kernel.cu | 19 + example/rcnn/rcnn/cython/setup.py | 17 + example/rcnn/rcnn/dataset/__init__.py | 17 + example/rcnn/rcnn/dataset/coco.py | 17 + example/rcnn/rcnn/dataset/ds_utils.py | 19 +- example/rcnn/rcnn/dataset/imdb.py | 17 + example/rcnn/rcnn/dataset/pascal_voc.py | 17 + example/rcnn/rcnn/dataset/pascal_voc_eval.py | 17 + example/rcnn/rcnn/io/image.py | 17 + example/rcnn/rcnn/io/rcnn.py | 17 + example/rcnn/rcnn/io/rpn.py | 17 + example/rcnn/rcnn/logger.py | 17 + .../rcnn/rcnn/processing/bbox_regression.py | 17 + .../rcnn/rcnn/processing/bbox_transform.py | 17 + .../rcnn/rcnn/processing/generate_anchor.py | 17 + example/rcnn/rcnn/processing/nms.py | 17 + example/rcnn/rcnn/pycocotools/__init__.py | 17 + example/rcnn/rcnn/pycocotools/coco.py | 19 +- example/rcnn/rcnn/pycocotools/cocoeval.py | 19 +- example/rcnn/rcnn/pycocotools/mask.py | 19 +- example/rcnn/rcnn/pycocotools/maskApi.h | 19 + example/rcnn/rcnn/pycocotools/setup.py | 17 + example/rcnn/rcnn/symbol/__init__.py | 17 + example/rcnn/rcnn/symbol/proposal.py | 17 + example/rcnn/rcnn/symbol/proposal_target.py | 17 + example/rcnn/rcnn/symbol/symbol_resnet.py | 17 + example/rcnn/rcnn/symbol/symbol_vgg.py | 19 +- example/rcnn/rcnn/tools/reeval.py | 17 + example/rcnn/rcnn/tools/test_rcnn.py | 17 + example/rcnn/rcnn/tools/test_rpn.py | 17 + example/rcnn/rcnn/tools/train_rcnn.py | 17 + example/rcnn/rcnn/tools/train_rpn.py | 17 + example/rcnn/rcnn/utils/combine_model.py | 17 + example/rcnn/rcnn/utils/load_data.py | 17 + example/rcnn/rcnn/utils/load_model.py | 17 + example/rcnn/rcnn/utils/save_model.py | 17 + example/rcnn/script/additional_deps.sh | 18 + example/rcnn/script/get_coco.sh | 18 + example/rcnn/script/get_pretrained_model.sh | 18 + example/rcnn/script/get_selective_search.sh | 18 + example/rcnn/script/get_voc.sh | 18 + example/rcnn/script/resnet_voc07.sh | 18 + example/rcnn/script/resnet_voc0712.sh | 18 + example/rcnn/script/vgg_alter_voc07.sh | 18 + example/rcnn/script/vgg_fast_rcnn.sh | 18 + example/rcnn/script/vgg_voc07.sh | 18 + example/rcnn/script/vgg_voc0712.sh | 18 + example/rcnn/test.py | 17 + example/rcnn/train_alternate.py | 17 + example/rcnn/train_end2end.py | 17 + example/recommenders/crossentropy.py | 24 +- example/recommenders/matrix_fact.py | 17 + example/recommenders/movielens_data.py | 23 +- example/recommenders/negativesample.py | 21 +- example/recommenders/randomproj.py | 25 +- example/recommenders/recotools.py | 19 +- example/recommenders/symbol_alexnet.py | 17 + example/reinforcement-learning/a3c/a3c.py | 21 +- .../reinforcement-learning/a3c/launcher.py | 17 + example/reinforcement-learning/a3c/rl_data.py | 17 + example/reinforcement-learning/a3c/sym.py | 17 + example/reinforcement-learning/ddpg/ddpg.py | 17 + .../reinforcement-learning/ddpg/policies.py | 25 +- example/reinforcement-learning/ddpg/qfuncs.py | 23 +- .../reinforcement-learning/ddpg/replay_mem.py | 19 +- example/reinforcement-learning/ddpg/run.py | 19 +- .../reinforcement-learning/ddpg/strategies.py | 25 +- example/reinforcement-learning/ddpg/utils.py | 17 + .../reinforcement-learning/dqn/atari_game.py | 17 + example/reinforcement-learning/dqn/base.py | 17 + .../reinforcement-learning/dqn/dqn_demo.py | 17 + .../dqn/dqn_run_test.py | 17 + example/reinforcement-learning/dqn/game.py | 19 +- .../reinforcement-learning/dqn/operators.py | 17 + .../dqn/replay_memory.py | 17 + example/reinforcement-learning/dqn/utils.py | 17 + .../parallel_actor_critic/config.py | 17 + .../parallel_actor_critic/envs.py | 17 + .../parallel_actor_critic/model.py | 17 + .../parallel_actor_critic/train.py | 17 + example/rnn-time-major/bucket_io.py | 21 +- example/rnn-time-major/get_ptb_data.sh | 18 + example/rnn-time-major/rnn_cell_demo.py | 21 +- example/rnn/cudnn_lstm_bucketing.py | 17 + example/rnn/get_ptb_data.sh | 18 + example/rnn/lstm_bucketing.py | 17 + example/rnn/old/bucket_io.py | 31 +- example/rnn/old/get_ptb_data.sh | 18 + example/rnn/old/gru.py | 17 + example/rnn/old/gru_bucketing.py | 17 + example/rnn/old/lstm.py | 17 + example/rnn/old/lstm_bucketing.py | 17 + example/rnn/old/rnn.py | 17 + example/rnn/old/rnn_cell_demo.py | 21 +- example/rnn/old/rnn_model.py | 17 + example/speech-demo/config_util.py | 17 + example/speech-demo/decode_mxnet.py | 25 +- example/speech-demo/decode_mxnet.sh | 24 +- example/speech-demo/io_func/convert2kaldi.py | 23 +- example/speech-demo/io_func/feat_io.py | 17 + .../io_func/feat_readers/common.py | 19 +- .../io_func/feat_readers/reader_atrack.py | 23 +- .../io_func/feat_readers/reader_bvec.py | 19 +- .../io_func/feat_readers/reader_htk.py | 21 +- .../io_func/feat_readers/reader_kaldi.py | 23 +- .../speech-demo/io_func/feat_readers/stats.py | 17 + .../io_func/feat_readers/writer_kaldi.py | 17 + example/speech-demo/io_func/info.py | 17 + example/speech-demo/io_func/kaldi_parser.py | 19 +- example/speech-demo/io_func/model_io.py | 57 +- example/speech-demo/io_func/regr_feat_io.py | 25 +- example/speech-demo/io_func/utils.py | 17 + example/speech-demo/io_util.py | 33 +- example/speech-demo/lstm_proj.py | 21 +- example/speech-demo/make_stats.py | 17 + example/speech-demo/python_wrap/ctypes.cc | 23 +- .../python_wrap/example_usage/example.py | 21 +- example/speech-demo/run_ami.sh | 18 + example/speech-demo/run_timit.sh | 18 + example/speech-demo/speechSGD.py | 19 +- example/speech-demo/tests/test_nothing.py | 19 +- example/speech-demo/tests/test_system.py | 23 +- example/speech-demo/train_lstm_proj.py | 19 +- example/speech_recognition/arch_deepspeech.py | 17 + example/speech_recognition/config_util.py | 17 + example/speech_recognition/flac_to_wav.sh | 33 +- example/speech_recognition/label_util.py | 17 + example/speech_recognition/log_util.py | 17 + example/speech_recognition/main.py | 719 +++++----- example/speech_recognition/singleton.py | 17 + .../stt_bi_graphemes_util.py | 17 + .../stt_bucketing_module.py | 43 +- .../speech_recognition/stt_datagenerator.py | 19 +- .../stt_io_bucketingiter.py | 313 +++-- example/speech_recognition/stt_io_iter.py | 17 + .../speech_recognition/stt_layer_batchnorm.py | 17 + example/speech_recognition/stt_layer_conv.py | 17 + example/speech_recognition/stt_layer_fc.py | 17 + example/speech_recognition/stt_layer_gru.py | 17 + example/speech_recognition/stt_layer_lstm.py | 17 + example/speech_recognition/stt_layer_slice.py | 17 + .../speech_recognition/stt_layer_warpctc.py | 17 + example/speech_recognition/stt_metric.py | 17 + example/speech_recognition/stt_utils.py | 19 +- example/speech_recognition/train.py | 17 + example/ssd/config/config.py | 17 + example/ssd/config/utils.py | 17 + example/ssd/data/demo/download_demo_images.py | 17 + example/ssd/dataset/concat_db.py | 17 + example/ssd/dataset/imdb.py | 17 + example/ssd/dataset/iterator.py | 17 + example/ssd/dataset/mscoco.py | 17 + example/ssd/dataset/pascal_voc.py | 17 + example/ssd/dataset/pycocotools/__init__.py | 17 + example/ssd/dataset/pycocotools/coco.py | 17 + example/ssd/dataset/testdb.py | 17 + example/ssd/dataset/yolo_format.py | 17 + example/ssd/demo.py | 17 + example/ssd/deploy.py | 17 + example/ssd/detect/detector.py | 17 + example/ssd/evaluate.py | 17 + example/ssd/evaluate/eval_metric.py | 17 + example/ssd/evaluate/eval_voc.py | 17 + example/ssd/evaluate/evaluate_net.py | 17 + example/ssd/symbol/common.py | 17 + example/ssd/symbol/inceptionv3.py | 17 + example/ssd/symbol/legacy_vgg16_ssd_300.py | 17 + example/ssd/symbol/legacy_vgg16_ssd_512.py | 17 + example/ssd/symbol/resnet.py | 17 + example/ssd/symbol/symbol_builder.py | 17 + example/ssd/symbol/symbol_factory.py | 17 + example/ssd/symbol/vgg16_reduced.py | 17 + .../caffe_parse/parse_from_protobuf.py | 17 + .../tools/caffe_converter/convert_model.py | 17 + .../tools/caffe_converter/convert_symbol.py | 17 + .../ssd/tools/caffe_converter/mean_image.py | 17 + example/ssd/tools/find_mxnet.py | 17 + example/ssd/tools/prepare_coco.sh | 18 + example/ssd/tools/prepare_dataset.py | 17 + example/ssd/tools/prepare_pascal.sh | 18 + example/ssd/tools/rand_sampler.py | 17 + example/ssd/tools/visualize_net.py | 17 + example/ssd/train.py | 17 + example/ssd/train/metric.py | 17 + example/ssd/train/train_net.py | Bin 9651 -> 10437 bytes example/stochastic-depth/sd_cifar10.py | 17 + example/stochastic-depth/sd_mnist.py | 17 + example/stochastic-depth/sd_module.py | 17 + example/svm_mnist/svm_mnist.py | 17 + example/torch/data.py | 17 + example/torch/torch_function.py | 17 + example/torch/torch_module.py | 17 + example/utils/get_data.py | 17 + example/warpctc/infer_ocr.py | 17 + example/warpctc/lstm.py | 17 + example/warpctc/lstm_model.py | 19 +- example/warpctc/lstm_ocr.py | 25 +- example/warpctc/ocr_predict.py | 18 + example/warpctc/toy_ctc.py | 27 +- include/mxnet/base.h | 20 +- include/mxnet/c_api.h | 20 +- include/mxnet/c_lapack_api.h | 20 +- include/mxnet/c_predict_api.h | 20 +- include/mxnet/engine.h | 20 +- include/mxnet/executor.h | 20 +- include/mxnet/io.h | 20 +- include/mxnet/kvstore.h | 20 +- include/mxnet/mxrtc.h | 20 +- include/mxnet/ndarray.h | 20 +- include/mxnet/op_attr_types.h | 20 +- include/mxnet/operator.h | 20 +- include/mxnet/operator_util.h | 20 +- include/mxnet/resource.h | 20 +- include/mxnet/storage.h | 20 +- include/mxnet/tensor_blob.h | 20 +- matlab/get_inception_model.sh | 18 + .../AI-MXNet/examples/get_ptb_data.sh | 18 + perl-package/AI-MXNet/lib/AI/MXNet.pm | 17 + perl-package/AI-MXNet/lib/AI/MXNet/Base.pm | 23 +- .../AI-MXNet/lib/AI/MXNet/CachedOp.pm | 17 + .../AI-MXNet/lib/AI/MXNet/Callback.pm | 21 +- perl-package/AI-MXNet/lib/AI/MXNet/Context.pm | 17 + perl-package/AI-MXNet/lib/AI/MXNet/Contrib.pm | 19 +- .../AI-MXNet/lib/AI/MXNet/Contrib/AutoGrad.pm | 19 +- .../AI-MXNet/lib/AI/MXNet/Contrib/NDArray.pm | 19 +- .../AI-MXNet/lib/AI/MXNet/Contrib/Symbol.pm | 19 +- .../AI-MXNet/lib/AI/MXNet/Executor.pm | 21 +- .../AI-MXNet/lib/AI/MXNet/Executor/Group.pm | 17 + .../lib/AI/MXNet/Function/Parameters.pm | 19 +- perl-package/AI-MXNet/lib/AI/MXNet/IO.pm | 21 +- perl-package/AI-MXNet/lib/AI/MXNet/Image.pm | 17 + .../AI-MXNet/lib/AI/MXNet/Initializer.pm | 19 +- perl-package/AI-MXNet/lib/AI/MXNet/KVStore.pm | 19 +- .../AI-MXNet/lib/AI/MXNet/KVStoreServer.pm | 19 +- .../AI-MXNet/lib/AI/MXNet/LRScheduler.pm | 19 +- perl-package/AI-MXNet/lib/AI/MXNet/Logging.pm | 17 + perl-package/AI-MXNet/lib/AI/MXNet/Metric.pm | 21 +- perl-package/AI-MXNet/lib/AI/MXNet/Module.pm | 21 +- .../AI-MXNet/lib/AI/MXNet/Module/Base.pm | 17 + .../AI-MXNet/lib/AI/MXNet/Module/Bucketing.pm | 17 + perl-package/AI-MXNet/lib/AI/MXNet/Monitor.pm | 19 +- perl-package/AI-MXNet/lib/AI/MXNet/NDArray.pm | 57 +- .../AI-MXNet/lib/AI/MXNet/NDArray/Base.pm | 19 +- .../AI-MXNet/lib/AI/MXNet/NDArray/Doc.pm | 17 + .../AI-MXNet/lib/AI/MXNet/NDArray/Slice.pm | 33 +- .../AI-MXNet/lib/AI/MXNet/Optimizer.pm | 29 +- .../AI-MXNet/lib/AI/MXNet/Profiler.pm | 17 + perl-package/AI-MXNet/lib/AI/MXNet/RNN.pm | 17 + .../AI-MXNet/lib/AI/MXNet/RNN/Cell.pm | 19 +- perl-package/AI-MXNet/lib/AI/MXNet/RNN/IO.pm | 19 +- perl-package/AI-MXNet/lib/AI/MXNet/Random.pm | 19 +- .../AI-MXNet/lib/AI/MXNet/RecordIO.pm | 21 +- perl-package/AI-MXNet/lib/AI/MXNet/Rtc.pm | 19 +- perl-package/AI-MXNet/lib/AI/MXNet/Symbol.pm | 23 +- .../AI-MXNet/lib/AI/MXNet/Symbol/AttrScope.pm | 17 + .../AI-MXNet/lib/AI/MXNet/Symbol/Base.pm | 23 +- .../AI-MXNet/lib/AI/MXNet/Symbol/Doc.pm | 17 + .../lib/AI/MXNet/Symbol/NameManager.pm | 17 + .../AI-MXNet/lib/AI/MXNet/TestUtils.pm | 19 +- perl-package/AI-MXNet/lib/AI/MXNet/Types.pm | 17 + .../AI-MXNet/lib/AI/MXNet/Util/Printable.pm | 19 +- .../AI-MXNet/lib/AI/MXNet/Visualization.pm | 21 +- perl-package/AI-MXNetCAPI/lib/AI/MXNetCAPI.pm | 17 + perl-package/AI-NNVMCAPI/lib/AI/NNVMCAPI.pm | 17 + perl-package/test.sh | 17 + plugin/caffe/caffe_blob.cc | 20 +- plugin/caffe/caffe_blob.h | 20 +- plugin/caffe/caffe_common.cc | 22 +- plugin/caffe/caffe_common.h | 20 +- plugin/caffe/caffe_data_iter.cc | 20 +- plugin/caffe/caffe_fieldentry.h | 20 +- plugin/caffe/caffe_loss-inl.h | 20 +- plugin/caffe/caffe_loss.cc | 24 +- plugin/caffe/caffe_loss.cu | 24 +- plugin/caffe/caffe_op-inl.h | 20 +- plugin/caffe/caffe_op.cc | 22 +- plugin/caffe/caffe_op.cu | 20 +- plugin/caffe/caffe_stream.cc | 22 +- plugin/caffe/caffe_stream.h | 22 +- plugin/opencv/__init__.py | 17 + plugin/opencv/cv_api.cc | 20 +- plugin/opencv/cv_api.h | 20 +- plugin/opencv/opencv.py | 17 + plugin/sframe/iter_sframe.cc | 20 +- plugin/torch/torch_base.cc | 20 +- plugin/torch/torch_base.h | 20 +- plugin/torch/torch_criterion-inl.h | 20 +- plugin/torch/torch_criterion.cc | 20 +- plugin/torch/torch_criterion.cu | 20 +- plugin/torch/torch_function.cc | 20 +- plugin/torch/torch_function.h | 20 +- plugin/torch/torch_module-inl.h | 20 +- plugin/torch/torch_module.cc | 20 +- plugin/torch/torch_module.cu | 20 +- plugin/warpctc/warpctc-inl.h | 20 +- plugin/warpctc/warpctc.cc | 20 +- plugin/warpctc/warpctc.cu | 20 +- prepare_mkl.sh | 18 + python/mxnet/__init__.py | 18 + python/mxnet/_ctypes/__init__.py | 17 + python/mxnet/_ctypes/ndarray.py | 17 + python/mxnet/_ctypes/symbol.py | 17 + python/mxnet/_cy2/__init__.py | 17 + python/mxnet/_cy3/__init__.py | 17 + python/mxnet/_ndarray_internal.py | 17 + python/mxnet/_symbol_internal.py | 17 + python/mxnet/attribute.py | 17 + python/mxnet/autograd.py | 17 + python/mxnet/base.py | 17 + python/mxnet/callback.py | 17 + python/mxnet/context.py | 17 + python/mxnet/contrib/__init__.py | 17 + python/mxnet/contrib/autograd.py | 17 + python/mxnet/contrib/ndarray.py | 17 + python/mxnet/contrib/symbol.py | 17 + python/mxnet/contrib/tensorboard.py | 17 + python/mxnet/executor.py | 17 + python/mxnet/executor_manager.py | 17 + python/mxnet/gluon/__init__.py | 17 + python/mxnet/gluon/block.py | 17 + python/mxnet/gluon/data/__init__.py | 17 + python/mxnet/gluon/data/dataloader.py | 17 + python/mxnet/gluon/data/dataset.py | 17 + python/mxnet/gluon/data/sampler.py | 17 + python/mxnet/gluon/data/vision.py | 17 + python/mxnet/gluon/loss.py | 17 + python/mxnet/gluon/model_zoo/__init__.py | 17 + python/mxnet/gluon/model_zoo/custom_layers.py | 17 + python/mxnet/gluon/model_zoo/model_store.py | 17 + .../mxnet/gluon/model_zoo/vision/__init__.py | 17 + .../mxnet/gluon/model_zoo/vision/alexnet.py | 17 + .../mxnet/gluon/model_zoo/vision/densenet.py | 17 + .../mxnet/gluon/model_zoo/vision/inception.py | 17 + python/mxnet/gluon/model_zoo/vision/resnet.py | 17 + .../gluon/model_zoo/vision/squeezenet.py | 17 + python/mxnet/gluon/model_zoo/vision/vgg.py | 17 + python/mxnet/gluon/nn/__init__.py | 17 + python/mxnet/gluon/nn/basic_layers.py | 17 + python/mxnet/gluon/nn/conv_layers.py | 19 +- python/mxnet/gluon/parameter.py | 17 + python/mxnet/gluon/rnn/__init__.py | 17 + python/mxnet/gluon/rnn/rnn_cell.py | 17 + python/mxnet/gluon/rnn/rnn_layer.py | 17 + python/mxnet/gluon/trainer.py | 17 + python/mxnet/gluon/utils.py | 17 + python/mxnet/image/__init__.py | 17 + python/mxnet/image/detection.py | 17 + python/mxnet/image/image.py | 17 + python/mxnet/initializer.py | 17 + python/mxnet/io.py | 17 + python/mxnet/kvstore.py | 17 + python/mxnet/kvstore_server.py | 17 + python/mxnet/libinfo.py | 17 + python/mxnet/log.py | 18 + python/mxnet/lr_scheduler.py | 17 + python/mxnet/metric.py | 17 + python/mxnet/misc.py | 17 + python/mxnet/model.py | 17 + python/mxnet/module/__init__.py | 17 + python/mxnet/module/base_module.py | 17 + python/mxnet/module/bucketing_module.py | 17 + python/mxnet/module/executor_group.py | 17 + python/mxnet/module/module.py | 17 + python/mxnet/module/python_module.py | 17 + python/mxnet/module/sequential_module.py | 17 + python/mxnet/monitor.py | 17 + python/mxnet/name.py | 17 + python/mxnet/ndarray.py | 21 +- python/mxnet/ndarray_doc.py | 17 + python/mxnet/notebook/__init__.py | 17 + python/mxnet/notebook/callback.py | 17 + python/mxnet/operator.py | 17 + python/mxnet/optimizer.py | 17 + python/mxnet/profiler.py | 17 + python/mxnet/random.py | 17 + python/mxnet/recordio.py | 17 + python/mxnet/registry.py | 17 + python/mxnet/rnn/__init__.py | 17 + python/mxnet/rnn/io.py | 17 + python/mxnet/rnn/rnn.py | 17 + python/mxnet/rnn/rnn_cell.py | 17 + python/mxnet/rtc.py | 17 + python/mxnet/symbol.py | 17 + python/mxnet/symbol_doc.py | 17 + python/mxnet/test_utils.py | 17 + python/mxnet/torch.py | 17 + python/mxnet/visualization.py | 17 + python/setup.py | 17 + scala-package/core/scripts/get_cifar_data.sh | 18 + scala-package/core/scripts/get_mnist_data.sh | 18 + .../examples/scripts/customop/run_customop.sh | 18 + .../scripts/customop/run_customopwithrtc.sh | 18 + .../examples/scripts/module/mnist_mlp.sh | 18 + .../scripts/module/run_sequential_module.sh | 18 + .../neuralstyle_end2end/run_test_end2end.sh | 22 +- .../neuralstyle_end2end/run_train_end2end.sh | 24 +- .../scripts/profiler/run_profiler_matmul.sh | 18 + .../scripts/profiler/run_profiler_ndarray.sh | 18 + .../examples/scripts/rnn/run_test_charrnn.sh | 18 + .../examples/scripts/rnn/run_train_charrnn.sh | 18 + .../scripts/run_cnntextclassification.sh | 24 +- .../examples/scripts/run_gan_mnist.sh | 20 +- .../examples/scripts/run_multitask.sh | 18 + .../examples/scripts/run_neuralstyle.sh | 24 +- .../examples/scripts/run_visualization.sh | 20 +- .../native/ml_dmlc_mxnet_init_native_c_api.cc | 20 +- .../native/src/main/native/jni_helper_func.h | 20 +- .../main/native/ml_dmlc_mxnet_native_c_api.cc | 20 +- scala-package/spark/bin/run-mnist-example.sh | 18 + setup-utils/install-mxnet-amz-linux.sh | 18 + setup-utils/install-mxnet-fedora-python.sh | 22 +- setup-utils/install-mxnet-osx-python.sh | 40 +- setup-utils/install-mxnet-ubuntu-python.sh | 18 + setup-utils/install-mxnet-ubuntu-r.sh | 20 +- src/c_api/c_api.cc | 20 +- src/c_api/c_api_common.h | 20 +- src/c_api/c_api_error.cc | 20 +- src/c_api/c_api_executor.cc | 20 +- src/c_api/c_api_ndarray.cc | 20 +- src/c_api/c_api_symbolic.cc | 20 +- src/c_api/c_predict_api.cc | 20 +- src/common/cuda_utils.h | 20 +- src/common/lazy_alloc_array.h | 20 +- src/common/mxrtc.cc | 20 +- src/common/object_pool.h | 20 +- src/common/utils.h | 20 +- src/engine/engine.cc | 20 +- src/engine/engine_impl.h | 20 +- src/engine/naive_engine.cc | 20 +- src/engine/profiler.cc | 20 +- src/engine/profiler.h | 20 +- src/engine/stream_manager.h | 20 +- src/engine/thread_pool.h | 20 +- src/engine/threaded_engine.cc | 20 +- src/engine/threaded_engine.h | 20 +- src/engine/threaded_engine_perdevice.cc | 20 +- src/engine/threaded_engine_pooled.cc | 20 +- src/executor/attach_op_execs_pass.cc | 20 +- src/executor/attach_op_resource_pass.cc | 20 +- src/executor/exec_pass.h | 20 +- src/executor/graph_executor.cc | 20 +- src/executor/graph_executor.h | 20 +- src/executor/inplace_addto_detect_pass.cc | 20 +- src/initialize.cc | 20 +- src/io/image_aug_default.cc | 20 +- src/io/image_augmenter.h | 20 +- src/io/image_det_aug_default.cc | 20 +- src/io/image_io.cc | 20 +- src/io/image_iter_common.h | 20 +- src/io/image_recordio.h | 26 +- src/io/inst_vector.h | 20 +- src/io/io.cc | 20 +- src/io/iter_batchloader.h | 20 +- src/io/iter_csv.cc | 20 +- src/io/iter_image_det_recordio.cc | 20 +- src/io/iter_image_recordio.cc | 20 +- src/io/iter_image_recordio_2.cc | 20 +- src/io/iter_mnist.cc | 20 +- src/io/iter_normalize.h | 20 +- src/io/iter_prefetcher.h | 20 +- src/kvstore/comm.h | 20 +- src/kvstore/kvstore.cc | 20 +- src/kvstore/kvstore_dist.h | 20 +- src/kvstore/kvstore_dist_server.h | 20 +- src/kvstore/kvstore_local.h | 20 +- src/ndarray/autograd.cc | 20 +- src/ndarray/autograd.h | 20 +- src/ndarray/ndarray.cc | 20 +- src/ndarray/ndarray_function-inl.h | 20 +- src/ndarray/ndarray_function.cc | 20 +- src/ndarray/ndarray_function.cu | 20 +- src/ndarray/ndarray_function.h | 20 +- src/operator/activation-inl.h | 20 +- src/operator/activation.cc | 20 +- src/operator/activation.cu | 20 +- src/operator/batch_norm-inl.h | 20 +- src/operator/batch_norm.cc | 20 +- src/operator/batch_norm.cu | 20 +- src/operator/batch_norm_v1-inl.h | 20 +- src/operator/batch_norm_v1.cc | 20 +- src/operator/batch_norm_v1.cu | 20 +- src/operator/bilinear_sampler-inl.h | 456 +++--- src/operator/bilinear_sampler.cc | 474 ++++--- src/operator/bilinear_sampler.cu | 432 +++--- src/operator/channel_op_common.h | 20 +- src/operator/concat-inl.h | 20 +- src/operator/concat.cc | 20 +- src/operator/concat.cu | 20 +- src/operator/contrib/count_sketch-inl.h | 20 +- src/operator/contrib/count_sketch.cc | 22 +- src/operator/contrib/count_sketch.cu | 20 +- src/operator/contrib/ctc_loss-inl.h | 20 +- src/operator/contrib/ctc_loss.cc | 40 +- src/operator/contrib/ctc_loss.cu | 20 +- .../contrib/deformable_convolution-inl.h | 19 + .../contrib/deformable_convolution.cc | 21 +- .../contrib/deformable_convolution.cu | 19 + .../contrib/deformable_psroi_pooling-inl.h | 19 + .../contrib/deformable_psroi_pooling.cc | 21 +- .../contrib/deformable_psroi_pooling.cu | 19 + src/operator/contrib/dequantize-inl.h | 20 +- src/operator/contrib/dequantize.cc | 20 +- src/operator/contrib/dequantize.cu | 20 +- src/operator/contrib/fft-inl.h | 20 +- src/operator/contrib/fft.cc | 20 +- src/operator/contrib/fft.cu | 20 +- src/operator/contrib/ifft-inl.h | 20 +- src/operator/contrib/ifft.cc | 20 +- src/operator/contrib/ifft.cu | 20 +- src/operator/contrib/krprod.h | 20 +- src/operator/contrib/multi_proposal-inl.h | 20 +- src/operator/contrib/multi_proposal.cc | 19 + src/operator/contrib/multi_proposal.cu | 20 +- src/operator/contrib/multibox_detection-inl.h | 20 +- src/operator/contrib/multibox_detection.cc | 20 +- src/operator/contrib/multibox_detection.cu | 20 +- src/operator/contrib/multibox_prior-inl.h | 20 +- src/operator/contrib/multibox_prior.cc | 20 +- src/operator/contrib/multibox_prior.cu | 20 +- src/operator/contrib/multibox_target-inl.h | 20 +- src/operator/contrib/multibox_target.cc | 20 +- src/operator/contrib/multibox_target.cu | 20 +- src/operator/contrib/nn/deformable_im2col.cuh | 53 +- src/operator/contrib/nn/deformable_im2col.h | 45 +- src/operator/contrib/proposal-inl.h | 20 +- src/operator/contrib/proposal.cc | 20 +- src/operator/contrib/proposal.cu | 20 +- src/operator/contrib/psroi_pooling-inl.h | 20 +- src/operator/contrib/psroi_pooling.cc | 20 +- src/operator/contrib/psroi_pooling.cu | 20 +- src/operator/contrib/quantize-inl.h | 20 +- src/operator/contrib/quantize.cc | 20 +- src/operator/contrib/quantize.cu | 20 +- src/operator/convolution-inl.h | 20 +- src/operator/convolution.cc | 20 +- src/operator/convolution.cu | 20 +- src/operator/convolution_v1-inl.h | 20 +- src/operator/convolution_v1.cc | 20 +- src/operator/convolution_v1.cu | 20 +- src/operator/correlation-inl.h | 490 +++---- src/operator/correlation.cc | 368 ++--- src/operator/correlation.cu | 1237 +++++++++-------- src/operator/crop-inl.h | 446 +++--- src/operator/crop.cc | 20 +- src/operator/crop.cu | 20 +- src/operator/cross_device_copy.cc | 20 +- src/operator/cudnn_activation-inl.h | 20 +- src/operator/cudnn_algoreg-inl.h | 20 +- src/operator/cudnn_algoreg.cc | 20 +- src/operator/cudnn_batch_norm-inl.h | 20 +- src/operator/cudnn_batch_norm.cc | 20 +- src/operator/cudnn_batch_norm.cu | 20 +- src/operator/cudnn_bilinear_sampler-inl.h | 352 ++--- src/operator/cudnn_convolution-inl.h | 20 +- src/operator/cudnn_deconvolution-inl.h | 20 +- src/operator/cudnn_lrn-inl.h | 20 +- src/operator/cudnn_pooling-inl.h | 20 +- src/operator/cudnn_rnn-inl.h | 20 +- src/operator/cudnn_softmax_activation-inl.h | 20 +- src/operator/cudnn_spatial_transformer-inl.h | 20 +- src/operator/custom/custom-inl.h | 20 +- src/operator/custom/custom.cc | 20 +- src/operator/custom/native_op-inl.h | 20 +- src/operator/custom/native_op.cc | 20 +- src/operator/custom/native_op.cu | 20 +- src/operator/custom/ndarray_op-inl.h | 20 +- src/operator/custom/ndarray_op.cc | 20 +- src/operator/deconvolution-inl.h | 20 +- src/operator/deconvolution.cc | 20 +- src/operator/deconvolution.cu | 20 +- src/operator/dropout-inl.h | 20 +- src/operator/dropout.cc | 20 +- src/operator/dropout.cu | 20 +- src/operator/elemwise_op_common.h | 20 +- src/operator/fully_connected-inl.h | 20 +- src/operator/fully_connected.cc | 20 +- src/operator/fully_connected.cu | 20 +- src/operator/grid_generator-inl.h | 654 ++++----- src/operator/grid_generator.cc | 90 +- src/operator/grid_generator.cu | 60 +- .../identity_attach_KL_sparse_reg-inl.h | 372 ++--- src/operator/identity_attach_KL_sparse_reg.cc | 92 +- src/operator/identity_attach_KL_sparse_reg.cu | 50 +- src/operator/instance_norm-inl.h | 20 +- src/operator/instance_norm.cc | 20 +- src/operator/instance_norm.cu | 20 +- src/operator/l2_normalization-inl.h | 20 +- src/operator/l2_normalization.cc | 20 +- src/operator/l2_normalization.cu | 20 +- src/operator/leaky_relu-inl.h | 20 +- src/operator/leaky_relu.cc | 22 +- src/operator/leaky_relu.cu | 20 +- src/operator/loss_binary_op-inl.h | 20 +- src/operator/loss_binary_op.cc | 20 +- src/operator/loss_binary_op.cu | 20 +- src/operator/lrn-inl.h | 20 +- src/operator/lrn.cc | 20 +- src/operator/lrn.cu | 20 +- src/operator/make_loss-inl.h | 20 +- src/operator/make_loss.cc | 20 +- src/operator/make_loss.cu | 20 +- src/operator/mshadow_op.h | 20 +- src/operator/mxnet_op.h | 20 +- src/operator/nn/im2col.cuh | 45 +- src/operator/nn/im2col.h | 20 +- src/operator/nn/pool.cuh | 19 + src/operator/nn/pool.h | 20 +- src/operator/nn/softmax-inl.h | 20 +- src/operator/nn/softmax.cc | 20 +- src/operator/nn/softmax.cu | 20 +- src/operator/nnpack/nnpack_convolution-inl.h | 20 +- .../nnpack/nnpack_fully_connected-inl.h | 20 +- src/operator/nnpack/nnpack_pooling-inl.h | 20 +- src/operator/nnpack/nnpack_util.cc | 20 +- src/operator/nnpack/nnpack_util.h | 20 +- src/operator/operator.cc | 20 +- src/operator/operator_common.h | 20 +- src/operator/operator_util.cc | 20 +- src/operator/optimizer_op-inl.h | 20 +- src/operator/optimizer_op.cc | 20 +- src/operator/optimizer_op.cu | 20 +- src/operator/pad-inl.h | 20 +- src/operator/pad.cc | 20 +- src/operator/pad.cu | 20 +- src/operator/pooling-inl.h | 20 +- src/operator/pooling.cc | 20 +- src/operator/pooling.cu | 20 +- src/operator/pooling_v1-inl.h | 20 +- src/operator/pooling_v1.cc | 20 +- src/operator/pooling_v1.cu | 20 +- src/operator/random/multisample_op.cc | 20 +- src/operator/random/multisample_op.h | 20 +- src/operator/random/sample_multinomial_op.cc | 20 +- src/operator/random/sample_multinomial_op.cu | 20 +- src/operator/random/sample_multinomial_op.h | 20 +- src/operator/random/sample_op.cc | 26 +- src/operator/random/sample_op.cu | 20 +- src/operator/random/sample_op.h | 20 +- src/operator/regression_output-inl.h | 20 +- src/operator/regression_output.cc | 20 +- src/operator/regression_output.cu | 20 +- src/operator/rnn-inl.h | 20 +- src/operator/rnn.cc | 20 +- src/operator/rnn.cu | 20 +- src/operator/roi_pooling-inl.h | 20 +- src/operator/roi_pooling.cc | 20 +- src/operator/roi_pooling.cu | 20 +- src/operator/sequence_last-inl.h | 20 +- src/operator/sequence_last.cc | 20 +- src/operator/sequence_last.cu | 20 +- src/operator/sequence_mask-inl.h | 20 +- src/operator/sequence_mask.cc | 20 +- src/operator/sequence_mask.cu | 20 +- src/operator/sequence_op_common.h | 20 +- src/operator/sequence_reverse-inl.h | 20 +- src/operator/sequence_reverse.cc | 20 +- src/operator/sequence_reverse.cu | 20 +- src/operator/slice_channel-inl.h | 20 +- src/operator/slice_channel.cc | 22 +- src/operator/slice_channel.cu | 20 +- src/operator/softmax_activation-inl.h | 20 +- src/operator/softmax_activation.cc | 20 +- src/operator/softmax_activation.cu | 20 +- src/operator/softmax_output-inl.h | 20 +- src/operator/softmax_output.cc | 20 +- src/operator/softmax_output.cu | 20 +- src/operator/spatial_transformer-inl.h | 20 +- src/operator/spatial_transformer.cc | 20 +- src/operator/spatial_transformer.cu | 20 +- src/operator/special_functions-inl.h | 20 +- src/operator/svm_output-inl.h | 20 +- src/operator/svm_output.cc | 20 +- src/operator/svm_output.cu | 20 +- src/operator/swapaxis-inl.h | 20 +- src/operator/swapaxis.cc | 20 +- src/operator/swapaxis.cu | 20 +- src/operator/tensor/broadcast_reduce-inl.cuh | 1229 ++++++++-------- src/operator/tensor/broadcast_reduce-inl.h | 20 +- src/operator/tensor/broadcast_reduce_op.h | 20 +- .../tensor/broadcast_reduce_op_index.cc | 20 +- .../tensor/broadcast_reduce_op_index.cu | 20 +- .../tensor/broadcast_reduce_op_value.cc | 20 +- .../tensor/broadcast_reduce_op_value.cu | 20 +- src/operator/tensor/control_flow_op.cc | 20 +- src/operator/tensor/control_flow_op.cu | 20 +- src/operator/tensor/control_flow_op.h | 20 +- .../tensor/elemwise_binary_broadcast_op.h | 20 +- .../elemwise_binary_broadcast_op_basic.cc | 20 +- .../elemwise_binary_broadcast_op_basic.cu | 20 +- .../elemwise_binary_broadcast_op_extended.cc | 20 +- .../elemwise_binary_broadcast_op_extended.cu | 20 +- .../elemwise_binary_broadcast_op_logic.cc | 20 +- .../elemwise_binary_broadcast_op_logic.cu | 20 +- src/operator/tensor/elemwise_binary_op.h | 20 +- .../tensor/elemwise_binary_op_basic.cc | 20 +- .../tensor/elemwise_binary_op_basic.cu | 20 +- .../tensor/elemwise_binary_op_extended.cc | 20 +- .../tensor/elemwise_binary_op_extended.cu | 20 +- .../tensor/elemwise_binary_op_logic.cc | 20 +- .../tensor/elemwise_binary_op_logic.cu | 20 +- .../tensor/elemwise_binary_scalar_op.h | 20 +- .../tensor/elemwise_binary_scalar_op_basic.cc | 20 +- .../tensor/elemwise_binary_scalar_op_basic.cu | 20 +- .../elemwise_binary_scalar_op_extended.cc | 20 +- .../elemwise_binary_scalar_op_extended.cu | 20 +- .../tensor/elemwise_binary_scalar_op_logic.cc | 20 +- .../tensor/elemwise_binary_scalar_op_logic.cu | 20 +- src/operator/tensor/elemwise_sum.cc | 20 +- src/operator/tensor/elemwise_sum.cu | 20 +- src/operator/tensor/elemwise_sum.h | 20 +- src/operator/tensor/elemwise_unary_op.cc | 20 +- src/operator/tensor/elemwise_unary_op.cu | 20 +- src/operator/tensor/elemwise_unary_op.h | 20 +- src/operator/tensor/indexing_op-inl.cuh | 605 ++++---- src/operator/tensor/indexing_op.cc | 20 +- src/operator/tensor/indexing_op.cu | 20 +- src/operator/tensor/indexing_op.h | 20 +- src/operator/tensor/init_op.cc | 20 +- src/operator/tensor/init_op.cu | 20 +- src/operator/tensor/init_op.h | 20 +- src/operator/tensor/la_op.cc | 22 +- src/operator/tensor/la_op.h | 20 +- src/operator/tensor/la_op_inline.h | 20 +- src/operator/tensor/matrix_op-inl.h | 20 +- src/operator/tensor/matrix_op.cc | 20 +- src/operator/tensor/matrix_op.cu | 20 +- src/operator/tensor/ordering_op-inl.h | 20 +- src/operator/tensor/ordering_op.cc | 20 +- src/operator/tensor/ordering_op.cu | 20 +- src/operator/tensor/sort_op-inl.cuh | 277 ++-- src/operator/tensor/sort_op.h | 192 +-- src/operator/upsampling-inl.h | 20 +- src/operator/upsampling.cc | 20 +- src/operator/upsampling.cu | 20 +- src/optimizer/sgd-inl.h | 20 +- src/resource.cc | 20 +- src/storage/cpu_device_storage.h | 20 +- src/storage/gpu_device_storage.h | 20 +- src/storage/naive_storage_manager.h | 20 +- src/storage/pinned_memory_storage.h | 20 +- src/storage/pooled_storage_manager.h | 20 +- src/storage/storage.cc | 20 +- src/storage/storage_manager.h | 20 +- tests/ci_build/ci_build.sh | 18 + tests/ci_build/install/install_julia.sh | 18 + tests/ci_build/install/install_library.sh | 18 + tests/ci_build/install/install_maven.sh | 18 + tests/ci_build/install/install_openblas.sh | 18 + tests/ci_build/install/install_opencv.sh | 18 + tests/ci_build/install/install_python2.sh | 18 + tests/ci_build/install/install_python3.sh | 18 + tests/ci_build/install/install_testdeps.sh | 18 + tests/ci_build/install/ubuntu_install_core.sh | 18 + tests/ci_build/install/ubuntu_install_perl.sh | 18 + .../ci_build/install/ubuntu_install_python.sh | 18 + tests/ci_build/install/ubuntu_install_r.sh | 18 + .../ci_build/install/ubuntu_install_scala.sh | 18 + tests/cpp/engine/threaded_engine_test.cc | 20 +- tests/cpp/include/test_op.h | 20 +- tests/cpp/include/test_perf.h | 20 +- tests/cpp/include/test_util.h | 20 +- tests/cpp/operator/batchnorm_test.cc | 20 +- tests/cpp/operator/krprod_test.cc | 20 +- tests/cpp/storage/storage_test.cc | 20 +- tests/cpp/test_main.cc | 20 +- tests/jenkins/run_as_user.sh | 18 + tests/jenkins/run_test.sh | 18 + tests/jenkins/run_test_amzn_linux_gpu.sh | 18 + tests/jenkins/run_test_installation_docs.sh | 22 +- tests/jenkins/run_test_pip_installations.sh | 18 + tests/jenkins/run_test_ubuntu.sh | 18 + tests/jenkins/set_user_permissions.sh | 18 + tests/nightly/TestDoc/doc_spell_checker.py | 17 + tests/nightly/TestDoc/doc_spell_grammar.sh | 18 + .../compilation_warnings.sh | 18 + .../compilation_warnings/process_output.py | 17 + tests/nightly/dist_lenet.py | 18 + tests/nightly/dist_sync_kvstore.py | 18 + tests/nightly/download.sh | 18 + tests/nightly/multi_lenet.py | 18 + .../assertion_util.py | 17 + .../model_util.py | 17 + .../mxnet_keras_integration_tests/profiler.py | 17 + .../test_mnist_mlp.py | 17 + tests/nightly/sh2ju.sh | 18 + tests/nightly/test_all.sh | 18 + tests/nightly/test_kvstore.py | 18 + .../test_mxnet_keras_integration_cpu.sh | 18 + .../test_mxnet_keras_integration_gpu.sh | 18 + tests/nightly/test_tutorial.py | 17 + tests/python/common/get_data.py | 17 + tests/python/common/models.py | 17 + tests/python/doctest/test_docstring.py | 17 + tests/python/gpu/test_forward.py | 17 + tests/python/gpu/test_operator_gpu.py | 17 + tests/python/gpu/test_rtc.py | 19 +- tests/python/predict/mxnet_predict_example.py | 17 + tests/python/train/common.py | 17 + tests/python/train/test_autograd.py | 17 + tests/python/train/test_bucketing.py | 17 + tests/python/train/test_conv.py | 17 + tests/python/train/test_dtype.py | 19 +- tests/python/train/test_mlp.py | 17 + tests/python/unittest/common.py | 17 + tests/python/unittest/test_attr.py | 17 + tests/python/unittest/test_autograd.py | 17 + .../python/unittest/test_contrib_autograd.py | 17 + tests/python/unittest/test_executor.py | 17 + tests/python/unittest/test_gluon.py | 17 + tests/python/unittest/test_gluon_data.py | 17 + tests/python/unittest/test_gluon_model_zoo.py | 17 + tests/python/unittest/test_gluon_rnn.py | 17 + tests/python/unittest/test_image.py | 17 + tests/python/unittest/test_infer_shape.py | 17 + tests/python/unittest/test_init.py | 17 + tests/python/unittest/test_io.py | 17 + tests/python/unittest/test_kvstore.py | 17 + tests/python/unittest/test_loss.py | 17 + tests/python/unittest/test_metric.py | 17 + tests/python/unittest/test_model_parallel.py | 17 + tests/python/unittest/test_module.py | 17 + .../python/unittest/test_multi_device_exec.py | 17 + tests/python/unittest/test_ndarray.py | 17 + tests/python/unittest/test_operator.py | 17 + tests/python/unittest/test_optimizer.py | 17 + tests/python/unittest/test_profiler.py | 17 + tests/python/unittest/test_random.py | 17 + tests/python/unittest/test_recordio.py | 19 +- tests/python/unittest/test_rnn.py | 17 + tests/python/unittest/test_symbol.py | 17 + tests/python/unittest/test_viz.py | 17 + tests/travis/is_core_changed.sh | 18 + tests/travis/run_test.sh | 18 + tests/travis/setup.sh | 18 + tests/travis/travis_after_failure.sh | 18 + tools/accnn/acc_conv.py | 171 ++- tools/accnn/acc_fc.py | 131 +- tools/accnn/accnn.py | 93 +- tools/accnn/rank_selection.py | 191 +-- tools/accnn/utils.py | 219 +-- tools/bandwidth/measure.py | 17 + tools/bandwidth/test_measure.py | 17 + tools/caffe_converter/caffe_parser.py | 17 + tools/caffe_converter/caffe_proto_utils.py | 17 + tools/caffe_converter/compare_layers.py | 17 + .../caffe_converter/convert_caffe_modelzoo.py | 17 + tools/caffe_converter/convert_mean.py | 17 + tools/caffe_converter/convert_model.py | 17 + tools/caffe_converter/convert_symbol.py | 17 + tools/caffe_converter/run.sh | 18 + tools/caffe_converter/test_converter.py | 17 + tools/im2rec.cc | 20 +- tools/im2rec.py | 631 +++++---- tools/ipynb2md.py | 18 + tools/kill-mxnet.py | 18 + tools/launch.py | 20 +- tools/license_header.py | 157 +++ tools/parse_log.py | 18 + tools/pip_package/make_pip_package.sh | 18 + tools/pip_package/setup.py | 17 + 1051 files changed, 24743 insertions(+), 6220 deletions(-) create mode 100644 tools/license_header.py diff --git a/Jenkinsfile b/Jenkinsfile index 49633d484981..370c2b397626 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -105,6 +105,7 @@ try { node('mxnetlinux') { ws('workspace/sanity') { init_git() + sh "python tools/license_header.py check" make('lint', 'cpplint rcpplint jnilint') make('lint', 'pylint') } diff --git a/amalgamation/amalgamation.py b/amalgamation/amalgamation.py index b33b81c62b4a..22b421d79fba 100644 --- a/amalgamation/amalgamation.py +++ b/amalgamation/amalgamation.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import sys import os.path, re, StringIO @@ -8,7 +25,7 @@ 'kvstore_dist.h', 'mach/clock.h', 'mach/mach.h', 'malloc.h', 'mkl.h', 'mkl_cblas.h', 'mkl_vsl.h', 'mkl_vsl_functions.h', 'nvml.h', 'opencv2/opencv.hpp', 'sys/stat.h', 'sys/types.h', 'cuda.h', 'cuda_fp16.h', - 'omp.h', 'execinfo.h', 'packet/sse-inl.h', 'emmintrin.h', 'thrust/device_vector.h', + 'omp.h', 'execinfo.h', 'packet/sse-inl.h', 'emmintrin.h', 'thrust/device_vector.h', 'cusolverDn.h' ] diff --git a/amalgamation/dmlc-minimum0.cc b/amalgamation/dmlc-minimum0.cc index bce61129ed2e..3f7a97bb0139 100644 --- a/amalgamation/dmlc-minimum0.cc +++ b/amalgamation/dmlc-minimum0.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright 2015 by Contributors. * \brief Mininum DMLC library Amalgamation, used for easy plugin of dmlc lib. * Normally this is not needed. */ diff --git a/amalgamation/jni/org_dmlc_mxnet_Predictor.h b/amalgamation/jni/org_dmlc_mxnet_Predictor.h index e5a68ed0d9d7..1bdf80d9ce6f 100644 --- a/amalgamation/jni/org_dmlc_mxnet_Predictor.h +++ b/amalgamation/jni/org_dmlc_mxnet_Predictor.h @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /* DO NOT EDIT THIS FILE - it is machine generated */ #include /* Header for class org_dmlc_mxnet_Predictor */ diff --git a/amalgamation/jni/predictor.cc b/amalgamation/jni/predictor.cc index b6cc9370b1f9..1936daf99f3d 100644 --- a/amalgamation/jni/predictor.cc +++ b/amalgamation/jni/predictor.cc @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + #include #include "org_dmlc_mxnet_Predictor.h" diff --git a/amalgamation/mxnet_predict0.cc b/amalgamation/mxnet_predict0.cc index ca1b581ce195..badf23771dbc 100644 --- a/amalgamation/mxnet_predict0.cc +++ b/amalgamation/mxnet_predict0.cc @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + // mxnet.cc #define MSHADOW_FORCE_STREAM diff --git a/amalgamation/python/mxnet_predict.py b/amalgamation/python/mxnet_predict.py index 684f23119d62..3dd6b387936f 100644 --- a/amalgamation/python/mxnet_predict.py +++ b/amalgamation/python/mxnet_predict.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 # pylint: disable=invalid-name, too-many-arguments """Lightweight API for mxnet prediction. diff --git a/cmake/Modules/FindAccelerate.cmake b/cmake/Modules/FindAccelerate.cmake index 8c9938246e54..695538ac924c 100644 --- a/cmake/Modules/FindAccelerate.cmake +++ b/cmake/Modules/FindAccelerate.cmake @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # Find the Apple Accelerate framework # # The following are set after configuration is done: diff --git a/cmake/Modules/FindAtlas.cmake b/cmake/Modules/FindAtlas.cmake index 350bbe9df5dc..27aaa0e856ab 100644 --- a/cmake/Modules/FindAtlas.cmake +++ b/cmake/Modules/FindAtlas.cmake @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # Find the Atlas (and Lapack) libraries # # The following variables are optionally searched for defaults diff --git a/cmake/Modules/FindJeMalloc.cmake b/cmake/Modules/FindJeMalloc.cmake index 8b965cf6c3bb..57f47448f0a0 100644 --- a/cmake/Modules/FindJeMalloc.cmake +++ b/cmake/Modules/FindJeMalloc.cmake @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # Copyright (c) 2014 Thomas Heller # Copyright (c) 2007-2012 Hartmut Kaiser diff --git a/cmake/Modules/FindMKL.cmake b/cmake/Modules/FindMKL.cmake index 9679f3d72e60..743a871ee7cd 100644 --- a/cmake/Modules/FindMKL.cmake +++ b/cmake/Modules/FindMKL.cmake @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # Find the MKL libraries # # Options: diff --git a/cmake/Modules/FindOpenBLAS.cmake b/cmake/Modules/FindOpenBLAS.cmake index b63817a29d3e..7c5272b7f779 100644 --- a/cmake/Modules/FindOpenBLAS.cmake +++ b/cmake/Modules/FindOpenBLAS.cmake @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + if(MKL_FOUND) message(ERROR " OpenBLAS is not required since MKL is enabled") endif() diff --git a/cmake/Utils.cmake b/cmake/Utils.cmake index c367edb75a74..ac6ce3926c37 100644 --- a/cmake/Utils.cmake +++ b/cmake/Utils.cmake @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # For cmake_parse_arguments include(CMakeParseArguments) diff --git a/cpp-package/example/feature_extract/run.sh b/cpp-package/example/feature_extract/run.sh index afac492b0a9d..dc6665604b1e 100755 --- a/cpp-package/example/feature_extract/run.sh +++ b/cpp-package/example/feature_extract/run.sh @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + ### To run the this example, ### ### 1. diff --git a/cpp-package/example/get_mnist.sh b/cpp-package/example/get_mnist.sh index 2bbe7a801872..40379621025d 100755 --- a/cpp-package/example/get_mnist.sh +++ b/cpp-package/example/get_mnist.sh @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + if [ ! -d "./mnist_data" ]; then mkdir mnist_data (cd mnist_data; wget http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz) diff --git a/cpp-package/example/run_lenet_with_mxdataiter.sh b/cpp-package/example/run_lenet_with_mxdataiter.sh index fffc355865bc..cafad3201635 100755 --- a/cpp-package/example/run_lenet_with_mxdataiter.sh +++ b/cpp-package/example/run_lenet_with_mxdataiter.sh @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + if [ ! -f "./mnist.zip" ]; then wget http://webdocs.cs.ualberta.ca/~bx3/data/mnist.zip unzip -u mnist.zip diff --git a/cpp-package/include/mxnet-cpp/MxNetCpp.h b/cpp-package/include/mxnet-cpp/MxNetCpp.h index 5d61b823baa2..882bbead51e5 100644 --- a/cpp-package/include/mxnet-cpp/MxNetCpp.h +++ b/cpp-package/include/mxnet-cpp/MxNetCpp.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file MxNetCpp.h * \brief meta include file for mxnet.cpp * \author Chuntao Hong, Zhang Chen diff --git a/cpp-package/include/mxnet-cpp/base.h b/cpp-package/include/mxnet-cpp/base.h index b684986a6f54..19375c0f81e8 100644 --- a/cpp-package/include/mxnet-cpp/base.h +++ b/cpp-package/include/mxnet-cpp/base.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! -* Copyright (c) 2016 by Contributors * \file base.h * \brief base definitions for mxnetcpp * \author Chuntao Hong, Zhang Chen diff --git a/cpp-package/include/mxnet-cpp/executor.h b/cpp-package/include/mxnet-cpp/executor.h index 67eec0100b65..7e45ef56ab95 100644 --- a/cpp-package/include/mxnet-cpp/executor.h +++ b/cpp-package/include/mxnet-cpp/executor.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! -* Copyright (c) 2016 by Contributors * \file executor.h * \brief executor definition * \author Chuntao Hong, Zhang Chen diff --git a/cpp-package/include/mxnet-cpp/initializer.h b/cpp-package/include/mxnet-cpp/initializer.h index f28656577482..e5bfa4da8eed 100644 --- a/cpp-package/include/mxnet-cpp/initializer.h +++ b/cpp-package/include/mxnet-cpp/initializer.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file initializer.h * \brief random initializer * \author Zhang Chen diff --git a/cpp-package/include/mxnet-cpp/io.h b/cpp-package/include/mxnet-cpp/io.h index 727a96467c63..7281416ae36a 100644 --- a/cpp-package/include/mxnet-cpp/io.h +++ b/cpp-package/include/mxnet-cpp/io.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! -* Copyright (c) 2016 by Contributors * \file operator.h * \brief definition of io, such as DataIter * \author Zhang Chen diff --git a/cpp-package/include/mxnet-cpp/kvstore.h b/cpp-package/include/mxnet-cpp/kvstore.h index 9bb33a4733dd..9c3c81f37ff7 100644 --- a/cpp-package/include/mxnet-cpp/kvstore.h +++ b/cpp-package/include/mxnet-cpp/kvstore.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! -* Copyright (c) 2016 by Contributors * \file kvstore.h * \brief definition of kvstore * \author Chuntao Hong diff --git a/cpp-package/include/mxnet-cpp/lr_scheduler.h b/cpp-package/include/mxnet-cpp/lr_scheduler.h index 4c56b7ab3f0b..b9381a830a88 100644 --- a/cpp-package/include/mxnet-cpp/lr_scheduler.h +++ b/cpp-package/include/mxnet-cpp/lr_scheduler.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! -* Copyright (c) 2017 by Contributors * \file lr_scheduler.h * \brief Scheduling learning rate */ diff --git a/cpp-package/include/mxnet-cpp/metric.h b/cpp-package/include/mxnet-cpp/metric.h index eda927199ca8..6dbb197dae49 100644 --- a/cpp-package/include/mxnet-cpp/metric.h +++ b/cpp-package/include/mxnet-cpp/metric.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! -* Copyright (c) 2016 by Contributors * \file base.h * \brief metrics defined * \author Zhang Chen diff --git a/cpp-package/include/mxnet-cpp/model.h b/cpp-package/include/mxnet-cpp/model.h index e4cb1a9aee95..c8af6a476a52 100644 --- a/cpp-package/include/mxnet-cpp/model.h +++ b/cpp-package/include/mxnet-cpp/model.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! -* Copyright (c) 2016 by Contributors * \file model.h * \brief MXNET.cpp model module * \author Zhang Chen diff --git a/cpp-package/include/mxnet-cpp/monitor.h b/cpp-package/include/mxnet-cpp/monitor.h index afe030cbd5d8..33ef4855c1a9 100644 --- a/cpp-package/include/mxnet-cpp/monitor.h +++ b/cpp-package/include/mxnet-cpp/monitor.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! -* Copyright (c) 2017 by Contributors * \file monitor.h * \brief monitor definition * \author Xin Li diff --git a/cpp-package/include/mxnet-cpp/ndarray.h b/cpp-package/include/mxnet-cpp/ndarray.h index 52451faa94cc..9e196d0730a8 100644 --- a/cpp-package/include/mxnet-cpp/ndarray.h +++ b/cpp-package/include/mxnet-cpp/ndarray.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! -* Copyright (c) 2016 by Contributors * \file ndarray.h * \brief definition of ndarray * \author Chuntao Hong, Zhang Chen diff --git a/cpp-package/include/mxnet-cpp/op_map.h b/cpp-package/include/mxnet-cpp/op_map.h index ea75a8ca7b4c..b54cc0ae2c01 100644 --- a/cpp-package/include/mxnet-cpp/op_map.h +++ b/cpp-package/include/mxnet-cpp/op_map.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! -* Copyright (c) 2016 by Contributors * \file op_map.h * \brief definition of OpMap * \author Chuntao Hong diff --git a/cpp-package/include/mxnet-cpp/op_suppl.h b/cpp-package/include/mxnet-cpp/op_suppl.h index b66521bc0654..52cdae772a68 100644 --- a/cpp-package/include/mxnet-cpp/op_suppl.h +++ b/cpp-package/include/mxnet-cpp/op_suppl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! -* Copyright (c) 2016 by Contributors * \file op_suppl.h * \brief A supplement and amendment of the operators from op.h * \author Zhang Chen, zhubuntu, Xin Li diff --git a/cpp-package/include/mxnet-cpp/op_util.h b/cpp-package/include/mxnet-cpp/op_util.h index 5a737480d469..20e06a851814 100644 --- a/cpp-package/include/mxnet-cpp/op_util.h +++ b/cpp-package/include/mxnet-cpp/op_util.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! -* Copyright (c) 2017 by Contributors * \file op_util.h * \brief operator helper functions * \author Chris Olivier diff --git a/cpp-package/include/mxnet-cpp/operator.h b/cpp-package/include/mxnet-cpp/operator.h index 4fc45bbc9f04..02bd21ebe8c9 100644 --- a/cpp-package/include/mxnet-cpp/operator.h +++ b/cpp-package/include/mxnet-cpp/operator.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! -* Copyright (c) 2016 by Contributors * \file operator.h * \brief definition of operator * \author Chuntao Hong, Zhang Chen diff --git a/cpp-package/include/mxnet-cpp/optimizer.h b/cpp-package/include/mxnet-cpp/optimizer.h index 1bc36d58fd1a..e57da5d95ceb 100644 --- a/cpp-package/include/mxnet-cpp/optimizer.h +++ b/cpp-package/include/mxnet-cpp/optimizer.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! -* Copyright (c) 2016 by Contributors * \file optimizer.h * \brief definition of optimizer * \author Chuntao Hong, Zhang Chen diff --git a/cpp-package/include/mxnet-cpp/shape.h b/cpp-package/include/mxnet-cpp/shape.h index d30ea9df2531..2793e436c072 100644 --- a/cpp-package/include/mxnet-cpp/shape.h +++ b/cpp-package/include/mxnet-cpp/shape.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! -* Copyright (c) 2016 by Contributors * \file shape.h * \brief definition of shape * \author Chuntao Hong, Zhang Chen diff --git a/cpp-package/include/mxnet-cpp/symbol.h b/cpp-package/include/mxnet-cpp/symbol.h index c04ae2a03d29..888aebd6f3ad 100644 --- a/cpp-package/include/mxnet-cpp/symbol.h +++ b/cpp-package/include/mxnet-cpp/symbol.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! -* Copyright (c) 2016 by Contributors * \file symbol.h * \brief definition of symbol * \author Chuntao Hong, Zhang Chen diff --git a/cpp-package/scripts/OpWrapperGenerator.py b/cpp-package/scripts/OpWrapperGenerator.py index 8f762368d0a4..83495febcc63 100644 --- a/cpp-package/scripts/OpWrapperGenerator.py +++ b/cpp-package/scripts/OpWrapperGenerator.py @@ -1,4 +1,21 @@ -# -*- coding: utf-8 -*- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# -*- coding: utf-8 -*- # This is a python script that generates operator wrappers such as FullyConnected, # based on current libmxnet.dll. This script is written so that we don't need to # write new operator wrappers when new ones are added to the library. diff --git a/cpp-package/scripts/lint.py b/cpp-package/scripts/lint.py index 89492eda4d82..f9f284ffc005 100644 --- a/cpp-package/scripts/lint.py +++ b/cpp-package/scripts/lint.py @@ -1,4 +1,22 @@ #!/usr/bin/env python + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: disable=protected-access, unused-variable, locally-disabled, redefined-variable-type """Lint helper to generate lint summary of source. Copyright by Contributors diff --git a/cpp-package/tests/ci_test.sh b/cpp-package/tests/ci_test.sh index 29d0a9df340a..3b2af35bf1be 100755 --- a/cpp-package/tests/ci_test.sh +++ b/cpp-package/tests/ci_test.sh @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + set -e # exit on the first error cd $(dirname $(readlink -f $0))/../example echo $PWD diff --git a/cpp-package/tests/travis/run_test.sh b/cpp-package/tests/travis/run_test.sh index 27506584f40c..4925b3526bf3 100755 --- a/cpp-package/tests/travis/run_test.sh +++ b/cpp-package/tests/travis/run_test.sh @@ -1,5 +1,23 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + if [ ${TASK} == "lint" ]; then make lint || exit -1 echo "Check documentations of c++ code..." diff --git a/cpp-package/tests/travis/setup.sh b/cpp-package/tests/travis/setup.sh index 4238c7654fe4..5a3813ee34eb 100755 --- a/cpp-package/tests/travis/setup.sh +++ b/cpp-package/tests/travis/setup.sh @@ -1,5 +1,23 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + if [ ${TASK} == "lint" ]; then pip install cpplint 'pylint==1.4.4' 'astroid==1.3.6' --user fi diff --git a/docker/Dockerfiles/Dockerfile.in.scala b/docker/Dockerfiles/Dockerfile.in.scala index 6898126c7cb2..1fe93652920b 100644 --- a/docker/Dockerfiles/Dockerfile.in.scala +++ b/docker/Dockerfiles/Dockerfile.in.scala @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + # -*- mode: dockerfile -*- # part of the dockerfile to install the scala binding diff --git a/docker/install/cpp.sh b/docker/install/cpp.sh index f30ab52f9a52..1aa55acfc977 100755 --- a/docker/install/cpp.sh +++ b/docker/install/cpp.sh @@ -1,4 +1,22 @@ #!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # libraries for building mxnet c++ core on ubuntu apt-get update && apt-get install -y \ diff --git a/docker/install/julia.sh b/docker/install/julia.sh index 604a1bc2c234..e6fe49bd5acc 100755 --- a/docker/install/julia.sh +++ b/docker/install/julia.sh @@ -1,4 +1,22 @@ #!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # install libraries for mxnet's julia package on ubuntu # the julia version shipped with ubuntu (version 0.4) is too low. so download a diff --git a/docker/install/perl.sh b/docker/install/perl.sh index da4df67a464a..a981746bc18d 100755 --- a/docker/install/perl.sh +++ b/docker/install/perl.sh @@ -1,4 +1,22 @@ #!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # install libraries for mxnet's perl package on ubuntu apt-get update && apt-get install -y libmouse-perl pdl cpanminus swig libgraphviz-perl cpanm -q Function::Parameters diff --git a/docker/install/python.sh b/docker/install/python.sh index 0459bb9198c4..763f27b8282f 100755 --- a/docker/install/python.sh +++ b/docker/install/python.sh @@ -1,4 +1,22 @@ #!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # install libraries for mxnet's python package on ubuntu apt-get update && apt-get install -y python-dev python3-dev diff --git a/docker/install/r.sh b/docker/install/r.sh index 9351763ddcee..a0fa27359ba5 100755 --- a/docker/install/r.sh +++ b/docker/install/r.sh @@ -1,4 +1,22 @@ #!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # install libraries for mxnet's r package on ubuntu echo "deb http://cran.rstudio.com/bin/linux/ubuntu trusty/" >> /etc/apt/sources.list diff --git a/docker/install/scala.sh b/docker/install/scala.sh index 8cbe91199463..bb0bb9c900d4 100755 --- a/docker/install/scala.sh +++ b/docker/install/scala.sh @@ -1,4 +1,22 @@ #!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # install libraries for mxnet's scala package on ubuntu apt-get install -y maven default-jdk diff --git a/docker/run.sh b/docker/run.sh index b13e13caa5fc..f570f706d9ec 100644 --- a/docker/run.sh +++ b/docker/run.sh @@ -1,4 +1,22 @@ #!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # Build and push all docker containers DEVICES=('cpu' 'gpu') diff --git a/docker/tool.sh b/docker/tool.sh index 222d428fb68b..d8ab9dba0f26 100755 --- a/docker/tool.sh +++ b/docker/tool.sh @@ -1,4 +1,22 @@ #!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # # Script to build, test and push a docker container # diff --git a/docs/build_version_doc/AddVersion.py b/docs/build_version_doc/AddVersion.py index ee46ef5ffd12..38ce48f63c2f 100644 --- a/docs/build_version_doc/AddVersion.py +++ b/docs/build_version_doc/AddVersion.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import os import argparse from bs4 import BeautifulSoup as bs @@ -55,4 +72,4 @@ navbar_mobile.append(version_str_mobile) outstr = str(content).replace('<', '<').replace('>', '>') with open(os.path.join(path, name), "w") as outf: - outf.write(outstr) \ No newline at end of file + outf.write(outstr) diff --git a/docs/build_version_doc/build_doc.sh b/docs/build_version_doc/build_doc.sh index 99b6bd81b517..c5b59ba1df92 100755 --- a/docs/build_version_doc/build_doc.sh +++ b/docs/build_version_doc/build_doc.sh @@ -1,12 +1,30 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + web_url="$1" web_folder="VersionedWeb" local_build="latest" web_branch="$2" git clone $web_url $web_folder cd $web_folder -git checkout $web_branch +git checkout $web_branch cd .. mkdir "$local_build" @@ -14,7 +32,7 @@ mkdir "$local_build" tag_list_file="tag_list.txt" cp "$web_folder/tag.txt" "$tag_list_file" tag_list=() -while read -r line +while read -r line do tag_list+=("$line") done < "$tag_list_file" diff --git a/docs/conf.py b/docs/conf.py index 7a1059e10948..ad51323f01e9 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # -*- coding: utf-8 -*- import sys, os, re, subprocess import mock diff --git a/docs/mxdoc.py b/docs/mxdoc.py index 25f6af779ef6..2726a1ca0676 100644 --- a/docs/mxdoc.py +++ b/docs/mxdoc.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """A sphnix-doc plugin to build mxnet docs""" import subprocess import re diff --git a/example/adversary/data.py b/example/adversary/data.py index d39821f52145..0ca8e1fd6653 100644 --- a/example/adversary/data.py +++ b/example/adversary/data.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: skip-file """ data iterator for mnist """ import sys diff --git a/example/autoencoder/autoencoder.py b/example/autoencoder/autoencoder.py index ca8db7a70289..a84b2718f748 100644 --- a/example/autoencoder/autoencoder.py +++ b/example/autoencoder/autoencoder.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: skip-file import mxnet as mx from mxnet import misc diff --git a/example/autoencoder/data.py b/example/autoencoder/data.py index ecd117d86218..d6a25edce886 100644 --- a/example/autoencoder/data.py +++ b/example/autoencoder/data.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import os import numpy as np from sklearn.datasets import fetch_mldata diff --git a/example/autoencoder/mnist_sae.py b/example/autoencoder/mnist_sae.py index 538d8b976d0c..552594823a93 100644 --- a/example/autoencoder/mnist_sae.py +++ b/example/autoencoder/mnist_sae.py @@ -1,27 +1,44 @@ -# pylint: skip-file -from __future__ import print_function -import mxnet as mx -import numpy as np -import logging -import data -from autoencoder import AutoEncoderModel - - -if __name__ == '__main__': - # set to INFO to see less information during training - logging.basicConfig(level=logging.DEBUG) - ae_model = AutoEncoderModel(mx.gpu(0), [784,500,500,2000,10], pt_dropout=0.2, - internal_act='relu', output_act='relu') - - X, _ = data.get_mnist() - train_X = X[:60000] - val_X = X[60000:] - - ae_model.layerwise_pretrain(train_X, 256, 50000, 'sgd', l_rate=0.1, decay=0.0, - lr_scheduler=mx.misc.FactorScheduler(20000,0.1)) - ae_model.finetune(train_X, 256, 100000, 'sgd', l_rate=0.1, decay=0.0, - lr_scheduler=mx.misc.FactorScheduler(20000,0.1)) - ae_model.save('mnist_pt.arg') - ae_model.load('mnist_pt.arg') - print("Training error:", ae_model.eval(train_X)) - print("Validation error:", ae_model.eval(val_X)) +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# pylint: skip-file +from __future__ import print_function +import mxnet as mx +import numpy as np +import logging +import data +from autoencoder import AutoEncoderModel + + +if __name__ == '__main__': + # set to INFO to see less information during training + logging.basicConfig(level=logging.DEBUG) + ae_model = AutoEncoderModel(mx.gpu(0), [784,500,500,2000,10], pt_dropout=0.2, + internal_act='relu', output_act='relu') + + X, _ = data.get_mnist() + train_X = X[:60000] + val_X = X[60000:] + + ae_model.layerwise_pretrain(train_X, 256, 50000, 'sgd', l_rate=0.1, decay=0.0, + lr_scheduler=mx.misc.FactorScheduler(20000,0.1)) + ae_model.finetune(train_X, 256, 100000, 'sgd', l_rate=0.1, decay=0.0, + lr_scheduler=mx.misc.FactorScheduler(20000,0.1)) + ae_model.save('mnist_pt.arg') + ae_model.load('mnist_pt.arg') + print("Training error:", ae_model.eval(train_X)) + print("Validation error:", ae_model.eval(val_X)) diff --git a/example/autoencoder/model.py b/example/autoencoder/model.py index 85fb48c5797b..1aaae1b5fdad 100644 --- a/example/autoencoder/model.py +++ b/example/autoencoder/model.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: skip-file import mxnet as mx import numpy as np diff --git a/example/autoencoder/solver.py b/example/autoencoder/solver.py index 21c5da2ed4c7..5589c5a14010 100644 --- a/example/autoencoder/solver.py +++ b/example/autoencoder/solver.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: skip-file import mxnet as mx import numpy as np diff --git a/example/bayesian-methods/algos.py b/example/bayesian-methods/algos.py index 81df9c5d8534..e47a18f398e9 100644 --- a/example/bayesian-methods/algos.py +++ b/example/bayesian-methods/algos.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from __future__ import print_function import mxnet as mx import mxnet.ndarray as nd diff --git a/example/bayesian-methods/bdk_demo.py b/example/bayesian-methods/bdk_demo.py index 77e9b945cb14..145dac10e2a6 100644 --- a/example/bayesian-methods/bdk_demo.py +++ b/example/bayesian-methods/bdk_demo.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from __future__ import print_function import mxnet as mx import mxnet.ndarray as nd diff --git a/example/bayesian-methods/data_loader.py b/example/bayesian-methods/data_loader.py index 90b01e0144cc..2649eb560b68 100644 --- a/example/bayesian-methods/data_loader.py +++ b/example/bayesian-methods/data_loader.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from __future__ import print_function import numpy import os diff --git a/example/bayesian-methods/utils.py b/example/bayesian-methods/utils.py index 4a2f41d7e149..a2744373e87d 100644 --- a/example/bayesian-methods/utils.py +++ b/example/bayesian-methods/utils.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx import mxnet.ndarray as nd import numpy @@ -169,4 +186,4 @@ def pred_test(testing_data, exe, param_list=None, save_path=""): ret[i, 1] = pred.std()**2 numpy.savetxt(save_path, ret) mse = numpy.square(ret[:, 0] - testing_data[:, 0] **3).mean() - return mse, ret \ No newline at end of file + return mse, ret diff --git a/example/bi-lstm-sort/infer_sort.py b/example/bi-lstm-sort/infer_sort.py index 0f5ef07a269b..b074c03d1159 100644 --- a/example/bi-lstm-sort/infer_sort.py +++ b/example/bi-lstm-sort/infer_sort.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: disable=C0111,too-many-arguments,too-many-instance-attributes,too-many-locals,redefined-outer-name,fixme # pylint: disable=superfluous-parens, no-member, invalid-name import sys @@ -31,7 +48,7 @@ def MakeInput(char, vocab, arr): rvocab = {} for k, v in vocab.items(): rvocab[v] = k - + _, arg_params, __ = mx.model.load_checkpoint("sort", 1) model = BiLSTMInferenceModel(5, len(vocab), @@ -42,9 +59,9 @@ def MakeInput(char, vocab, arr): data = np.zeros((1, len(tks))) for k in range(len(tks)): data[0][k] = vocab[tks[k]] - + data = mx.nd.array(data) prob = model.forward(data) - for k in range(len(tks)): + for k in range(len(tks)): print(rvocab[np.argmax(prob, axis = 1)[k]]) - + diff --git a/example/bi-lstm-sort/lstm.py b/example/bi-lstm-sort/lstm.py index 61687124266d..a082092b0351 100644 --- a/example/bi-lstm-sort/lstm.py +++ b/example/bi-lstm-sort/lstm.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint:skip-file import sys sys.path.insert(0, "../../python") @@ -64,7 +81,7 @@ def bi_lstm_unroll(seq_len, input_size, embed = mx.sym.Embedding(data=data, input_dim=input_size, weight=embed_weight, output_dim=num_embed, name='embed') wordvec = mx.sym.SliceChannel(data=embed, num_outputs=seq_len, squeeze_axis=1) - + forward_hidden = [] for seqidx in range(seq_len): hidden = wordvec[seqidx] @@ -87,7 +104,7 @@ def bi_lstm_unroll(seq_len, input_size, hidden = next_state.h last_states[1] = next_state backward_hidden.insert(0, hidden) - + hidden_all = [] for i in range(seq_len): hidden_all.append(mx.sym.Concat(*[forward_hidden[i], backward_hidden[i]], dim=1)) @@ -109,7 +126,7 @@ def bi_lstm_inference_symbol(input_size, seq_len, embed_weight=mx.sym.Variable("embed_weight") cls_weight = mx.sym.Variable("cls_weight") cls_bias = mx.sym.Variable("cls_bias") - last_states = [LSTMState(c = mx.sym.Variable("l0_init_c"), h = mx.sym.Variable("l0_init_h")), + last_states = [LSTMState(c = mx.sym.Variable("l0_init_c"), h = mx.sym.Variable("l0_init_h")), LSTMState(c = mx.sym.Variable("l1_init_c"), h = mx.sym.Variable("l1_init_h"))] forward_param = LSTMParam(i2h_weight=mx.sym.Variable("l0_i2h_weight"), i2h_bias=mx.sym.Variable("l0_i2h_bias"), @@ -143,7 +160,7 @@ def bi_lstm_inference_symbol(input_size, seq_len, hidden = next_state.h last_states[1] = next_state backward_hidden.insert(0, hidden) - + hidden_all = [] for i in range(seq_len): hidden_all.append(mx.sym.Concat(*[forward_hidden[i], backward_hidden[i]], dim=1)) diff --git a/example/bi-lstm-sort/lstm_sort.py b/example/bi-lstm-sort/lstm_sort.py index fe8c38b559bd..aef88b899ce3 100644 --- a/example/bi-lstm-sort/lstm_sort.py +++ b/example/bi-lstm-sort/lstm_sort.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: disable=C0111,too-many-arguments,too-many-instance-attributes,too-many-locals,redefined-outer-name,fixme # pylint: disable=superfluous-parens, no-member, invalid-name import sys diff --git a/example/bi-lstm-sort/rnn_model.py b/example/bi-lstm-sort/rnn_model.py index a253e862fcce..202aae608726 100644 --- a/example/bi-lstm-sort/rnn_model.py +++ b/example/bi-lstm-sort/rnn_model.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: disable=C0111,too-many-arguments,too-many-instance-attributes,too-many-locals,redefined-outer-name,fixme # pylint: disable=superfluous-parens, no-member, invalid-name import sys @@ -25,7 +42,7 @@ def __init__(self, batch_size = 1 init_c = [('l%d_init_c'%l, (batch_size, num_hidden)) for l in range(2)] init_h = [('l%d_init_h'%l, (batch_size, num_hidden)) for l in range(2)] - + data_shape = [("data", (batch_size, seq_len, ))] input_shapes = dict(init_c + init_h + data_shape) diff --git a/example/bi-lstm-sort/sort_io.py b/example/bi-lstm-sort/sort_io.py index 8e1152173ac7..8cb44c678a72 100644 --- a/example/bi-lstm-sort/sort_io.py +++ b/example/bi-lstm-sort/sort_io.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: disable=C0111,too-many-arguments,too-many-instance-attributes,too-many-locals,redefined-outer-name,fixme # pylint: disable=superfluous-parens, no-member, invalid-name from __future__ import print_function diff --git a/example/caffe/caffe_net.py b/example/caffe/caffe_net.py index c91d37bcbecb..0dc4770a24f0 100644 --- a/example/caffe/caffe_net.py +++ b/example/caffe/caffe_net.py @@ -1,5 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx -from data import get_iterator +from data import get_iterator import argparse import train_model diff --git a/example/caffe/data.py b/example/caffe/data.py index 723e7da02b85..fac8e11989dc 100644 --- a/example/caffe/data.py +++ b/example/caffe/data.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import sys import os # code to automatically download dataset diff --git a/example/caffe/train_model.py b/example/caffe/train_model.py index 9a51f07bda87..2eadd869e70e 100644 --- a/example/caffe/train_model.py +++ b/example/caffe/train_model.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx import logging import os diff --git a/example/cnn_text_classification/data_helpers.py b/example/cnn_text_classification/data_helpers.py index b3ece2d4825b..3812683e7a5d 100644 --- a/example/cnn_text_classification/data_helpers.py +++ b/example/cnn_text_classification/data_helpers.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import numpy as np import re import itertools diff --git a/example/cnn_text_classification/old/text_cnn.py b/example/cnn_text_classification/old/text_cnn.py index e41af36cf2ff..8d82d6ef7945 100644 --- a/example/cnn_text_classification/old/text_cnn.py +++ b/example/cnn_text_classification/old/text_cnn.py @@ -1,4 +1,22 @@ #!/usr/bin/env python + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # -*- coding: utf-8 -*- from __future__ import print_function import sys,os @@ -242,7 +260,7 @@ def train_without_pretrained_embedding(): print('train shape:', x_train.shape) print('dev shape:', x_dev.shape) print('vocab_size', vocab_size) - + batch_size = 50 num_embed = 300 sentence_size = x_train.shape[1] diff --git a/example/cnn_text_classification/text_cnn.py b/example/cnn_text_classification/text_cnn.py index 16d3dca260fd..d88a8e699420 100644 --- a/example/cnn_text_classification/text_cnn.py +++ b/example/cnn_text_classification/text_cnn.py @@ -1,4 +1,22 @@ #!/usr/bin/env python + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # -*- coding: utf-8 -*- import sys @@ -77,7 +95,7 @@ def data_iter(batch_size, num_embed, pre_trained_word2vec=False): x_train, y_train, batch_size, shuffle=True) valid = mx.io.NDArrayIter( x_dev, y_dev, batch_size) - + return (train, valid, sentence_size, embed_size, vocab_size) def sym_gen(batch_size, sentence_size, num_embed, vocab_size, @@ -121,7 +139,7 @@ def sym_gen(batch_size, sentence_size, num_embed, vocab_size, # softmax output sm = mx.sym.SoftmaxOutput(data=fc, label=input_y, name='softmax') - return sm, ('data',), ('softmax_label',) + return sm, ('data',), ('softmax_label',) def train(symbol, train_iter, valid_iter, data_names, label_names): devs = mx.cpu() if args.gpus is None or args.gpus is '' else [ diff --git a/example/ctc/lstm.py b/example/ctc/lstm.py index 9c493bbfb500..7e18c8699492 100644 --- a/example/ctc/lstm.py +++ b/example/ctc/lstm.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint:skip-file import sys diff --git a/example/ctc/lstm_ocr.py b/example/ctc/lstm_ocr.py index 7d437bfdc424..c9928aa43ab8 100644 --- a/example/ctc/lstm_ocr.py +++ b/example/ctc/lstm_ocr.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: disable=C0111,too-many-arguments,too-many-instance-attributes,too-many-locals,redefined-outer-name,fixme # pylint: disable=superfluous-parens, no-member, invalid-name from __future__ import print_function diff --git a/example/ctc/ocr_predict.py b/example/ctc/ocr_predict.py index a07733ef55e0..3096a664a20f 100644 --- a/example/ctc/ocr_predict.py +++ b/example/ctc/ocr_predict.py @@ -1,4 +1,22 @@ #!/usr/bin/env python2.7 + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding=utf-8 from __future__ import print_function import sys, os diff --git a/example/dec/dec.py b/example/dec/dec.py index d8a45149d2e8..ac6545abb1de 100644 --- a/example/dec/dec.py +++ b/example/dec/dec.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: skip-file from __future__ import print_function import sys @@ -154,4 +171,4 @@ def mnist_exp(xpu): if __name__ == '__main__': logging.basicConfig(level=logging.INFO) mnist_exp(mx.gpu(0)) - + diff --git a/example/dsd/mlp.py b/example/dsd/mlp.py index ccb094062f58..767e5924b294 100644 --- a/example/dsd/mlp.py +++ b/example/dsd/mlp.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx import os import logging diff --git a/example/dsd/sparse_sgd.py b/example/dsd/sparse_sgd.py index f11a2395c4c0..b21e9b9b89fc 100644 --- a/example/dsd/sparse_sgd.py +++ b/example/dsd/sparse_sgd.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from mxnet.ndarray import NDArray, topk, abs as NDabs from mxnet.optimizer import SGD, register import logging diff --git a/example/fcn-xs/data.py b/example/fcn-xs/data.py index 9de0d8d31c69..685b6f7da8f4 100644 --- a/example/fcn-xs/data.py +++ b/example/fcn-xs/data.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: skip-file """ file iterator for pasval voc 2012""" import mxnet as mx diff --git a/example/fcn-xs/fcn_xs.py b/example/fcn-xs/fcn_xs.py index 85961d92c694..53244a1759c3 100644 --- a/example/fcn-xs/fcn_xs.py +++ b/example/fcn-xs/fcn_xs.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: skip-file import sys, os import argparse diff --git a/example/fcn-xs/image_segmentaion.py b/example/fcn-xs/image_segmentaion.py index 6d619c198c0b..ddd850fe4e9d 100644 --- a/example/fcn-xs/image_segmentaion.py +++ b/example/fcn-xs/image_segmentaion.py @@ -1,58 +1,75 @@ -# pylint: skip-file -import numpy as np -import mxnet as mx -from PIL import Image - -def getpallete(num_cls): - # this function is to get the colormap for visualizing the segmentation mask - n = num_cls - pallete = [0]*(n*3) - for j in xrange(0,n): - lab = j - pallete[j*3+0] = 0 - pallete[j*3+1] = 0 - pallete[j*3+2] = 0 - i = 0 - while (lab > 0): - pallete[j*3+0] |= (((lab >> 0) & 1) << (7-i)) - pallete[j*3+1] |= (((lab >> 1) & 1) << (7-i)) - pallete[j*3+2] |= (((lab >> 2) & 1) << (7-i)) - i = i + 1 - lab >>= 3 - return pallete - -pallete = getpallete(256) -img = "./person_bicycle.jpg" -seg = img.replace("jpg", "png") -model_previx = "FCN8s_VGG16" -epoch = 19 -ctx = mx.gpu(0) - -def get_data(img_path): - """get the (1, 3, h, w) np.array data for the img_path""" - mean = np.array([123.68, 116.779, 103.939]) # (R,G,B) - img = Image.open(img_path) - img = np.array(img, dtype=np.float32) - reshaped_mean = mean.reshape(1, 1, 3) - img = img - reshaped_mean - img = np.swapaxes(img, 0, 2) - img = np.swapaxes(img, 1, 2) - img = np.expand_dims(img, axis=0) - return img - -def main(): - fcnxs, fcnxs_args, fcnxs_auxs = mx.model.load_checkpoint(model_previx, epoch) - fcnxs_args["data"] = mx.nd.array(get_data(img), ctx) - data_shape = fcnxs_args["data"].shape - label_shape = (1, data_shape[2]*data_shape[3]) - fcnxs_args["softmax_label"] = mx.nd.empty(label_shape, ctx) - exector = fcnxs.bind(ctx, fcnxs_args ,args_grad=None, grad_req="null", aux_states=fcnxs_args) - exector.forward(is_train=False) - output = exector.outputs[0] - out_img = np.uint8(np.squeeze(output.asnumpy().argmax(axis=1))) - out_img = Image.fromarray(out_img) - out_img.putpalette(pallete) - out_img.save(seg) - -if __name__ == "__main__": - main() +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# pylint: skip-file +import numpy as np +import mxnet as mx +from PIL import Image + +def getpallete(num_cls): + # this function is to get the colormap for visualizing the segmentation mask + n = num_cls + pallete = [0]*(n*3) + for j in xrange(0,n): + lab = j + pallete[j*3+0] = 0 + pallete[j*3+1] = 0 + pallete[j*3+2] = 0 + i = 0 + while (lab > 0): + pallete[j*3+0] |= (((lab >> 0) & 1) << (7-i)) + pallete[j*3+1] |= (((lab >> 1) & 1) << (7-i)) + pallete[j*3+2] |= (((lab >> 2) & 1) << (7-i)) + i = i + 1 + lab >>= 3 + return pallete + +pallete = getpallete(256) +img = "./person_bicycle.jpg" +seg = img.replace("jpg", "png") +model_previx = "FCN8s_VGG16" +epoch = 19 +ctx = mx.gpu(0) + +def get_data(img_path): + """get the (1, 3, h, w) np.array data for the img_path""" + mean = np.array([123.68, 116.779, 103.939]) # (R,G,B) + img = Image.open(img_path) + img = np.array(img, dtype=np.float32) + reshaped_mean = mean.reshape(1, 1, 3) + img = img - reshaped_mean + img = np.swapaxes(img, 0, 2) + img = np.swapaxes(img, 1, 2) + img = np.expand_dims(img, axis=0) + return img + +def main(): + fcnxs, fcnxs_args, fcnxs_auxs = mx.model.load_checkpoint(model_previx, epoch) + fcnxs_args["data"] = mx.nd.array(get_data(img), ctx) + data_shape = fcnxs_args["data"].shape + label_shape = (1, data_shape[2]*data_shape[3]) + fcnxs_args["softmax_label"] = mx.nd.empty(label_shape, ctx) + exector = fcnxs.bind(ctx, fcnxs_args ,args_grad=None, grad_req="null", aux_states=fcnxs_args) + exector.forward(is_train=False) + output = exector.outputs[0] + out_img = np.uint8(np.squeeze(output.asnumpy().argmax(axis=1))) + out_img = Image.fromarray(out_img) + out_img.putpalette(pallete) + out_img.save(seg) + +if __name__ == "__main__": + main() diff --git a/example/fcn-xs/init_fcnxs.py b/example/fcn-xs/init_fcnxs.py index c90a45bb4358..ede46b80d56c 100644 --- a/example/fcn-xs/init_fcnxs.py +++ b/example/fcn-xs/init_fcnxs.py @@ -1,89 +1,106 @@ -# pylint: skip-file -import mxnet as mx -import numpy as np -import sys -import logging - -logger = logging.getLogger() -logger.setLevel(logging.INFO) - -# make a bilinear interpolation kernel, return a numpy.ndarray -def upsample_filt(size): - factor = (size + 1) // 2 - if size % 2 == 1: - center = factor - 1.0 - else: - center = factor - 0.5 - og = np.ogrid[:size, :size] - return (1 - abs(og[0] - center) / factor) * \ - (1 - abs(og[1] - center) / factor) - -def init_from_vgg16(ctx, fcnxs_symbol, vgg16fc_args, vgg16fc_auxs): - fcnxs_args = vgg16fc_args.copy() - fcnxs_auxs = vgg16fc_auxs.copy() - for k,v in fcnxs_args.items(): - if(v.context != ctx): - fcnxs_args[k] = mx.nd.zeros(v.shape, ctx) - v.copyto(fcnxs_args[k]) - for k,v in fcnxs_auxs.items(): - if(v.context != ctx): - fcnxs_auxs[k] = mx.nd.zeros(v.shape, ctx) - v.copyto(fcnxs_auxs[k]) - data_shape=(1,3,500,500) - arg_names = fcnxs_symbol.list_arguments() - arg_shapes, _, _ = fcnxs_symbol.infer_shape(data=data_shape) - rest_params = dict([(x[0], mx.nd.zeros(x[1], ctx)) for x in zip(arg_names, arg_shapes) - if x[0] in ['score_weight', 'score_bias', 'score_pool4_weight', 'score_pool4_bias', \ - 'score_pool3_weight', 'score_pool3_bias']]) - fcnxs_args.update(rest_params) - deconv_params = dict([(x[0], x[1]) for x in zip(arg_names, arg_shapes) - if x[0] in ["bigscore_weight", 'score2_weight', 'score4_weight']]) - for k, v in deconv_params.items(): - filt = upsample_filt(v[3]) - initw = np.zeros(v) - initw[range(v[0]), range(v[1]), :, :] = filt # becareful here is the slice assing - fcnxs_args[k] = mx.nd.array(initw, ctx) - return fcnxs_args, fcnxs_auxs - -def init_from_fcnxs(ctx, fcnxs_symbol, fcnxs_args_from, fcnxs_auxs_from): - """ use zero initialization for better convergence, because it tends to oputut 0, - and the label 0 stands for background, which may occupy most size of one image. - """ - fcnxs_args = fcnxs_args_from.copy() - fcnxs_auxs = fcnxs_auxs_from.copy() - for k,v in fcnxs_args.items(): - if(v.context != ctx): - fcnxs_args[k] = mx.nd.zeros(v.shape, ctx) - v.copyto(fcnxs_args[k]) - for k,v in fcnxs_auxs.items(): - if(v.context != ctx): - fcnxs_auxs[k] = mx.nd.zeros(v.shape, ctx) - v.copyto(fcnxs_auxs[k]) - data_shape=(1,3,500,500) - arg_names = fcnxs_symbol.list_arguments() - arg_shapes, _, _ = fcnxs_symbol.infer_shape(data=data_shape) - rest_params = {} - deconv_params = {} - # this is fcn8s init from fcn16s - if 'score_pool3_weight' in arg_names: - rest_params = dict([(x[0], mx.nd.zeros(x[1], ctx)) for x in zip(arg_names, arg_shapes) - if x[0] in ['score_pool3_bias', 'score_pool3_weight']]) - deconv_params = dict([(x[0], x[1]) for x in zip(arg_names, arg_shapes) if x[0] \ - in ["bigscore_weight", 'score4_weight']]) - # this is fcn16s init from fcn32s - elif 'score_pool4_weight' in arg_names: - rest_params = dict([(x[0], mx.nd.zeros(x[1], ctx)) for x in zip(arg_names, arg_shapes) - if x[0] in ['score_pool4_weight', 'score_pool4_bias']]) - deconv_params = dict([(x[0], x[1]) for x in zip(arg_names, arg_shapes) if x[0] \ - in ["bigscore_weight", 'score2_weight']]) - # this is fcn32s init - else: - logging.error("you are init the fcn32s model, so you should use init_from_vgg16()") - sys.exit() - fcnxs_args.update(rest_params) - for k, v in deconv_params.items(): - filt = upsample_filt(v[3]) - initw = np.zeros(v) - initw[range(v[0]), range(v[1]), :, :] = filt # becareful here is the slice assing - fcnxs_args[k] = mx.nd.array(initw, ctx) - return fcnxs_args, fcnxs_auxs +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# pylint: skip-file +import mxnet as mx +import numpy as np +import sys +import logging + +logger = logging.getLogger() +logger.setLevel(logging.INFO) + +# make a bilinear interpolation kernel, return a numpy.ndarray +def upsample_filt(size): + factor = (size + 1) // 2 + if size % 2 == 1: + center = factor - 1.0 + else: + center = factor - 0.5 + og = np.ogrid[:size, :size] + return (1 - abs(og[0] - center) / factor) * \ + (1 - abs(og[1] - center) / factor) + +def init_from_vgg16(ctx, fcnxs_symbol, vgg16fc_args, vgg16fc_auxs): + fcnxs_args = vgg16fc_args.copy() + fcnxs_auxs = vgg16fc_auxs.copy() + for k,v in fcnxs_args.items(): + if(v.context != ctx): + fcnxs_args[k] = mx.nd.zeros(v.shape, ctx) + v.copyto(fcnxs_args[k]) + for k,v in fcnxs_auxs.items(): + if(v.context != ctx): + fcnxs_auxs[k] = mx.nd.zeros(v.shape, ctx) + v.copyto(fcnxs_auxs[k]) + data_shape=(1,3,500,500) + arg_names = fcnxs_symbol.list_arguments() + arg_shapes, _, _ = fcnxs_symbol.infer_shape(data=data_shape) + rest_params = dict([(x[0], mx.nd.zeros(x[1], ctx)) for x in zip(arg_names, arg_shapes) + if x[0] in ['score_weight', 'score_bias', 'score_pool4_weight', 'score_pool4_bias', \ + 'score_pool3_weight', 'score_pool3_bias']]) + fcnxs_args.update(rest_params) + deconv_params = dict([(x[0], x[1]) for x in zip(arg_names, arg_shapes) + if x[0] in ["bigscore_weight", 'score2_weight', 'score4_weight']]) + for k, v in deconv_params.items(): + filt = upsample_filt(v[3]) + initw = np.zeros(v) + initw[range(v[0]), range(v[1]), :, :] = filt # becareful here is the slice assing + fcnxs_args[k] = mx.nd.array(initw, ctx) + return fcnxs_args, fcnxs_auxs + +def init_from_fcnxs(ctx, fcnxs_symbol, fcnxs_args_from, fcnxs_auxs_from): + """ use zero initialization for better convergence, because it tends to oputut 0, + and the label 0 stands for background, which may occupy most size of one image. + """ + fcnxs_args = fcnxs_args_from.copy() + fcnxs_auxs = fcnxs_auxs_from.copy() + for k,v in fcnxs_args.items(): + if(v.context != ctx): + fcnxs_args[k] = mx.nd.zeros(v.shape, ctx) + v.copyto(fcnxs_args[k]) + for k,v in fcnxs_auxs.items(): + if(v.context != ctx): + fcnxs_auxs[k] = mx.nd.zeros(v.shape, ctx) + v.copyto(fcnxs_auxs[k]) + data_shape=(1,3,500,500) + arg_names = fcnxs_symbol.list_arguments() + arg_shapes, _, _ = fcnxs_symbol.infer_shape(data=data_shape) + rest_params = {} + deconv_params = {} + # this is fcn8s init from fcn16s + if 'score_pool3_weight' in arg_names: + rest_params = dict([(x[0], mx.nd.zeros(x[1], ctx)) for x in zip(arg_names, arg_shapes) + if x[0] in ['score_pool3_bias', 'score_pool3_weight']]) + deconv_params = dict([(x[0], x[1]) for x in zip(arg_names, arg_shapes) if x[0] \ + in ["bigscore_weight", 'score4_weight']]) + # this is fcn16s init from fcn32s + elif 'score_pool4_weight' in arg_names: + rest_params = dict([(x[0], mx.nd.zeros(x[1], ctx)) for x in zip(arg_names, arg_shapes) + if x[0] in ['score_pool4_weight', 'score_pool4_bias']]) + deconv_params = dict([(x[0], x[1]) for x in zip(arg_names, arg_shapes) if x[0] \ + in ["bigscore_weight", 'score2_weight']]) + # this is fcn32s init + else: + logging.error("you are init the fcn32s model, so you should use init_from_vgg16()") + sys.exit() + fcnxs_args.update(rest_params) + for k, v in deconv_params.items(): + filt = upsample_filt(v[3]) + initw = np.zeros(v) + initw[range(v[0]), range(v[1]), :, :] = filt # becareful here is the slice assing + fcnxs_args[k] = mx.nd.array(initw, ctx) + return fcnxs_args, fcnxs_auxs diff --git a/example/fcn-xs/run_fcnxs.sh b/example/fcn-xs/run_fcnxs.sh index 926f3f840415..df9a880b8396 100755 --- a/example/fcn-xs/run_fcnxs.sh +++ b/example/fcn-xs/run_fcnxs.sh @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # train fcn-32s model python -u fcn_xs.py --model=fcn32s --prefix=VGG_FC_ILSVRC_16_layers \ --epoch=74 --init-type=vgg16 diff --git a/example/fcn-xs/solver.py b/example/fcn-xs/solver.py index dd78e73b9b84..cf7298b83c8c 100644 --- a/example/fcn-xs/solver.py +++ b/example/fcn-xs/solver.py @@ -1,126 +1,143 @@ -# pylint: skip-file -import numpy as np -import mxnet as mx -import time -import logging -from collections import namedtuple -from mxnet import optimizer as opt -from mxnet.optimizer import get_updater -from mxnet import metric - -# Parameter to pass to batch_end_callback -BatchEndParam = namedtuple('BatchEndParams', ['epoch', 'nbatch', 'eval_metric']) -class Solver(object): - def __init__(self, symbol, ctx=None, - begin_epoch=0, num_epoch=None, - arg_params=None, aux_params=None, - optimizer='sgd', **kwargs): - self.symbol = symbol - if ctx is None: - ctx = mx.cpu() - self.ctx = ctx - self.begin_epoch = begin_epoch - self.num_epoch = num_epoch - self.arg_params = arg_params - self.aux_params = aux_params - self.optimizer = optimizer - self.kwargs = kwargs.copy() - - def fit(self, train_data, eval_data=None, - eval_metric='acc', - grad_req='write', - epoch_end_callback=None, - batch_end_callback=None, - kvstore='local', - logger=None): - if logger is None: - logger = logging - logging.info('Start training with %s', str(self.ctx)) - arg_shapes, out_shapes, aux_shapes = self.symbol.infer_shape(data=train_data.provide_data[0][1]) - arg_names = self.symbol.list_arguments() - if grad_req != 'null': - self.grad_params = {} - for name, shape in zip(arg_names, arg_shapes): - if not (name.endswith('data') or name.endswith('label')): - self.grad_params[name] = mx.nd.zeros(shape, self.ctx) - else: - self.grad_params = None - aux_names = self.symbol.list_auxiliary_states() - self.aux_params = {k : nd.zeros(s) for k, s in zip(aux_names, aux_shapes)} - data_name = train_data.data_name - label_name = train_data.label_name - input_names = [data_name, label_name] - self.optimizer = opt.create(self.optimizer, rescale_grad=(1.0/train_data.get_batch_size()), **(self.kwargs)) - self.updater = get_updater(self.optimizer) - eval_metric = metric.create(eval_metric) - # begin training - for epoch in range(self.begin_epoch, self.num_epoch): - nbatch = 0 - train_data.reset() - eval_metric.reset() - for data in train_data: - nbatch += 1 - label_shape = data[label_name].shape - self.arg_params[data_name] = mx.nd.array(data[data_name], self.ctx) - self.arg_params[label_name] = mx.nd.array(data[label_name].reshape(label_shape[0], \ - label_shape[1]*label_shape[2]), self.ctx) - output_names = self.symbol.list_outputs() - self.exector = self.symbol.bind(self.ctx, self.arg_params, - args_grad=self.grad_params, - grad_req=grad_req, - aux_states=self.aux_params) - assert len(self.symbol.list_arguments()) == len(self.exector.grad_arrays) - update_dict = {name: nd for name, nd in zip(self.symbol.list_arguments(), \ - self.exector.grad_arrays) if nd is not None} - output_dict = {} - output_buff = {} - for key, arr in zip(self.symbol.list_outputs(), self.exector.outputs): - output_dict[key] = arr - output_buff[key] = mx.nd.empty(arr.shape, ctx=mx.cpu()) - self.exector.forward(is_train=True) - for key in output_dict: - output_dict[key].copyto(output_buff[key]) - self.exector.backward() - for key, arr in update_dict.items(): - if key != "bigscore_weight": - self.updater(key, arr, self.arg_params[key]) - pred_shape = self.exector.outputs[0].shape - label = mx.nd.array(data[label_name].reshape(label_shape[0], label_shape[1]*label_shape[2])) - pred = mx.nd.array(output_buff["softmax_output"].asnumpy().reshape(pred_shape[0], \ - pred_shape[1], pred_shape[2]*pred_shape[3])) - eval_metric.update([label], [pred]) - self.exector.outputs[0].wait_to_read() - batch_end_params = BatchEndParam(epoch=epoch, nbatch=nbatch, eval_metric=eval_metric) - batch_end_callback(batch_end_params) - if epoch_end_callback is not None: - epoch_end_callback(epoch, self.symbol, self.arg_params, self.aux_params) - name, value = eval_metric.get() - logger.info(" --->Epoch[%d] Train-%s=%f", epoch, name, value) - # evaluation - if eval_data: - logger.info(" in eval process...") - nbatch = 0 - eval_data.reset() - eval_metric.reset() - for data in eval_data: - nbatch += 1 - label_shape = data[label_name].shape - self.arg_params[data_name] = mx.nd.array(data[data_name], self.ctx) - self.arg_params[label_name] = mx.nd.array(data[label_name].reshape(label_shape[0], \ - label_shape[1]*label_shape[2]), self.ctx) - exector = self.symbol.bind(self.ctx, self.arg_params, - args_grad=self.grad_params, - grad_req=grad_req, - aux_states=self.aux_params) - cpu_output_array = mx.nd.zeros(exector.outputs[0].shape) - exector.forward(is_train=False) - exector.outputs[0].copyto(cpu_output_array) - pred_shape = cpu_output_array.shape - label = mx.nd.array(data[label_name].reshape(label_shape[0], \ - label_shape[1]*label_shape[2])) - pred = mx.nd.array(cpu_output_array.asnumpy().reshape(pred_shape[0], \ - pred_shape[1], pred_shape[2]*pred_shape[3])) - eval_metric.update([label], [pred]) - exector.outputs[0].wait_to_read() - name, value = eval_metric.get() - logger.info('batch[%d] Validation-%s=%f', nbatch, name, value) +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# pylint: skip-file +import numpy as np +import mxnet as mx +import time +import logging +from collections import namedtuple +from mxnet import optimizer as opt +from mxnet.optimizer import get_updater +from mxnet import metric + +# Parameter to pass to batch_end_callback +BatchEndParam = namedtuple('BatchEndParams', ['epoch', 'nbatch', 'eval_metric']) +class Solver(object): + def __init__(self, symbol, ctx=None, + begin_epoch=0, num_epoch=None, + arg_params=None, aux_params=None, + optimizer='sgd', **kwargs): + self.symbol = symbol + if ctx is None: + ctx = mx.cpu() + self.ctx = ctx + self.begin_epoch = begin_epoch + self.num_epoch = num_epoch + self.arg_params = arg_params + self.aux_params = aux_params + self.optimizer = optimizer + self.kwargs = kwargs.copy() + + def fit(self, train_data, eval_data=None, + eval_metric='acc', + grad_req='write', + epoch_end_callback=None, + batch_end_callback=None, + kvstore='local', + logger=None): + if logger is None: + logger = logging + logging.info('Start training with %s', str(self.ctx)) + arg_shapes, out_shapes, aux_shapes = self.symbol.infer_shape(data=train_data.provide_data[0][1]) + arg_names = self.symbol.list_arguments() + if grad_req != 'null': + self.grad_params = {} + for name, shape in zip(arg_names, arg_shapes): + if not (name.endswith('data') or name.endswith('label')): + self.grad_params[name] = mx.nd.zeros(shape, self.ctx) + else: + self.grad_params = None + aux_names = self.symbol.list_auxiliary_states() + self.aux_params = {k : nd.zeros(s) for k, s in zip(aux_names, aux_shapes)} + data_name = train_data.data_name + label_name = train_data.label_name + input_names = [data_name, label_name] + self.optimizer = opt.create(self.optimizer, rescale_grad=(1.0/train_data.get_batch_size()), **(self.kwargs)) + self.updater = get_updater(self.optimizer) + eval_metric = metric.create(eval_metric) + # begin training + for epoch in range(self.begin_epoch, self.num_epoch): + nbatch = 0 + train_data.reset() + eval_metric.reset() + for data in train_data: + nbatch += 1 + label_shape = data[label_name].shape + self.arg_params[data_name] = mx.nd.array(data[data_name], self.ctx) + self.arg_params[label_name] = mx.nd.array(data[label_name].reshape(label_shape[0], \ + label_shape[1]*label_shape[2]), self.ctx) + output_names = self.symbol.list_outputs() + self.exector = self.symbol.bind(self.ctx, self.arg_params, + args_grad=self.grad_params, + grad_req=grad_req, + aux_states=self.aux_params) + assert len(self.symbol.list_arguments()) == len(self.exector.grad_arrays) + update_dict = {name: nd for name, nd in zip(self.symbol.list_arguments(), \ + self.exector.grad_arrays) if nd is not None} + output_dict = {} + output_buff = {} + for key, arr in zip(self.symbol.list_outputs(), self.exector.outputs): + output_dict[key] = arr + output_buff[key] = mx.nd.empty(arr.shape, ctx=mx.cpu()) + self.exector.forward(is_train=True) + for key in output_dict: + output_dict[key].copyto(output_buff[key]) + self.exector.backward() + for key, arr in update_dict.items(): + if key != "bigscore_weight": + self.updater(key, arr, self.arg_params[key]) + pred_shape = self.exector.outputs[0].shape + label = mx.nd.array(data[label_name].reshape(label_shape[0], label_shape[1]*label_shape[2])) + pred = mx.nd.array(output_buff["softmax_output"].asnumpy().reshape(pred_shape[0], \ + pred_shape[1], pred_shape[2]*pred_shape[3])) + eval_metric.update([label], [pred]) + self.exector.outputs[0].wait_to_read() + batch_end_params = BatchEndParam(epoch=epoch, nbatch=nbatch, eval_metric=eval_metric) + batch_end_callback(batch_end_params) + if epoch_end_callback is not None: + epoch_end_callback(epoch, self.symbol, self.arg_params, self.aux_params) + name, value = eval_metric.get() + logger.info(" --->Epoch[%d] Train-%s=%f", epoch, name, value) + # evaluation + if eval_data: + logger.info(" in eval process...") + nbatch = 0 + eval_data.reset() + eval_metric.reset() + for data in eval_data: + nbatch += 1 + label_shape = data[label_name].shape + self.arg_params[data_name] = mx.nd.array(data[data_name], self.ctx) + self.arg_params[label_name] = mx.nd.array(data[label_name].reshape(label_shape[0], \ + label_shape[1]*label_shape[2]), self.ctx) + exector = self.symbol.bind(self.ctx, self.arg_params, + args_grad=self.grad_params, + grad_req=grad_req, + aux_states=self.aux_params) + cpu_output_array = mx.nd.zeros(exector.outputs[0].shape) + exector.forward(is_train=False) + exector.outputs[0].copyto(cpu_output_array) + pred_shape = cpu_output_array.shape + label = mx.nd.array(data[label_name].reshape(label_shape[0], \ + label_shape[1]*label_shape[2])) + pred = mx.nd.array(cpu_output_array.asnumpy().reshape(pred_shape[0], \ + pred_shape[1], pred_shape[2]*pred_shape[3])) + eval_metric.update([label], [pred]) + exector.outputs[0].wait_to_read() + name, value = eval_metric.get() + logger.info('batch[%d] Validation-%s=%f', nbatch, name, value) diff --git a/example/fcn-xs/symbol_fcnxs.py b/example/fcn-xs/symbol_fcnxs.py index a9c4f3b712ec..56888fc94822 100644 --- a/example/fcn-xs/symbol_fcnxs.py +++ b/example/fcn-xs/symbol_fcnxs.py @@ -1,189 +1,206 @@ -# pylint: skip-file -import mxnet as mx - -def filter_map(kernel=1, stride=1, pad=0): - return (stride, (kernel-stride)/2-pad) - -def compose_fp(fp_first, fp_second): - return (fp_first[0]*fp_second[0], fp_first[0]*fp_second[1]+fp_first[1]) - -def compose_fp_list(fp_list): - fp_out = (1.0, 0.0) - for fp in fp_list: - fp_out = compose_fp(fp_out, fp) - return fp_out - -def inv_fp(fp_in): - return (1.0/fp_in[0], -1.0*fp_in[1]/fp_in[0]) - -def offset(): - conv1_1_fp = filter_map(kernel=3, pad=100) - conv1_2_fp = conv2_1_fp = conv2_2_fp = conv3_1_fp = conv3_2_fp = conv3_3_fp \ - = conv4_1_fp = conv4_2_fp = conv4_3_fp = conv5_1_fp = conv5_2_fp \ - = conv5_3_fp = filter_map(kernel=3, pad=1) - pool1_fp = pool2_fp = pool3_fp = pool4_fp = pool5_fp = filter_map(kernel=2, stride=2) - fc6_fp = filter_map(kernel=7) - fc7_fp = score_fp = score_pool4_fp = score_pool3_fp = filter_map() - # for fcn-32s - fcn32s_upscore_fp = inv_fp(filter_map(kernel=64, stride=32)) - fcn32s_upscore_list = [conv1_1_fp, conv1_2_fp, pool1_fp, conv2_1_fp, conv2_2_fp, - pool2_fp, conv3_1_fp, conv3_2_fp, conv3_3_fp, pool3_fp, - conv4_1_fp, conv4_2_fp, conv4_3_fp, pool4_fp, conv5_1_fp, - conv5_2_fp, conv5_3_fp, pool5_fp, fc6_fp, fc7_fp, score_fp, - fcn32s_upscore_fp] - crop = {} - crop["fcn32s_upscore"] = (-int(round(compose_fp_list(fcn32s_upscore_list)[1])), - -int(round(compose_fp_list(fcn32s_upscore_list)[1]))) - # for fcn-16s - score2_fp = inv_fp(filter_map(kernel=4, stride=2)) - fcn16s_upscore_fp = inv_fp(filter_map(kernel=32, stride=16)) - score_pool4c_fp_list = [inv_fp(score2_fp), inv_fp(score_fp), inv_fp(fc7_fp), inv_fp(fc6_fp), - inv_fp(pool5_fp), inv_fp(conv5_3_fp), inv_fp(conv5_2_fp), - inv_fp(conv5_1_fp), score_pool4_fp] - crop["score_pool4c"] = (-int(round(compose_fp_list(score_pool4c_fp_list)[1])), - -int(round(compose_fp_list(score_pool4c_fp_list)[1]))) - fcn16s_upscore_list = [conv1_1_fp, conv1_2_fp, pool1_fp, conv2_1_fp, conv2_2_fp, - pool2_fp, conv3_1_fp, conv3_2_fp, conv3_3_fp, pool3_fp, - conv4_1_fp, conv4_2_fp, conv4_3_fp, pool4_fp, score_pool4_fp, - inv_fp((1, -crop["score_pool4c"][0])), fcn16s_upscore_fp] - crop["fcn16s_upscore"] = (-int(round(compose_fp_list(fcn16s_upscore_list)[1])), - -int(round(compose_fp_list(fcn16s_upscore_list)[1]))) - # for fcn-8s - score4_fp = inv_fp(filter_map(kernel=4, stride=2)) - fcn8s_upscore_fp = inv_fp(filter_map(kernel=16, stride=8)) - score_pool3c_fp_list = [inv_fp(score4_fp), (1, -crop["score_pool4c"][0]), inv_fp(score_pool4_fp), - inv_fp(pool4_fp), inv_fp(conv4_3_fp), inv_fp(conv4_2_fp), - inv_fp(conv4_1_fp), score_pool3_fp, score_pool3_fp] - crop["score_pool3c"] = (-int(round(compose_fp_list(score_pool3c_fp_list)[1])), - -int(round(compose_fp_list(score_pool3c_fp_list)[1]))) - fcn8s_upscore_list = [conv1_1_fp, conv1_2_fp, pool1_fp, conv2_1_fp, conv2_2_fp, pool2_fp, - conv3_1_fp, conv3_2_fp, conv3_3_fp, pool3_fp, score_pool3_fp, - inv_fp((1, -crop["score_pool3c"][0])), fcn8s_upscore_fp] - crop["fcn8s_upscore"] = (-int(round(compose_fp_list(fcn8s_upscore_list)[1])), - -int(round(compose_fp_list(fcn8s_upscore_list)[1]))) - return crop - -def vgg16_pool3(input, workspace_default=1024): - # group 1 - conv1_1 = mx.symbol.Convolution(data=input, kernel=(3, 3), pad=(100, 100), num_filter=64, - workspace=workspace_default, name="conv1_1") - relu1_1 = mx.symbol.Activation(data=conv1_1, act_type="relu", name="relu1_1") - conv1_2 = mx.symbol.Convolution(data=relu1_1, kernel=(3, 3), pad=(1, 1), num_filter=64, - workspace=workspace_default, name="conv1_2") - relu1_2 = mx.symbol.Activation(data=conv1_2, act_type="relu", name="relu1_2") - pool1 = mx.symbol.Pooling(data=relu1_2, pool_type="max", kernel=(2, 2), stride=(2,2), name="pool1") - # group 2 - conv2_1 = mx.symbol.Convolution(data=pool1, kernel=(3, 3), pad=(1, 1), num_filter=128, - workspace=workspace_default, name="conv2_1") - relu2_1 = mx.symbol.Activation(data=conv2_1, act_type="relu", name="relu2_1") - conv2_2 = mx.symbol.Convolution(data=relu2_1, kernel=(3, 3), pad=(1, 1), num_filter=128, - workspace=workspace_default, name="conv2_2") - relu2_2 = mx.symbol.Activation(data=conv2_2, act_type="relu", name="relu2_2") - pool2 = mx.symbol.Pooling(data=relu2_2, pool_type="max", kernel=(2, 2), stride=(2,2), name="pool2") - # group 3 - conv3_1 = mx.symbol.Convolution(data=pool2, kernel=(3, 3), pad=(1, 1), num_filter=256, - workspace=workspace_default, name="conv3_1") - relu3_1 = mx.symbol.Activation(data=conv3_1, act_type="relu", name="relu3_1") - conv3_2 = mx.symbol.Convolution(data=relu3_1, kernel=(3, 3), pad=(1, 1), num_filter=256, - workspace=workspace_default, name="conv3_2") - relu3_2 = mx.symbol.Activation(data=conv3_2, act_type="relu", name="relu3_2") - conv3_3 = mx.symbol.Convolution(data=relu3_2, kernel=(3, 3), pad=(1, 1), num_filter=256, - workspace=workspace_default, name="conv3_3") - relu3_3 = mx.symbol.Activation(data=conv3_3, act_type="relu", name="relu3_3") - pool3 = mx.symbol.Pooling(data=relu3_3, pool_type="max", kernel=(2, 2), stride=(2,2), name="pool3") - return pool3 - -def vgg16_pool4(input, workspace_default=1024): - # group 4 - conv4_1 = mx.symbol.Convolution(data=input, kernel=(3, 3), pad=(1, 1), num_filter=512, - workspace=workspace_default, name="conv4_1") - relu4_1 = mx.symbol.Activation(data=conv4_1, act_type="relu", name="relu4_1") - conv4_2 = mx.symbol.Convolution(data=relu4_1, kernel=(3, 3), pad=(1, 1), num_filter=512, - workspace=workspace_default, name="conv4_2") - relu4_2 = mx.symbol.Activation(data=conv4_2, act_type="relu", name="relu4_2") - conv4_3 = mx.symbol.Convolution(data=relu4_2, kernel=(3, 3), pad=(1, 1), num_filter=512, - workspace=workspace_default, name="conv4_3") - relu4_3 = mx.symbol.Activation(data=conv4_3, act_type="relu", name="relu4_3") - pool4 = mx.symbol.Pooling(data=relu4_3, pool_type="max", kernel=(2, 2), stride=(2,2), name="pool4") - return pool4 - -def vgg16_score(input, numclass, workspace_default=1024): - # group 5 - conv5_1 = mx.symbol.Convolution(data=input, kernel=(3, 3), pad=(1, 1), num_filter=512, - workspace=workspace_default, name="conv5_1") - relu5_1 = mx.symbol.Activation(data=conv5_1, act_type="relu", name="relu5_1") - conv5_2 = mx.symbol.Convolution(data=relu5_1, kernel=(3, 3), pad=(1, 1), num_filter=512, - workspace=workspace_default, name="conv5_2") - relu5_2 = mx.symbol.Activation(data=conv5_2, act_type="relu", name="relu5_2") - conv5_3 = mx.symbol.Convolution(data=relu5_2, kernel=(3, 3), pad=(1, 1), num_filter=512, - workspace=workspace_default, name="conv5_3") - relu5_3 = mx.symbol.Activation(data=conv5_3, act_type="relu", name="relu5_3") - pool5 = mx.symbol.Pooling(data=relu5_3, pool_type="max", kernel=(2, 2), stride=(2,2), name="pool5") - # group 6 - fc6 = mx.symbol.Convolution(data=pool5, kernel=(7, 7), num_filter=4096, - workspace=workspace_default, name="fc6") - relu6 = mx.symbol.Activation(data=fc6, act_type="relu", name="relu6") - drop6 = mx.symbol.Dropout(data=relu6, p=0.5, name="drop6") - # group 7 - fc7 = mx.symbol.Convolution(data=drop6, kernel=(1, 1), num_filter=4096, - workspace=workspace_default, name="fc7") - relu7 = mx.symbol.Activation(data=fc7, act_type="relu", name="relu7") - drop7 = mx.symbol.Dropout(data=relu7, p=0.5, name="drop7") - # group 8 - score = mx.symbol.Convolution(data=drop7, kernel=(1, 1), num_filter=numclass, - workspace=workspace_default, name="score") - return score - -def fcnxs_score(input, crop, offset, kernel=(64,64), stride=(32,32), numclass=21, workspace_default=1024): - # score out - bigscore = mx.symbol.Deconvolution(data=input, kernel=kernel, stride=stride, adj=(stride[0]-1, stride[1]-1), - num_filter=numclass, workspace=workspace_default, name="bigscore") - upscore = mx.symbol.Crop(*[bigscore, crop], offset=offset, name="upscore") - # upscore = mx.symbol.Crop(*[input, crop], offset=offset, name="upscore") - softmax = mx.symbol.SoftmaxOutput(data=upscore, multi_output=True, use_ignore=True, ignore_label=255, name="softmax") - return softmax - -def get_fcn32s_symbol(numclass=21, workspace_default=1024): - data = mx.symbol.Variable(name="data") - pool3 = vgg16_pool3(data, workspace_default) - pool4 = vgg16_pool4(pool3, workspace_default) - score = vgg16_score(pool4, numclass, workspace_default) - softmax = fcnxs_score(score, data, offset()["fcn32s_upscore"], (64,64), (32,32), numclass, workspace_default) - return softmax - -def get_fcn16s_symbol(numclass=21, workspace_default=1024): - data = mx.symbol.Variable(name="data") - pool3 = vgg16_pool3(data, workspace_default) - pool4 = vgg16_pool4(pool3, workspace_default) - score = vgg16_score(pool4, numclass, workspace_default) - # score 2X - score2 = mx.symbol.Deconvolution(data=score, kernel=(4, 4), stride=(2, 2), num_filter=numclass, - adj=(1, 1), workspace=workspace_default, name="score2") # 2X - score_pool4 = mx.symbol.Convolution(data=pool4, kernel=(1, 1), num_filter=numclass, - workspace=workspace_default, name="score_pool4") - score_pool4c = mx.symbol.Crop(*[score_pool4, score2], offset=offset()["score_pool4c"], name="score_pool4c") - score_fused = score2 + score_pool4c - softmax = fcnxs_score(score_fused, data, offset()["fcn16s_upscore"], (32, 32), (16, 16), numclass, workspace_default) - return softmax - -def get_fcn8s_symbol(numclass=21, workspace_default=1024): - data = mx.symbol.Variable(name="data") - pool3 = vgg16_pool3(data, workspace_default) - pool4 = vgg16_pool4(pool3, workspace_default) - score = vgg16_score(pool4, numclass, workspace_default) - # score 2X - score2 = mx.symbol.Deconvolution(data=score, kernel=(4, 4), stride=(2, 2),num_filter=numclass, - adj=(1, 1), workspace=workspace_default, name="score2") # 2X - score_pool4 = mx.symbol.Convolution(data=pool4, kernel=(1, 1), num_filter=numclass, - workspace=workspace_default, name="score_pool4") - score_pool4c = mx.symbol.Crop(*[score_pool4, score2], offset=offset()["score_pool4c"], name="score_pool4c") - score_fused = score2 + score_pool4c - # score 4X - score4 = mx.symbol.Deconvolution(data=score_fused, kernel=(4, 4), stride=(2, 2),num_filter=numclass, - adj=(1, 1), workspace=workspace_default, name="score4") # 4X - score_pool3 = mx.symbol.Convolution(data=pool3, kernel=(1, 1), num_filter=numclass, - workspace=workspace_default, name="score_pool3") - score_pool3c = mx.symbol.Crop(*[score_pool3, score4], offset=offset()["score_pool3c"], name="score_pool3c") - score_final = score4 + score_pool3c - softmax = fcnxs_score(score_final, data, offset()["fcn8s_upscore"], (16, 16), (8, 8), numclass, workspace_default) - return softmax +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# pylint: skip-file +import mxnet as mx + +def filter_map(kernel=1, stride=1, pad=0): + return (stride, (kernel-stride)/2-pad) + +def compose_fp(fp_first, fp_second): + return (fp_first[0]*fp_second[0], fp_first[0]*fp_second[1]+fp_first[1]) + +def compose_fp_list(fp_list): + fp_out = (1.0, 0.0) + for fp in fp_list: + fp_out = compose_fp(fp_out, fp) + return fp_out + +def inv_fp(fp_in): + return (1.0/fp_in[0], -1.0*fp_in[1]/fp_in[0]) + +def offset(): + conv1_1_fp = filter_map(kernel=3, pad=100) + conv1_2_fp = conv2_1_fp = conv2_2_fp = conv3_1_fp = conv3_2_fp = conv3_3_fp \ + = conv4_1_fp = conv4_2_fp = conv4_3_fp = conv5_1_fp = conv5_2_fp \ + = conv5_3_fp = filter_map(kernel=3, pad=1) + pool1_fp = pool2_fp = pool3_fp = pool4_fp = pool5_fp = filter_map(kernel=2, stride=2) + fc6_fp = filter_map(kernel=7) + fc7_fp = score_fp = score_pool4_fp = score_pool3_fp = filter_map() + # for fcn-32s + fcn32s_upscore_fp = inv_fp(filter_map(kernel=64, stride=32)) + fcn32s_upscore_list = [conv1_1_fp, conv1_2_fp, pool1_fp, conv2_1_fp, conv2_2_fp, + pool2_fp, conv3_1_fp, conv3_2_fp, conv3_3_fp, pool3_fp, + conv4_1_fp, conv4_2_fp, conv4_3_fp, pool4_fp, conv5_1_fp, + conv5_2_fp, conv5_3_fp, pool5_fp, fc6_fp, fc7_fp, score_fp, + fcn32s_upscore_fp] + crop = {} + crop["fcn32s_upscore"] = (-int(round(compose_fp_list(fcn32s_upscore_list)[1])), + -int(round(compose_fp_list(fcn32s_upscore_list)[1]))) + # for fcn-16s + score2_fp = inv_fp(filter_map(kernel=4, stride=2)) + fcn16s_upscore_fp = inv_fp(filter_map(kernel=32, stride=16)) + score_pool4c_fp_list = [inv_fp(score2_fp), inv_fp(score_fp), inv_fp(fc7_fp), inv_fp(fc6_fp), + inv_fp(pool5_fp), inv_fp(conv5_3_fp), inv_fp(conv5_2_fp), + inv_fp(conv5_1_fp), score_pool4_fp] + crop["score_pool4c"] = (-int(round(compose_fp_list(score_pool4c_fp_list)[1])), + -int(round(compose_fp_list(score_pool4c_fp_list)[1]))) + fcn16s_upscore_list = [conv1_1_fp, conv1_2_fp, pool1_fp, conv2_1_fp, conv2_2_fp, + pool2_fp, conv3_1_fp, conv3_2_fp, conv3_3_fp, pool3_fp, + conv4_1_fp, conv4_2_fp, conv4_3_fp, pool4_fp, score_pool4_fp, + inv_fp((1, -crop["score_pool4c"][0])), fcn16s_upscore_fp] + crop["fcn16s_upscore"] = (-int(round(compose_fp_list(fcn16s_upscore_list)[1])), + -int(round(compose_fp_list(fcn16s_upscore_list)[1]))) + # for fcn-8s + score4_fp = inv_fp(filter_map(kernel=4, stride=2)) + fcn8s_upscore_fp = inv_fp(filter_map(kernel=16, stride=8)) + score_pool3c_fp_list = [inv_fp(score4_fp), (1, -crop["score_pool4c"][0]), inv_fp(score_pool4_fp), + inv_fp(pool4_fp), inv_fp(conv4_3_fp), inv_fp(conv4_2_fp), + inv_fp(conv4_1_fp), score_pool3_fp, score_pool3_fp] + crop["score_pool3c"] = (-int(round(compose_fp_list(score_pool3c_fp_list)[1])), + -int(round(compose_fp_list(score_pool3c_fp_list)[1]))) + fcn8s_upscore_list = [conv1_1_fp, conv1_2_fp, pool1_fp, conv2_1_fp, conv2_2_fp, pool2_fp, + conv3_1_fp, conv3_2_fp, conv3_3_fp, pool3_fp, score_pool3_fp, + inv_fp((1, -crop["score_pool3c"][0])), fcn8s_upscore_fp] + crop["fcn8s_upscore"] = (-int(round(compose_fp_list(fcn8s_upscore_list)[1])), + -int(round(compose_fp_list(fcn8s_upscore_list)[1]))) + return crop + +def vgg16_pool3(input, workspace_default=1024): + # group 1 + conv1_1 = mx.symbol.Convolution(data=input, kernel=(3, 3), pad=(100, 100), num_filter=64, + workspace=workspace_default, name="conv1_1") + relu1_1 = mx.symbol.Activation(data=conv1_1, act_type="relu", name="relu1_1") + conv1_2 = mx.symbol.Convolution(data=relu1_1, kernel=(3, 3), pad=(1, 1), num_filter=64, + workspace=workspace_default, name="conv1_2") + relu1_2 = mx.symbol.Activation(data=conv1_2, act_type="relu", name="relu1_2") + pool1 = mx.symbol.Pooling(data=relu1_2, pool_type="max", kernel=(2, 2), stride=(2,2), name="pool1") + # group 2 + conv2_1 = mx.symbol.Convolution(data=pool1, kernel=(3, 3), pad=(1, 1), num_filter=128, + workspace=workspace_default, name="conv2_1") + relu2_1 = mx.symbol.Activation(data=conv2_1, act_type="relu", name="relu2_1") + conv2_2 = mx.symbol.Convolution(data=relu2_1, kernel=(3, 3), pad=(1, 1), num_filter=128, + workspace=workspace_default, name="conv2_2") + relu2_2 = mx.symbol.Activation(data=conv2_2, act_type="relu", name="relu2_2") + pool2 = mx.symbol.Pooling(data=relu2_2, pool_type="max", kernel=(2, 2), stride=(2,2), name="pool2") + # group 3 + conv3_1 = mx.symbol.Convolution(data=pool2, kernel=(3, 3), pad=(1, 1), num_filter=256, + workspace=workspace_default, name="conv3_1") + relu3_1 = mx.symbol.Activation(data=conv3_1, act_type="relu", name="relu3_1") + conv3_2 = mx.symbol.Convolution(data=relu3_1, kernel=(3, 3), pad=(1, 1), num_filter=256, + workspace=workspace_default, name="conv3_2") + relu3_2 = mx.symbol.Activation(data=conv3_2, act_type="relu", name="relu3_2") + conv3_3 = mx.symbol.Convolution(data=relu3_2, kernel=(3, 3), pad=(1, 1), num_filter=256, + workspace=workspace_default, name="conv3_3") + relu3_3 = mx.symbol.Activation(data=conv3_3, act_type="relu", name="relu3_3") + pool3 = mx.symbol.Pooling(data=relu3_3, pool_type="max", kernel=(2, 2), stride=(2,2), name="pool3") + return pool3 + +def vgg16_pool4(input, workspace_default=1024): + # group 4 + conv4_1 = mx.symbol.Convolution(data=input, kernel=(3, 3), pad=(1, 1), num_filter=512, + workspace=workspace_default, name="conv4_1") + relu4_1 = mx.symbol.Activation(data=conv4_1, act_type="relu", name="relu4_1") + conv4_2 = mx.symbol.Convolution(data=relu4_1, kernel=(3, 3), pad=(1, 1), num_filter=512, + workspace=workspace_default, name="conv4_2") + relu4_2 = mx.symbol.Activation(data=conv4_2, act_type="relu", name="relu4_2") + conv4_3 = mx.symbol.Convolution(data=relu4_2, kernel=(3, 3), pad=(1, 1), num_filter=512, + workspace=workspace_default, name="conv4_3") + relu4_3 = mx.symbol.Activation(data=conv4_3, act_type="relu", name="relu4_3") + pool4 = mx.symbol.Pooling(data=relu4_3, pool_type="max", kernel=(2, 2), stride=(2,2), name="pool4") + return pool4 + +def vgg16_score(input, numclass, workspace_default=1024): + # group 5 + conv5_1 = mx.symbol.Convolution(data=input, kernel=(3, 3), pad=(1, 1), num_filter=512, + workspace=workspace_default, name="conv5_1") + relu5_1 = mx.symbol.Activation(data=conv5_1, act_type="relu", name="relu5_1") + conv5_2 = mx.symbol.Convolution(data=relu5_1, kernel=(3, 3), pad=(1, 1), num_filter=512, + workspace=workspace_default, name="conv5_2") + relu5_2 = mx.symbol.Activation(data=conv5_2, act_type="relu", name="relu5_2") + conv5_3 = mx.symbol.Convolution(data=relu5_2, kernel=(3, 3), pad=(1, 1), num_filter=512, + workspace=workspace_default, name="conv5_3") + relu5_3 = mx.symbol.Activation(data=conv5_3, act_type="relu", name="relu5_3") + pool5 = mx.symbol.Pooling(data=relu5_3, pool_type="max", kernel=(2, 2), stride=(2,2), name="pool5") + # group 6 + fc6 = mx.symbol.Convolution(data=pool5, kernel=(7, 7), num_filter=4096, + workspace=workspace_default, name="fc6") + relu6 = mx.symbol.Activation(data=fc6, act_type="relu", name="relu6") + drop6 = mx.symbol.Dropout(data=relu6, p=0.5, name="drop6") + # group 7 + fc7 = mx.symbol.Convolution(data=drop6, kernel=(1, 1), num_filter=4096, + workspace=workspace_default, name="fc7") + relu7 = mx.symbol.Activation(data=fc7, act_type="relu", name="relu7") + drop7 = mx.symbol.Dropout(data=relu7, p=0.5, name="drop7") + # group 8 + score = mx.symbol.Convolution(data=drop7, kernel=(1, 1), num_filter=numclass, + workspace=workspace_default, name="score") + return score + +def fcnxs_score(input, crop, offset, kernel=(64,64), stride=(32,32), numclass=21, workspace_default=1024): + # score out + bigscore = mx.symbol.Deconvolution(data=input, kernel=kernel, stride=stride, adj=(stride[0]-1, stride[1]-1), + num_filter=numclass, workspace=workspace_default, name="bigscore") + upscore = mx.symbol.Crop(*[bigscore, crop], offset=offset, name="upscore") + # upscore = mx.symbol.Crop(*[input, crop], offset=offset, name="upscore") + softmax = mx.symbol.SoftmaxOutput(data=upscore, multi_output=True, use_ignore=True, ignore_label=255, name="softmax") + return softmax + +def get_fcn32s_symbol(numclass=21, workspace_default=1024): + data = mx.symbol.Variable(name="data") + pool3 = vgg16_pool3(data, workspace_default) + pool4 = vgg16_pool4(pool3, workspace_default) + score = vgg16_score(pool4, numclass, workspace_default) + softmax = fcnxs_score(score, data, offset()["fcn32s_upscore"], (64,64), (32,32), numclass, workspace_default) + return softmax + +def get_fcn16s_symbol(numclass=21, workspace_default=1024): + data = mx.symbol.Variable(name="data") + pool3 = vgg16_pool3(data, workspace_default) + pool4 = vgg16_pool4(pool3, workspace_default) + score = vgg16_score(pool4, numclass, workspace_default) + # score 2X + score2 = mx.symbol.Deconvolution(data=score, kernel=(4, 4), stride=(2, 2), num_filter=numclass, + adj=(1, 1), workspace=workspace_default, name="score2") # 2X + score_pool4 = mx.symbol.Convolution(data=pool4, kernel=(1, 1), num_filter=numclass, + workspace=workspace_default, name="score_pool4") + score_pool4c = mx.symbol.Crop(*[score_pool4, score2], offset=offset()["score_pool4c"], name="score_pool4c") + score_fused = score2 + score_pool4c + softmax = fcnxs_score(score_fused, data, offset()["fcn16s_upscore"], (32, 32), (16, 16), numclass, workspace_default) + return softmax + +def get_fcn8s_symbol(numclass=21, workspace_default=1024): + data = mx.symbol.Variable(name="data") + pool3 = vgg16_pool3(data, workspace_default) + pool4 = vgg16_pool4(pool3, workspace_default) + score = vgg16_score(pool4, numclass, workspace_default) + # score 2X + score2 = mx.symbol.Deconvolution(data=score, kernel=(4, 4), stride=(2, 2),num_filter=numclass, + adj=(1, 1), workspace=workspace_default, name="score2") # 2X + score_pool4 = mx.symbol.Convolution(data=pool4, kernel=(1, 1), num_filter=numclass, + workspace=workspace_default, name="score_pool4") + score_pool4c = mx.symbol.Crop(*[score_pool4, score2], offset=offset()["score_pool4c"], name="score_pool4c") + score_fused = score2 + score_pool4c + # score 4X + score4 = mx.symbol.Deconvolution(data=score_fused, kernel=(4, 4), stride=(2, 2),num_filter=numclass, + adj=(1, 1), workspace=workspace_default, name="score4") # 4X + score_pool3 = mx.symbol.Convolution(data=pool3, kernel=(1, 1), num_filter=numclass, + workspace=workspace_default, name="score_pool3") + score_pool3c = mx.symbol.Crop(*[score_pool3, score4], offset=offset()["score_pool3c"], name="score_pool3c") + score_final = score4 + score_pool3c + softmax = fcnxs_score(score_final, data, offset()["fcn8s_upscore"], (16, 16), (8, 8), numclass, workspace_default) + return softmax diff --git a/example/gan/dcgan.py b/example/gan/dcgan.py index 5faff9aa5d9f..981f4a4778e3 100644 --- a/example/gan/dcgan.py +++ b/example/gan/dcgan.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from __future__ import print_function import mxnet as mx import numpy as np diff --git a/example/gluon/actor_critic.py b/example/gluon/actor_critic.py index 9c475ce15017..6d4474b4f239 100644 --- a/example/gluon/actor_critic.py +++ b/example/gluon/actor_critic.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from __future__ import print_function import argparse diff --git a/example/gluon/data.py b/example/gluon/data.py index 80a50bdce66b..c5ddd0af302b 100644 --- a/example/gluon/data.py +++ b/example/gluon/data.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: skip-file """ data iterator for mnist """ import os diff --git a/example/gluon/dcgan.py b/example/gluon/dcgan.py index f643b28f4bc1..ed814df61e99 100644 --- a/example/gluon/dcgan.py +++ b/example/gluon/dcgan.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import matplotlib as mpl mpl.use('Agg') from matplotlib import pyplot as plt diff --git a/example/gluon/image_classification.py b/example/gluon/image_classification.py index bb1fa8da8b2a..3f84ff8602ed 100644 --- a/example/gluon/image_classification.py +++ b/example/gluon/image_classification.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from __future__ import division import argparse, time diff --git a/example/gluon/lstm_crf.py b/example/gluon/lstm_crf.py index 8344789faa6e..40c8c2be2784 100644 --- a/example/gluon/lstm_crf.py +++ b/example/gluon/lstm_crf.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx from mxnet import autograd as ag, ndarray as nd, gluon from mxnet.gluon import Block, nn, rnn diff --git a/example/gluon/mnist.py b/example/gluon/mnist.py index 9d567d5011cb..198d7ca5ab2a 100644 --- a/example/gluon/mnist.py +++ b/example/gluon/mnist.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: skip-file from __future__ import print_function diff --git a/example/gluon/super_resolution.py b/example/gluon/super_resolution.py index d61fb160e197..acc5ffa01d4a 100644 --- a/example/gluon/super_resolution.py +++ b/example/gluon/super_resolution.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from __future__ import print_function import argparse, tarfile import math diff --git a/example/gluon/tree_lstm/dataset.py b/example/gluon/tree_lstm/dataset.py index f9cfce5c2bff..4a836ddb0eaf 100644 --- a/example/gluon/tree_lstm/dataset.py +++ b/example/gluon/tree_lstm/dataset.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import os import logging logging.basicConfig(level=logging.INFO) diff --git a/example/gluon/tree_lstm/fetch_and_preprocess.sh b/example/gluon/tree_lstm/fetch_and_preprocess.sh index dfbf82a872c1..f372392830d0 100755 --- a/example/gluon/tree_lstm/fetch_and_preprocess.sh +++ b/example/gluon/tree_lstm/fetch_and_preprocess.sh @@ -1,7 +1,25 @@ #!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + set -e python2.7 scripts/download.py CLASSPATH="lib:lib/stanford-parser/stanford-parser.jar:lib/stanford-parser/stanford-parser-3.5.1-models.jar" javac -cp $CLASSPATH lib/*.java -python2.7 scripts/preprocess-sick.py \ No newline at end of file +python2.7 scripts/preprocess-sick.py diff --git a/example/gluon/tree_lstm/main.py b/example/gluon/tree_lstm/main.py index 7903e67a1441..f04a69f2671f 100644 --- a/example/gluon/tree_lstm/main.py +++ b/example/gluon/tree_lstm/main.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # This example is inspired by https://github.com/dasguptar/treelstm.pytorch import argparse, cPickle, math, os, random import logging diff --git a/example/gluon/tree_lstm/scripts/download.py b/example/gluon/tree_lstm/scripts/download.py index d38b3a46ecd8..7ea930370175 100644 --- a/example/gluon/tree_lstm/scripts/download.py +++ b/example/gluon/tree_lstm/scripts/download.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ Downloads the following: - Stanford parser diff --git a/example/gluon/tree_lstm/scripts/preprocess-sick.py b/example/gluon/tree_lstm/scripts/preprocess-sick.py index fd28b58a5f7f..abbcc5fac844 100644 --- a/example/gluon/tree_lstm/scripts/preprocess-sick.py +++ b/example/gluon/tree_lstm/scripts/preprocess-sick.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ Preprocessing script for SICK data. diff --git a/example/gluon/tree_lstm/tree_lstm.py b/example/gluon/tree_lstm/tree_lstm.py index ced7f7eeadf5..e96fe26bf9b6 100644 --- a/example/gluon/tree_lstm/tree_lstm.py +++ b/example/gluon/tree_lstm/tree_lstm.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx from mxnet.gluon import Block, nn from mxnet.gluon.parameter import Parameter diff --git a/example/gluon/word_language_model/data.py b/example/gluon/word_language_model/data.py index e3a283b64285..913963ec20cb 100644 --- a/example/gluon/word_language_model/data.py +++ b/example/gluon/word_language_model/data.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import os import numpy as np import mxnet as mx diff --git a/example/gluon/word_language_model/get_ptb_data.sh b/example/gluon/word_language_model/get_ptb_data.sh index 1ec009aa2f99..d2641cb32b81 100755 --- a/example/gluon/word_language_model/get_ptb_data.sh +++ b/example/gluon/word_language_model/get_ptb_data.sh @@ -1,5 +1,23 @@ #!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + RNN_DIR=$(cd `dirname $0`; pwd) DATA_DIR="${RNN_DIR}/data/" diff --git a/example/gluon/word_language_model/model.py b/example/gluon/word_language_model/model.py index 91378cee3cb4..40e7926ef8d6 100644 --- a/example/gluon/word_language_model/model.py +++ b/example/gluon/word_language_model/model.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx from mxnet import gluon from mxnet.gluon import nn, rnn diff --git a/example/gluon/word_language_model/train.py b/example/gluon/word_language_model/train.py index 5b34c00ecea0..0b504998bed2 100644 --- a/example/gluon/word_language_model/train.py +++ b/example/gluon/word_language_model/train.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import argparse import time import math diff --git a/example/image-classification/benchmark.py b/example/image-classification/benchmark.py index 5b040f3b09ff..3096fae07897 100644 --- a/example/image-classification/benchmark.py +++ b/example/image-classification/benchmark.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from __future__ import print_function import logging import argparse diff --git a/example/image-classification/benchmark_score.py b/example/image-classification/benchmark_score.py index f54b6aebb750..aeacffa82b78 100644 --- a/example/image-classification/benchmark_score.py +++ b/example/image-classification/benchmark_score.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ Benchmark the scoring performance on various CNNs """ diff --git a/example/image-classification/common/data.py b/example/image-classification/common/data.py index fe27ec26b939..eb694a45dc27 100755 --- a/example/image-classification/common/data.py +++ b/example/image-classification/common/data.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx import random from mxnet.io import DataBatch, DataIter diff --git a/example/image-classification/common/find_mxnet.py b/example/image-classification/common/find_mxnet.py index 24dcaf809a38..2ce07130a361 100644 --- a/example/image-classification/common/find_mxnet.py +++ b/example/image-classification/common/find_mxnet.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import os, sys try: import mxnet as mx diff --git a/example/image-classification/common/fit.py b/example/image-classification/common/fit.py index 69baed1ef4f6..73235fc2e4ef 100755 --- a/example/image-classification/common/fit.py +++ b/example/image-classification/common/fit.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx import logging import os diff --git a/example/image-classification/common/modelzoo.py b/example/image-classification/common/modelzoo.py index c2944cdc2827..1fe14ca4fcd2 100644 --- a/example/image-classification/common/modelzoo.py +++ b/example/image-classification/common/modelzoo.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import os from util import download_file diff --git a/example/image-classification/common/util.py b/example/image-classification/common/util.py index a25e2181be92..5f70411ab084 100644 --- a/example/image-classification/common/util.py +++ b/example/image-classification/common/util.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import subprocess import os import errno diff --git a/example/image-classification/data/caltech256.sh b/example/image-classification/data/caltech256.sh index 3befdac0c73a..3fc329a9b835 100755 --- a/example/image-classification/data/caltech256.sh +++ b/example/image-classification/data/caltech256.sh @@ -1,5 +1,23 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + # This file download the caltech 256 dataset # (http://www.vision.caltech.edu/Image_Datasets/Caltech256/), and split it into # the train and val rec files. diff --git a/example/image-classification/data/imagenet1k-val.sh b/example/image-classification/data/imagenet1k-val.sh index 51f8130103fd..13cb551140f8 100755 --- a/example/image-classification/data/imagenet1k-val.sh +++ b/example/image-classification/data/imagenet1k-val.sh @@ -1,5 +1,23 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + # This file download the imagnet-1k validation dataset and convert it into a rec # file. One need to provide the URL for the ILSVRC2012_img_val.tar, which can be # find at http://www.image-net.org/download-images diff --git a/example/image-classification/fine-tune.py b/example/image-classification/fine-tune.py index 5a2a04d2c73b..a5fb2434d958 100644 --- a/example/image-classification/fine-tune.py +++ b/example/image-classification/fine-tune.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import os import argparse import logging diff --git a/example/image-classification/predict-cpp/image-classification-predict.cc b/example/image-classification/predict-cpp/image-classification-predict.cc index a8652c4f0815..fb74ed9d7170 100644 --- a/example/image-classification/predict-cpp/image-classification-predict.cc +++ b/example/image-classification/predict-cpp/image-classification-predict.cc @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! * Copyright (c) 2015 by Xiao Liu, pertusa, caprice-j * \file image_classification-predict.cpp @@ -196,7 +215,7 @@ int main(int argc, char* argv[]) { const mx_uint input_shape_indptr[2] = { 0, 4 }; const mx_uint input_shape_data[4] = { 1, static_cast(channels), - static_cast(height), + static_cast(height), static_cast(width)}; PredictorHandle pred_hnd = 0; diff --git a/example/image-classification/score.py b/example/image-classification/score.py index d26ddddf9b83..f40e649f1f42 100644 --- a/example/image-classification/score.py +++ b/example/image-classification/score.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import argparse from common import modelzoo, find_mxnet import mxnet as mx diff --git a/example/image-classification/symbols/alexnet.py b/example/image-classification/symbols/alexnet.py index e2b512b195c8..f945b9f87cd9 100755 --- a/example/image-classification/symbols/alexnet.py +++ b/example/image-classification/symbols/alexnet.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ Reference: diff --git a/example/image-classification/symbols/googlenet.py b/example/image-classification/symbols/googlenet.py index cc8c7adc6540..05f33da5d884 100644 --- a/example/image-classification/symbols/googlenet.py +++ b/example/image-classification/symbols/googlenet.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """References: Szegedy, Christian, Wei Liu, Yangqing Jia, Pierre Sermanet, Scott Reed, Dragomir diff --git a/example/image-classification/symbols/inception-bn.py b/example/image-classification/symbols/inception-bn.py index 7dae9ad053e0..84934a5f72aa 100644 --- a/example/image-classification/symbols/inception-bn.py +++ b/example/image-classification/symbols/inception-bn.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ Inception + BN, suitable for images with around 224 x 224 diff --git a/example/image-classification/symbols/inception-resnet-v2.py b/example/image-classification/symbols/inception-resnet-v2.py index b2b0c6023ac1..5f313351eab2 100644 --- a/example/image-classification/symbols/inception-resnet-v2.py +++ b/example/image-classification/symbols/inception-resnet-v2.py @@ -1,9 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ -Contains the definition of the Inception Resnet V2 architecture. -As described in http://arxiv.org/abs/1602.07261. -Inception-v4, Inception-ResNet and the Impact of Residual Connections -on Learning -Christian Szegedy, Sergey Ioffe, Vincent Vanhoucke, Alex Alemi +Contains the definition of the Inception Resnet V2 architecture. +As described in http://arxiv.org/abs/1602.07261. +Inception-v4, Inception-ResNet and the Impact of Residual Connections +on Learning +Christian Szegedy, Sergey Ioffe, Vincent Vanhoucke, Alex Alemi """ import mxnet as mx diff --git a/example/image-classification/symbols/inception-v3.py b/example/image-classification/symbols/inception-v3.py index 35562d663745..5108579ffd3a 100644 --- a/example/image-classification/symbols/inception-v3.py +++ b/example/image-classification/symbols/inception-v3.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ Inception V3, suitable for images with around 299 x 299 diff --git a/example/image-classification/symbols/inception-v4.py b/example/image-classification/symbols/inception-v4.py index eead5f7c7ea0..2b4fe6fbb0c7 100644 --- a/example/image-classification/symbols/inception-v4.py +++ b/example/image-classification/symbols/inception-v4.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # -*- coding:utf-8 -*- __author__ = 'zhangshuai' modified_date = '16/7/5' diff --git a/example/image-classification/symbols/lenet.py b/example/image-classification/symbols/lenet.py index 6df02991e4ad..f2cc106f60ac 100644 --- a/example/image-classification/symbols/lenet.py +++ b/example/image-classification/symbols/lenet.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ LeCun, Yann, Leon Bottou, Yoshua Bengio, and Patrick Haffner. Gradient-based learning applied to document recognition. diff --git a/example/image-classification/symbols/mlp.py b/example/image-classification/symbols/mlp.py index cc569bc10917..4b190b29db9e 100644 --- a/example/image-classification/symbols/mlp.py +++ b/example/image-classification/symbols/mlp.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ a simple multilayer perceptron """ diff --git a/example/image-classification/symbols/mobilenet.py b/example/image-classification/symbols/mobilenet.py index 8ad584a50768..42b963626164 100644 --- a/example/image-classification/symbols/mobilenet.py +++ b/example/image-classification/symbols/mobilenet.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx def Conv(data, num_filter=1, kernel=(1, 1), stride=(1, 1), pad=(0, 0), num_group=1, name=None, suffix=''): diff --git a/example/image-classification/symbols/resnet-v1.py b/example/image-classification/symbols/resnet-v1.py index 0d7bee1d16cd..e5752f775447 100755 --- a/example/image-classification/symbols/resnet-v1.py +++ b/example/image-classification/symbols/resnet-v1.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + ''' Adapted from https://github.com/tornadomeet/ResNet/blob/master/symbol_resnet.py (Original author Wei Wu) by Antti-Pekka Hynninen diff --git a/example/image-classification/symbols/resnet.py b/example/image-classification/symbols/resnet.py index 41cbc82bcb45..be498602f0b7 100644 --- a/example/image-classification/symbols/resnet.py +++ b/example/image-classification/symbols/resnet.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + ''' Adapted from https://github.com/tornadomeet/ResNet/blob/master/symbol_resnet.py Original author Wei Wu diff --git a/example/image-classification/symbols/resnext.py b/example/image-classification/symbols/resnext.py index bd5b65621722..59749430c76c 100644 --- a/example/image-classification/symbols/resnext.py +++ b/example/image-classification/symbols/resnext.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + ''' Adapted from https://github.com/tornadomeet/ResNet/blob/master/symbol_resnet.py Original author Wei Wu @@ -29,19 +46,19 @@ def residual_unit(data, num_filter, stride, dim_match, name, bottle_neck=True, n """ if bottle_neck: # the same as https://github.com/facebook/fb.resnet.torch#notes, a bit difference with origin paper - + conv1 = mx.sym.Convolution(data=data, num_filter=int(num_filter*0.5), kernel=(1,1), stride=(1,1), pad=(0,0), no_bias=True, workspace=workspace, name=name + '_conv1') bn1 = mx.sym.BatchNorm(data=conv1, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn1') act1 = mx.sym.Activation(data=bn1, act_type='relu', name=name + '_relu1') - + conv2 = mx.sym.Convolution(data=act1, num_filter=int(num_filter*0.5), num_group=num_group, kernel=(3,3), stride=stride, pad=(1,1), no_bias=True, workspace=workspace, name=name + '_conv2') bn2 = mx.sym.BatchNorm(data=conv2, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn2') act2 = mx.sym.Activation(data=bn2, act_type='relu', name=name + '_relu2') - + conv3 = mx.sym.Convolution(data=act2, num_filter=num_filter, kernel=(1,1), stride=(1,1), pad=(0,0), no_bias=True, workspace=workspace, name=name + '_conv3') bn3 = mx.sym.BatchNorm(data=conv3, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn3') @@ -58,13 +75,13 @@ def residual_unit(data, num_filter, stride, dim_match, name, bottle_neck=True, n eltwise = bn3 + shortcut return mx.sym.Activation(data=eltwise, act_type='relu', name=name + '_relu') else: - + conv1 = mx.sym.Convolution(data=data, num_filter=num_filter, kernel=(3,3), stride=stride, pad=(1,1), no_bias=True, workspace=workspace, name=name + '_conv1') bn1 = mx.sym.BatchNorm(data=conv1, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_bn1') act1 = mx.sym.Activation(data=bn1, act_type='relu', name=name + '_relu1') - + conv2 = mx.sym.Convolution(data=act1, num_filter=num_filter, kernel=(3,3), stride=(1,1), pad=(1,1), no_bias=True, workspace=workspace, name=name + '_conv2') bn2 = mx.sym.BatchNorm(data=conv2, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_bn2') @@ -124,12 +141,12 @@ def resnext(units, num_stages, filter_list, num_classes, num_group, image_shape, for i in range(num_stages): body = residual_unit(body, filter_list[i+1], (1 if i==0 else 2, 1 if i==0 else 2), False, - name='stage%d_unit%d' % (i + 1, 1), bottle_neck=bottle_neck, num_group=num_group, + name='stage%d_unit%d' % (i + 1, 1), bottle_neck=bottle_neck, num_group=num_group, bn_mom=bn_mom, workspace=workspace, memonger=memonger) for j in range(units[i]-1): body = residual_unit(body, filter_list[i+1], (1,1), True, name='stage%d_unit%d' % (i + 1, j + 2), bottle_neck=bottle_neck, num_group=num_group, bn_mom=bn_mom, workspace=workspace, memonger=memonger) - + pool1 = mx.sym.Pooling(data=body, global_pool=True, kernel=(7, 7), pool_type='avg', name='pool1') flat = mx.sym.Flatten(data=pool1) fc1 = mx.sym.FullyConnected(data=flat, num_hidden=num_classes, name='fc1') @@ -186,7 +203,7 @@ def get_symbol(num_classes, num_layers, image_shape, num_group=32, conv_workspac num_stages = num_stages, filter_list = filter_list, num_classes = num_classes, - num_group = num_group, + num_group = num_group, image_shape = image_shape, bottle_neck = bottle_neck, workspace = conv_workspace, diff --git a/example/image-classification/symbols/vgg.py b/example/image-classification/symbols/vgg.py index 8dae74f1c94f..ca1013621863 100644 --- a/example/image-classification/symbols/vgg.py +++ b/example/image-classification/symbols/vgg.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """References: Simonyan, Karen, and Andrew Zisserman. "Very deep convolutional networks for diff --git a/example/image-classification/test_score.py b/example/image-classification/test_score.py index 19a1d3072664..0789c9270fff 100644 --- a/example/image-classification/test_score.py +++ b/example/image-classification/test_score.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ test pretrained models """ diff --git a/example/image-classification/train_cifar10.py b/example/image-classification/train_cifar10.py index 0186233d6ea2..7eb56ebce36d 100644 --- a/example/image-classification/train_cifar10.py +++ b/example/image-classification/train_cifar10.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import os import argparse import logging diff --git a/example/image-classification/train_imagenet.py b/example/image-classification/train_imagenet.py index 760ea6b22391..5760a9af3782 100644 --- a/example/image-classification/train_imagenet.py +++ b/example/image-classification/train_imagenet.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import os import argparse import logging diff --git a/example/image-classification/train_mnist.py b/example/image-classification/train_mnist.py index 31ecbfb463c4..2bc4289318d9 100644 --- a/example/image-classification/train_mnist.py +++ b/example/image-classification/train_mnist.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ Train mnist, see more explanation at http://mxnet.io/tutorials/python/mnist.html """ @@ -53,9 +70,9 @@ def get_mnist_iter(args, kv): help='the number of classes') parser.add_argument('--num-examples', type=int, default=60000, help='the number of training examples') - + parser.add_argument('--add_stn', action="store_true", default=False, help='Add Spatial Transformer Network Layer (lenet only)') - + fit.add_fit_args(parser) parser.set_defaults( # network diff --git a/example/kaggle-ndsb1/gen_img_list.py b/example/kaggle-ndsb1/gen_img_list.py index 2da5d7097d96..adfc4fe09d68 100644 --- a/example/kaggle-ndsb1/gen_img_list.py +++ b/example/kaggle-ndsb1/gen_img_list.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from __future__ import print_function import csv import os @@ -26,7 +43,7 @@ fo_name=os.path.join(args.out_folder+args.out_file) fo = csv.writer(open(fo_name, "w"), delimiter='\t', lineterminator='\n') - + if args.train: tr_fo_name=os.path.join(args.out_folder+"tr.lst") va_fo_name=os.path.join(args.out_folder+"va.lst") @@ -58,7 +75,7 @@ #write for item in img_lst: fo.writerow(item) - + ## If training, split into train and validation lists (tr.lst and va.lst) diff --git a/example/kaggle-ndsb1/predict_dsb.py b/example/kaggle-ndsb1/predict_dsb.py index 483243a430a7..2be2eccb2c88 100644 --- a/example/kaggle-ndsb1/predict_dsb.py +++ b/example/kaggle-ndsb1/predict_dsb.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from __future__ import print_function import find_mxnet import submission_dsb diff --git a/example/kaggle-ndsb1/submission_dsb.py b/example/kaggle-ndsb1/submission_dsb.py index a2644f8d2a74..2695c1abb7ce 100644 --- a/example/kaggle-ndsb1/submission_dsb.py +++ b/example/kaggle-ndsb1/submission_dsb.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from __future__ import print_function import pandas as pd import os @@ -14,9 +31,9 @@ def gen_sub(predictions,test_lst_path="test.lst",submission_path="submission.csv ## check sampleSubmission.csv from kaggle website to view submission format header = "acantharia_protist_big_center,acantharia_protist_halo,acantharia_protist,amphipods,appendicularian_fritillaridae,appendicularian_s_shape,appendicularian_slight_curve,appendicularian_straight,artifacts_edge,artifacts,chaetognath_non_sagitta,chaetognath_other,chaetognath_sagitta,chordate_type1,copepod_calanoid_eggs,copepod_calanoid_eucalanus,copepod_calanoid_flatheads,copepod_calanoid_frillyAntennae,copepod_calanoid_large_side_antennatucked,copepod_calanoid_large,copepod_calanoid_octomoms,copepod_calanoid_small_longantennae,copepod_calanoid,copepod_cyclopoid_copilia,copepod_cyclopoid_oithona_eggs,copepod_cyclopoid_oithona,copepod_other,crustacean_other,ctenophore_cestid,ctenophore_cydippid_no_tentacles,ctenophore_cydippid_tentacles,ctenophore_lobate,decapods,detritus_blob,detritus_filamentous,detritus_other,diatom_chain_string,diatom_chain_tube,echinoderm_larva_pluteus_brittlestar,echinoderm_larva_pluteus_early,echinoderm_larva_pluteus_typeC,echinoderm_larva_pluteus_urchin,echinoderm_larva_seastar_bipinnaria,echinoderm_larva_seastar_brachiolaria,echinoderm_seacucumber_auricularia_larva,echinopluteus,ephyra,euphausiids_young,euphausiids,fecal_pellet,fish_larvae_deep_body,fish_larvae_leptocephali,fish_larvae_medium_body,fish_larvae_myctophids,fish_larvae_thin_body,fish_larvae_very_thin_body,heteropod,hydromedusae_aglaura,hydromedusae_bell_and_tentacles,hydromedusae_h15,hydromedusae_haliscera_small_sideview,hydromedusae_haliscera,hydromedusae_liriope,hydromedusae_narco_dark,hydromedusae_narco_young,hydromedusae_narcomedusae,hydromedusae_other,hydromedusae_partial_dark,hydromedusae_shapeA_sideview_small,hydromedusae_shapeA,hydromedusae_shapeB,hydromedusae_sideview_big,hydromedusae_solmaris,hydromedusae_solmundella,hydromedusae_typeD_bell_and_tentacles,hydromedusae_typeD,hydromedusae_typeE,hydromedusae_typeF,invertebrate_larvae_other_A,invertebrate_larvae_other_B,jellies_tentacles,polychaete,protist_dark_center,protist_fuzzy_olive,protist_noctiluca,protist_other,protist_star,pteropod_butterfly,pteropod_theco_dev_seq,pteropod_triangle,radiolarian_chain,radiolarian_colony,shrimp_caridean,shrimp_sergestidae,shrimp_zoea,shrimp-like_other,siphonophore_calycophoran_abylidae,siphonophore_calycophoran_rocketship_adult,siphonophore_calycophoran_rocketship_young,siphonophore_calycophoran_sphaeronectes_stem,siphonophore_calycophoran_sphaeronectes_young,siphonophore_calycophoran_sphaeronectes,siphonophore_other_parts,siphonophore_partial,siphonophore_physonect_young,siphonophore_physonect,stomatopod,tornaria_acorn_worm_larvae,trichodesmium_bowtie,trichodesmium_multiple,trichodesmium_puff,trichodesmium_tuft,trochophore_larvae,tunicate_doliolid_nurse,tunicate_doliolid,tunicate_partial,tunicate_salp_chains,tunicate_salp,unknown_blobs_and_smudges,unknown_sticks,unknown_unclassified".split(',') - + # read first line to know the number of columns and column to use - img_lst = pd.read_csv(test_lst_path,sep="/",header=None, nrows=1) + img_lst = pd.read_csv(test_lst_path,sep="/",header=None, nrows=1) columns = img_lst.columns.tolist() # get the columns cols_to_use = columns[len(columns)-1] # drop the last one cols_to_use= map(int, str(cols_to_use)) ## convert scalar to list @@ -28,15 +45,15 @@ def gen_sub(predictions,test_lst_path="test.lst",submission_path="submission.csv df = pd.DataFrame(predictions,columns = header, index=img_lst) df.index.name = 'image' - + print("Saving csv to %s" % submission_path) df.to_csv(submission_path) - + print("Compress with gzip") os.system("gzip -f %s" % submission_path) - + print(" stored in %s.gz" % submission_path) - + diff --git a/example/kaggle-ndsb1/symbol_dsb.py b/example/kaggle-ndsb1/symbol_dsb.py index 43898a106c00..0a4db8f19ad5 100644 --- a/example/kaggle-ndsb1/symbol_dsb.py +++ b/example/kaggle-ndsb1/symbol_dsb.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import find_mxnet import mxnet as mx diff --git a/example/kaggle-ndsb1/train_dsb.py b/example/kaggle-ndsb1/train_dsb.py index 19beb022c8fa..5cec0f6d4fd4 100644 --- a/example/kaggle-ndsb1/train_dsb.py +++ b/example/kaggle-ndsb1/train_dsb.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import find_mxnet import mxnet as mx import logging @@ -23,7 +40,7 @@ help='clip min/max gradient to prevent extreme value') parser.add_argument('--num-epochs', type=int, default=100, help='the number of training epochs') -parser.add_argument('--load-epoch', type=int, +parser.add_argument('--load-epoch', type=int, help="load the model on an epoch using the model-prefix") parser.add_argument('--batch-size', type=int, default=64, help='the batch size') @@ -35,7 +52,7 @@ help='the number of training examples') parser.add_argument('--num-classes', type=int, default=121, help='the number of classes') -parser.add_argument('--log-file', type=str, +parser.add_argument('--log-file', type=str, help='the name of log file') parser.add_argument('--log-dir', type=str, default="/tmp/", help='directory of the log file') @@ -63,7 +80,7 @@ def get_iterator(args, kv): rand_crop = True, rand_mirror = True, ) - + # validate data iterator val = mx.io.ImageRecordIter( path_imgrec = args.data_dir + "va.rec", diff --git a/example/kaggle-ndsb1/training_curves.py b/example/kaggle-ndsb1/training_curves.py index e4ffd94081da..67f25f0042f6 100644 --- a/example/kaggle-ndsb1/training_curves.py +++ b/example/kaggle-ndsb1/training_curves.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + ## based on https://github.com/dmlc/mxnet/issues/1302 ## Parses the model fit log file and generates a train/val vs epoch plot import matplotlib.pyplot as plt diff --git a/example/kaggle-ndsb2/Preprocessing.py b/example/kaggle-ndsb2/Preprocessing.py index 64d15e036d6d..29b4ba009a9a 100644 --- a/example/kaggle-ndsb2/Preprocessing.py +++ b/example/kaggle-ndsb2/Preprocessing.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Preprocessing script. This script walks over the directories and dump the frames into a csv file diff --git a/example/kaggle-ndsb2/Train.py b/example/kaggle-ndsb2/Train.py index 22aa3ed721e1..51e308a2e21c 100644 --- a/example/kaggle-ndsb2/Train.py +++ b/example/kaggle-ndsb2/Train.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Training script, this is converted from a ipython notebook """ diff --git a/example/memcost/inception_memcost.py b/example/memcost/inception_memcost.py index 45e85901714b..c539e73b3c24 100644 --- a/example/memcost/inception_memcost.py +++ b/example/memcost/inception_memcost.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: skip-file import sys sys.path.append('../../python/') diff --git a/example/model-parallel-lstm/get_ptb_data.sh b/example/model-parallel-lstm/get_ptb_data.sh index 1ec009aa2f99..d2641cb32b81 100755 --- a/example/model-parallel-lstm/get_ptb_data.sh +++ b/example/model-parallel-lstm/get_ptb_data.sh @@ -1,5 +1,23 @@ #!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + RNN_DIR=$(cd `dirname $0`; pwd) DATA_DIR="${RNN_DIR}/data/" diff --git a/example/model-parallel-lstm/lstm.py b/example/model-parallel-lstm/lstm.py index 795eb6ef3bc1..c24017ff0d9c 100644 --- a/example/model-parallel-lstm/lstm.py +++ b/example/model-parallel-lstm/lstm.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint:skip-file import sys sys.path.insert(0, "../../python") @@ -152,7 +169,7 @@ def setup_rnn_model(default_ctx, models = {} buckets.reverse() for bucket_key in buckets: - # bind max_len first + # bind max_len first rnn_sym = lstm_unroll(num_lstm_layer=num_lstm_layer, num_hidden=num_hidden, seq_len=seq_len, @@ -190,7 +207,7 @@ def setup_rnn_model(default_ctx, args_grad[name] = mx.nd.zeros(shape, ctx) if not name.startswith("t"): print("%s group=%s, ctx=%s" % (name, group, str(ctx))) - + #bind with shared executor rnn_exec = None if max_len == bucket_key: @@ -220,7 +237,7 @@ def setup_rnn_model(default_ctx, h=arg_dict["l%d_init_h" % i]) for i in range(num_lstm_layer)] seq_data = [rnn_exec.arg_dict["t%d_data" % i] for i in range(seq_len)] - # we don't need to store the last state + # we don't need to store the last state last_states = None if concat_decode: @@ -235,7 +252,7 @@ def setup_rnn_model(default_ctx, seq_data=seq_data, seq_labels=seq_labels, seq_outputs=seq_outputs, param_blocks=param_blocks) models[bucket_key] = model - buckets.reverse() + buckets.reverse() return models @@ -256,7 +273,7 @@ def set_rnn_inputs(m, X, begin): def set_rnn_inputs_from_batch(m, batch, batch_seq_length, batch_size): X = batch.data for seqidx in range(batch_seq_length): - idx = seqidx + idx = seqidx next_idx = (seqidx + 1) % batch_seq_length x = X[idx, :] y = X[next_idx, :] @@ -295,20 +312,20 @@ def train_lstm(model, X_train_batch, X_val_batch, nbatch = 0 train_nll = 0 tic = time.time() - for data_batch in X_train_batch: + for data_batch in X_train_batch: batch_seq_length = data_batch.bucket_key m = model[batch_seq_length] # reset init state for state in m.init_states: state.c[:] = 0.0 state.h[:] = 0.0 - + head_grad = [] if use_loss: ctx = m.seq_outputs[0].context head_grad = [mx.nd.ones((1,), ctx) for x in m.seq_outputs] - set_rnn_inputs_from_batch(m, data_batch, batch_seq_length, batch_size) + set_rnn_inputs_from_batch(m, data_batch, batch_seq_length, batch_size) m.rnn_exec.forward(is_train=True) # probability of each label class, used to evaluate nll @@ -390,7 +407,7 @@ def train_lstm(model, X_train_batch, X_val_batch, else: val_nll += sum([x.asscalar() for x in seq_loss]) / batch_size nbatch += batch_size - + perp = np.exp(val_nll / nbatch) print("Iter [%d] Val: NLL=%.3f, Perp=%.3f" % ( iteration, val_nll / nbatch, np.exp(val_nll / nbatch))) @@ -401,7 +418,7 @@ def train_lstm(model, X_train_batch, X_val_batch, X_val_batch.reset() X_train_batch.reset() -# is this function being used? +# is this function being used? def setup_rnn_sample_model(ctx, params, num_lstm_layer, diff --git a/example/model-parallel-lstm/lstm_ptb.py b/example/model-parallel-lstm/lstm_ptb.py index 20ce89653fd8..0141338329e4 100644 --- a/example/model-parallel-lstm/lstm_ptb.py +++ b/example/model-parallel-lstm/lstm_ptb.py @@ -1,10 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint:skip-file import lstm import sys sys.path.insert(0, "../../python") import mxnet as mx import numpy as np -# reuse the bucket_io library +# reuse the bucket_io library sys.path.insert(0, "../rnn") from bucket_io import BucketSentenceIter, default_build_vocab diff --git a/example/module/lstm_bucketing.py b/example/module/lstm_bucketing.py index dc00ef55aa7d..ecc7e7be0bca 100644 --- a/example/module/lstm_bucketing.py +++ b/example/module/lstm_bucketing.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: disable=C0111,too-many-arguments,too-many-instance-attributes,too-many-locals,redefined-outer-name,fixme # pylint: disable=superfluous-parens, no-member, invalid-name import sys diff --git a/example/module/mnist_mlp.py b/example/module/mnist_mlp.py index 6d9d6bff4cb5..d2737dc12af7 100644 --- a/example/module/mnist_mlp.py +++ b/example/module/mnist_mlp.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: skip-file import os, sys sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) diff --git a/example/module/python_loss.py b/example/module/python_loss.py index d139789fdf33..9680ac6cb091 100644 --- a/example/module/python_loss.py +++ b/example/module/python_loss.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: skip-file import numpy as np import mxnet as mx diff --git a/example/module/sequential_module.py b/example/module/sequential_module.py index 4659457c5113..48e1046a2067 100644 --- a/example/module/sequential_module.py +++ b/example/module/sequential_module.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: skip-file import os, sys sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) diff --git a/example/module/train_cifar10.py b/example/module/train_cifar10.py index ec3be57edb66..a96e8d92846b 100644 --- a/example/module/train_cifar10.py +++ b/example/module/train_cifar10.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Train CIFAR-10 classifier in MXNet. Demonstrates using the Module class. """ @@ -132,7 +149,7 @@ def do_train(args, callback_args=None): (train, val) = get_iterator(args, kv) if args.gpus is None or args.gpus == '': - devs = mx.cpu() + devs = mx.cpu() elif type(args.gpus) == str: devs = [mx.gpu(int(i)) for i in args.gpus.split(',')] else: diff --git a/example/multi-task/data.py b/example/multi-task/data.py index d39821f52145..0ca8e1fd6653 100644 --- a/example/multi-task/data.py +++ b/example/multi-task/data.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: skip-file """ data iterator for mnist """ import sys diff --git a/example/multi-task/example_multi_task.py b/example/multi-task/example_multi_task.py index 8ee396f0daf4..853d435bbf0b 100644 --- a/example/multi-task/example_multi_task.py +++ b/example/multi-task/example_multi_task.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: skip-file import sys sys.path.insert(0, "../../python/") diff --git a/example/nce-loss/get_text8.sh b/example/nce-loss/get_text8.sh index ccd4a08e69bb..e1390eb6fe90 100755 --- a/example/nce-loss/get_text8.sh +++ b/example/nce-loss/get_text8.sh @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + mkdir -p ./data/ cd ./data/ wget http://mattmahoney.net/dc/text8.zip diff --git a/example/nce-loss/lstm_word.py b/example/nce-loss/lstm_word.py index 3b39207b58a3..23729917d939 100644 --- a/example/nce-loss/lstm_word.py +++ b/example/nce-loss/lstm_word.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint:skip-file from __future__ import print_function import logging @@ -55,7 +72,7 @@ def get_net(vocab_size, seq_len, num_label, num_lstm_layer, num_hidden): state = LSTMState(c=mx.sym.Variable("l%d_init_c" % i), h=mx.sym.Variable("l%d_init_h" % i)) last_states.append(state) - + data = mx.sym.Variable('data') label = mx.sym.Variable('label') label_weight = mx.sym.Variable('label_weight') @@ -76,7 +93,7 @@ def get_net(vocab_size, seq_len, num_label, num_lstm_layer, num_hidden): probs = [] for seqidx in range(seq_len): hidden = datavec[seqidx] - + for i in range(num_lstm_layer): next_state = lstm(num_hidden, indata = hidden, prev_state = last_states[i], @@ -84,7 +101,7 @@ def get_net(vocab_size, seq_len, num_label, num_lstm_layer, num_hidden): seqidx = seqidx, layeridx = i) hidden = next_state.h last_states[i] = next_state - + probs.append(nce_loss(data = hidden, label = labelvec[seqidx], label_weight = labelweightvec[seqidx], @@ -149,7 +166,7 @@ def __init__(self, name, batch_size, seq_len, num_label, init_states): self.provide_data = [('data', (batch_size, seq_len))] + init_states self.provide_label = [('label', (self.batch_size, seq_len, num_label)), ('label_weight', (self.batch_size, seq_len, num_label))] - + def sample_ne(self): return self.negative[random.randint(0, len(self.negative) - 1)] @@ -203,7 +220,7 @@ def reset(self): data_train = DataIter("./data/text8", batch_size, seq_len, num_label, init_states) - + network = get_net(data_train.vocab_size, seq_len, num_label, num_lstm_layer, num_hidden) options, args = parser.parse_args() devs = mx.cpu() @@ -216,7 +233,7 @@ def reset(self): momentum = 0.9, wd = 0.0000, initializer=mx.init.Xavier(factor_type="in", magnitude=2.34)) - + metric = NceLSTMAuc() model.fit(X = data_train, eval_metric = metric, diff --git a/example/nce-loss/nce.py b/example/nce-loss/nce.py index abe4135ef367..7f57dfdb751d 100644 --- a/example/nce-loss/nce.py +++ b/example/nce-loss/nce.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint:skip-file import sys sys.path.insert(0, "../../python") diff --git a/example/nce-loss/toy_nce.py b/example/nce-loss/toy_nce.py index 9770be093fbe..39da7c779031 100644 --- a/example/nce-loss/toy_nce.py +++ b/example/nce-loss/toy_nce.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint:skip-file import logging import sys, random, time @@ -19,7 +36,7 @@ def get_net(vocab_size, num_label): embed_weight = embed_weight, vocab_size = vocab_size, num_hidden = 100, - num_label = num_label) + num_label = num_label) return ret class SimpleBatch(object): @@ -91,10 +108,10 @@ def reset(self): vocab_size = 10000 feature_size = 100 num_label = 6 - + data_train = DataIter(100000, batch_size, vocab_size, num_label, feature_size) data_test = DataIter(1000, batch_size, vocab_size, num_label, feature_size) - + network = get_net(vocab_size, num_label) devs = [mx.cpu()] model = mx.model.FeedForward(ctx = devs, @@ -104,7 +121,7 @@ def reset(self): momentum = 0.9, wd = 0.00001, initializer=mx.init.Xavier(factor_type="in", magnitude=2.34)) - + metric = NceAccuracy() model.fit(X = data_train, eval_data = data_test, eval_metric = metric, diff --git a/example/nce-loss/toy_softmax.py b/example/nce-loss/toy_softmax.py index 66f9cdc0e113..ff6ff4327c8e 100644 --- a/example/nce-loss/toy_softmax.py +++ b/example/nce-loss/toy_softmax.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint:skip-file import logging import sys, random, time @@ -83,7 +100,7 @@ def reset(self): data_train = DataIter(100000, batch_size, vocab_size, num_label, feature_size) data_test = DataIter(1000, batch_size, vocab_size, num_label, feature_size) - + network = get_net(vocab_size) devs = mx.cpu() model = mx.model.FeedForward(ctx = devs, @@ -93,7 +110,7 @@ def reset(self): momentum = 0.9, wd = 0.0000, initializer=mx.init.Xavier(factor_type="in", magnitude=2.34)) - + model.fit(X = data_train, eval_data = data_test, batch_end_callback = mx.callback.Speedometer(batch_size, 50),) diff --git a/example/nce-loss/wordvec.py b/example/nce-loss/wordvec.py index 24b78305210d..887d586ff342 100644 --- a/example/nce-loss/wordvec.py +++ b/example/nce-loss/wordvec.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint:skip-file from __future__ import print_function import logging @@ -30,7 +47,7 @@ def get_net(vocab_size, num_input, num_label): embed_weight = embed_weight, vocab_size = vocab_size, num_hidden = 100, - num_label = num_label) + num_label = num_label) def load_data(name): buf = open(name).read() @@ -82,7 +99,7 @@ def __init__(self, name, batch_size, num_label): self.provide_data = [('data', (batch_size, num_label - 1))] self.provide_label = [('label', (self.batch_size, num_label)), ('label_weight', (self.batch_size, num_label))] - + def sample_ne(self): return self.negative[random.randint(0, len(self.negative) - 1)] @@ -126,11 +143,11 @@ def reset(self): help = "use gpu") batch_size = 256 num_label = 5 - + data_train = DataIter("./data/text8", batch_size, num_label) - + network = get_net(data_train.vocab_size, num_label - 1, num_label) - + options, args = parser.parse_args() devs = mx.cpu() if options.gpu == True: @@ -143,7 +160,7 @@ def reset(self): wd = 0.0000, initializer=mx.init.Xavier(factor_type="in", magnitude=2.34)) - + metric = NceAuc() model.fit(X = data_train, eval_metric = metric, diff --git a/example/nce-loss/wordvec_subwords.py b/example/nce-loss/wordvec_subwords.py index 049dc9d6ffcb..c8d46a1aeb3a 100644 --- a/example/nce-loss/wordvec_subwords.py +++ b/example/nce-loss/wordvec_subwords.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint:skip-file import logging import sys, random, time, math diff --git a/example/neural-style/download.sh b/example/neural-style/download.sh index a58640aa8142..d5303a72c5a6 100755 --- a/example/neural-style/download.sh +++ b/example/neural-style/download.sh @@ -1,5 +1,23 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + mkdir -p model cd model wget https://github.com/dmlc/web-data/raw/master/mxnet/neural-style/model/vgg19.params diff --git a/example/neural-style/end_to_end/basic.py b/example/neural-style/end_to_end/basic.py index ed9d3f601554..1763e884b984 100644 --- a/example/neural-style/end_to_end/basic.py +++ b/example/neural-style/end_to_end/basic.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import sys sys.path.insert(0, "../../mxnet/python/") diff --git a/example/neural-style/end_to_end/boost_inference.py b/example/neural-style/end_to_end/boost_inference.py index 72427bedc7a6..0ec8308f3054 100644 --- a/example/neural-style/end_to_end/boost_inference.py +++ b/example/neural-style/end_to_end/boost_inference.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import sys sys.path.insert(0, "../mxnet/python") diff --git a/example/neural-style/end_to_end/boost_train.py b/example/neural-style/end_to_end/boost_train.py index 9100cc1875a2..fa525e7e52c0 100644 --- a/example/neural-style/end_to_end/boost_train.py +++ b/example/neural-style/end_to_end/boost_train.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import sys sys.path.insert(0, "../../mxnet/python") diff --git a/example/neural-style/end_to_end/data_processing.py b/example/neural-style/end_to_end/data_processing.py index 80f1bcd5cfcd..1c1ab493edec 100644 --- a/example/neural-style/end_to_end/data_processing.py +++ b/example/neural-style/end_to_end/data_processing.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import numpy as np from skimage import io, transform from skimage.restoration import denoise_tv_chambolle diff --git a/example/neural-style/end_to_end/gen_v3.py b/example/neural-style/end_to_end/gen_v3.py index dbc83b1ea004..7962e68da2fd 100644 --- a/example/neural-style/end_to_end/gen_v3.py +++ b/example/neural-style/end_to_end/gen_v3.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 diff --git a/example/neural-style/end_to_end/gen_v4.py b/example/neural-style/end_to_end/gen_v4.py index 379e904b9690..fb4e6d1e1647 100644 --- a/example/neural-style/end_to_end/gen_v4.py +++ b/example/neural-style/end_to_end/gen_v4.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 diff --git a/example/neural-style/end_to_end/model_vgg19.py b/example/neural-style/end_to_end/model_vgg19.py index 6e287b55b2fa..0d369ae08f58 100644 --- a/example/neural-style/end_to_end/model_vgg19.py +++ b/example/neural-style/end_to_end/model_vgg19.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx import os, sys from collections import namedtuple diff --git a/example/neural-style/find_mxnet.py b/example/neural-style/find_mxnet.py index 2e3970ddd85d..b919a2a78715 100644 --- a/example/neural-style/find_mxnet.py +++ b/example/neural-style/find_mxnet.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + try: import mxnet as mx except ImportError: diff --git a/example/neural-style/model_vgg19.py b/example/neural-style/model_vgg19.py index 3344a274e827..aa83bc362e5c 100644 --- a/example/neural-style/model_vgg19.py +++ b/example/neural-style/model_vgg19.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import find_mxnet import mxnet as mx import os, sys diff --git a/example/neural-style/nstyle.py b/example/neural-style/nstyle.py index 3eec33d4cbf1..e3bc8bcc5684 100644 --- a/example/neural-style/nstyle.py +++ b/example/neural-style/nstyle.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import find_mxnet import mxnet as mx import numpy as np @@ -196,7 +213,7 @@ def train_nstyle(args, callback=None): img = mx.nd.zeros(content_np.shape, ctx=dev) img[:] = mx.rnd.uniform(-0.1, 0.1, img.shape) - lr = mx.lr_scheduler.FactorScheduler(step=args.lr_sched_delay, + lr = mx.lr_scheduler.FactorScheduler(step=args.lr_sched_delay, factor=args.lr_sched_factor) optimizer = mx.optimizer.NAG( diff --git a/example/numpy-ops/custom_softmax.py b/example/numpy-ops/custom_softmax.py index cbd9a027d7a6..162215f3b0d1 100644 --- a/example/numpy-ops/custom_softmax.py +++ b/example/numpy-ops/custom_softmax.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: skip-file import os from data import mnist_iterator @@ -23,7 +40,7 @@ def backward(self, req, out_grad, in_data, out_data, in_grad, aux): class SoftmaxProp(mx.operator.CustomOpProp): def __init__(self): super(SoftmaxProp, self).__init__(need_top_grad=False) - + def list_arguments(self): return ['data', 'label'] diff --git a/example/numpy-ops/data.py b/example/numpy-ops/data.py index d39821f52145..0ca8e1fd6653 100644 --- a/example/numpy-ops/data.py +++ b/example/numpy-ops/data.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: skip-file """ data iterator for mnist """ import sys diff --git a/example/numpy-ops/ndarray_softmax.py b/example/numpy-ops/ndarray_softmax.py index 5c3176833b63..aa8555e5ad3e 100644 --- a/example/numpy-ops/ndarray_softmax.py +++ b/example/numpy-ops/ndarray_softmax.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: skip-file from data import mnist_iterator import mxnet as mx @@ -10,7 +27,7 @@ def __init__(self): super(NDArraySoftmax, self).__init__(False) self.fwd_kernel = None self.bwd_kernel = None - + def list_arguments(self): return ['data', 'label'] diff --git a/example/numpy-ops/numpy_softmax.py b/example/numpy-ops/numpy_softmax.py index 3f9f6c8be849..f90783b494b4 100644 --- a/example/numpy-ops/numpy_softmax.py +++ b/example/numpy-ops/numpy_softmax.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: skip-file from data import mnist_iterator import mxnet as mx @@ -8,7 +25,7 @@ class NumpySoftmax(mx.operator.NumpyOp): def __init__(self): super(NumpySoftmax, self).__init__(False) - + def list_arguments(self): return ['data', 'label'] diff --git a/example/numpy-ops/weighted_logistic_regression.py b/example/numpy-ops/weighted_logistic_regression.py index 7094b3aca969..26b5fb2fda84 100644 --- a/example/numpy-ops/weighted_logistic_regression.py +++ b/example/numpy-ops/weighted_logistic_regression.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import os import numpy as np import mxnet as mx diff --git a/example/profiler/profiler_executor.py b/example/profiler/profiler_executor.py index e70574d41cb5..26e3e1ba2acd 100644 --- a/example/profiler/profiler_executor.py +++ b/example/profiler/profiler_executor.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx import argparse import os, sys diff --git a/example/profiler/profiler_imageiter.py b/example/profiler/profiler_imageiter.py index af4c5d11aee8..e16b9b7de45f 100644 --- a/example/profiler/profiler_imageiter.py +++ b/example/profiler/profiler_imageiter.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import os # uncomment to set the number of worker threads. # os.environ["MXNET_CPU_WORKER_NTHREADS"] = "4" @@ -8,7 +25,7 @@ def run_imageiter(path_rec, n, batch_size = 32): - + data = mx.img.ImageIter(batch_size=batch_size, data_shape=(3, 224, 224), path_imgrec=path_rec, @@ -26,4 +43,4 @@ def run_imageiter(path_rec, n, batch_size = 32): mx.profiler.profiler_set_config(mode='all', filename='profile_imageiter.json') mx.profiler.profiler_set_state('run') run_imageiter('test.rec', 20) # See http://mxnet.io/tutorials/python/image_io.html for how to create .rec files. - mx.profiler.profiler_set_state('stop') \ No newline at end of file + mx.profiler.profiler_set_state('stop') diff --git a/example/profiler/profiler_matmul.py b/example/profiler/profiler_matmul.py index baa962307461..1b1cf74f4187 100644 --- a/example/profiler/profiler_matmul.py +++ b/example/profiler/profiler_matmul.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from __future__ import print_function import mxnet as mx import argparse diff --git a/example/profiler/profiler_ndarray.py b/example/profiler/profiler_ndarray.py index bb4d658275c0..67ea87b1ed62 100644 --- a/example/profiler/profiler_ndarray.py +++ b/example/profiler/profiler_ndarray.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import os import mxnet as mx import numpy as np diff --git a/example/python-howto/data.py b/example/python-howto/data.py index d39821f52145..0ca8e1fd6653 100644 --- a/example/python-howto/data.py +++ b/example/python-howto/data.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: skip-file """ data iterator for mnist """ import sys diff --git a/example/python-howto/data_iter.py b/example/python-howto/data_iter.py index 34e9a4181cd1..81c8988a8e51 100644 --- a/example/python-howto/data_iter.py +++ b/example/python-howto/data_iter.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Create a Cifar data iterator. This example shows how to create a iterator reading from recordio, diff --git a/example/python-howto/debug_conv.py b/example/python-howto/debug_conv.py index 3c38d20b89fa..9de421d8e88e 100644 --- a/example/python-howto/debug_conv.py +++ b/example/python-howto/debug_conv.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx data_shape = (1,3,5,5) @@ -19,4 +36,4 @@ def __init__(self, data): input_data = mx.nd.ones(data_shape) mod.forward(data_batch=SimpleData([input_data])) res = mod.get_outputs()[0].asnumpy() -print(res) \ No newline at end of file +print(res) diff --git a/example/python-howto/monitor_weights.py b/example/python-howto/monitor_weights.py index 8dcca1fba70b..c54e64954535 100644 --- a/example/python-howto/monitor_weights.py +++ b/example/python-howto/monitor_weights.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: skip-file from data import mnist_iterator import mxnet as mx @@ -27,6 +44,6 @@ def norm_stat(d): return mx.nd.norm(d)/np.sqrt(d.size) mon = mx.mon.Monitor(100, norm_stat) -model.fit(X=train, eval_data=val, monitor=mon, +model.fit(X=train, eval_data=val, monitor=mon, batch_end_callback = mx.callback.Speedometer(100, 100)) diff --git a/example/python-howto/multiple_outputs.py b/example/python-howto/multiple_outputs.py index 97ce469d58a2..43b4538d1d79 100644 --- a/example/python-howto/multiple_outputs.py +++ b/example/python-howto/multiple_outputs.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Create a Multiple output configuration. This example shows how to create a multiple output configuration. diff --git a/example/rcnn/demo.py b/example/rcnn/demo.py index 34ea327cffac..b59403379ddd 100644 --- a/example/rcnn/demo.py +++ b/example/rcnn/demo.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import argparse import os import cv2 diff --git a/example/rcnn/rcnn/config.py b/example/rcnn/rcnn/config.py index 445c2439b91e..17738f054b33 100644 --- a/example/rcnn/rcnn/config.py +++ b/example/rcnn/rcnn/config.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import numpy as np from easydict import EasyDict as edict diff --git a/example/rcnn/rcnn/core/callback.py b/example/rcnn/rcnn/core/callback.py index 5d48b9fc75d1..bacff9665e91 100644 --- a/example/rcnn/rcnn/core/callback.py +++ b/example/rcnn/rcnn/core/callback.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import time import logging import mxnet as mx diff --git a/example/rcnn/rcnn/core/loader.py b/example/rcnn/rcnn/core/loader.py index 3f5cf3c6c011..826ee20f080c 100644 --- a/example/rcnn/rcnn/core/loader.py +++ b/example/rcnn/rcnn/core/loader.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx import numpy as np from mxnet.executor_manager import _split_input_slice diff --git a/example/rcnn/rcnn/core/metric.py b/example/rcnn/rcnn/core/metric.py index 5808190e9d80..d33edb65beda 100644 --- a/example/rcnn/rcnn/core/metric.py +++ b/example/rcnn/rcnn/core/metric.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx import numpy as np diff --git a/example/rcnn/rcnn/core/module.py b/example/rcnn/rcnn/core/module.py index bf28f8ee56f1..337f0f35852b 100644 --- a/example/rcnn/rcnn/core/module.py +++ b/example/rcnn/rcnn/core/module.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """A `MutableModule` implement the `BaseModule` API, and allows input shape varying with training iterations. If shapes vary, executors will rebind, using shared arrays from the initial module binded with maximum shape. diff --git a/example/rcnn/rcnn/core/tester.py b/example/rcnn/rcnn/core/tester.py index 0ccc47df71eb..651b2a945e71 100644 --- a/example/rcnn/rcnn/core/tester.py +++ b/example/rcnn/rcnn/core/tester.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import cPickle import os import time diff --git a/example/rcnn/rcnn/cython/nms_kernel.cu b/example/rcnn/rcnn/cython/nms_kernel.cu index 038a59012f60..047a5e0c6d9f 100644 --- a/example/rcnn/rcnn/cython/nms_kernel.cu +++ b/example/rcnn/rcnn/cython/nms_kernel.cu @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + // ------------------------------------------------------------------ // Faster R-CNN // Copyright (c) 2015 Microsoft diff --git a/example/rcnn/rcnn/cython/setup.py b/example/rcnn/rcnn/cython/setup.py index 786460798fd2..e50478b2d967 100644 --- a/example/rcnn/rcnn/cython/setup.py +++ b/example/rcnn/rcnn/cython/setup.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # -------------------------------------------------------- # Fast R-CNN # Copyright (c) 2015 Microsoft diff --git a/example/rcnn/rcnn/dataset/__init__.py b/example/rcnn/rcnn/dataset/__init__.py index 266f344ede75..1a706e9e0c15 100644 --- a/example/rcnn/rcnn/dataset/__init__.py +++ b/example/rcnn/rcnn/dataset/__init__.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from imdb import IMDB from pascal_voc import PascalVOC from coco import coco diff --git a/example/rcnn/rcnn/dataset/coco.py b/example/rcnn/rcnn/dataset/coco.py index 00c4c41cf3ce..9ca5a74cc461 100644 --- a/example/rcnn/rcnn/dataset/coco.py +++ b/example/rcnn/rcnn/dataset/coco.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import cPickle import cv2 import os diff --git a/example/rcnn/rcnn/dataset/ds_utils.py b/example/rcnn/rcnn/dataset/ds_utils.py index 131644b726fc..e6f839b8fdb9 100644 --- a/example/rcnn/rcnn/dataset/ds_utils.py +++ b/example/rcnn/rcnn/dataset/ds_utils.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import numpy as np @@ -13,4 +30,4 @@ def filter_small_boxes(boxes, min_size): w = boxes[:, 2] - boxes[:, 0] h = boxes[:, 3] - boxes[:, 1] keep = np.where((w >= min_size) & (h > min_size))[0] - return keep \ No newline at end of file + return keep diff --git a/example/rcnn/rcnn/dataset/imdb.py b/example/rcnn/rcnn/dataset/imdb.py index acdcd50f8208..b9038c5da0a0 100644 --- a/example/rcnn/rcnn/dataset/imdb.py +++ b/example/rcnn/rcnn/dataset/imdb.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ General image database An image database creates a list of relative image path called image_set_index and diff --git a/example/rcnn/rcnn/dataset/pascal_voc.py b/example/rcnn/rcnn/dataset/pascal_voc.py index 2135971faadf..091c4e8ea17b 100644 --- a/example/rcnn/rcnn/dataset/pascal_voc.py +++ b/example/rcnn/rcnn/dataset/pascal_voc.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ Pascal VOC database This class loads ground truth notations from standard Pascal VOC XML data formats diff --git a/example/rcnn/rcnn/dataset/pascal_voc_eval.py b/example/rcnn/rcnn/dataset/pascal_voc_eval.py index 54fa12ddccd8..e584ed750304 100644 --- a/example/rcnn/rcnn/dataset/pascal_voc_eval.py +++ b/example/rcnn/rcnn/dataset/pascal_voc_eval.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ given a pascal voc imdb, compute mAP """ diff --git a/example/rcnn/rcnn/io/image.py b/example/rcnn/rcnn/io/image.py index d4155a069d3e..e468e4647b97 100644 --- a/example/rcnn/rcnn/io/image.py +++ b/example/rcnn/rcnn/io/image.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import numpy as np import cv2 import os diff --git a/example/rcnn/rcnn/io/rcnn.py b/example/rcnn/rcnn/io/rcnn.py index 807447c61baa..f9613d68bda5 100644 --- a/example/rcnn/rcnn/io/rcnn.py +++ b/example/rcnn/rcnn/io/rcnn.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ Fast R-CNN: data = diff --git a/example/rcnn/rcnn/io/rpn.py b/example/rcnn/rcnn/io/rpn.py index 52fe1a50c276..20cd1ce4e744 100644 --- a/example/rcnn/rcnn/io/rpn.py +++ b/example/rcnn/rcnn/io/rpn.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ RPN: data = diff --git a/example/rcnn/rcnn/logger.py b/example/rcnn/rcnn/logger.py index 2806e1add180..e82201797942 100644 --- a/example/rcnn/rcnn/logger.py +++ b/example/rcnn/rcnn/logger.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import logging # set up logger diff --git a/example/rcnn/rcnn/processing/bbox_regression.py b/example/rcnn/rcnn/processing/bbox_regression.py index d5b48a71b754..d5330f409875 100644 --- a/example/rcnn/rcnn/processing/bbox_regression.py +++ b/example/rcnn/rcnn/processing/bbox_regression.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ This file has functions about generating bounding box regression targets """ diff --git a/example/rcnn/rcnn/processing/bbox_transform.py b/example/rcnn/rcnn/processing/bbox_transform.py index 7a8667e14fe2..04fa81feda78 100644 --- a/example/rcnn/rcnn/processing/bbox_transform.py +++ b/example/rcnn/rcnn/processing/bbox_transform.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import numpy as np from ..cython.bbox import bbox_overlaps_cython diff --git a/example/rcnn/rcnn/processing/generate_anchor.py b/example/rcnn/rcnn/processing/generate_anchor.py index 8996a3aaab48..0e97d6ef2ba6 100644 --- a/example/rcnn/rcnn/processing/generate_anchor.py +++ b/example/rcnn/rcnn/processing/generate_anchor.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ Generate base anchors on index 0 """ diff --git a/example/rcnn/rcnn/processing/nms.py b/example/rcnn/rcnn/processing/nms.py index 230139c413ec..eca8d58626d3 100644 --- a/example/rcnn/rcnn/processing/nms.py +++ b/example/rcnn/rcnn/processing/nms.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import numpy as np from ..cython.cpu_nms import cpu_nms try: diff --git a/example/rcnn/rcnn/pycocotools/__init__.py b/example/rcnn/rcnn/pycocotools/__init__.py index 3f7d85bba884..2f4e0d430df9 100644 --- a/example/rcnn/rcnn/pycocotools/__init__.py +++ b/example/rcnn/rcnn/pycocotools/__init__.py @@ -1 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + __author__ = 'tylin' diff --git a/example/rcnn/rcnn/pycocotools/coco.py b/example/rcnn/rcnn/pycocotools/coco.py index ca35cc0b053b..5cc835a05633 100644 --- a/example/rcnn/rcnn/pycocotools/coco.py +++ b/example/rcnn/rcnn/pycocotools/coco.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + __author__ = 'tylin' __version__ = '2.0' # Interface for accessing the Microsoft COCO dataset. @@ -425,4 +442,4 @@ def annToMask(self, ann): """ rle = self.annToRLE(ann) m = maskUtils.decode(rle) - return m \ No newline at end of file + return m diff --git a/example/rcnn/rcnn/pycocotools/cocoeval.py b/example/rcnn/rcnn/pycocotools/cocoeval.py index a5dd1852912d..8b78026d39e4 100644 --- a/example/rcnn/rcnn/pycocotools/cocoeval.py +++ b/example/rcnn/rcnn/pycocotools/cocoeval.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + __author__ = 'tsungyi' import numpy as np @@ -525,4 +542,4 @@ def __init__(self, iouType='segm'): raise Exception('iouType not supported') self.iouType = iouType # useSegm is deprecated - self.useSegm = None \ No newline at end of file + self.useSegm = None diff --git a/example/rcnn/rcnn/pycocotools/mask.py b/example/rcnn/rcnn/pycocotools/mask.py index f49b8736b280..48c050c594b6 100644 --- a/example/rcnn/rcnn/pycocotools/mask.py +++ b/example/rcnn/rcnn/pycocotools/mask.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + __author__ = 'tsungyi' import _mask @@ -100,4 +117,4 @@ def toBbox(rleObjs): if type(rleObjs) == list: return _mask.toBbox(rleObjs) else: - return _mask.toBbox([rleObjs])[0] \ No newline at end of file + return _mask.toBbox([rleObjs])[0] diff --git a/example/rcnn/rcnn/pycocotools/maskApi.h b/example/rcnn/rcnn/pycocotools/maskApi.h index ebc7892da382..56b4c0c4c704 100644 --- a/example/rcnn/rcnn/pycocotools/maskApi.h +++ b/example/rcnn/rcnn/pycocotools/maskApi.h @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /************************************************************************** * Microsoft COCO Toolbox. version 2.0 * Data, paper, and tutorials available at: http://mscoco.org/ diff --git a/example/rcnn/rcnn/pycocotools/setup.py b/example/rcnn/rcnn/pycocotools/setup.py index 5e836f1b4b6b..d7074e910ee5 100644 --- a/example/rcnn/rcnn/pycocotools/setup.py +++ b/example/rcnn/rcnn/pycocotools/setup.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from distutils.core import setup from Cython.Build import cythonize from distutils.extension import Extension diff --git a/example/rcnn/rcnn/symbol/__init__.py b/example/rcnn/rcnn/symbol/__init__.py index f359ed821b8c..113b52c98abd 100644 --- a/example/rcnn/rcnn/symbol/__init__.py +++ b/example/rcnn/rcnn/symbol/__init__.py @@ -1,2 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from symbol_vgg import * from symbol_resnet import * diff --git a/example/rcnn/rcnn/symbol/proposal.py b/example/rcnn/rcnn/symbol/proposal.py index dd0bb15f5168..64981513980b 100644 --- a/example/rcnn/rcnn/symbol/proposal.py +++ b/example/rcnn/rcnn/symbol/proposal.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ Proposal Operator transform anchor coordinates into ROI coordinates with prediction results on classification probability and bounding box prediction results, and image size and scale information. diff --git a/example/rcnn/rcnn/symbol/proposal_target.py b/example/rcnn/rcnn/symbol/proposal_target.py index 6f1a6ffbc440..e0444f978b33 100644 --- a/example/rcnn/rcnn/symbol/proposal_target.py +++ b/example/rcnn/rcnn/symbol/proposal_target.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ Proposal Target Operator selects foreground and background roi and assigns label, bbox_transform to them. """ diff --git a/example/rcnn/rcnn/symbol/symbol_resnet.py b/example/rcnn/rcnn/symbol/symbol_resnet.py index ad60de9e854d..f914d117eb18 100644 --- a/example/rcnn/rcnn/symbol/symbol_resnet.py +++ b/example/rcnn/rcnn/symbol/symbol_resnet.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx import proposal import proposal_target diff --git a/example/rcnn/rcnn/symbol/symbol_vgg.py b/example/rcnn/rcnn/symbol/symbol_vgg.py index 34860a49e883..f04ba89dc1d4 100644 --- a/example/rcnn/rcnn/symbol/symbol_vgg.py +++ b/example/rcnn/rcnn/symbol/symbol_vgg.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx import proposal import proposal_target @@ -130,7 +147,7 @@ def get_vgg_rcnn_test(num_classes=config.NUM_CLASSES): # shared convolutional layer relu5_3 = get_vgg_conv(data) - + # Fast R-CNN pool5 = mx.symbol.ROIPooling( name='roi_pool5', data=relu5_3, rois=rois, pooled_size=(7, 7), spatial_scale=1.0 / config.RCNN_FEAT_STRIDE) diff --git a/example/rcnn/rcnn/tools/reeval.py b/example/rcnn/rcnn/tools/reeval.py index 22e5e206f4d0..a7ae898f41bd 100644 --- a/example/rcnn/rcnn/tools/reeval.py +++ b/example/rcnn/rcnn/tools/reeval.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import argparse import cPickle import os diff --git a/example/rcnn/rcnn/tools/test_rcnn.py b/example/rcnn/rcnn/tools/test_rcnn.py index 83a9fac03e67..2c5c22223f14 100644 --- a/example/rcnn/rcnn/tools/test_rcnn.py +++ b/example/rcnn/rcnn/tools/test_rcnn.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import argparse import pprint import mxnet as mx diff --git a/example/rcnn/rcnn/tools/test_rpn.py b/example/rcnn/rcnn/tools/test_rpn.py index 09f6af74368f..f2244a568d6a 100644 --- a/example/rcnn/rcnn/tools/test_rpn.py +++ b/example/rcnn/rcnn/tools/test_rpn.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import argparse import pprint import mxnet as mx diff --git a/example/rcnn/rcnn/tools/train_rcnn.py b/example/rcnn/rcnn/tools/train_rcnn.py index 3f1cde380e8c..c5417b34c2dc 100644 --- a/example/rcnn/rcnn/tools/train_rcnn.py +++ b/example/rcnn/rcnn/tools/train_rcnn.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import argparse import pprint import mxnet as mx diff --git a/example/rcnn/rcnn/tools/train_rpn.py b/example/rcnn/rcnn/tools/train_rpn.py index 87b92c8229ef..aaaf570a1fc3 100644 --- a/example/rcnn/rcnn/tools/train_rpn.py +++ b/example/rcnn/rcnn/tools/train_rpn.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import argparse import pprint import mxnet as mx diff --git a/example/rcnn/rcnn/utils/combine_model.py b/example/rcnn/rcnn/utils/combine_model.py index 5518dda4a989..eabe937be20c 100644 --- a/example/rcnn/rcnn/utils/combine_model.py +++ b/example/rcnn/rcnn/utils/combine_model.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from load_model import load_checkpoint from save_model import save_checkpoint diff --git a/example/rcnn/rcnn/utils/load_data.py b/example/rcnn/rcnn/utils/load_data.py index 4700229e65af..816b3b3a405e 100644 --- a/example/rcnn/rcnn/utils/load_data.py +++ b/example/rcnn/rcnn/utils/load_data.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import numpy as np from ..logger import logger from ..config import config diff --git a/example/rcnn/rcnn/utils/load_model.py b/example/rcnn/rcnn/utils/load_model.py index 6f8354869b66..0dc0752600c4 100644 --- a/example/rcnn/rcnn/utils/load_model.py +++ b/example/rcnn/rcnn/utils/load_model.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx diff --git a/example/rcnn/rcnn/utils/save_model.py b/example/rcnn/rcnn/utils/save_model.py index 1c9886973bd6..f27fb61b0f7a 100644 --- a/example/rcnn/rcnn/utils/save_model.py +++ b/example/rcnn/rcnn/utils/save_model.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx diff --git a/example/rcnn/script/additional_deps.sh b/example/rcnn/script/additional_deps.sh index 17e3d0bb0610..0e6599c77fd2 100755 --- a/example/rcnn/script/additional_deps.sh +++ b/example/rcnn/script/additional_deps.sh @@ -1,5 +1,23 @@ #!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + # install additional depts sudo apt install python-pip python-dev unzip python-matplotlib sudo pip install cython scikit-image easydict diff --git a/example/rcnn/script/get_coco.sh b/example/rcnn/script/get_coco.sh index d49046cc6c19..a2f8f90e8a6d 100755 --- a/example/rcnn/script/get_coco.sh +++ b/example/rcnn/script/get_coco.sh @@ -1,5 +1,23 @@ #!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + # make a data folder if ! [ -e data ] then diff --git a/example/rcnn/script/get_pretrained_model.sh b/example/rcnn/script/get_pretrained_model.sh index f63128d800da..746be0b7ddd8 100755 --- a/example/rcnn/script/get_pretrained_model.sh +++ b/example/rcnn/script/get_pretrained_model.sh @@ -1,5 +1,23 @@ #!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + # make a model folder if ! [ -e model ] then diff --git a/example/rcnn/script/get_selective_search.sh b/example/rcnn/script/get_selective_search.sh index 728bd8fffc60..487c653b23a7 100755 --- a/example/rcnn/script/get_selective_search.sh +++ b/example/rcnn/script/get_selective_search.sh @@ -1,5 +1,23 @@ #!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + # make a data folder if ! [ -e data ] then diff --git a/example/rcnn/script/get_voc.sh b/example/rcnn/script/get_voc.sh index c0cefa9a5f77..060b79336619 100755 --- a/example/rcnn/script/get_voc.sh +++ b/example/rcnn/script/get_voc.sh @@ -1,5 +1,23 @@ #!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + # make a data folder if ! [ -e data ] then diff --git a/example/rcnn/script/resnet_voc07.sh b/example/rcnn/script/resnet_voc07.sh index a5d16ec250ac..3cb421f016c0 100755 --- a/example/rcnn/script/resnet_voc07.sh +++ b/example/rcnn/script/resnet_voc07.sh @@ -1,5 +1,23 @@ #!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + # run this experiment with # nohup bash script/resnet_voc07.sh 0,1 &> resnet_voc07.log & # to use gpu 0,1 to train, gpu 0 to test and write logs to resnet_voc07.log diff --git a/example/rcnn/script/resnet_voc0712.sh b/example/rcnn/script/resnet_voc0712.sh index c993d49589b5..aa2bd39499a4 100755 --- a/example/rcnn/script/resnet_voc0712.sh +++ b/example/rcnn/script/resnet_voc0712.sh @@ -1,5 +1,23 @@ #!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + # run this experiment with # nohup bash script/resnet_voc00712.sh 0,1 &> resnet_voc0712.log & # to use gpu 0,1 to train, gpu 0 to test and write logs to resnet_voc0712.log diff --git a/example/rcnn/script/vgg_alter_voc07.sh b/example/rcnn/script/vgg_alter_voc07.sh index 1345f4fd2a43..72ee0cddea2f 100755 --- a/example/rcnn/script/vgg_alter_voc07.sh +++ b/example/rcnn/script/vgg_alter_voc07.sh @@ -1,5 +1,23 @@ #!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + # run this experiment with # nohup bash script/vgg_alter_voc07.sh 0,1 &> vgg_voc07.log & # to use gpu 0,1 to train, gpu 0 to test and write logs to vgg_voc07.log diff --git a/example/rcnn/script/vgg_fast_rcnn.sh b/example/rcnn/script/vgg_fast_rcnn.sh index 7d863b5f73ad..cafd2ea66b3e 100755 --- a/example/rcnn/script/vgg_fast_rcnn.sh +++ b/example/rcnn/script/vgg_fast_rcnn.sh @@ -1,5 +1,23 @@ #!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + # run this experiment with # nohup bash script/vgg_fast_rcnn.sh 0,1 &> vgg_fast_rcnn.log & # to use gpu 0,1 to train, gpu 0 to test and write logs to vgg_fast_rcnn.log diff --git a/example/rcnn/script/vgg_voc07.sh b/example/rcnn/script/vgg_voc07.sh index 4b70f7c9705f..22249e153838 100755 --- a/example/rcnn/script/vgg_voc07.sh +++ b/example/rcnn/script/vgg_voc07.sh @@ -1,5 +1,23 @@ #!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + # run this experiment with # nohup bash script/vgg_voc07.sh 0,1 &> vgg_voc07.log & # to use gpu 0,1 to train, gpu 0 to test and write logs to vgg_voc07.log diff --git a/example/rcnn/script/vgg_voc0712.sh b/example/rcnn/script/vgg_voc0712.sh index ff2490492b2e..22416dad4878 100755 --- a/example/rcnn/script/vgg_voc0712.sh +++ b/example/rcnn/script/vgg_voc0712.sh @@ -1,5 +1,23 @@ #!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + # run this experiment with # nohup bash script/vgg_voc00712.sh 0,1 &> vgg_voc0712.log & # to use gpu 0,1 to train, gpu 0 to test and write logs to vgg_voc0712.log diff --git a/example/rcnn/test.py b/example/rcnn/test.py index 12fe6973fbcf..2989bc02a4f7 100644 --- a/example/rcnn/test.py +++ b/example/rcnn/test.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import argparse import mxnet as mx from rcnn.logger import logger diff --git a/example/rcnn/train_alternate.py b/example/rcnn/train_alternate.py index 74f16b9980aa..715816087a61 100644 --- a/example/rcnn/train_alternate.py +++ b/example/rcnn/train_alternate.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import argparse import mxnet as mx diff --git a/example/rcnn/train_end2end.py b/example/rcnn/train_end2end.py index b8b1c5c3a410..5c942936aa4c 100644 --- a/example/rcnn/train_end2end.py +++ b/example/rcnn/train_end2end.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import argparse import pprint import mxnet as mx diff --git a/example/recommenders/crossentropy.py b/example/recommenders/crossentropy.py index 79fee7439438..51648b0eb157 100644 --- a/example/recommenders/crossentropy.py +++ b/example/recommenders/crossentropy.py @@ -1,4 +1,22 @@ #!/usr/bin/env python + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Cross-entropy loss layer for MXNet. """ import os @@ -12,13 +30,13 @@ class CrossEntropyLoss(mx.operator.CustomOp): """An output layer that calculates gradient for cross-entropy loss y * log(p) + (1-y) * log(p) for label "y" and prediction "p". - However, the output of this layer is the original prediction -- same as + However, the output of this layer is the original prediction -- same as the "data" input, making it useful for tasks like "predict". If you actually want to use the calculated loss, see CrossEntropyLoss op. This is useful for multi-label prediction where each possible output label is considered independently. - Cross-entropy loss provides a very large penalty for guessing + Cross-entropy loss provides a very large penalty for guessing the wrong answer (0 or 1) confidently. The gradient calculation is optimized for y only being 0 or 1. """ @@ -93,7 +111,7 @@ def infer_shape(self, in_shape): print("Simple test of cross-entropy") data = mx.symbol.Variable('data') labs = mx.symbol.Variable('labs') - net = mx.symbol.Custom(data=data, label=labs, name='ce', + net = mx.symbol.Custom(data=data, label=labs, name='ce', op_type='CrossEntropyLoss') rand = np.random.RandomState(seed=123) for i in range(20): diff --git a/example/recommenders/matrix_fact.py b/example/recommenders/matrix_fact.py index 90be41ed7f36..73f561a87959 100644 --- a/example/recommenders/matrix_fact.py +++ b/example/recommenders/matrix_fact.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import math import mxnet as mx import numpy as np diff --git a/example/recommenders/movielens_data.py b/example/recommenders/movielens_data.py index 157e8c209978..3d664fbb02aa 100644 --- a/example/recommenders/movielens_data.py +++ b/example/recommenders/movielens_data.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """MovieLens data handling: download, parse, and expose as DataIter """ @@ -22,13 +39,13 @@ def load_mldata_iter(filename, batch_size): user = mx.nd.array(user) item = mx.nd.array(item) score = mx.nd.array(score) - return mx.io.NDArrayIter(data={'user':user,'item':item},label={'score':score}, + return mx.io.NDArrayIter(data={'user':user,'item':item},label={'score':score}, batch_size=batch_size, shuffle=True) def ensure_local_data(prefix): if not os.path.exists("%s.zip" % prefix): print("Downloading MovieLens data: %s" % prefix) - os.system("wget http://files.grouplens.org/datasets/movielens/%s.zip" % prefix) + os.system("wget http://files.grouplens.org/datasets/movielens/%s.zip" % prefix) os.system("unzip %s.zip" % prefix) @@ -36,7 +53,7 @@ def get_data_iter(batch_size, prefix='ml-100k'): """Returns a pair of NDArrayDataIter, one for train, one for test. """ ensure_local_data(prefix) - return (load_mldata_iter('./%s/u1.base' % prefix, batch_size), + return (load_mldata_iter('./%s/u1.base' % prefix, batch_size), load_mldata_iter('./%s/u1.test' % prefix, batch_size)) def max_id(fname): diff --git a/example/recommenders/negativesample.py b/example/recommenders/negativesample.py index ecbd85e6f407..0b4ea8477b6f 100644 --- a/example/recommenders/negativesample.py +++ b/example/recommenders/negativesample.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """DataIter for negative sampling. """ import mxnet as mx @@ -8,7 +25,7 @@ class NegativeSamplingDataIter(mx.io.DataIter): Assumes that all the relevant inputs are in data, not labels. Drops (replaces) any labels in the original DataIter. - It only shuffles one of the input data columns, specified in the + It only shuffles one of the input data columns, specified in the constructor as shuffle_data_idx. So if the original input data has three columns, ('item_ids', 'item_words', 'users') and you want to keep the two "item_*" together, then set `shuffle_data_idx=2` @@ -46,7 +63,7 @@ def _clear_queue(self): self._sampled_queue = [] def _push_queue(self, data_list, labels): - """Takes a list of numpy arrays for data, + """Takes a list of numpy arrays for data, and a numpy array for labels. Converts to minibatches and puts it on the queue. """ diff --git a/example/recommenders/randomproj.py b/example/recommenders/randomproj.py index 539f50e0f647..ba080a07ec38 100644 --- a/example/recommenders/randomproj.py +++ b/example/recommenders/randomproj.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Random projection layers in MXNet as custom python ops. Currently slow and memory-inefficient, but functional. """ @@ -34,7 +51,7 @@ def random_unit_vecs(self, num_vecs, num_dims, rs): def _get_mask(self, idx, in_data): """Returns the mask by which to multiply the parts of the embedding layer. - In this version, we have no weights to apply. + In this version, we have no weights to apply. """ mask = idx >= 0 # bool False for -1 values that should be removed. shape=(b,mnz) mask = np.expand_dims(mask,2) # shape = (b,mnz,1) @@ -98,7 +115,7 @@ def _get_mask(self, idx, in_data): mask = np.expand_dims(mask,2) # shape = (b,mnz,1) mask = np.repeat(mask, self._proj_dim, axis=2) # shape = (b,mnz,d) return mask - + @mx.operator.register("SparseRandomProjection") class SparseRandomProjectionProp(RandomBagOfWordsProjectionProp): @@ -121,8 +138,8 @@ def infer_shape(self, in_shape): print("Simple test of proj layer") data = mx.symbol.Variable('data') vals = mx.symbol.Variable('vals') - net = mx.symbol.Custom(indexes=data, values=vals, name='rproj', - op_type='SparseRandomProjection', + net = mx.symbol.Custom(indexes=data, values=vals, name='rproj', + op_type='SparseRandomProjection', vocab_size=999, output_dim=29) d = mx.nd.zeros(shape=(3,100)) v = mx.nd.ones(shape=(3,100)) diff --git a/example/recommenders/recotools.py b/example/recommenders/recotools.py index f3681709db02..250baa5c07cf 100644 --- a/example/recommenders/recotools.py +++ b/example/recommenders/recotools.py @@ -1,7 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx from negativesample import NegativeSamplingDataIter -import randomproj +import randomproj import crossentropy def CosineLoss(a, b, label): diff --git a/example/recommenders/symbol_alexnet.py b/example/recommenders/symbol_alexnet.py index 20a3547db460..e5d02f0412cd 100644 --- a/example/recommenders/symbol_alexnet.py +++ b/example/recommenders/symbol_alexnet.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ Reference: diff --git a/example/reinforcement-learning/a3c/a3c.py b/example/reinforcement-learning/a3c/a3c.py index 19ab2305fb5e..4d89a24852c8 100644 --- a/example/reinforcement-learning/a3c/a3c.py +++ b/example/reinforcement-learning/a3c/a3c.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from __future__ import print_function import mxnet as mx import numpy as np @@ -76,7 +93,7 @@ def train(): if args.kv_store == 'dist_sync': epoch_size /= kv.num_workers - + # disable kvstore for single device if 'local' in kv.type and ( args.gpus is None or len(args.gpus.split(',')) is 1): @@ -164,7 +181,7 @@ def train(): print('h', h[0].asnumpy()) err += (adv**2).mean() score += r[i] - final_score *= (1-D[i]) + final_score *= (1-D[i]) final_score += score * D[i] score *= 1-D[i] T += D[i].sum() diff --git a/example/reinforcement-learning/a3c/launcher.py b/example/reinforcement-learning/a3c/launcher.py index 8a4a7d17c73e..e0bda21891f0 100644 --- a/example/reinforcement-learning/a3c/launcher.py +++ b/example/reinforcement-learning/a3c/launcher.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Submission job for local jobs.""" # pylint: disable=invalid-name from __future__ import absolute_import diff --git a/example/reinforcement-learning/a3c/rl_data.py b/example/reinforcement-learning/a3c/rl_data.py index 0d16bca793a4..ad78975753bd 100644 --- a/example/reinforcement-learning/a3c/rl_data.py +++ b/example/reinforcement-learning/a3c/rl_data.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from __future__ import print_function import mxnet as mx import numpy as np diff --git a/example/reinforcement-learning/a3c/sym.py b/example/reinforcement-learning/a3c/sym.py index d3e1767ea5b2..c48d752d2d0c 100644 --- a/example/reinforcement-learning/a3c/sym.py +++ b/example/reinforcement-learning/a3c/sym.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx def get_symbol_atari(act_dim): diff --git a/example/reinforcement-learning/ddpg/ddpg.py b/example/reinforcement-learning/ddpg/ddpg.py index 4ded9b952273..aa34e4d92804 100644 --- a/example/reinforcement-learning/ddpg/ddpg.py +++ b/example/reinforcement-learning/ddpg/ddpg.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from replay_mem import ReplayMem from utils import discount_return, sample_rewards import rllab.misc.logger as logger diff --git a/example/reinforcement-learning/ddpg/policies.py b/example/reinforcement-learning/ddpg/policies.py index 2a625c8872b3..2bae8f68cf0c 100644 --- a/example/reinforcement-learning/ddpg/policies.py +++ b/example/reinforcement-learning/ddpg/policies.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from utils import define_policy import mxnet as mx @@ -40,7 +57,7 @@ def __init__( self.obs = mx.symbol.Variable("obs") self.act = define_policy( - self.obs, + self.obs, self.env_spec.action_space.flat_dim) def get_output_symbol(self): @@ -59,7 +76,7 @@ def define_loss(self, loss_exp): raise NotImplementedError - def define_exe(self, ctx, init, updater, input_shapes=None, args=None, + def define_exe(self, ctx, init, updater, input_shapes=None, args=None, grad_req=None): # define an executor, initializer and updater for batch version @@ -71,7 +88,7 @@ def define_exe(self, ctx, init, updater, input_shapes=None, args=None, for name, arr in self.arg_dict.items(): if name not in input_shapes: init(name, arr) - + self.updater = updater # define an executor for sampled single observation @@ -110,4 +127,4 @@ def get_action(self, obs): - \ No newline at end of file + diff --git a/example/reinforcement-learning/ddpg/qfuncs.py b/example/reinforcement-learning/ddpg/qfuncs.py index 21957c47e351..7dbc1d601d30 100644 --- a/example/reinforcement-learning/ddpg/qfuncs.py +++ b/example/reinforcement-learning/ddpg/qfuncs.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from utils import define_qfunc import mxnet as mx @@ -47,7 +64,7 @@ def define_loss(self, loss_exp): self.loss = mx.symbol.MakeLoss(loss_exp, name="qfunc_loss") self.loss = mx.symbol.Group([self.loss, mx.symbol.BlockGrad(self.qval)]) - def define_exe(self, ctx, init, updater, input_shapes=None, args=None, + def define_exe(self, ctx, init, updater, input_shapes=None, args=None, grad_req=None): # define an executor, initializer and updater for batch version loss @@ -55,11 +72,11 @@ def define_exe(self, ctx, init, updater, input_shapes=None, args=None, self.arg_arrays = self.exe.arg_arrays self.grad_arrays = self.exe.grad_arrays self.arg_dict = self.exe.arg_dict - + for name, arr in self.arg_dict.items(): if name not in input_shapes: init(name, arr) - + self.updater = updater def update_params(self, obs, act, yval): diff --git a/example/reinforcement-learning/ddpg/replay_mem.py b/example/reinforcement-learning/ddpg/replay_mem.py index 885d7da301f0..47e9bc843ac0 100644 --- a/example/reinforcement-learning/ddpg/replay_mem.py +++ b/example/reinforcement-learning/ddpg/replay_mem.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from __future__ import print_function import numpy as np @@ -5,7 +22,7 @@ class ReplayMem(object): def __init__( - self, + self, obs_dim, act_dim, memory_size=1000000): diff --git a/example/reinforcement-learning/ddpg/run.py b/example/reinforcement-learning/ddpg/run.py index 0cd162ff5e9e..043cd997e647 100644 --- a/example/reinforcement-learning/ddpg/run.py +++ b/example/reinforcement-learning/ddpg/run.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from ddpg import DDPG from rllab.envs.box2d.cartpole_env import CartpoleEnv from rllab.envs.normalized_env import normalize @@ -32,4 +49,4 @@ policy_lr=1e-4, seed=SEED) -algo.train() \ No newline at end of file +algo.train() diff --git a/example/reinforcement-learning/ddpg/strategies.py b/example/reinforcement-learning/ddpg/strategies.py index c346e9e2a133..d73ad060cc87 100644 --- a/example/reinforcement-learning/ddpg/strategies.py +++ b/example/reinforcement-learning/ddpg/strategies.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import numpy as np @@ -28,7 +45,7 @@ def __init__(self, env_spec, mu=0, theta=0.15, sigma=0.3): self.sigma = sigma self.action_space = env_spec.action_space self.state = np.ones(self.action_space.flat_dim) * self.mu - + def evolve_state(self): x = self.state @@ -47,9 +64,9 @@ def get_action(self, obs, policy): obs = obs.reshape((1, -1)) action = policy.get_action(obs) increment = self.evolve_state() - - return np.clip(action + increment, - self.action_space.low, + + return np.clip(action + increment, + self.action_space.low, self.action_space.high) diff --git a/example/reinforcement-learning/ddpg/utils.py b/example/reinforcement-learning/ddpg/utils.py index 8c063db76506..a9a445632fd2 100644 --- a/example/reinforcement-learning/ddpg/utils.py +++ b/example/reinforcement-learning/ddpg/utils.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx import numpy as np diff --git a/example/reinforcement-learning/dqn/atari_game.py b/example/reinforcement-learning/dqn/atari_game.py index 369016fe134f..5c1314ffcf6c 100644 --- a/example/reinforcement-learning/dqn/atari_game.py +++ b/example/reinforcement-learning/dqn/atari_game.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + __author__ = 'sxjscience' import mxnet as mx diff --git a/example/reinforcement-learning/dqn/base.py b/example/reinforcement-learning/dqn/base.py index f433d5d7cbc0..ce82f2b1ad6c 100644 --- a/example/reinforcement-learning/dqn/base.py +++ b/example/reinforcement-learning/dqn/base.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from __future__ import absolute_import, division, print_function import mxnet as mx diff --git a/example/reinforcement-learning/dqn/dqn_demo.py b/example/reinforcement-learning/dqn/dqn_demo.py index 000a796b5821..750da7a69a7c 100644 --- a/example/reinforcement-learning/dqn/dqn_demo.py +++ b/example/reinforcement-learning/dqn/dqn_demo.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx import mxnet.ndarray as nd import numpy diff --git a/example/reinforcement-learning/dqn/dqn_run_test.py b/example/reinforcement-learning/dqn/dqn_run_test.py index 1a918eb92aab..2abf273978fa 100644 --- a/example/reinforcement-learning/dqn/dqn_run_test.py +++ b/example/reinforcement-learning/dqn/dqn_run_test.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx import mxnet.ndarray as nd import numpy diff --git a/example/reinforcement-learning/dqn/game.py b/example/reinforcement-learning/dqn/game.py index 0e1b7f1bd651..e17cc6f03f41 100644 --- a/example/reinforcement-learning/dqn/game.py +++ b/example/reinforcement-learning/dqn/game.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + DEFAULT_MAX_EPISODE_STEP = 1000000 @@ -29,4 +46,4 @@ def current_state(self): return self.replay_memory.latest_slice() def play(self, a): - raise NotImplementedError \ No newline at end of file + raise NotImplementedError diff --git a/example/reinforcement-learning/dqn/operators.py b/example/reinforcement-learning/dqn/operators.py index e8180c4d3f10..0c9b588f1685 100644 --- a/example/reinforcement-learning/dqn/operators.py +++ b/example/reinforcement-learning/dqn/operators.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx import mxnet.ndarray as nd import numpy diff --git a/example/reinforcement-learning/dqn/replay_memory.py b/example/reinforcement-learning/dqn/replay_memory.py index 42f4866c2a08..02691a01888a 100644 --- a/example/reinforcement-learning/dqn/replay_memory.py +++ b/example/reinforcement-learning/dqn/replay_memory.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from __future__ import absolute_import, division, print_function import mxnet as mx diff --git a/example/reinforcement-learning/dqn/utils.py b/example/reinforcement-learning/dqn/utils.py index 7d84bba74524..bae11e18021d 100644 --- a/example/reinforcement-learning/dqn/utils.py +++ b/example/reinforcement-learning/dqn/utils.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from __future__ import absolute_import, division, print_function import os diff --git a/example/reinforcement-learning/parallel_actor_critic/config.py b/example/reinforcement-learning/parallel_actor_critic/config.py index 48ef1d0c5a38..e962cf99be1f 100644 --- a/example/reinforcement-learning/parallel_actor_critic/config.py +++ b/example/reinforcement-learning/parallel_actor_critic/config.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx diff --git a/example/reinforcement-learning/parallel_actor_critic/envs.py b/example/reinforcement-learning/parallel_actor_critic/envs.py index 09f30d73cf2d..a537df6a7630 100644 --- a/example/reinforcement-learning/parallel_actor_critic/envs.py +++ b/example/reinforcement-learning/parallel_actor_critic/envs.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import numpy as np diff --git a/example/reinforcement-learning/parallel_actor_critic/model.py b/example/reinforcement-learning/parallel_actor_critic/model.py index 8fba78f7b950..b90af6790544 100644 --- a/example/reinforcement-learning/parallel_actor_critic/model.py +++ b/example/reinforcement-learning/parallel_actor_critic/model.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from itertools import chain import numpy as np import scipy.signal diff --git a/example/reinforcement-learning/parallel_actor_critic/train.py b/example/reinforcement-learning/parallel_actor_critic/train.py index d559ff346774..128a55030258 100644 --- a/example/reinforcement-learning/parallel_actor_critic/train.py +++ b/example/reinforcement-learning/parallel_actor_critic/train.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Trains an `Agent` using trajectories from multiple environments.""" import argparse diff --git a/example/rnn-time-major/bucket_io.py b/example/rnn-time-major/bucket_io.py index 5cf2c81967a8..950b0c05cfca 100644 --- a/example/rnn-time-major/bucket_io.py +++ b/example/rnn-time-major/bucket_io.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: disable=C0111,too-many-arguments,too-many-instance-attributes,too-many-locals,redefined-outer-name,fixme # pylint: disable=superfluous-parens, no-member, invalid-name from __future__ import print_function @@ -58,7 +75,7 @@ def default_gen_buckets(sentences, batch_size, the_vocab): tl = 0 buckets = [] - for l, n in len_dict.items(): # TODO: There are better heuristic ways to do this + for l, n in len_dict.items(): # TODO: There are better heuristic ways to do this if n + tl >= batch_size: buckets.append(l) tl = 0 @@ -217,7 +234,7 @@ def __iter__(self): i_idx = self.bucket_curr_idx[i_bucket] idx = self.bucket_idx_all[i_bucket][i_idx:i_idx+self.batch_size] self.bucket_curr_idx[i_bucket] += self.batch_size - + init_state_names = [x[0] for x in self.init_states] if self.time_major: diff --git a/example/rnn-time-major/get_ptb_data.sh b/example/rnn-time-major/get_ptb_data.sh index 1ec009aa2f99..d2641cb32b81 100755 --- a/example/rnn-time-major/get_ptb_data.sh +++ b/example/rnn-time-major/get_ptb_data.sh @@ -1,5 +1,23 @@ #!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + RNN_DIR=$(cd `dirname $0`; pwd) DATA_DIR="${RNN_DIR}/data/" diff --git a/example/rnn-time-major/rnn_cell_demo.py b/example/rnn-time-major/rnn_cell_demo.py index cb69c55770e0..c29d1ddea4e3 100644 --- a/example/rnn-time-major/rnn_cell_demo.py +++ b/example/rnn-time-major/rnn_cell_demo.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """A simple demo of new RNN cell with PTB language model.""" ################################################################################ @@ -100,7 +117,7 @@ def sym_gen(seq_len): # RNN cell takes input of shape (time, batch, feature) rnn = mx.sym.RNN(data=embed, state_size=num_hidden, num_layers=num_lstm_layer, mode='lstm', - name='LSTM', + name='LSTM', # The following params can be omitted # provided we do not need to apply the # workarounds mentioned above @@ -134,7 +151,7 @@ def sym_gen(seq_len): if len(buckets) == 1: mod = mx.mod.Module(*sym_gen(buckets[0]), context=contexts) else: - mod = mx.mod.BucketingModule(sym_gen, + mod = mx.mod.BucketingModule(sym_gen, default_bucket_key=data_train.default_bucket_key, context=contexts) diff --git a/example/rnn/cudnn_lstm_bucketing.py b/example/rnn/cudnn_lstm_bucketing.py index fbf32bbacb42..e9c3237f26fc 100644 --- a/example/rnn/cudnn_lstm_bucketing.py +++ b/example/rnn/cudnn_lstm_bucketing.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import numpy as np import mxnet as mx import argparse diff --git a/example/rnn/get_ptb_data.sh b/example/rnn/get_ptb_data.sh index 1ec009aa2f99..d2641cb32b81 100755 --- a/example/rnn/get_ptb_data.sh +++ b/example/rnn/get_ptb_data.sh @@ -1,5 +1,23 @@ #!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + RNN_DIR=$(cd `dirname $0`; pwd) DATA_DIR="${RNN_DIR}/data/" diff --git a/example/rnn/lstm_bucketing.py b/example/rnn/lstm_bucketing.py index 609276a11f19..2e7bc65d437a 100644 --- a/example/rnn/lstm_bucketing.py +++ b/example/rnn/lstm_bucketing.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import numpy as np import mxnet as mx import argparse diff --git a/example/rnn/old/bucket_io.py b/example/rnn/old/bucket_io.py index f515e348c4f4..21f96ef196fa 100644 --- a/example/rnn/old/bucket_io.py +++ b/example/rnn/old/bucket_io.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: disable=C0111,too-many-arguments,too-many-instance-attributes,too-many-locals,redefined-outer-name,fixme # pylint: disable=superfluous-parens, no-member, invalid-name from __future__ import print_function @@ -58,7 +75,7 @@ def default_gen_buckets(sentences, batch_size, the_vocab): tl = 0 buckets = [] - for l, n in len_dict.items(): # TODO: There are better heuristic ways to do this + for l, n in len_dict.items(): # TODO: There are better heuristic ways to do this if n + tl >= batch_size: buckets.append(l) tl = 0 @@ -210,7 +227,7 @@ def make_data_iter_plan(self): self.data_buffer.append(data) if self.model_parallel: - # Transpose data if model parallel + # Transpose data if model parallel for i in range(len(self.data)): bucket_data = self.data[i] self.data[i] = np.transpose(bucket_data) @@ -222,8 +239,8 @@ def __iter__(self): i_idx = self.bucket_curr_idx[i_bucket] idx = self.bucket_idx_all[i_bucket][i_idx:i_idx+self.batch_size] self.bucket_curr_idx[i_bucket] += self.batch_size - - # Model parallelism + + # Model parallelism if self.model_parallel: if self.data[i_bucket][:, idx].shape[1] == 0: print("WARNING: detected shape " + str(self.data[i_bucket][:, idx].shape)) @@ -231,7 +248,7 @@ def __iter__(self): data[:] = self.data[i_bucket][:, idx] data_batch = ModelParallelBatch(data, self.buckets[i_bucket]) yield data_batch - + # Data parallelism else: init_state_names = [x[0] for x in self.init_states] @@ -239,7 +256,7 @@ def __iter__(self): for sentence in data: assert len(sentence) == self.buckets[i_bucket] - + label = self.label_buffer[i_bucket] label[:, :-1] = data[:, 1:] label[:, -1] = 0 @@ -255,4 +272,4 @@ def __iter__(self): def reset(self): - self.bucket_curr_idx = [0 for x in self.data] \ No newline at end of file + self.bucket_curr_idx = [0 for x in self.data] diff --git a/example/rnn/old/get_ptb_data.sh b/example/rnn/old/get_ptb_data.sh index 1ec009aa2f99..d2641cb32b81 100755 --- a/example/rnn/old/get_ptb_data.sh +++ b/example/rnn/old/get_ptb_data.sh @@ -1,5 +1,23 @@ #!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + RNN_DIR=$(cd `dirname $0`; pwd) DATA_DIR="${RNN_DIR}/data/" diff --git a/example/rnn/old/gru.py b/example/rnn/old/gru.py index 5b5138bd0388..e6ec0952334b 100644 --- a/example/rnn/old/gru.py +++ b/example/rnn/old/gru.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: disable=C0111,too-many-arguments,too-many-instance-attributes,too-many-locals,redefined-outer-name,fixme # pylint: disable=superfluous-parens, no-member, invalid-name import sys diff --git a/example/rnn/old/gru_bucketing.py b/example/rnn/old/gru_bucketing.py index 859d449121a2..226018c02685 100644 --- a/example/rnn/old/gru_bucketing.py +++ b/example/rnn/old/gru_bucketing.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: disable=C0111,too-many-arguments,too-many-instance-attributes,too-many-locals,redefined-outer-name,fixme # pylint: disable=superfluous-parens, no-member, invalid-name import sys diff --git a/example/rnn/old/lstm.py b/example/rnn/old/lstm.py index d67b0dbe5790..84509a3daf3d 100644 --- a/example/rnn/old/lstm.py +++ b/example/rnn/old/lstm.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint:skip-file import sys sys.path.insert(0, "../../python") diff --git a/example/rnn/old/lstm_bucketing.py b/example/rnn/old/lstm_bucketing.py index 78fa4f89480e..3e3494776dc3 100644 --- a/example/rnn/old/lstm_bucketing.py +++ b/example/rnn/old/lstm_bucketing.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: disable=C0111,too-many-arguments,too-many-instance-attributes,too-many-locals,redefined-outer-name,fixme # pylint: disable=superfluous-parens, no-member, invalid-name import sys diff --git a/example/rnn/old/rnn.py b/example/rnn/old/rnn.py index 136c40380b53..fe7bdbd922fa 100644 --- a/example/rnn/old/rnn.py +++ b/example/rnn/old/rnn.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import sys sys.path.insert(0, "../../python/") import mxnet as mx diff --git a/example/rnn/old/rnn_cell_demo.py b/example/rnn/old/rnn_cell_demo.py index 2c798e2c9c13..3223e936c37f 100644 --- a/example/rnn/old/rnn_cell_demo.py +++ b/example/rnn/old/rnn_cell_demo.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """A simple demo of new RNN cell with PTB language model.""" import os @@ -87,12 +104,12 @@ def sym_gen(seq_len): # RNN cell takes input of shape (time, batch, feature) rnn = mx.sym.RNN(data=embed_tm, state_size=num_hidden, num_layers=num_lstm_layer, mode='lstm', - name='LSTM', + name='LSTM', # The following params can be omitted # provided we do not need to apply the # workarounds mentioned above state=rnn_h_init, - state_cell=rnn_c_init, + state_cell=rnn_c_init, parameters=rnn_params) # the RNN cell output is of shape (time, batch, dim) diff --git a/example/rnn/old/rnn_model.py b/example/rnn/old/rnn_model.py index 2135abd357c9..6fe0d22ef3c9 100644 --- a/example/rnn/old/rnn_model.py +++ b/example/rnn/old/rnn_model.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: disable=C0111,too-many-arguments,too-many-instance-attributes,too-many-locals,redefined-outer-name,fixme # pylint: disable=superfluous-parens, no-member, invalid-name diff --git a/example/speech-demo/config_util.py b/example/speech-demo/config_util.py index 9e2ecc45abd7..6fd6a50a19fb 100644 --- a/example/speech-demo/config_util.py +++ b/example/speech-demo/config_util.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import re import os import sys diff --git a/example/speech-demo/decode_mxnet.py b/example/speech-demo/decode_mxnet.py index 4680fbe904df..deb9c30d79c7 100644 --- a/example/speech-demo/decode_mxnet.py +++ b/example/speech-demo/decode_mxnet.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import re import sys sys.path.insert(0, "../../python") @@ -80,7 +97,7 @@ def prepare_data(args): num_epoch = args.config.getint('train', 'num_epoch') model_name = get_checkpoint_path(args) logging.basicConfig(level=logging.DEBUG, format='%(asctime)-15s %(message)s') - + # load the model sym, arg_params, aux_params = mx.model.load_checkpoint(model_name, num_epoch) @@ -89,7 +106,7 @@ def prepare_data(args): buckets = list(map(int, re.split(r'\W+', buckets))) data_test = BucketSentenceIter(test_sets, buckets, batch_size, init_states, feat_dim=feat_dim, has_label=False) def sym_gen(seq_len): - sym = lstm_unroll(num_lstm_layer, seq_len, feat_dim, num_hidden=num_hidden, + sym = lstm_unroll(num_lstm_layer, seq_len, feat_dim, num_hidden=num_hidden, num_label=label_dim, take_softmax=True, num_hidden_proj=num_hidden_proj) data_names = ['data'] + state_names label_names = ['softmax_label'] @@ -102,7 +119,7 @@ def sym_gen(seq_len): data_test = SimpleIter(test_sets, batch_size, init_states, feat_dim=feat_dim, label_dim=label_dim, label_mean_sets=label_mean_sets, has_label=False) def sym_gen(seq_len): - sym = lstm_unroll(num_lstm_layer, seq_len, feat_dim, num_hidden=num_hidden, + sym = lstm_unroll(num_lstm_layer, seq_len, feat_dim, num_hidden=num_hidden, num_label=label_dim, take_softmax=False, num_hidden_proj=num_hidden_proj) data_names = ['data'] + state_names label_names = [] @@ -127,7 +144,7 @@ def sym_gen(seq_len): # set the parameters module.bind(data_shapes=data_test.provide_data, label_shapes=None, for_training=False) module.set_params(arg_params=arg_params, aux_params=aux_params) - + kaldiWriter = KaldiWriteOut(None, out_file) kaldiWriter.open_or_fd() for preds, i_batch, batch in module.iter_predict(data_test): diff --git a/example/speech-demo/decode_mxnet.sh b/example/speech-demo/decode_mxnet.sh index e5209b8c76d5..d300d0e91c40 100755 --- a/example/speech-demo/decode_mxnet.sh +++ b/example/speech-demo/decode_mxnet.sh @@ -1,14 +1,32 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + # Copyright 2012-2013 Karel Vesely, Daniel Povey # 2015 Yu Zhang # Apache 2.0 -# Begin configuration section. +# Begin configuration section. nnet= # Optionally pre-select network to use for getting state-likelihoods feature_transform= # Optionally pre-select feature transform (in front of nnet) model= # Optionally pre-select transition model -class_frame_counts= # Optionally pre-select class-counts used to compute PDF priors +class_frame_counts= # Optionally pre-select class-counts used to compute PDF priors stage=0 # stage=1 skips lattice generation nj=4 @@ -44,7 +62,7 @@ mkdir -p $dir/log echo $nj > $dir/num_jobs if [ -z "$model" ]; then # if --model was not specified on the command line... - if [ -z $iter ]; then model=$srcdir/final.mdl; + if [ -z $iter ]; then model=$srcdir/final.mdl; else model=$srcdir/$iter.mdl; fi fi diff --git a/example/speech-demo/io_func/convert2kaldi.py b/example/speech-demo/io_func/convert2kaldi.py index ffa8c4d6b3a9..eac8ee695a9b 100644 --- a/example/speech-demo/io_func/convert2kaldi.py +++ b/example/speech-demo/io_func/convert2kaldi.py @@ -1,4 +1,21 @@ -# Copyright 2013 Yajie Miao Carnegie Mellon University +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Copyright 2013 Yajie Miao Carnegie Mellon University # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -28,7 +45,7 @@ def _nnet2kaldi(nnet_spec, set_layer_num = -1, filein='nnet.in', _nnet2kaldi_main(nnet_spec, set_layer_num=set_layer_num, filein=filein, fileout=fileout, activation=activation, withfinal=withfinal, maxout=False) -def _nnet2kaldi_maxout(nnet_spec, pool_size = 1, set_layer_num = -1, +def _nnet2kaldi_maxout(nnet_spec, pool_size = 1, set_layer_num = -1, filein='nnet.in', fileout='nnet.out', activation='sigmoid', withfinal=True): _nnet2kaldi_main(nnet_spec, set_layer_num=set_layer_num, filein=filein, fileout=fileout, activation=activation, withfinal=withfinal, @@ -110,4 +127,4 @@ def _nnet2kaldi_main(nnet_spec, set_layer_num = -1, filein='nnet.in', fout.write('[ ' + b_layer.strip() + ' ]' + '\n') fout.write(' ' + str(output_size) + ' ' + str(output_size) + '\n') - fout.close(); \ No newline at end of file + fout.close(); diff --git a/example/speech-demo/io_func/feat_io.py b/example/speech-demo/io_func/feat_io.py index 83d417eb0ffb..6a7e424d1e65 100644 --- a/example/speech-demo/io_func/feat_io.py +++ b/example/speech-demo/io_func/feat_io.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from __future__ import print_function import os import sys diff --git a/example/speech-demo/io_func/feat_readers/common.py b/example/speech-demo/io_func/feat_readers/common.py index a7b6413082ce..742d3e25a1c7 100644 --- a/example/speech-demo/io_func/feat_readers/common.py +++ b/example/speech-demo/io_func/feat_readers/common.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import numpy import os @@ -55,4 +72,4 @@ def getReader(fileformat, featureFile, labelFile): return reader_kaldi.kaldiReader(featureFile, labelFile) else: msg = "Error: Specified format '{}' is not supported".format(fileformat) - raise Exception(msg) \ No newline at end of file + raise Exception(msg) diff --git a/example/speech-demo/io_func/feat_readers/reader_atrack.py b/example/speech-demo/io_func/feat_readers/reader_atrack.py index 0bf1deeac95e..e8db0fd14da2 100644 --- a/example/speech-demo/io_func/feat_readers/reader_atrack.py +++ b/example/speech-demo/io_func/feat_readers/reader_atrack.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import numpy import numpy as num import stats @@ -30,11 +47,11 @@ def Read(self): -1.677172 -1076449904 -1867655489 -1.562828 -1077409088 -1073035073 """ - + f = open(self.featureFile, "rb") header = num.fromfile(f, dtype=num.dtype('>i4'), count=7) self.checkHeader(header) - + frameSize = header[1] numSamples = header[2] @@ -46,4 +63,4 @@ def Read(self): self._markDone() - return a, ReadLabel(self.labelFile) \ No newline at end of file + return a, ReadLabel(self.labelFile) diff --git a/example/speech-demo/io_func/feat_readers/reader_bvec.py b/example/speech-demo/io_func/feat_readers/reader_bvec.py index ac68bf477c05..3a0f745b92ea 100644 --- a/example/speech-demo/io_func/feat_readers/reader_bvec.py +++ b/example/speech-demo/io_func/feat_readers/reader_bvec.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from __future__ import print_function import struct import array @@ -22,7 +39,7 @@ def Read(self): print('Num samples = {}'.format(numSamples)) print('dim = {}'.format(dim)) - dt = numpy.dtype([('sample',(numpy.float32,dim))]) + dt = numpy.dtype([('sample',(numpy.float32,dim))]) samples = numpy.fromfile(f,dt.newbyteorder('>'),count=numSamples) self._markDone() diff --git a/example/speech-demo/io_func/feat_readers/reader_htk.py b/example/speech-demo/io_func/feat_readers/reader_htk.py index b04d6f3e52ee..dca24d9bd35c 100644 --- a/example/speech-demo/io_func/feat_readers/reader_htk.py +++ b/example/speech-demo/io_func/feat_readers/reader_htk.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import numpy import stats from common import * @@ -24,7 +41,7 @@ def Read(self): # print 'Sample period = {}'.format(sampPeriod) # print 'Sample size = {}'.format(sampSize) # print 'Sample kind = {}'.format(sampKind) - dt = numpy.dtype([('sample',(numpy.float32,sampSize/4))]) + dt = numpy.dtype([('sample',(numpy.float32,sampSize/4))]) samples = numpy.fromfile(f,dt.newbyteorder('>' if self.byteOrder==ByteOrder.BigEndian else '<'),count=numSamples) self._markDone() @@ -33,5 +50,5 @@ def Read(self): labels = None else: labels = ReadLabel(self.labelFile) - + return samples[:]['sample'], labels diff --git a/example/speech-demo/io_func/feat_readers/reader_kaldi.py b/example/speech-demo/io_func/feat_readers/reader_kaldi.py index bad6a9d3e989..345934a91790 100644 --- a/example/speech-demo/io_func/feat_readers/reader_kaldi.py +++ b/example/speech-demo/io_func/feat_readers/reader_kaldi.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from common import * import random @@ -95,7 +112,7 @@ def Read(self): feat_rows = kaldi.MatrixF_NumRows(feat_value) feat_cols = kaldi.MatrixF_NumCols(feat_value) feat_data = kaldi.MatrixF_Data(feat_value) - + # never use numpy.ndarray(buf=) or numpy.ctypeslib.as_array # because you don't know if Python or C owns buffer # (even if you numpy.copy() resulting array) @@ -114,7 +131,7 @@ def Read(self): if self.targets_rspecifier is not None: if kaldi.RAPReader_HasKey(self.targets_reader, utt): tgt_value = kaldi.RAPReader_Value(self.targets_reader, utt) - + tgts = numpy.empty((feat_rows,), dtype=numpy.int32) # ok to use memmove because this is 1-dimensional array I made in C (no stride) tgts_numpy_ptr = ctypes.cast(tgts.ctypes.data, c_int_ptr) @@ -125,7 +142,7 @@ def Read(self): tgts = None else: tgts = None - + kaldi.SBFMReader_Next(self.feature_reader) #print "FEATS:", feats[0:5][0:5] diff --git a/example/speech-demo/io_func/feat_readers/stats.py b/example/speech-demo/io_func/feat_readers/stats.py index 70033ebae456..a2c847359dc9 100644 --- a/example/speech-demo/io_func/feat_readers/stats.py +++ b/example/speech-demo/io_func/feat_readers/stats.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from __future__ import print_function import numpy diff --git a/example/speech-demo/io_func/feat_readers/writer_kaldi.py b/example/speech-demo/io_func/feat_readers/writer_kaldi.py index f331160a4f3d..0f8fb938087f 100644 --- a/example/speech-demo/io_func/feat_readers/writer_kaldi.py +++ b/example/speech-demo/io_func/feat_readers/writer_kaldi.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import sys import numpy import struct diff --git a/example/speech-demo/io_func/info.py b/example/speech-demo/io_func/info.py index 64bb77d49736..eaf95ab983bb 100644 --- a/example/speech-demo/io_func/info.py +++ b/example/speech-demo/io_func/info.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import os _mydir = os.path.dirname(__file__) or '.' diff --git a/example/speech-demo/io_func/kaldi_parser.py b/example/speech-demo/io_func/kaldi_parser.py index 8b1d67893b79..10a373d7138f 100644 --- a/example/speech-demo/io_func/kaldi_parser.py +++ b/example/speech-demo/io_func/kaldi_parser.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from __future__ import print_function import struct import numpy as num @@ -199,4 +216,4 @@ def file2nnet_binary(filename): fout.write(' ' + str(int(layers[i + 1])) + ' ' + str(output_size) + '\n') else: fout.write(' ' + str(output_size) + ' ' + str(output_size) + '\n') -""" \ No newline at end of file +""" diff --git a/example/speech-demo/io_func/model_io.py b/example/speech-demo/io_func/model_io.py index 18496634e462..8b6e0436c22b 100755 --- a/example/speech-demo/io_func/model_io.py +++ b/example/speech-demo/io_func/model_io.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from __future__ import print_function import numpy as np import os @@ -59,7 +76,7 @@ def _nnet2file(layers, set_layer_num = -1, filename='nnet.out', activation='sigm nnet_dict[dict_a] = array_2_string(layers[i].delta_params[0].get_value()) dict_a = str(i) + ' ' + activation + ' db' nnet_dict[dict_a] = array_2_string(layers[i].delta_params[1].get_value()) - + if layers[i].kahan: logger.info("Loading hidden kahan") dict_a = str(i) + ' ' + activation + ' W_carry' @@ -71,9 +88,9 @@ def _nnet2file(layers, set_layer_num = -1, filename='nnet.out', activation='sigm #dict_a = str(i) + ' ' + activation + ' db_carry' #nnet_dict[dict_a] = array_2_string(layers[i].delta_params_carry[1].get_value()) - if withfinal: + if withfinal: logger.info("Saving final layer ") - + dict_a = 'logreg W' nnet_dict[dict_a] = array_2_string((1.0 - factor[-1]) * layers[-1].params[0].get_value()) dict_a = 'logreg b' @@ -96,7 +113,7 @@ def _nnet2file(layers, set_layer_num = -1, filename='nnet.out', activation='sigm #dict_a = 'logreg db_carry' #nnet_dict[dict_a] = array_2_string(layers[-1].delta_params_carry[1].get_value()) - utils.pickle_save(nnet_dict, filename) + utils.pickle_save(nnet_dict, filename) def zero(x): x.set_value(np.zeros_like(x.get_value(borrow=True), dtype=theano.config.floatX)) @@ -147,14 +164,14 @@ def _file2nnet(layers, set_layer_num = -1, filename='nnet.in', activation='sigmo dict_key = str(i) + ' ' + activation + ' W' layers[i].params[0].set_value(factors[i] * factor * np.asarray(string_2_array(nnet_dict[dict_key]), dtype=theano.config.floatX)) - dict_key = str(i) + ' ' + activation + ' b' + dict_key = str(i) + ' ' + activation + ' b' layers[i].params[1].set_value(np.asarray(string_2_array(nnet_dict[dict_key]), dtype=theano.config.floatX)) if gradients: dict_key = str(i) + ' ' + activation + ' dW' layers[i].delta_params[0].set_value(np.asarray(string_2_array(nnet_dict[dict_key]), dtype=theano.config.floatX)) - dict_key = str(i) + ' ' + activation + ' db' - layers[i].delta_params[1].set_value(np.asarray(string_2_array(nnet_dict[dict_key]), dtype=theano.config.floatX)) + dict_key = str(i) + ' ' + activation + ' db' + layers[i].delta_params[1].set_value(np.asarray(string_2_array(nnet_dict[dict_key]), dtype=theano.config.floatX)) else: zero(layers[i].delta_params[0]) zero(layers[i].delta_params[1]) @@ -164,12 +181,12 @@ def _file2nnet(layers, set_layer_num = -1, filename='nnet.in', activation='sigmo logger.info("Loading hidden kahan") dict_key = str(i) + ' ' + activation + ' W_carry' layers[i].params_carry[0].set_value(np.asarray(string_2_array(nnet_dict[dict_key]), dtype=theano.config.floatX)) - dict_key = str(i) + ' ' + activation + ' b_carry' - layers[i].params_carry[1].set_value(np.asarray(string_2_array(nnet_dict[dict_key]), dtype=theano.config.floatX)) + dict_key = str(i) + ' ' + activation + ' b_carry' + layers[i].params_carry[1].set_value(np.asarray(string_2_array(nnet_dict[dict_key]), dtype=theano.config.floatX)) #dict_key = str(i) + ' ' + activation + ' dW_carry' #layers[i].delta_params_carry[0].set_value(np.asarray(string_2_array(nnet_dict[dict_key]), dtype=theano.config.floatX)) - #dict_key = str(i) + ' ' + activation + ' db_carry' - #layers[i].delta_params_carry[1].set_value(np.asarray(string_2_array(nnet_dict[dict_key]), dtype=theano.config.floatX)) + #dict_key = str(i) + ' ' + activation + ' db_carry' + #layers[i].delta_params_carry[1].set_value(np.asarray(string_2_array(nnet_dict[dict_key]), dtype=theano.config.floatX)) if layers[i].sync: layers[i].params_sync[0].set_value(layers[i].params[0].get_value().astype('float32')) @@ -197,12 +214,12 @@ def _file2nnet(layers, set_layer_num = -1, filename='nnet.in', activation='sigmo logger.info("Loading softmax kahan") dict_key = 'logreg W_carry' layers[-1].params_carry[0].set_value(np.asarray(string_2_array(nnet_dict[dict_key]), dtype=theano.config.floatX)) - dict_key = 'logreg b_carry' - layers[-1].params_carry[1].set_value(np.asarray(string_2_array(nnet_dict[dict_key]), dtype=theano.config.floatX)) + dict_key = 'logreg b_carry' + layers[-1].params_carry[1].set_value(np.asarray(string_2_array(nnet_dict[dict_key]), dtype=theano.config.floatX)) #dict_key = 'logreg dW_carry' #layers[-1].delta_params_carry[0].set_value(np.asarray(string_2_array(nnet_dict[dict_key]), dtype=theano.config.floatX)) - #dict_key = 'logreg db_carry' - #layers[-1].delta_params_carry[1].set_value(np.asarray(string_2_array(nnet_dict[dict_key]), dtype=theano.config.floatX)) + #dict_key = 'logreg db_carry' + #layers[-1].delta_params_carry[1].set_value(np.asarray(string_2_array(nnet_dict[dict_key]), dtype=theano.config.floatX)) if layers[-1].sync: layers[-1].params_sync[0].set_value(layers[-1].params[0].get_value().astype('float32')) @@ -220,10 +237,10 @@ def _cnn2file(conv_layers, filename='nnet.out', activation='sigmoid', withfinal= for i in xrange(n_layers): conv_layer = conv_layers[i] filter_shape = conv_layer.filter_shape - + for next_X in xrange(filter_shape[0]): for this_X in xrange(filter_shape[1]): - dict_a = 'W ' + str(i) + ' ' + str(next_X) + ' ' + str(this_X) + dict_a = 'W ' + str(i) + ' ' + str(next_X) + ' ' + str(this_X) if i == 0: nnet_dict[dict_a] = array_2_string(input_factor * (conv_layer.W.get_value())[next_X, this_X]) else: @@ -231,7 +248,7 @@ def _cnn2file(conv_layers, filename='nnet.out', activation='sigmoid', withfinal= dict_a = 'b ' + str(i) nnet_dict[dict_a] = array_2_string(conv_layer.b.get_value()) - + with open(filename, 'wb') as fp: json.dump(nnet_dict, fp, indent=2, sort_keys = True) fp.flush() @@ -252,7 +269,7 @@ def _file2cnn(conv_layers, filename='nnet.in', activation='sigmoid', withfinal=T dict_a = 'W ' + str(i) + ' ' + str(next_X) + ' ' + str(this_X) W_array[next_X, this_X, :, :] = factor * np.asarray(string_2_array(nnet_dict[dict_a])) - conv_layer.W.set_value(W_array) + conv_layer.W.set_value(W_array) dict_a = 'b ' + str(i) - conv_layer.b.set_value(np.asarray(string_2_array(nnet_dict[dict_a]), dtype=theano.config.floatX)) + conv_layer.b.set_value(np.asarray(string_2_array(nnet_dict[dict_a]), dtype=theano.config.floatX)) diff --git a/example/speech-demo/io_func/regr_feat_io.py b/example/speech-demo/io_func/regr_feat_io.py index 2f3c4ec9ffd0..a1737bf9ab32 100644 --- a/example/speech-demo/io_func/regr_feat_io.py +++ b/example/speech-demo/io_func/regr_feat_io.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import os import sys import random @@ -36,7 +53,7 @@ def read_by_matrix(self): def make_shared(self): self.input.make_shared() self.output.make_shared() - + def get_shared(self): iret = self.input.get_shared() oret = self.output.get_shared() @@ -56,13 +73,13 @@ def current_utt_id(self): def load_next_block(self): a = self.input.load_next_block() - b = self.output.load_next_block() + b = self.output.load_next_block() assert(a == b) return a def get_state(self): a = self.input.get_state() - b = self.output.get_state() + b = self.output.get_state() assert(a[0] == b[0]) assert(a[2] == b[2]) assert(a[3] == b[3]) @@ -72,4 +89,4 @@ def get_state(self): def set_state(self, state): self.input.set_state(state) - self.output.set_state(state) + self.output.set_state(state) diff --git a/example/speech-demo/io_func/utils.py b/example/speech-demo/io_func/utils.py index 513261ea6f4f..4ba8496c7fb7 100644 --- a/example/speech-demo/io_func/utils.py +++ b/example/speech-demo/io_func/utils.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import sys, subprocess, pickle, os, json, logging, socket import logging.config import datetime diff --git a/example/speech-demo/io_util.py b/example/speech-demo/io_util.py index 926f20fbb58d..e5bd74cb6fa7 100644 --- a/example/speech-demo/io_util.py +++ b/example/speech-demo/io_util.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx import numpy as np import sys @@ -310,7 +327,7 @@ def __init__(self, train_sets, batch_size, init_states, truncate_len=20, delay=5 self.data = [mx.nd.zeros((batch_size, truncate_len, feat_dim))] if has_label: self.label = [mx.nd.zeros((batch_size, truncate_len))] - + self.init_state_names = [x[0] for x in init_states] self.init_state_arrays = [mx.nd.zeros(x[1]) for x in init_states] @@ -368,14 +385,14 @@ def __iter__(self): next_utt_idx = self.batch_size is_pad = [False] * self.batch_size pad = 0 - + if self.time_major: np_data_buffer = np.zeros((self.truncate_len, self.batch_size, self.feat_dim)) np_label_buffer = np.zeros((self.truncate_len, self.batch_size)) else: np_data_buffer = np.zeros((self.batch_size, self.truncate_len, self.feat_dim)) np_label_buffer = np.zeros((self.batch_size, self.truncate_len)) - + utt_id_buffer = [None] * self.batch_size data_names = [self.data_name] + self.init_state_names @@ -432,7 +449,7 @@ def __iter__(self): else: np_data_buffer[i, :n_take, :] = fea_utt[idx_take] np_label_buffer[i, :n_take] = self.labels[idx][idx_take] - + if n_take < self.truncate_len: if self.time_major: np_data_buffer[n_take:, i, :] = 0 @@ -440,7 +457,7 @@ def __iter__(self): else: np_data_buffer[i, n_take:, :] = 0 np_label_buffer[i, n_take:] = 0 - + effective_sample_count -= self.truncate_len - n_take utt_inside_idx[i] += n_take @@ -450,11 +467,11 @@ def __iter__(self): if pad == self.batch_size: # finished all the senteces break - + self.data[0][:] = np_data_buffer self.label[0][:] = np_label_buffer - - data_batch = SimpleBatch(data_names, + + data_batch = SimpleBatch(data_names, self.data + self.init_state_arrays, label_names, self.label, bucket_key=None, utt_id=utt_id_buffer, diff --git a/example/speech-demo/lstm_proj.py b/example/speech-demo/lstm_proj.py index ae2271c800b7..a27518c604b0 100644 --- a/example/speech-demo/lstm_proj.py +++ b/example/speech-demo/lstm_proj.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint:skip-file import mxnet as mx import numpy as np @@ -17,7 +34,7 @@ def lstm(num_hidden, indata, prev_state, param, seqidx, layeridx, dropout=0., nu """LSTM Cell symbol""" if dropout > 0.: indata = mx.sym.Dropout(data=indata, p=dropout) - + i2h = mx.sym.FullyConnected(data=indata, weight=param.i2h_weight, bias=param.i2h_bias, @@ -115,7 +132,7 @@ def lstm_unroll(num_lstm_layer, seq_len, input_size, pred = mx.sym.Reshape(pred, shape=(-1, num_label)) label = mx.sym.Reshape(label, shape=(-1,)) if take_softmax: - sm = mx.sym.SoftmaxOutput(data=pred, label=label, ignore_label=0, + sm = mx.sym.SoftmaxOutput(data=pred, label=label, ignore_label=0, use_ignore=True, name='softmax') else: sm = pred diff --git a/example/speech-demo/make_stats.py b/example/speech-demo/make_stats.py index 440f514729d0..64991db20ad8 100644 --- a/example/speech-demo/make_stats.py +++ b/example/speech-demo/make_stats.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import re import sys sys.path.insert(0, "../../python") diff --git a/example/speech-demo/python_wrap/ctypes.cc b/example/speech-demo/python_wrap/ctypes.cc index cd77d471ba08..a2c79468ed30 100644 --- a/example/speech-demo/python_wrap/ctypes.cc +++ b/example/speech-demo/python_wrap/ctypes.cc @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + #include #include "util/table-types.h" @@ -143,7 +162,7 @@ extern "C" { RAPReader* RAPReader_new_char(char * rspecifier) { return new RAPReader(rspecifier); - } + } //bool HasKey (const std::string &key) int RAPReader_HasKey(RAPReader* r, char * key) { @@ -178,7 +197,7 @@ extern "C" { } vals[row] = pair.first; } - + return vals; } diff --git a/example/speech-demo/python_wrap/example_usage/example.py b/example/speech-demo/python_wrap/example_usage/example.py index 766bb6ebb3b5..d930327f196d 100644 --- a/example/speech-demo/python_wrap/example_usage/example.py +++ b/example/speech-demo/python_wrap/example_usage/example.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from __future__ import print_function import ctypes import numpy @@ -53,7 +70,7 @@ def decl(f, restype, argtypes): print("-------- Kaldi SBFMReader and MatrixF class example --------") reader = kaldi.SBFMReader_new_char("scp:data.scp") - + # data.scp has exactly one utterance, assert it's there assert(not kaldi.SBFMReader_Done(reader)) @@ -63,7 +80,7 @@ def decl(f, restype, argtypes): feat_rows = kaldi.MatrixF_NumRows(feat_value) feat_cols = kaldi.MatrixF_NumCols(feat_value) feat_data = kaldi.MatrixF_Data(feat_value) - + # never use numpy.ndarray(buf=) or numpy.ctypeslib.as_array # because you don't know if Python or C owns buffer # (even if you numpy.copy() resulting array) diff --git a/example/speech-demo/run_ami.sh b/example/speech-demo/run_ami.sh index 6c4dc13bf0ff..0103fd1832ac 100755 --- a/example/speech-demo/run_ami.sh +++ b/example/speech-demo/run_ami.sh @@ -1,5 +1,23 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + # This script trains and evaluate LSTM models. There is no # discriminative training yet. # In this recipe, MXNet directly read Kaldi features and labels, diff --git a/example/speech-demo/run_timit.sh b/example/speech-demo/run_timit.sh index 4bc037dd62de..023ae6f2291f 100755 --- a/example/speech-demo/run_timit.sh +++ b/example/speech-demo/run_timit.sh @@ -1,5 +1,23 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + # This script trains and evaluate LSTM models. There is no # discriminative training yet. # In this recipe, MXNet directly read Kaldi features and labels, diff --git a/example/speech-demo/speechSGD.py b/example/speech-demo/speechSGD.py index 37fb89d51ead..931f40afc062 100644 --- a/example/speech-demo/speechSGD.py +++ b/example/speech-demo/speechSGD.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx from mxnet.ndarray import NDArray, zeros, clip, sqrt @@ -44,7 +61,7 @@ def create_state(self, index, weight): return None else: return zeros(weight.shape, weight.context, dtype=weight.dtype) - + def _get_lr(self, index): """get learning rate for index. diff --git a/example/speech-demo/tests/test_nothing.py b/example/speech-demo/tests/test_nothing.py index 1436522acd3a..d6e810f6e9e1 100644 --- a/example/speech-demo/tests/test_nothing.py +++ b/example/speech-demo/tests/test_nothing.py @@ -1,2 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + def test_nothing(): - pass \ No newline at end of file + pass diff --git a/example/speech-demo/tests/test_system.py b/example/speech-demo/tests/test_system.py index 9d2a4b9a0f18..a64879ae44ba 100644 --- a/example/speech-demo/tests/test_system.py +++ b/example/speech-demo/tests/test_system.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from __future__ import print_function from pdnn.run_DNN import run_DNN from pdnn.run_RBM import run_RBM @@ -37,7 +54,7 @@ def test_rbm_dnn(): "with_final": 1 } mnist_conf["train_rbm"]["max_iters"] = 0 - run_RBM(mnist_conf) + run_RBM(mnist_conf) def test_sda_dnn(): banner("sda dnn") @@ -60,7 +77,7 @@ def test_sda_dnn(): "with_final": 1 } mnist_conf["train_sda"]["max_iters"] = 1 - run_SDA(mnist_conf) + run_SDA(mnist_conf) def test_dnn_eval(): banner("dnn cv") @@ -82,7 +99,7 @@ def test_dnn_eval(): eval_DNN(mnist_conf) mnist_conf["eval_dnn"] = {"mode": "per-feat", "batch_size": 1024} - eval_DNN(mnist_conf) + eval_DNN(mnist_conf) def test_dropout(): banner("dropout") diff --git a/example/speech-demo/train_lstm_proj.py b/example/speech-demo/train_lstm_proj.py index d2a7a2744253..5749b0c39df7 100644 --- a/example/speech-demo/train_lstm_proj.py +++ b/example/speech-demo/train_lstm_proj.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import re import sys sys.path.insert(0, "../../python") @@ -129,7 +146,7 @@ def do_training(training_method, args, module, data_train, data_val): mkpath(os.path.dirname(get_checkpoint_path(args))) batch_size = data_train.batch_size - batch_end_callbacks = [mx.callback.Speedometer(batch_size, + batch_end_callbacks = [mx.callback.Speedometer(batch_size, args.config.getint('train', 'show_every'))] eval_allow_extra = True if training_method == METHOD_TBPTT else False eval_metric = [mx.metric.np(CrossEntropy, allow_extra_outputs=eval_allow_extra), diff --git a/example/speech_recognition/arch_deepspeech.py b/example/speech_recognition/arch_deepspeech.py index 4288b246f3e5..e5b3d43ac07d 100644 --- a/example/speech_recognition/arch_deepspeech.py +++ b/example/speech_recognition/arch_deepspeech.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: disable=C0111, too-many-statements, too-many-locals # pylint: too-many-arguments,too-many-instance-attributes,too-many-locals,redefined-outer-name,fixme # pylint: disable=superfluous-parens, no-member, invalid-name diff --git a/example/speech_recognition/config_util.py b/example/speech_recognition/config_util.py index e8b7fda23506..09733917f924 100644 --- a/example/speech_recognition/config_util.py +++ b/example/speech_recognition/config_util.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import argparse import os import re diff --git a/example/speech_recognition/flac_to_wav.sh b/example/speech_recognition/flac_to_wav.sh index a622b60963e5..7fd53f8719e1 100644 --- a/example/speech_recognition/flac_to_wav.sh +++ b/example/speech_recognition/flac_to_wav.sh @@ -1,8 +1,25 @@ -# Convert all .flac files within this folder to .wav files - -find . -iname "*.flac" | wc - -for flacfile in `find . -iname "*.flac"` -do - sox "${flacfile%.*}.flac" -e signed -b 16 -c 1 -r 16000 "${flacfile%.*}.wav" -done +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Convert all .flac files within this folder to .wav files + +find . -iname "*.flac" | wc + +for flacfile in `find . -iname "*.flac"` +do + sox "${flacfile%.*}.flac" -e signed -b 16 -c 1 -r 16000 "${flacfile%.*}.wav" +done diff --git a/example/speech_recognition/label_util.py b/example/speech_recognition/label_util.py index 3eb56c516e04..dab1d1ef1b40 100644 --- a/example/speech_recognition/label_util.py +++ b/example/speech_recognition/label_util.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # -*- coding: utf-8 -*- import csv diff --git a/example/speech_recognition/log_util.py b/example/speech_recognition/log_util.py index 097cfbd1a00a..e61407f5f4d5 100644 --- a/example/speech_recognition/log_util.py +++ b/example/speech_recognition/log_util.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import logging import logging.handlers diff --git a/example/speech_recognition/main.py b/example/speech_recognition/main.py index 4ecb1168b51f..e45026343de7 100644 --- a/example/speech_recognition/main.py +++ b/example/speech_recognition/main.py @@ -1,351 +1,368 @@ -import json -import os -import sys -from collections import namedtuple -from datetime import datetime -from config_util import parse_args, parse_contexts, generate_file_path -from train import do_training -import mxnet as mx -from stt_io_iter import STTIter -from label_util import LabelUtil -from log_util import LogUtil -import numpy as np -from stt_datagenerator import DataGenerator -from stt_metric import STTMetric -from stt_bi_graphemes_util import generate_bi_graphemes_dictionary -from stt_bucketing_module import STTBucketingModule -from stt_io_bucketingiter import BucketSTTIter -sys.path.insert(0, "../../python") - -# os.environ['MXNET_ENGINE_TYPE'] = "NaiveEngine" -os.environ['MXNET_ENGINE_TYPE'] = "ThreadedEnginePerDevice" -os.environ['MXNET_ENABLE_GPU_P2P'] = "0" - -class WHCS: - width = 0 - height = 0 - channel = 0 - stride = 0 - -class ConfigLogger(object): - def __init__(self, log): - self.__log = log - - def __call__(self, config): - self.__log.info("Config:") - config.write(self) - - def write(self, data): - # stripping the data makes the output nicer and avoids empty lines - line = data.strip() - self.__log.info(line) - -def load_labelutil(labelUtil, is_bi_graphemes, language="en"): - if language == "en": - if is_bi_graphemes: - try: - labelUtil.load_unicode_set("resources/unicodemap_en_baidu_bi_graphemes.csv") - except: - raise Exception("There is no resources/unicodemap_en_baidu_bi_graphemes.csv." + - " Please set overwrite_bi_graphemes_dictionary True at train section") - else: - labelUtil.load_unicode_set("resources/unicodemap_en_baidu.csv") - else: - raise Exception("Error: Language Type: %s" % language) - - - -def load_data(args): - mode = args.config.get('common', 'mode') - if mode not in ['train', 'predict', 'load']: - raise Exception('mode must be the one of the followings - train,predict,load') - batch_size = args.config.getint('common', 'batch_size') - - whcs = WHCS() - whcs.width = args.config.getint('data', 'width') - whcs.height = args.config.getint('data', 'height') - whcs.channel = args.config.getint('data', 'channel') - whcs.stride = args.config.getint('data', 'stride') - save_dir = 'checkpoints' - model_name = args.config.get('common', 'prefix') - is_bi_graphemes = args.config.getboolean('common', 'is_bi_graphemes') - overwrite_meta_files = args.config.getboolean('train', 'overwrite_meta_files') - overwrite_bi_graphemes_dictionary = args.config.getboolean('train', 'overwrite_bi_graphemes_dictionary') - max_duration = args.config.getfloat('data', 'max_duration') - language = args.config.get('data', 'language') - - log = LogUtil().getlogger() - labelUtil = LabelUtil.getInstance() - if mode == "train" or mode == "load": - data_json = args.config.get('data', 'train_json') - val_json = args.config.get('data', 'val_json') - datagen = DataGenerator(save_dir=save_dir, model_name=model_name) - datagen.load_train_data(data_json, max_duration=max_duration) - datagen.load_validation_data(val_json, max_duration=max_duration) - if is_bi_graphemes: - if not os.path.isfile("resources/unicodemap_en_baidu_bi_graphemes.csv") or overwrite_bi_graphemes_dictionary: - load_labelutil(labelUtil=labelUtil, is_bi_graphemes=False, language=language) - generate_bi_graphemes_dictionary(datagen.train_texts+datagen.val_texts) - load_labelutil(labelUtil=labelUtil, is_bi_graphemes=is_bi_graphemes, language=language) - args.config.set('arch', 'n_classes', str(labelUtil.get_count())) - - if mode == "train": - if overwrite_meta_files: - log.info("Generate mean and std from samples") - normalize_target_k = args.config.getint('train', 'normalize_target_k') - datagen.sample_normalize(normalize_target_k, True) - else: - log.info("Read mean and std from meta files") - datagen.get_meta_from_file( - np.loadtxt(generate_file_path(save_dir, model_name, 'feats_mean')), - np.loadtxt(generate_file_path(save_dir, model_name, 'feats_std'))) - elif mode == "load": - # get feat_mean and feat_std to normalize dataset - datagen.get_meta_from_file( - np.loadtxt(generate_file_path(save_dir, model_name, 'feats_mean')), - np.loadtxt(generate_file_path(save_dir, model_name, 'feats_std'))) - - elif mode == 'predict': - test_json = args.config.get('data', 'test_json') - datagen = DataGenerator(save_dir=save_dir, model_name=model_name) - datagen.load_train_data(test_json, max_duration=max_duration) - labelutil = load_labelutil(labelUtil, is_bi_graphemes, language="en") - args.config.set('arch', 'n_classes', str(labelUtil.get_count())) - datagen.get_meta_from_file( - np.loadtxt(generate_file_path(save_dir, model_name, 'feats_mean')), - np.loadtxt(generate_file_path(save_dir, model_name, 'feats_std'))) - - is_batchnorm = args.config.getboolean('arch', 'is_batchnorm') - if batch_size == 1 and is_batchnorm and (mode == 'train' or mode == 'load'): - raise Warning('batch size 1 is too small for is_batchnorm') - - # sort file paths by its duration in ascending order to implement sortaGrad - if mode == "train" or mode == "load": - max_t_count = datagen.get_max_seq_length(partition="train") - max_label_length = \ - datagen.get_max_label_length(partition="train", is_bi_graphemes=is_bi_graphemes) - elif mode == "predict": - max_t_count = datagen.get_max_seq_length(partition="test") - max_label_length = \ - datagen.get_max_label_length(partition="test", is_bi_graphemes=is_bi_graphemes) - - args.config.set('arch', 'max_t_count', str(max_t_count)) - args.config.set('arch', 'max_label_length', str(max_label_length)) - from importlib import import_module - prepare_data_template = import_module(args.config.get('arch', 'arch_file')) - init_states = prepare_data_template.prepare_data(args) - sort_by_duration = (mode == "train") - is_bucketing = args.config.getboolean('arch', 'is_bucketing') - save_feature_as_csvfile = args.config.getboolean('train', 'save_feature_as_csvfile') - if is_bucketing: - buckets = json.loads(args.config.get('arch', 'buckets')) - data_loaded = BucketSTTIter(partition="train", - count=datagen.count, - datagen=datagen, - batch_size=batch_size, - num_label=max_label_length, - init_states=init_states, - seq_length=max_t_count, - width=whcs.width, - height=whcs.height, - sort_by_duration=sort_by_duration, - is_bi_graphemes=is_bi_graphemes, - buckets=buckets, - save_feature_as_csvfile=save_feature_as_csvfile) - else: - data_loaded = STTIter(partition="train", - count=datagen.count, - datagen=datagen, - batch_size=batch_size, - num_label=max_label_length, - init_states=init_states, - seq_length=max_t_count, - width=whcs.width, - height=whcs.height, - sort_by_duration=sort_by_duration, - is_bi_graphemes=is_bi_graphemes, - save_feature_as_csvfile=save_feature_as_csvfile) - - if mode == 'train' or mode == 'load': - if is_bucketing: - validation_loaded = BucketSTTIter(partition="validation", - count=datagen.val_count, - datagen=datagen, - batch_size=batch_size, - num_label=max_label_length, - init_states=init_states, - seq_length=max_t_count, - width=whcs.width, - height=whcs.height, - sort_by_duration=False, - is_bi_graphemes=is_bi_graphemes, - buckets=buckets, - save_feature_as_csvfile=save_feature_as_csvfile) - else: - validation_loaded = STTIter(partition="validation", - count=datagen.val_count, - datagen=datagen, - batch_size=batch_size, - num_label=max_label_length, - init_states=init_states, - seq_length=max_t_count, - width=whcs.width, - height=whcs.height, - sort_by_duration=False, - is_bi_graphemes=is_bi_graphemes, - save_feature_as_csvfile=save_feature_as_csvfile) - return data_loaded, validation_loaded, args - elif mode == 'predict': - return data_loaded, args - - -def load_model(args, contexts, data_train): - # load model from model_name prefix and epoch of model_num_epoch with gpu contexts of contexts - mode = args.config.get('common', 'mode') - load_optimizer_states = args.config.getboolean('load', 'load_optimizer_states') - is_start_from_batch = args.config.getboolean('load', 'is_start_from_batch') - - from importlib import import_module - symbol_template = import_module(args.config.get('arch', 'arch_file')) - is_bucketing = args.config.getboolean('arch', 'is_bucketing') - - if mode == 'train': - if is_bucketing: - bucketing_arch = symbol_template.BucketingArch(args) - model_loaded = bucketing_arch.get_sym_gen() - else: - model_loaded = symbol_template.arch(args) - model_num_epoch = None - elif mode == 'load' or mode == 'predict': - model_file = args.config.get('common', 'model_file') - model_name = os.path.splitext(model_file)[0] - model_num_epoch = int(model_name[-4:]) - if is_bucketing: - bucketing_arch = symbol_template.BucketingArch(args) - model_loaded = bucketing_arch.get_sym_gen() - else: - model_path = 'checkpoints/' + str(model_name[:-5]) - - data_names = [x[0] for x in data_train.provide_data] - label_names = [x[0] for x in data_train.provide_label] - - model_loaded = mx.module.Module.load( - prefix=model_path, epoch=model_num_epoch, context=contexts, - data_names=data_names, label_names=label_names, - load_optimizer_states=load_optimizer_states) - if is_start_from_batch: - import re - model_num_epoch = int(re.findall('\d+', model_file)[0]) - - return model_loaded, model_num_epoch - - -if __name__ == '__main__': - if len(sys.argv) <= 1: - raise Exception('cfg file path must be provided. ' + - 'ex)python main.py --configfile examplecfg.cfg') - args = parse_args(sys.argv[1]) - # set parameters from cfg file - # give random seed - random_seed = args.config.getint('common', 'random_seed') - mx_random_seed = args.config.getint('common', 'mx_random_seed') - # random seed for shuffling data list - if random_seed != -1: - np.random.seed(random_seed) - # set mx.random.seed to give seed for parameter initialization - if mx_random_seed != -1: - mx.random.seed(mx_random_seed) - else: - mx.random.seed(hash(datetime.now())) - # set log file name - log_filename = args.config.get('common', 'log_filename') - log = LogUtil(filename=log_filename).getlogger() - - # set parameters from data section(common) - mode = args.config.get('common', 'mode') - if mode not in ['train', 'predict', 'load']: - raise Exception( - 'Define mode in the cfg file first. ' + - 'train or predict or load can be the candidate for the mode.') - - # get meta file where character to number conversions are defined - - contexts = parse_contexts(args) - num_gpu = len(contexts) - batch_size = args.config.getint('common', 'batch_size') - # check the number of gpus is positive divisor of the batch size for data parallel - if batch_size % num_gpu != 0: - raise Exception('num_gpu should be positive divisor of batch_size') - if mode == "train" or mode == "load": - data_train, data_val, args = load_data(args) - elif mode == "predict": - data_train, args = load_data(args) - is_batchnorm = args.config.getboolean('arch', 'is_batchnorm') - is_bucketing = args.config.getboolean('arch', 'is_bucketing') - - # log current config - config_logger = ConfigLogger(log) - config_logger(args.config) - - # load model - model_loaded, model_num_epoch = load_model(args, contexts, data_train) - # if mode is 'train', it trains the model - if mode == 'train': - if is_bucketing: - module = STTBucketingModule( - sym_gen=model_loaded, - default_bucket_key=data_train.default_bucket_key, - context=contexts - ) - else: - data_names = [x[0] for x in data_train.provide_data] - label_names = [x[0] for x in data_train.provide_label] - module = mx.mod.Module(model_loaded, context=contexts, - data_names=data_names, label_names=label_names) - do_training(args=args, module=module, data_train=data_train, data_val=data_val) - # if mode is 'load', it loads model from the checkpoint and continues the training. - elif mode == 'load': - do_training(args=args, module=model_loaded, data_train=data_train, data_val=data_val, - begin_epoch=model_num_epoch + 1) - # if mode is 'predict', it predict label from the input by the input model - elif mode == 'predict': - # predict through data - if is_bucketing: - max_t_count = args.config.getint('arch', 'max_t_count') - load_optimizer_states = args.config.getboolean('load', 'load_optimizer_states') - model_file = args.config.get('common', 'model_file') - model_name = os.path.splitext(model_file)[0] - model_num_epoch = int(model_name[-4:]) - - model_path = 'checkpoints/' + str(model_name[:-5]) - model = STTBucketingModule( - sym_gen=model_loaded, - default_bucket_key=data_train.default_bucket_key, - context=contexts - ) - - model.bind(data_shapes=data_train.provide_data, - label_shapes=data_train.provide_label, - for_training=True) - _, arg_params, aux_params = mx.model.load_checkpoint(model_path, model_num_epoch) - model.set_params(arg_params, aux_params) - model_loaded = model - else: - model_loaded.bind(for_training=False, data_shapes=data_train.provide_data, - label_shapes=data_train.provide_label) - max_t_count = args.config.getint('arch', 'max_t_count') - eval_metric = STTMetric(batch_size=batch_size, num_gpu=num_gpu) - if is_batchnorm: - for nbatch, data_batch in enumerate(data_train): - model_loaded.forward(data_batch, is_train=False) - model_loaded.update_metric(eval_metric, data_batch.label) - else: - #model_loaded.score(eval_data=data_train, num_batch=None, - # eval_metric=eval_metric, reset=True) - for nbatch, data_batch in enumerate(data_train): - model_loaded.forward(data_batch, is_train=False) - model_loaded.update_metric(eval_metric, data_batch.label) - else: - raise Exception( - 'Define mode in the cfg file first. ' + - 'train or predict or load can be the candidate for the mode') +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import json +import os +import sys +from collections import namedtuple +from datetime import datetime +from config_util import parse_args, parse_contexts, generate_file_path +from train import do_training +import mxnet as mx +from stt_io_iter import STTIter +from label_util import LabelUtil +from log_util import LogUtil +import numpy as np +from stt_datagenerator import DataGenerator +from stt_metric import STTMetric +from stt_bi_graphemes_util import generate_bi_graphemes_dictionary +from stt_bucketing_module import STTBucketingModule +from stt_io_bucketingiter import BucketSTTIter +sys.path.insert(0, "../../python") + +# os.environ['MXNET_ENGINE_TYPE'] = "NaiveEngine" +os.environ['MXNET_ENGINE_TYPE'] = "ThreadedEnginePerDevice" +os.environ['MXNET_ENABLE_GPU_P2P'] = "0" + +class WHCS: + width = 0 + height = 0 + channel = 0 + stride = 0 + +class ConfigLogger(object): + def __init__(self, log): + self.__log = log + + def __call__(self, config): + self.__log.info("Config:") + config.write(self) + + def write(self, data): + # stripping the data makes the output nicer and avoids empty lines + line = data.strip() + self.__log.info(line) + +def load_labelutil(labelUtil, is_bi_graphemes, language="en"): + if language == "en": + if is_bi_graphemes: + try: + labelUtil.load_unicode_set("resources/unicodemap_en_baidu_bi_graphemes.csv") + except: + raise Exception("There is no resources/unicodemap_en_baidu_bi_graphemes.csv." + + " Please set overwrite_bi_graphemes_dictionary True at train section") + else: + labelUtil.load_unicode_set("resources/unicodemap_en_baidu.csv") + else: + raise Exception("Error: Language Type: %s" % language) + + + +def load_data(args): + mode = args.config.get('common', 'mode') + if mode not in ['train', 'predict', 'load']: + raise Exception('mode must be the one of the followings - train,predict,load') + batch_size = args.config.getint('common', 'batch_size') + + whcs = WHCS() + whcs.width = args.config.getint('data', 'width') + whcs.height = args.config.getint('data', 'height') + whcs.channel = args.config.getint('data', 'channel') + whcs.stride = args.config.getint('data', 'stride') + save_dir = 'checkpoints' + model_name = args.config.get('common', 'prefix') + is_bi_graphemes = args.config.getboolean('common', 'is_bi_graphemes') + overwrite_meta_files = args.config.getboolean('train', 'overwrite_meta_files') + overwrite_bi_graphemes_dictionary = args.config.getboolean('train', 'overwrite_bi_graphemes_dictionary') + max_duration = args.config.getfloat('data', 'max_duration') + language = args.config.get('data', 'language') + + log = LogUtil().getlogger() + labelUtil = LabelUtil.getInstance() + if mode == "train" or mode == "load": + data_json = args.config.get('data', 'train_json') + val_json = args.config.get('data', 'val_json') + datagen = DataGenerator(save_dir=save_dir, model_name=model_name) + datagen.load_train_data(data_json, max_duration=max_duration) + datagen.load_validation_data(val_json, max_duration=max_duration) + if is_bi_graphemes: + if not os.path.isfile("resources/unicodemap_en_baidu_bi_graphemes.csv") or overwrite_bi_graphemes_dictionary: + load_labelutil(labelUtil=labelUtil, is_bi_graphemes=False, language=language) + generate_bi_graphemes_dictionary(datagen.train_texts+datagen.val_texts) + load_labelutil(labelUtil=labelUtil, is_bi_graphemes=is_bi_graphemes, language=language) + args.config.set('arch', 'n_classes', str(labelUtil.get_count())) + + if mode == "train": + if overwrite_meta_files: + log.info("Generate mean and std from samples") + normalize_target_k = args.config.getint('train', 'normalize_target_k') + datagen.sample_normalize(normalize_target_k, True) + else: + log.info("Read mean and std from meta files") + datagen.get_meta_from_file( + np.loadtxt(generate_file_path(save_dir, model_name, 'feats_mean')), + np.loadtxt(generate_file_path(save_dir, model_name, 'feats_std'))) + elif mode == "load": + # get feat_mean and feat_std to normalize dataset + datagen.get_meta_from_file( + np.loadtxt(generate_file_path(save_dir, model_name, 'feats_mean')), + np.loadtxt(generate_file_path(save_dir, model_name, 'feats_std'))) + + elif mode == 'predict': + test_json = args.config.get('data', 'test_json') + datagen = DataGenerator(save_dir=save_dir, model_name=model_name) + datagen.load_train_data(test_json, max_duration=max_duration) + labelutil = load_labelutil(labelUtil, is_bi_graphemes, language="en") + args.config.set('arch', 'n_classes', str(labelUtil.get_count())) + datagen.get_meta_from_file( + np.loadtxt(generate_file_path(save_dir, model_name, 'feats_mean')), + np.loadtxt(generate_file_path(save_dir, model_name, 'feats_std'))) + + is_batchnorm = args.config.getboolean('arch', 'is_batchnorm') + if batch_size == 1 and is_batchnorm and (mode == 'train' or mode == 'load'): + raise Warning('batch size 1 is too small for is_batchnorm') + + # sort file paths by its duration in ascending order to implement sortaGrad + if mode == "train" or mode == "load": + max_t_count = datagen.get_max_seq_length(partition="train") + max_label_length = \ + datagen.get_max_label_length(partition="train", is_bi_graphemes=is_bi_graphemes) + elif mode == "predict": + max_t_count = datagen.get_max_seq_length(partition="test") + max_label_length = \ + datagen.get_max_label_length(partition="test", is_bi_graphemes=is_bi_graphemes) + + args.config.set('arch', 'max_t_count', str(max_t_count)) + args.config.set('arch', 'max_label_length', str(max_label_length)) + from importlib import import_module + prepare_data_template = import_module(args.config.get('arch', 'arch_file')) + init_states = prepare_data_template.prepare_data(args) + sort_by_duration = (mode == "train") + is_bucketing = args.config.getboolean('arch', 'is_bucketing') + save_feature_as_csvfile = args.config.getboolean('train', 'save_feature_as_csvfile') + if is_bucketing: + buckets = json.loads(args.config.get('arch', 'buckets')) + data_loaded = BucketSTTIter(partition="train", + count=datagen.count, + datagen=datagen, + batch_size=batch_size, + num_label=max_label_length, + init_states=init_states, + seq_length=max_t_count, + width=whcs.width, + height=whcs.height, + sort_by_duration=sort_by_duration, + is_bi_graphemes=is_bi_graphemes, + buckets=buckets, + save_feature_as_csvfile=save_feature_as_csvfile) + else: + data_loaded = STTIter(partition="train", + count=datagen.count, + datagen=datagen, + batch_size=batch_size, + num_label=max_label_length, + init_states=init_states, + seq_length=max_t_count, + width=whcs.width, + height=whcs.height, + sort_by_duration=sort_by_duration, + is_bi_graphemes=is_bi_graphemes, + save_feature_as_csvfile=save_feature_as_csvfile) + + if mode == 'train' or mode == 'load': + if is_bucketing: + validation_loaded = BucketSTTIter(partition="validation", + count=datagen.val_count, + datagen=datagen, + batch_size=batch_size, + num_label=max_label_length, + init_states=init_states, + seq_length=max_t_count, + width=whcs.width, + height=whcs.height, + sort_by_duration=False, + is_bi_graphemes=is_bi_graphemes, + buckets=buckets, + save_feature_as_csvfile=save_feature_as_csvfile) + else: + validation_loaded = STTIter(partition="validation", + count=datagen.val_count, + datagen=datagen, + batch_size=batch_size, + num_label=max_label_length, + init_states=init_states, + seq_length=max_t_count, + width=whcs.width, + height=whcs.height, + sort_by_duration=False, + is_bi_graphemes=is_bi_graphemes, + save_feature_as_csvfile=save_feature_as_csvfile) + return data_loaded, validation_loaded, args + elif mode == 'predict': + return data_loaded, args + + +def load_model(args, contexts, data_train): + # load model from model_name prefix and epoch of model_num_epoch with gpu contexts of contexts + mode = args.config.get('common', 'mode') + load_optimizer_states = args.config.getboolean('load', 'load_optimizer_states') + is_start_from_batch = args.config.getboolean('load', 'is_start_from_batch') + + from importlib import import_module + symbol_template = import_module(args.config.get('arch', 'arch_file')) + is_bucketing = args.config.getboolean('arch', 'is_bucketing') + + if mode == 'train': + if is_bucketing: + bucketing_arch = symbol_template.BucketingArch(args) + model_loaded = bucketing_arch.get_sym_gen() + else: + model_loaded = symbol_template.arch(args) + model_num_epoch = None + elif mode == 'load' or mode == 'predict': + model_file = args.config.get('common', 'model_file') + model_name = os.path.splitext(model_file)[0] + model_num_epoch = int(model_name[-4:]) + if is_bucketing: + bucketing_arch = symbol_template.BucketingArch(args) + model_loaded = bucketing_arch.get_sym_gen() + else: + model_path = 'checkpoints/' + str(model_name[:-5]) + + data_names = [x[0] for x in data_train.provide_data] + label_names = [x[0] for x in data_train.provide_label] + + model_loaded = mx.module.Module.load( + prefix=model_path, epoch=model_num_epoch, context=contexts, + data_names=data_names, label_names=label_names, + load_optimizer_states=load_optimizer_states) + if is_start_from_batch: + import re + model_num_epoch = int(re.findall('\d+', model_file)[0]) + + return model_loaded, model_num_epoch + + +if __name__ == '__main__': + if len(sys.argv) <= 1: + raise Exception('cfg file path must be provided. ' + + 'ex)python main.py --configfile examplecfg.cfg') + args = parse_args(sys.argv[1]) + # set parameters from cfg file + # give random seed + random_seed = args.config.getint('common', 'random_seed') + mx_random_seed = args.config.getint('common', 'mx_random_seed') + # random seed for shuffling data list + if random_seed != -1: + np.random.seed(random_seed) + # set mx.random.seed to give seed for parameter initialization + if mx_random_seed != -1: + mx.random.seed(mx_random_seed) + else: + mx.random.seed(hash(datetime.now())) + # set log file name + log_filename = args.config.get('common', 'log_filename') + log = LogUtil(filename=log_filename).getlogger() + + # set parameters from data section(common) + mode = args.config.get('common', 'mode') + if mode not in ['train', 'predict', 'load']: + raise Exception( + 'Define mode in the cfg file first. ' + + 'train or predict or load can be the candidate for the mode.') + + # get meta file where character to number conversions are defined + + contexts = parse_contexts(args) + num_gpu = len(contexts) + batch_size = args.config.getint('common', 'batch_size') + # check the number of gpus is positive divisor of the batch size for data parallel + if batch_size % num_gpu != 0: + raise Exception('num_gpu should be positive divisor of batch_size') + if mode == "train" or mode == "load": + data_train, data_val, args = load_data(args) + elif mode == "predict": + data_train, args = load_data(args) + is_batchnorm = args.config.getboolean('arch', 'is_batchnorm') + is_bucketing = args.config.getboolean('arch', 'is_bucketing') + + # log current config + config_logger = ConfigLogger(log) + config_logger(args.config) + + # load model + model_loaded, model_num_epoch = load_model(args, contexts, data_train) + # if mode is 'train', it trains the model + if mode == 'train': + if is_bucketing: + module = STTBucketingModule( + sym_gen=model_loaded, + default_bucket_key=data_train.default_bucket_key, + context=contexts + ) + else: + data_names = [x[0] for x in data_train.provide_data] + label_names = [x[0] for x in data_train.provide_label] + module = mx.mod.Module(model_loaded, context=contexts, + data_names=data_names, label_names=label_names) + do_training(args=args, module=module, data_train=data_train, data_val=data_val) + # if mode is 'load', it loads model from the checkpoint and continues the training. + elif mode == 'load': + do_training(args=args, module=model_loaded, data_train=data_train, data_val=data_val, + begin_epoch=model_num_epoch + 1) + # if mode is 'predict', it predict label from the input by the input model + elif mode == 'predict': + # predict through data + if is_bucketing: + max_t_count = args.config.getint('arch', 'max_t_count') + load_optimizer_states = args.config.getboolean('load', 'load_optimizer_states') + model_file = args.config.get('common', 'model_file') + model_name = os.path.splitext(model_file)[0] + model_num_epoch = int(model_name[-4:]) + + model_path = 'checkpoints/' + str(model_name[:-5]) + model = STTBucketingModule( + sym_gen=model_loaded, + default_bucket_key=data_train.default_bucket_key, + context=contexts + ) + + model.bind(data_shapes=data_train.provide_data, + label_shapes=data_train.provide_label, + for_training=True) + _, arg_params, aux_params = mx.model.load_checkpoint(model_path, model_num_epoch) + model.set_params(arg_params, aux_params) + model_loaded = model + else: + model_loaded.bind(for_training=False, data_shapes=data_train.provide_data, + label_shapes=data_train.provide_label) + max_t_count = args.config.getint('arch', 'max_t_count') + eval_metric = STTMetric(batch_size=batch_size, num_gpu=num_gpu) + if is_batchnorm: + for nbatch, data_batch in enumerate(data_train): + model_loaded.forward(data_batch, is_train=False) + model_loaded.update_metric(eval_metric, data_batch.label) + else: + #model_loaded.score(eval_data=data_train, num_batch=None, + # eval_metric=eval_metric, reset=True) + for nbatch, data_batch in enumerate(data_train): + model_loaded.forward(data_batch, is_train=False) + model_loaded.update_metric(eval_metric, data_batch.label) + else: + raise Exception( + 'Define mode in the cfg file first. ' + + 'train or predict or load can be the candidate for the mode') diff --git a/example/speech_recognition/singleton.py b/example/speech_recognition/singleton.py index 16f129b41017..aa9531b9443c 100644 --- a/example/speech_recognition/singleton.py +++ b/example/speech_recognition/singleton.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import logging as log class Singleton: diff --git a/example/speech_recognition/stt_bi_graphemes_util.py b/example/speech_recognition/stt_bi_graphemes_util.py index b8246a09c137..7ac83142b7cf 100644 --- a/example/speech_recognition/stt_bi_graphemes_util.py +++ b/example/speech_recognition/stt_bi_graphemes_util.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import csv from collections import Counter diff --git a/example/speech_recognition/stt_bucketing_module.py b/example/speech_recognition/stt_bucketing_module.py index 796a3368a6ad..073f6bf649bf 100644 --- a/example/speech_recognition/stt_bucketing_module.py +++ b/example/speech_recognition/stt_bucketing_module.py @@ -1,13 +1,30 @@ -import mxnet as mx - - -class STTBucketingModule(mx.mod.BucketingModule): - - def save_checkpoint(self, prefix, epoch, save_optimizer_states=False): - symbol, data_names, label_names = self._sym_gen(self._default_bucket_key) - symbol.save('%s-symbol.json' % prefix) - param_name = '%s-%04d.params' % (prefix, epoch) - self.save_params(param_name) - if save_optimizer_states: - state_name = '%s-%04d.states' % (prefix, epoch) - self._curr_module.save_optimizer_states(state_name) \ No newline at end of file +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import mxnet as mx + + +class STTBucketingModule(mx.mod.BucketingModule): + + def save_checkpoint(self, prefix, epoch, save_optimizer_states=False): + symbol, data_names, label_names = self._sym_gen(self._default_bucket_key) + symbol.save('%s-symbol.json' % prefix) + param_name = '%s-%04d.params' % (prefix, epoch) + self.save_params(param_name) + if save_optimizer_states: + state_name = '%s-%04d.states' % (prefix, epoch) + self._curr_module.save_optimizer_states(state_name) diff --git a/example/speech_recognition/stt_datagenerator.py b/example/speech_recognition/stt_datagenerator.py index d2a7b4b5cbae..8fafa7909377 100644 --- a/example/speech_recognition/stt_datagenerator.py +++ b/example/speech_recognition/stt_datagenerator.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from __future__ import absolute_import, division, print_function import json @@ -55,7 +72,7 @@ def featurize(self, audio_clip, overwrite=False, save_feature_as_csvfile=False): """ return spectrogram_from_file( audio_clip, step=self.step, window=self.window, - max_freq=self.max_freq, overwrite=overwrite, + max_freq=self.max_freq, overwrite=overwrite, save_feature_as_csvfile=save_feature_as_csvfile) def load_metadata_from_desc_file(self, desc_file, partition='train', diff --git a/example/speech_recognition/stt_io_bucketingiter.py b/example/speech_recognition/stt_io_bucketingiter.py index 9655688f214f..41b93f3bb9e5 100644 --- a/example/speech_recognition/stt_io_bucketingiter.py +++ b/example/speech_recognition/stt_io_bucketingiter.py @@ -1,148 +1,165 @@ -from __future__ import print_function -import mxnet as mx -import sys -sys.path.insert(0, "../../python") - -import bisect -import random -import numpy as np - -BATCH_SIZE = 1 -SEQ_LENGTH = 0 -NUM_GPU = 1 - - -def get_label(buf, num_lable): - ret = np.zeros(num_lable) - for i in range(len(buf)): - ret[i] = int(buf[i]) - return ret - - -class BucketSTTIter(mx.io.DataIter): - def __init__(self, count, datagen, batch_size, num_label, init_states, seq_length, width, height, - sort_by_duration=True, - is_bi_graphemes=False, - partition="train", - buckets=[], - save_feature_as_csvfile=False - ): - super(BucketSTTIter, self).__init__() - - self.maxLabelLength = num_label - # global param - self.batch_size = batch_size - self.count = count - self.num_label = num_label - self.init_states = init_states - self.init_state_arrays = [mx.nd.zeros(x[1]) for x in init_states] - self.width = width - self.height = height - self.datagen = datagen - self.label = None - self.is_bi_graphemes = is_bi_graphemes - # self.partition = datagen.partition - if partition == 'train': - durations = datagen.train_durations - audio_paths = datagen.train_audio_paths - texts = datagen.train_texts - elif partition == 'validation': - durations = datagen.val_durations - audio_paths = datagen.val_audio_paths - texts = datagen.val_texts - elif partition == 'test': - durations = datagen.test_durations - audio_paths = datagen.test_audio_paths - texts = datagen.test_texts - else: - raise Exception("Invalid partition to load metadata. " - "Must be train/validation/test") - # if sortagrad - if sort_by_duration: - durations, audio_paths, texts = datagen.sort_by_duration(durations, - audio_paths, - texts) - else: - durations = durations - audio_paths = audio_paths - texts = texts - self.trainDataList = zip(durations, audio_paths, texts) - - self.trainDataIter = iter(self.trainDataList) - self.is_first_epoch = True - - data_lengths = [int(d*100) for d in durations] - if len(buckets) == 0: - buckets = [i for i, j in enumerate(np.bincount(data_lengths)) - if j >= batch_size] - if len(buckets) == 0: - raise Exception('There is no valid buckets. It may occured by large batch_size for each buckets. max bincount:%d batch_size:%d' % (max(np.bincount(data_lengths)), batch_size)) - buckets.sort() - ndiscard = 0 - self.data = [[] for _ in buckets] - for i, sent in enumerate(data_lengths): - buck = bisect.bisect_left(buckets, sent) - if buck == len(buckets): - ndiscard += 1 - continue - self.data[buck].append(self.trainDataList[i]) - if ndiscard != 0: - print("WARNING: discarded %d sentences longer than the largest bucket."% ndiscard) - - self.buckets = buckets - self.nddata = [] - self.ndlabel = [] - self.default_bucket_key = max(buckets) - - self.idx = [] - for i, buck in enumerate(self.data): - self.idx.extend([(i, j) for j in range(0, len(buck) - batch_size + 1, batch_size)]) - self.curr_idx = 0 - - self.provide_data = [('data', (self.batch_size, self.default_bucket_key , width * height))] + init_states - self.provide_label = [('label', (self.batch_size, self.maxLabelLength))] - self.save_feature_as_csvfile=save_feature_as_csvfile - - #self.reset() - - def reset(self): - """Resets the iterator to the beginning of the data.""" - self.curr_idx = 0 - random.shuffle(self.idx) - for buck in self.data: - np.random.shuffle(buck) - - def next(self): - """Returns the next batch of data.""" - if self.curr_idx == len(self.idx): - raise StopIteration - i, j = self.idx[self.curr_idx] - self.curr_idx += 1 - - audio_paths = [] - texts = [] - for duration, audio_path, text in self.data[i][j:j+self.batch_size]: - audio_paths.append(audio_path) - texts.append(text) - - if self.is_first_epoch: - data_set = self.datagen.prepare_minibatch(audio_paths, texts, overwrite=True, - is_bi_graphemes=self.is_bi_graphemes, - seq_length=self.buckets[i], - save_feature_as_csvfile=self.save_feature_as_csvfile) - else: - data_set = self.datagen.prepare_minibatch(audio_paths, texts, overwrite=False, - is_bi_graphemes=self.is_bi_graphemes, - seq_length=self.buckets[i], - save_feature_as_csvfile=self.save_feature_as_csvfile) - - data_all = [mx.nd.array(data_set['x'])] + self.init_state_arrays - label_all = [mx.nd.array(data_set['y'])] - - self.label = label_all - provide_data = [('data', (self.batch_size, self.buckets[i], self.width * self.height))] + self.init_states - - return mx.io.DataBatch(data_all, label_all, pad=0, - bucket_key=self.buckets[i], - provide_data=provide_data, - provide_label=self.provide_label) +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import print_function +import mxnet as mx +import sys +sys.path.insert(0, "../../python") + +import bisect +import random +import numpy as np + +BATCH_SIZE = 1 +SEQ_LENGTH = 0 +NUM_GPU = 1 + + +def get_label(buf, num_lable): + ret = np.zeros(num_lable) + for i in range(len(buf)): + ret[i] = int(buf[i]) + return ret + + +class BucketSTTIter(mx.io.DataIter): + def __init__(self, count, datagen, batch_size, num_label, init_states, seq_length, width, height, + sort_by_duration=True, + is_bi_graphemes=False, + partition="train", + buckets=[], + save_feature_as_csvfile=False + ): + super(BucketSTTIter, self).__init__() + + self.maxLabelLength = num_label + # global param + self.batch_size = batch_size + self.count = count + self.num_label = num_label + self.init_states = init_states + self.init_state_arrays = [mx.nd.zeros(x[1]) for x in init_states] + self.width = width + self.height = height + self.datagen = datagen + self.label = None + self.is_bi_graphemes = is_bi_graphemes + # self.partition = datagen.partition + if partition == 'train': + durations = datagen.train_durations + audio_paths = datagen.train_audio_paths + texts = datagen.train_texts + elif partition == 'validation': + durations = datagen.val_durations + audio_paths = datagen.val_audio_paths + texts = datagen.val_texts + elif partition == 'test': + durations = datagen.test_durations + audio_paths = datagen.test_audio_paths + texts = datagen.test_texts + else: + raise Exception("Invalid partition to load metadata. " + "Must be train/validation/test") + # if sortagrad + if sort_by_duration: + durations, audio_paths, texts = datagen.sort_by_duration(durations, + audio_paths, + texts) + else: + durations = durations + audio_paths = audio_paths + texts = texts + self.trainDataList = zip(durations, audio_paths, texts) + + self.trainDataIter = iter(self.trainDataList) + self.is_first_epoch = True + + data_lengths = [int(d*100) for d in durations] + if len(buckets) == 0: + buckets = [i for i, j in enumerate(np.bincount(data_lengths)) + if j >= batch_size] + if len(buckets) == 0: + raise Exception('There is no valid buckets. It may occured by large batch_size for each buckets. max bincount:%d batch_size:%d' % (max(np.bincount(data_lengths)), batch_size)) + buckets.sort() + ndiscard = 0 + self.data = [[] for _ in buckets] + for i, sent in enumerate(data_lengths): + buck = bisect.bisect_left(buckets, sent) + if buck == len(buckets): + ndiscard += 1 + continue + self.data[buck].append(self.trainDataList[i]) + if ndiscard != 0: + print("WARNING: discarded %d sentences longer than the largest bucket."% ndiscard) + + self.buckets = buckets + self.nddata = [] + self.ndlabel = [] + self.default_bucket_key = max(buckets) + + self.idx = [] + for i, buck in enumerate(self.data): + self.idx.extend([(i, j) for j in range(0, len(buck) - batch_size + 1, batch_size)]) + self.curr_idx = 0 + + self.provide_data = [('data', (self.batch_size, self.default_bucket_key , width * height))] + init_states + self.provide_label = [('label', (self.batch_size, self.maxLabelLength))] + self.save_feature_as_csvfile=save_feature_as_csvfile + + #self.reset() + + def reset(self): + """Resets the iterator to the beginning of the data.""" + self.curr_idx = 0 + random.shuffle(self.idx) + for buck in self.data: + np.random.shuffle(buck) + + def next(self): + """Returns the next batch of data.""" + if self.curr_idx == len(self.idx): + raise StopIteration + i, j = self.idx[self.curr_idx] + self.curr_idx += 1 + + audio_paths = [] + texts = [] + for duration, audio_path, text in self.data[i][j:j+self.batch_size]: + audio_paths.append(audio_path) + texts.append(text) + + if self.is_first_epoch: + data_set = self.datagen.prepare_minibatch(audio_paths, texts, overwrite=True, + is_bi_graphemes=self.is_bi_graphemes, + seq_length=self.buckets[i], + save_feature_as_csvfile=self.save_feature_as_csvfile) + else: + data_set = self.datagen.prepare_minibatch(audio_paths, texts, overwrite=False, + is_bi_graphemes=self.is_bi_graphemes, + seq_length=self.buckets[i], + save_feature_as_csvfile=self.save_feature_as_csvfile) + + data_all = [mx.nd.array(data_set['x'])] + self.init_state_arrays + label_all = [mx.nd.array(data_set['y'])] + + self.label = label_all + provide_data = [('data', (self.batch_size, self.buckets[i], self.width * self.height))] + self.init_states + + return mx.io.DataBatch(data_all, label_all, pad=0, + bucket_key=self.buckets[i], + provide_data=provide_data, + provide_label=self.provide_label) diff --git a/example/speech_recognition/stt_io_iter.py b/example/speech_recognition/stt_io_iter.py index 5ae65191c840..6c9bacd1a526 100644 --- a/example/speech_recognition/stt_io_iter.py +++ b/example/speech_recognition/stt_io_iter.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from __future__ import print_function import sys diff --git a/example/speech_recognition/stt_layer_batchnorm.py b/example/speech_recognition/stt_layer_batchnorm.py index 5b73f4f9f890..eb61ba6e597e 100644 --- a/example/speech_recognition/stt_layer_batchnorm.py +++ b/example/speech_recognition/stt_layer_batchnorm.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx diff --git a/example/speech_recognition/stt_layer_conv.py b/example/speech_recognition/stt_layer_conv.py index ab0035e4803b..c34ddf21844d 100644 --- a/example/speech_recognition/stt_layer_conv.py +++ b/example/speech_recognition/stt_layer_conv.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx diff --git a/example/speech_recognition/stt_layer_fc.py b/example/speech_recognition/stt_layer_fc.py index f435922426c5..b3db1b163ffa 100644 --- a/example/speech_recognition/stt_layer_fc.py +++ b/example/speech_recognition/stt_layer_fc.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx from stt_layer_batchnorm import batchnorm diff --git a/example/speech_recognition/stt_layer_gru.py b/example/speech_recognition/stt_layer_gru.py index 89af1c72216d..0dd132825235 100644 --- a/example/speech_recognition/stt_layer_gru.py +++ b/example/speech_recognition/stt_layer_gru.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from collections import namedtuple import mxnet as mx diff --git a/example/speech_recognition/stt_layer_lstm.py b/example/speech_recognition/stt_layer_lstm.py index 93b4ca09b908..4adbbd4bec1f 100644 --- a/example/speech_recognition/stt_layer_lstm.py +++ b/example/speech_recognition/stt_layer_lstm.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint:skip-file from collections import namedtuple diff --git a/example/speech_recognition/stt_layer_slice.py b/example/speech_recognition/stt_layer_slice.py index 6b434ec1049b..ac7eae9ae884 100644 --- a/example/speech_recognition/stt_layer_slice.py +++ b/example/speech_recognition/stt_layer_slice.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx diff --git a/example/speech_recognition/stt_layer_warpctc.py b/example/speech_recognition/stt_layer_warpctc.py index 9f97adfe5de1..c821f9c666ab 100644 --- a/example/speech_recognition/stt_layer_warpctc.py +++ b/example/speech_recognition/stt_layer_warpctc.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx diff --git a/example/speech_recognition/stt_metric.py b/example/speech_recognition/stt_metric.py index 1c5f4408a60e..fc1916b40c38 100644 --- a/example/speech_recognition/stt_metric.py +++ b/example/speech_recognition/stt_metric.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx import numpy as np diff --git a/example/speech_recognition/stt_utils.py b/example/speech_recognition/stt_utils.py index 3c7ffce0f980..0539d59f37af 100644 --- a/example/speech_recognition/stt_utils.py +++ b/example/speech_recognition/stt_utils.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import logging import os import os.path @@ -104,7 +121,7 @@ def spectrogram_from_file(filename, step=10, window=20, max_freq=None, """ csvfilename = filename.replace(".wav", ".csv") - if (os.path.isfile(csvfilename) is False) or overwrite: + if (os.path.isfile(csvfilename) is False) or overwrite: with soundfile.SoundFile(filename) as sound_file: audio = sound_file.read(dtype='float32') sample_rate = sound_file.samplerate diff --git a/example/speech_recognition/train.py b/example/speech_recognition/train.py index f3a7555529e3..0d04e4e47a5f 100644 --- a/example/speech_recognition/train.py +++ b/example/speech_recognition/train.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import sys sys.path.insert(0, "../../python") diff --git a/example/ssd/config/config.py b/example/ssd/config/config.py index 278b770febe9..38a07b5e655d 100644 --- a/example/ssd/config/config.py +++ b/example/ssd/config/config.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import os from utils import DotDict, namedtuple_with_defaults, zip_namedtuple, config_as_dict diff --git a/example/ssd/config/utils.py b/example/ssd/config/utils.py index 1d66655e8bee..5c8af6a4dd93 100644 --- a/example/ssd/config/utils.py +++ b/example/ssd/config/utils.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import collections class DotDict(dict): diff --git a/example/ssd/data/demo/download_demo_images.py b/example/ssd/data/demo/download_demo_images.py index 8546aa5e939e..554ba7e4e1aa 100755 --- a/example/ssd/data/demo/download_demo_images.py +++ b/example/ssd/data/demo/download_demo_images.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import os wd = os.path.dirname(os.path.realpath(__file__)) diff --git a/example/ssd/dataset/concat_db.py b/example/ssd/dataset/concat_db.py index da9e151054c3..cb6c99e34fc1 100644 --- a/example/ssd/dataset/concat_db.py +++ b/example/ssd/dataset/concat_db.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from imdb import Imdb import random diff --git a/example/ssd/dataset/imdb.py b/example/ssd/dataset/imdb.py index 279fe9cab4a1..4fbb5d85c873 100644 --- a/example/ssd/dataset/imdb.py +++ b/example/ssd/dataset/imdb.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import numpy as np import os.path as osp diff --git a/example/ssd/dataset/iterator.py b/example/ssd/dataset/iterator.py index 5cefece1c147..8b6857b94edf 100644 --- a/example/ssd/dataset/iterator.py +++ b/example/ssd/dataset/iterator.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx import numpy as np import cv2 diff --git a/example/ssd/dataset/mscoco.py b/example/ssd/dataset/mscoco.py index b46b22745413..ff2a753ddc6f 100644 --- a/example/ssd/dataset/mscoco.py +++ b/example/ssd/dataset/mscoco.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import os import numpy as np from imdb import Imdb diff --git a/example/ssd/dataset/pascal_voc.py b/example/ssd/dataset/pascal_voc.py index 31e287e3cbac..d9868905514c 100644 --- a/example/ssd/dataset/pascal_voc.py +++ b/example/ssd/dataset/pascal_voc.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from __future__ import print_function import os import numpy as np diff --git a/example/ssd/dataset/pycocotools/__init__.py b/example/ssd/dataset/pycocotools/__init__.py index 3f7d85bba884..2f4e0d430df9 100755 --- a/example/ssd/dataset/pycocotools/__init__.py +++ b/example/ssd/dataset/pycocotools/__init__.py @@ -1 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + __author__ = 'tylin' diff --git a/example/ssd/dataset/pycocotools/coco.py b/example/ssd/dataset/pycocotools/coco.py index a8939f64a38c..4dd54ad69d15 100755 --- a/example/ssd/dataset/pycocotools/coco.py +++ b/example/ssd/dataset/pycocotools/coco.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + __author__ = 'tylin' __version__ = '2.0' # Interface for accessing the Microsoft COCO dataset. diff --git a/example/ssd/dataset/testdb.py b/example/ssd/dataset/testdb.py index 7477d77c0aef..9a4b985d8e6b 100644 --- a/example/ssd/dataset/testdb.py +++ b/example/ssd/dataset/testdb.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import os from imdb import Imdb diff --git a/example/ssd/dataset/yolo_format.py b/example/ssd/dataset/yolo_format.py index ce6605f8c637..f1b73d032293 100644 --- a/example/ssd/dataset/yolo_format.py +++ b/example/ssd/dataset/yolo_format.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import os import numpy as np from imdb import Imdb diff --git a/example/ssd/demo.py b/example/ssd/demo.py index bda4606543d7..521267550b41 100644 --- a/example/ssd/demo.py +++ b/example/ssd/demo.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import argparse import tools.find_mxnet import mxnet as mx diff --git a/example/ssd/deploy.py b/example/ssd/deploy.py index aa70cac548c4..415f334fdc2d 100644 --- a/example/ssd/deploy.py +++ b/example/ssd/deploy.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from __future__ import print_function import argparse import tools.find_mxnet diff --git a/example/ssd/detect/detector.py b/example/ssd/detect/detector.py index 19b78f63f561..b6adac110cf7 100644 --- a/example/ssd/detect/detector.py +++ b/example/ssd/detect/detector.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from __future__ import print_function import mxnet as mx import numpy as np diff --git a/example/ssd/evaluate.py b/example/ssd/evaluate.py index 65e0b30b985c..4e7f0a4b9173 100644 --- a/example/ssd/evaluate.py +++ b/example/ssd/evaluate.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import argparse import tools.find_mxnet import mxnet as mx diff --git a/example/ssd/evaluate/eval_metric.py b/example/ssd/evaluate/eval_metric.py index bb03e0133ece..bb2b77b3d52b 100644 --- a/example/ssd/evaluate/eval_metric.py +++ b/example/ssd/evaluate/eval_metric.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx import numpy as np diff --git a/example/ssd/evaluate/eval_voc.py b/example/ssd/evaluate/eval_voc.py index d16856e35009..0ba7f7eaf843 100644 --- a/example/ssd/evaluate/eval_voc.py +++ b/example/ssd/evaluate/eval_voc.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ given a pascal voc imdb, compute mAP """ diff --git a/example/ssd/evaluate/evaluate_net.py b/example/ssd/evaluate/evaluate_net.py index 4c629f892bd6..7f1a32dea518 100644 --- a/example/ssd/evaluate/evaluate_net.py +++ b/example/ssd/evaluate/evaluate_net.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from __future__ import print_function import os import sys diff --git a/example/ssd/symbol/common.py b/example/ssd/symbol/common.py index 474d3ea944b9..ea58c1599add 100644 --- a/example/ssd/symbol/common.py +++ b/example/ssd/symbol/common.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx import numpy as np diff --git a/example/ssd/symbol/inceptionv3.py b/example/ssd/symbol/inceptionv3.py index 1c38ae6d57c9..6022ce505a8f 100644 --- a/example/ssd/symbol/inceptionv3.py +++ b/example/ssd/symbol/inceptionv3.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ Inception V3, suitable for images with around 299 x 299 diff --git a/example/ssd/symbol/legacy_vgg16_ssd_300.py b/example/ssd/symbol/legacy_vgg16_ssd_300.py index 257fdd6acb7a..c1f8ea7cb88e 100644 --- a/example/ssd/symbol/legacy_vgg16_ssd_300.py +++ b/example/ssd/symbol/legacy_vgg16_ssd_300.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx from common import legacy_conv_act_layer from common import multibox_layer diff --git a/example/ssd/symbol/legacy_vgg16_ssd_512.py b/example/ssd/symbol/legacy_vgg16_ssd_512.py index 15424f93e643..6cc3aa274a73 100644 --- a/example/ssd/symbol/legacy_vgg16_ssd_512.py +++ b/example/ssd/symbol/legacy_vgg16_ssd_512.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx from common import legacy_conv_act_layer from common import multibox_layer diff --git a/example/ssd/symbol/resnet.py b/example/ssd/symbol/resnet.py index 9c121e8f9da7..d7dc3cc5bd76 100644 --- a/example/ssd/symbol/resnet.py +++ b/example/ssd/symbol/resnet.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + ''' Adapted from https://github.com/tornadomeet/ResNet/blob/master/symbol_resnet.py Original author Wei Wu diff --git a/example/ssd/symbol/symbol_builder.py b/example/ssd/symbol/symbol_builder.py index 48c9d1d53f34..4cd7f88ea312 100644 --- a/example/ssd/symbol/symbol_builder.py +++ b/example/ssd/symbol/symbol_builder.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx from common import multi_layer_feature, multibox_layer diff --git a/example/ssd/symbol/symbol_factory.py b/example/ssd/symbol/symbol_factory.py index ef156f194c88..c451cd61ab83 100644 --- a/example/ssd/symbol/symbol_factory.py +++ b/example/ssd/symbol/symbol_factory.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Presets for various network configurations""" import logging import symbol_builder diff --git a/example/ssd/symbol/vgg16_reduced.py b/example/ssd/symbol/vgg16_reduced.py index c6a55b117cf8..16535e6dc22d 100644 --- a/example/ssd/symbol/vgg16_reduced.py +++ b/example/ssd/symbol/vgg16_reduced.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx def get_symbol(num_classes=1000, **kwargs): diff --git a/example/ssd/tools/caffe_converter/caffe_parse/parse_from_protobuf.py b/example/ssd/tools/caffe_converter/caffe_parse/parse_from_protobuf.py index b828ca28dff0..862049a770b1 100644 --- a/example/ssd/tools/caffe_converter/caffe_parse/parse_from_protobuf.py +++ b/example/ssd/tools/caffe_converter/caffe_parse/parse_from_protobuf.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from google.protobuf import text_format import numpy as np import caffe_parse.caffe_pb2 as caffe_pb2 diff --git a/example/ssd/tools/caffe_converter/convert_model.py b/example/ssd/tools/caffe_converter/convert_model.py index a06b655b53d9..f17a3f250ecf 100644 --- a/example/ssd/tools/caffe_converter/convert_model.py +++ b/example/ssd/tools/caffe_converter/convert_model.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from __future__ import print_function import sys import os.path as osp diff --git a/example/ssd/tools/caffe_converter/convert_symbol.py b/example/ssd/tools/caffe_converter/convert_symbol.py index 63b044a46c97..10510aa92569 100644 --- a/example/ssd/tools/caffe_converter/convert_symbol.py +++ b/example/ssd/tools/caffe_converter/convert_symbol.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from __future__ import print_function from google.protobuf import text_format import argparse diff --git a/example/ssd/tools/caffe_converter/mean_image.py b/example/ssd/tools/caffe_converter/mean_image.py index d28a750271e0..e07c6fb281c0 100644 --- a/example/ssd/tools/caffe_converter/mean_image.py +++ b/example/ssd/tools/caffe_converter/mean_image.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx import numpy as np import argparse diff --git a/example/ssd/tools/find_mxnet.py b/example/ssd/tools/find_mxnet.py index 66545f38ba0e..0ad64cca01d7 100644 --- a/example/ssd/tools/find_mxnet.py +++ b/example/ssd/tools/find_mxnet.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + try: import mxnet as mx except ImportError: diff --git a/example/ssd/tools/prepare_coco.sh b/example/ssd/tools/prepare_coco.sh index 1b11f9384946..fd34bd55513a 100644 --- a/example/ssd/tools/prepare_coco.sh +++ b/example/ssd/tools/prepare_coco.sh @@ -1,4 +1,22 @@ #!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" python $DIR/prepare_dataset.py --dataset coco --set train2014,valminusminival2014 --target $DIR/../data/train.lst --root $DIR/../data/coco python $DIR/prepare_dataset.py --dataset coco --set minival2014 --target $DIR/../data/val.lst --shuffle False --root $DIR/../data/coco diff --git a/example/ssd/tools/prepare_dataset.py b/example/ssd/tools/prepare_dataset.py index 12b090c27841..9b4fceb221ca 100644 --- a/example/ssd/tools/prepare_dataset.py +++ b/example/ssd/tools/prepare_dataset.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from __future__ import print_function import sys, os import argparse diff --git a/example/ssd/tools/prepare_pascal.sh b/example/ssd/tools/prepare_pascal.sh index 954327aeefa6..1c23cd5b85c9 100644 --- a/example/ssd/tools/prepare_pascal.sh +++ b/example/ssd/tools/prepare_pascal.sh @@ -1,4 +1,22 @@ #!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" python $DIR/prepare_dataset.py --dataset pascal --year 2007,2012 --set trainval --target $DIR/../data/train.lst python $DIR/prepare_dataset.py --dataset pascal --year 2007 --set test --target $DIR/../data/val.lst --shuffle False diff --git a/example/ssd/tools/rand_sampler.py b/example/ssd/tools/rand_sampler.py index d2ed3ad9afe7..7f0cb6f8ba3d 100644 --- a/example/ssd/tools/rand_sampler.py +++ b/example/ssd/tools/rand_sampler.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import numpy as np import math diff --git a/example/ssd/tools/visualize_net.py b/example/ssd/tools/visualize_net.py index f72d6a6678fa..b3b714a7f49b 100644 --- a/example/ssd/tools/visualize_net.py +++ b/example/ssd/tools/visualize_net.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from __future__ import print_function import find_mxnet import mxnet as mx diff --git a/example/ssd/train.py b/example/ssd/train.py index 62ad2c4e0a78..f08aafb97b8f 100644 --- a/example/ssd/train.py +++ b/example/ssd/train.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import argparse import tools.find_mxnet import mxnet as mx diff --git a/example/ssd/train/metric.py b/example/ssd/train/metric.py index c7d007832c4f..731f8fcc19f4 100644 --- a/example/ssd/train/metric.py +++ b/example/ssd/train/metric.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx import numpy as np diff --git a/example/ssd/train/train_net.py b/example/ssd/train/train_net.py index 54cae05a4421bf0587e445174ace680f90aa73f5..767e3244d4060e4c54b4377fcec8e7fa0d12ee0a 100644 GIT binary patch delta 800 zcmZWnO^%x|6yE0qT|OnMG$W-+)n&V)Gl`03MAVq+XqFB>)%m9ci5lQ-lmrl^$}KEf2fmzHU7I-N_ejHa7N>~9H2wq!If zR%9$S29GkMy;Y~q@BSm9OwdIqHr5 zNd;EPJXZl()}%Bpnvn+vEtk^fN|!)x&sLR{C(~=Tu2L=F(jY|aFI?QTcSeQKv*|`1 zgT2-~Td}4eSQG?*+I*xNm;%J*{BwExWt?v0nY8iLxgZRr!*b%+j9U+~;MKIXk~#~+ zvoyY#4TdjZ(8%FT=Hw;b%TWoLVk9(!k@Sld=7}or+t1r=-}f6i2DCA@+|E|J?ITQ0 z#O?zKj{{G-X6InjuTEKbdpeVB!6%a^u&iYtl0!fPPr-vgZ(+4A*MM`E?DMO5rr1M6 zF61Ek`T_isIye;xip7$?Mj{q#dXCd?ho_XDqvJ73(s(E6aHP9Kav!Jha7b|6Q9ckG%%7sK7 YDCTHIqx@KaxY>KN5c6hL02P1(wEzGB diff --git a/example/stochastic-depth/sd_cifar10.py b/example/stochastic-depth/sd_cifar10.py index 9c6f2736600d..c123562cf7ef 100644 --- a/example/stochastic-depth/sd_cifar10.py +++ b/example/stochastic-depth/sd_cifar10.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + ########################################################################################### # Implementation of the stochastic depth algorithm described in the paper # diff --git a/example/stochastic-depth/sd_mnist.py b/example/stochastic-depth/sd_mnist.py index 8a13d4bb532d..7eb93741ff5a 100644 --- a/example/stochastic-depth/sd_mnist.py +++ b/example/stochastic-depth/sd_mnist.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + ################################################################################ # A sanity check mainly for debugging purpose. See sd_cifar10.py for a non-trivial # example of stochastic depth on cifar10. diff --git a/example/stochastic-depth/sd_module.py b/example/stochastic-depth/sd_module.py index ae8cfe0ba255..f30913d550e2 100644 --- a/example/stochastic-depth/sd_module.py +++ b/example/stochastic-depth/sd_module.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import logging import mxnet as mx import numpy as np diff --git a/example/svm_mnist/svm_mnist.py b/example/svm_mnist/svm_mnist.py index ac2702e1260d..679540198d28 100644 --- a/example/svm_mnist/svm_mnist.py +++ b/example/svm_mnist/svm_mnist.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + ############################################################# ## Please read the README.md document for better reference ## diff --git a/example/torch/data.py b/example/torch/data.py index d39821f52145..0ca8e1fd6653 100644 --- a/example/torch/data.py +++ b/example/torch/data.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: skip-file """ data iterator for mnist """ import sys diff --git a/example/torch/torch_function.py b/example/torch/torch_function.py index 4ea4558475ec..af285de22713 100644 --- a/example/torch/torch_function.py +++ b/example/torch/torch_function.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from __future__ import print_function import mxnet as mx x = mx.th.randn(2, 2, ctx=mx.cpu(0)) diff --git a/example/torch/torch_module.py b/example/torch/torch_module.py index 02eacc311d73..1595173b02d4 100644 --- a/example/torch/torch_module.py +++ b/example/torch/torch_module.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: skip-file from data import mnist_iterator import mxnet as mx diff --git a/example/utils/get_data.py b/example/utils/get_data.py index 64a753218225..861d16cdbad9 100644 --- a/example/utils/get_data.py +++ b/example/utils/get_data.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import os import mxnet as mx diff --git a/example/warpctc/infer_ocr.py b/example/warpctc/infer_ocr.py index 2d496f06b1f4..d469990ff937 100644 --- a/example/warpctc/infer_ocr.py +++ b/example/warpctc/infer_ocr.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding=utf-8 # pylint: disable=C0111,too-many-arguments,too-many-instance-attributes,too-many-locals,redefined-outer-name,fixme # pylint: disable=superfluous-parens, no-member, invalid-name diff --git a/example/warpctc/lstm.py b/example/warpctc/lstm.py index 4be4a0d914f1..9e0e05c9011d 100644 --- a/example/warpctc/lstm.py +++ b/example/warpctc/lstm.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint:skip-file import sys sys.path.insert(0, "../../python") diff --git a/example/warpctc/lstm_model.py b/example/warpctc/lstm_model.py index e9c8aa74365f..d359f1ae5a90 100644 --- a/example/warpctc/lstm_model.py +++ b/example/warpctc/lstm_model.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: disable=C0111,too-many-arguments,too-many-instance-attributes,too-many-locals,redefined-outer-name,fixme # pylint: disable=superfluous-parens, no-member, invalid-name @@ -51,4 +68,4 @@ def forward(self, input_data, new_seq=False): for key in self.states_dict.keys(): self.states_dict[key].copyto(self.executor.arg_dict[key]) prob = self.executor.outputs[0].asnumpy() - return prob \ No newline at end of file + return prob diff --git a/example/warpctc/lstm_ocr.py b/example/warpctc/lstm_ocr.py index 540c676f53e7..49df98a77236 100644 --- a/example/warpctc/lstm_ocr.py +++ b/example/warpctc/lstm_ocr.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: disable=C0111,too-many-arguments,too-many-instance-attributes,too-many-locals,redefined-outer-name,fixme # pylint: disable=superfluous-parens, no-member, invalid-name from __future__ import print_function @@ -80,8 +97,8 @@ def __iter__(self): label_all = [mx.nd.array(label)] data_names = ['data'] + init_state_names label_names = ['label'] - - + + data_batch = SimpleBatch(data_names, data_all, label_names, label_all) yield data_batch @@ -198,14 +215,14 @@ def sym_gen(seq_len): import logging head = '%(asctime)-15s %(message)s' logging.basicConfig(level=logging.DEBUG, format=head) - + print('begin fit') prefix = 'ocr' model.fit(X=data_train, eval_data=data_val, eval_metric = mx.metric.np(Accuracy), # Use the following eval_metric if your num_label >= 10, or varies in a wide range - # eval_metric = mx.metric.np(Accuracy_LCS), + # eval_metric = mx.metric.np(Accuracy_LCS), batch_end_callback=mx.callback.Speedometer(BATCH_SIZE, 50), epoch_end_callback = mx.callback.do_checkpoint(prefix, 1)) diff --git a/example/warpctc/ocr_predict.py b/example/warpctc/ocr_predict.py index a07733ef55e0..3096a664a20f 100644 --- a/example/warpctc/ocr_predict.py +++ b/example/warpctc/ocr_predict.py @@ -1,4 +1,22 @@ #!/usr/bin/env python2.7 + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding=utf-8 from __future__ import print_function import sys, os diff --git a/example/warpctc/toy_ctc.py b/example/warpctc/toy_ctc.py index 46bab5776018..c7b0ccc3df3d 100644 --- a/example/warpctc/toy_ctc.py +++ b/example/warpctc/toy_ctc.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: disable=C0111,too-many-arguments,too-many-instance-attributes,too-many-locals,redefined-outer-name,fixme # pylint: disable=superfluous-parens, no-member, invalid-name from __future__ import print_function @@ -68,13 +85,13 @@ def __iter__(self): num, img = gen_rand() data.append(img) label.append(get_label(num)) - + data_all = [mx.nd.array(data)] + self.init_state_arrays label_all = [mx.nd.array(label)] data_names = ['data'] + init_state_names label_names = ['label'] - - + + data_batch = SimpleBatch(data_names, data_all, label_names, label_all) yield data_batch @@ -94,7 +111,7 @@ def ctc_label(p): continue ret.append(c2) return ret - + def Accuracy(label, pred): global BATCH_SIZE @@ -154,7 +171,7 @@ def sym_gen(seq_len): import logging head = '%(asctime)-15s %(message)s' logging.basicConfig(level=logging.DEBUG, format=head) - + print('begin fit') model.fit(X=data_train, eval_data=data_val, diff --git a/include/mxnet/base.h b/include/mxnet/base.h index 739105b388bc..50642049b8f8 100644 --- a/include/mxnet/base.h +++ b/include/mxnet/base.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file base.h * \brief configuation of mxnet as well as basic data structure. */ diff --git a/include/mxnet/c_api.h b/include/mxnet/c_api.h index 3b8d54ce9725..7a45099b8da0 100644 --- a/include/mxnet/c_api.h +++ b/include/mxnet/c_api.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file c_api.h * \brief C API of mxnet */ diff --git a/include/mxnet/c_lapack_api.h b/include/mxnet/c_lapack_api.h index 440d284a8636..1ae90a9396d5 100644 --- a/include/mxnet/c_lapack_api.h +++ b/include/mxnet/c_lapack_api.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * \file c_lapack_api.h * \brief Unified interface for LAPACK calls from within mxnet. * Purpose is to hide the platform specific differences. diff --git a/include/mxnet/c_predict_api.h b/include/mxnet/c_predict_api.h index df60c84c7dfa..8cf153e7cae1 100644 --- a/include/mxnet/c_predict_api.h +++ b/include/mxnet/c_predict_api.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file c_predict_api.h * \brief C predict API of mxnet, contains a minimum API to run prediction. * This file is self-contained, and do not dependent on any other files. diff --git a/include/mxnet/engine.h b/include/mxnet/engine.h index ed46c84cfe83..e997a2b27253 100644 --- a/include/mxnet/engine.h +++ b/include/mxnet/engine.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file engine.h * \brief Engine that schedules all the operations according to dependency. */ diff --git a/include/mxnet/executor.h b/include/mxnet/executor.h index 9308587c8d72..a74d3b07b5be 100644 --- a/include/mxnet/executor.h +++ b/include/mxnet/executor.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file executor.h * \brief Symbolic executor interface of mxnet. * \author Min Lin, Bing Xu diff --git a/include/mxnet/io.h b/include/mxnet/io.h index b4429a951920..68c1ede65ada 100644 --- a/include/mxnet/io.h +++ b/include/mxnet/io.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file io.h * \brief mxnet io data structure and data iterator */ diff --git a/include/mxnet/kvstore.h b/include/mxnet/kvstore.h index 11db28e6cd20..d2924ecea1b5 100644 --- a/include/mxnet/kvstore.h +++ b/include/mxnet/kvstore.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file kvstore.h * \brief key-value store interface for mxnet */ diff --git a/include/mxnet/mxrtc.h b/include/mxnet/mxrtc.h index 9de59f63da2a..8d7facc5b82a 100644 --- a/include/mxnet/mxrtc.h +++ b/include/mxnet/mxrtc.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file mxrtc.h * \brief Wrapper for NVRTC * \author Junyuan Xie diff --git a/include/mxnet/ndarray.h b/include/mxnet/ndarray.h index e349b3091c56..d40b549d8740 100644 --- a/include/mxnet/ndarray.h +++ b/include/mxnet/ndarray.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file ndarray.h * \brief NDArray interface that handles array arithematics. */ diff --git a/include/mxnet/op_attr_types.h b/include/mxnet/op_attr_types.h index dbf9a07e0bcb..1ba07374d894 100644 --- a/include/mxnet/op_attr_types.h +++ b/include/mxnet/op_attr_types.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file op_attr_types.h * \brief Additional operator attributes * beside the ones provided by NNVM diff --git a/include/mxnet/operator.h b/include/mxnet/operator.h index 09a643390342..2245db0dbb93 100644 --- a/include/mxnet/operator.h +++ b/include/mxnet/operator.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file operator.h * \brief Operator interface of mxnet. * \author Naiyan Wang diff --git a/include/mxnet/operator_util.h b/include/mxnet/operator_util.h index 0f27b10368cf..92ef2ecc58f6 100644 --- a/include/mxnet/operator_util.h +++ b/include/mxnet/operator_util.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file operator_util.h * \brief Utility functions and registries to help quickly build new operators. * [Deprecated] diff --git a/include/mxnet/resource.h b/include/mxnet/resource.h index 93b8352b2617..1ca1fc6fa707 100644 --- a/include/mxnet/resource.h +++ b/include/mxnet/resource.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file resource.h * \brief Global resource allocation handling. */ diff --git a/include/mxnet/storage.h b/include/mxnet/storage.h index 1b765233947d..bfb42de8771a 100644 --- a/include/mxnet/storage.h +++ b/include/mxnet/storage.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file storage.h * \brief Storage manager across multiple devices. */ diff --git a/include/mxnet/tensor_blob.h b/include/mxnet/tensor_blob.h index 1928aa48c85f..18bf4fa780d9 100755 --- a/include/mxnet/tensor_blob.h +++ b/include/mxnet/tensor_blob.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2014 by Contributors * \file tensor_blob.h * \brief TBlob class that holds common representation of * arbirary dimension tensor, can be used to transformed diff --git a/matlab/get_inception_model.sh b/matlab/get_inception_model.sh index aa0092deb6d8..af2479b33b83 100755 --- a/matlab/get_inception_model.sh +++ b/matlab/get_inception_model.sh @@ -1,5 +1,23 @@ #!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + MATLAB_DIR=$(cd `dirname $0`; pwd) DATA_DIR="${MATLAB_DIR}/data/" diff --git a/perl-package/AI-MXNet/examples/get_ptb_data.sh b/perl-package/AI-MXNet/examples/get_ptb_data.sh index 1ec009aa2f99..d2641cb32b81 100755 --- a/perl-package/AI-MXNet/examples/get_ptb_data.sh +++ b/perl-package/AI-MXNet/examples/get_ptb_data.sh @@ -1,5 +1,23 @@ #!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + RNN_DIR=$(cd `dirname $0`; pwd) DATA_DIR="${RNN_DIR}/data/" diff --git a/perl-package/AI-MXNet/lib/AI/MXNet.pm b/perl-package/AI-MXNet/lib/AI/MXNet.pm index 54fb6b31e81f..1d2125354019 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet.pm @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + package AI::MXNet; use v5.14.0; use strict; diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Base.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Base.pm index 69f8e43af30c..d5ff0dd08ecc 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Base.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Base.pm @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + package AI::MXNet::Base; use strict; use warnings; @@ -12,7 +29,7 @@ use Exporter; use base qw(Exporter); use List::Util qw(shuffle); -@AI::MXNet::Base::EXPORT = qw(product enumerate assert zip check_call build_param_doc +@AI::MXNet::Base::EXPORT = qw(product enumerate assert zip check_call build_param_doc pdl cat dog svd bisect_left pdl_shuffle DTYPE_STR_TO_MX DTYPE_MX_TO_STR DTYPE_MX_TO_PDL DTYPE_PDL_TO_MX DTYPE_MX_TO_PERL GRAD_REQ_MAP); @@ -239,12 +256,12 @@ sub build_param_doc $remove_dup //= 1; my %param_keys; my @param_str; - zip(sub { + zip(sub { my ($key, $type_info, $desc) = @_; return if exists $param_keys{$key} and $remove_dup; $param_keys{$key} = 1; my $ret = sprintf("%s : %s", $key, $type_info); - $ret .= "\n ".$desc if length($desc); + $ret .= "\n ".$desc if length($desc); push @param_str, $ret; }, $arg_names, $arg_types, $arg_descs diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/CachedOp.pm b/perl-package/AI-MXNet/lib/AI/MXNet/CachedOp.pm index ede48265970f..f3c21ed17f30 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/CachedOp.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/CachedOp.pm @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + package AI::MXNet::CachedOp; =head1 NAME diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Callback.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Callback.pm index 04aaea06c47d..da3309700394 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Callback.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Callback.pm @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + package AI::MXNet::Callback; use strict; use warnings; @@ -92,7 +109,7 @@ extends 'AI::MXNet::Callback'; =head1 NAME - AI::MXNet::Speedometer - A callback that logs training speed + AI::MXNet::Speedometer - A callback that logs training speed =cut =head1 DESCRIPTION @@ -244,4 +261,4 @@ method LogValidationMetricsCallback() AI::MXNet::LogValidationMetricsCallback->new } -1; \ No newline at end of file +1; diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Context.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Context.pm index 68628a8cc14c..2eca42436dc7 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Context.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Context.pm @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + package AI::MXNet::Context; use strict; use warnings; diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Contrib.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Contrib.pm index cb6bc01008f9..a81030bdc6e0 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Contrib.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Contrib.pm @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + package AI::MXNet::Contrib; use strict; use warnings; @@ -9,4 +26,4 @@ sub symbol { 'AI::MXNet::Contrib::Symbol' } sub nd { 'AI::MXNet::Contrib::NDArray' } sub autograd { 'AI::MXNet::Contrib::AutoGrad' } -1; \ No newline at end of file +1; diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Contrib/AutoGrad.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Contrib/AutoGrad.pm index 6d9c10340939..ff659982b813 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Contrib/AutoGrad.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Contrib/AutoGrad.pm @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + package AI::MXNet::Contrib::AutoGrad; use strict; use warnings; @@ -224,4 +241,4 @@ method test_section(CodeRef $sub) __PACKAGE__->set_is_training(1) if $prev; } -1; \ No newline at end of file +1; diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Contrib/NDArray.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Contrib/NDArray.pm index 239f1c48e81f..78aed8fd59cc 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Contrib/NDArray.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Contrib/NDArray.pm @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + package AI::MXNet::Contrib::NDArray; use strict; use warnings; @@ -10,4 +27,4 @@ sub AUTOLOAD { return AI::MXNet::NDArray->$sub(@_); } -1; \ No newline at end of file +1; diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Contrib/Symbol.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Contrib/Symbol.pm index c67cdad4baa5..efe785d181f7 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Contrib/Symbol.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Contrib/Symbol.pm @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + package AI::MXNet::Contrib::Symbol; use strict; use warnings; @@ -10,4 +27,4 @@ sub AUTOLOAD { return AI::MXNet::Symbol->$sub(@_); } -1; \ No newline at end of file +1; diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Executor.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Executor.pm index a2ab786a62d1..20a6f580a3db 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Executor.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Executor.pm @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + package AI::MXNet::Executor; use strict; use warnings; @@ -9,7 +26,7 @@ use AI::MXNet::Function::Parameters; has 'handle' => (is => 'ro', isa => 'ExecutorHandle', required => 1); has 'arg_arrays' => (is => 'rw', isa => 'Maybe[ArrayRef[AI::MXNet::NDArray]]'); -has 'grad_arrays' => (is => 'rw', isa => 'Maybe[ArrayRef[Undef|AI::MXNet::NDArray]]'); +has 'grad_arrays' => (is => 'rw', isa => 'Maybe[ArrayRef[Undef|AI::MXNet::NDArray]]'); has 'aux_arrays' => (is => 'rw', isa => 'Maybe[ArrayRef[AI::MXNet::NDArray]]'); has '_symbol' => (is => 'rw', init_arg => 'symbol', isa => 'AI::MXNet::Symbol'); has '_ctx' => (is => 'rw', init_arg => 'ctx', isa => 'AI::MXNet::Context' ); @@ -420,7 +437,7 @@ method copy_params_from( method reshape(HashRef[Shape] $kwargs, Int :$partial_shaping=0, Int :$allow_up_sizing=0) { my ($arg_shapes, undef, $aux_shapes) = $self->_symbol->infer_shape(%{ $kwargs }); - confess("Insufficient argument shapes provided.") + confess("Insufficient argument shapes provided.") unless defined $arg_shapes; my %new_arg_dict; my %new_grad_dict; diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Executor/Group.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Executor/Group.pm index 35f1b57ee5fa..611c93148f25 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Executor/Group.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Executor/Group.pm @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + package AI::MXNet::Executor::Group; use strict; use warnings; diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Function/Parameters.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Function/Parameters.pm index 021252063c49..e4bbc90ca0a3 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Function/Parameters.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Function/Parameters.pm @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + package AI::MXNet::Function::Parameters; use strict; use warnings; @@ -32,4 +49,4 @@ sub import { }; } -1; \ No newline at end of file +1; diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/IO.pm b/perl-package/AI-MXNet/lib/AI/MXNet/IO.pm index 73a672eb4a5c..7a61cd9f1f1f 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/IO.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/IO.pm @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + package AI::MXNet::IO; use strict; use warnings; @@ -784,7 +801,7 @@ method _init_io_module() no strict 'refs'; { *{__PACKAGE__."::$name"} = $data_iter; - } + } } } } @@ -792,4 +809,4 @@ method _init_io_module() # Initialize the io in startups __PACKAGE__->_init_io_module; -1; \ No newline at end of file +1; diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Image.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Image.pm index 50e4a41f5cc1..b996b028dff2 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Image.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Image.pm @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + package AI::MXNet::Image; use strict; use warnings; diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Initializer.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Initializer.pm index e6beffb78372..182327dfccfe 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Initializer.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Initializer.pm @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + package AI::MXNet::InitDesc; use Mouse; use AI::MXNet::Function::Parameters; @@ -104,7 +121,7 @@ method register() { my $existing = $init_registry{ $name }; warn( - "WARNING: New initializer $self.$name" + "WARNING: New initializer $self.$name" ."is overriding existing initializer $existing.$name" ); } diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/KVStore.pm b/perl-package/AI-MXNet/lib/AI/MXNet/KVStore.pm index 465cfd65cf07..eff57a31dc53 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/KVStore.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/KVStore.pm @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + package AI::MXNet::KVStore; use strict; use warnings; @@ -13,7 +30,7 @@ use AI::MXNet::Function::Parameters; AI::MXNet::KVStore - Key value store interface of MXNet. -=head1 DESCRIPTION +=head1 DESCRIPTION Key value store interface of MXNet for parameter synchronization, over multiple devices. =cut diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/KVStoreServer.pm b/perl-package/AI-MXNet/lib/AI/MXNet/KVStoreServer.pm index 6b018afaf6e5..4c274b92c71f 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/KVStoreServer.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/KVStoreServer.pm @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + package AI::MXNet::KVStoreServer; use strict; use warnings; @@ -30,7 +47,7 @@ has 'init_logging' => (is => 'rw', isa => 'Int', default => 0); # return the server controller method _controller() { - return sub { + return sub { my ($cmd_id, $cmd_body) = @_; if (not $self->init_logging) { diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/LRScheduler.pm b/perl-package/AI-MXNet/lib/AI/MXNet/LRScheduler.pm index a9ffb37d6a69..27420f45167d 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/LRScheduler.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/LRScheduler.pm @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + package AI::MXNet::LRScheduler; use strict; use warnings; @@ -173,4 +190,4 @@ method call(Int $num_update) return $self->base_lr; } -1; \ No newline at end of file +1; diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Logging.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Logging.pm index d6d3744ef37f..f3039cc09bfd 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Logging.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Logging.pm @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + package AI::MXNet::Logging; ## TODO use Mouse; diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Metric.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Metric.pm index fbb93b006a6f..6504481ba8ea 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Metric.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Metric.pm @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + package AI::MXNet::Metric; use strict; use warnings; @@ -484,7 +501,7 @@ method update(ArrayRef[AI::MXNet::NDArray] $labels, ArrayRef[AI::MXNet::NDArray] my $label_shape = $label->shape->at(0); my $pred_shape = $pred->shape->at(-1); confess( - "Size of label $label_shape and + "Size of label $label_shape and .first dimension of pred $pred_shape do not match" ) unless $label_shape == $pred_shape; my $prob = $pred->index($label); @@ -599,4 +616,4 @@ method create(Metric|ArrayRef[Metric] $metric, %kwargs) } } -1; \ No newline at end of file +1; diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Module.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Module.pm index ba70fd059fbe..967a51142aac 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Module.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Module.pm @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + ## TODO ## this class is here because of https://github.com/gfx/p5-Mouse/pull/67 ## once 2.4.7 version of Mouse in Ubuntu for affected Perl version @@ -173,7 +190,7 @@ has 'state_names' => (is => 'rw', isa => 'Maybe[ArrayRef[Str]]'); has 'logger' => (is => 'ro', default => sub { AI::MXNet::Logging->get_logger }); has '_p' => (is => 'rw', init_arg => undef); has 'context' => ( - is => 'ro', + is => 'ro', isa => 'AI::MXNet::Context|ArrayRef[AI::MXNet::Context]', default => sub { AI::MXNet::Context->cpu } ); @@ -952,4 +969,4 @@ method _kvstore() $self->_p->_kvstore; } -1; \ No newline at end of file +1; diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Module/Base.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Module/Base.pm index 293696db218f..7a9e3de090db 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Module/Base.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Module/Base.pm @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + package AI::MXNet::BatchEndParam; use Mouse; use AI::MXNet::Function::Parameters; diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Module/Bucketing.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Module/Bucketing.pm index af768f087025..531f41d58a3a 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Module/Bucketing.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Module/Bucketing.pm @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + package AI::MXNet::Module::Bucketing; use Mouse; use AI::MXNet::Function::Parameters; diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Monitor.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Monitor.pm index 7ac989c6e27f..993461713cb6 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Monitor.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Monitor.pm @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + package AI::MXNet::Monitor; use Mouse; use AI::MXNet::Function::Parameters; @@ -153,7 +170,7 @@ method toc() my $s = ''; for my $v (@{ $v_list }) { - confess("the argument must be NDArray") + confess("the argument must be NDArray") unless blessed($v) and $v->isa('AI::MXNet::NDArray'); if($v->size == 1) { diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/NDArray.pm b/perl-package/AI-MXNet/lib/AI/MXNet/NDArray.pm index edeb9b1ba1a2..1f58a74e2bba 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/NDArray.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/NDArray.pm @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + package AI::MXNet::NDArray; =head1 NAME @@ -68,7 +85,7 @@ method at(Index @indices) my $isize = @indices; confess("Dimensions size $dsize < indexes size $isize") if $dsize < $isize; - confess("Dimensions size $dsize = indexes size $isize, + confess("Dimensions size $dsize = indexes size $isize, ndarray only supports either ->at on dimension 0 or full crop") if $isize > 1 and $dsize != $isize; @@ -78,7 +95,7 @@ method at(Index @indices) confess("Dimension $i mismatch Idx: $idx >= Dim Size: $dim_size") if $idx >= $dim_size or ($idx + $dim_size) < 0; ++$i; - }, \@indices, $shape); + }, \@indices, $shape); $i = 0; for my $v (@indices) { @@ -181,7 +198,7 @@ method _sync_copyfrom(ArrayRef|PDL|PDL::Matrix $source_array) my $convert_func = $pdl_type->convertfunc; $source_array = $source_array->$convert_func; } - $source_array = pdl($pdl_type, [@{ $source_array->unpdl } ? $source_array->unpdl->[0] : 0 ]) + $source_array = pdl($pdl_type, [@{ $source_array->unpdl } ? $source_array->unpdl->[0] : 0 ]) unless @{ $source_array->shape->unpdl }; my $pdl_shape = $source_array->shape->unpdl; my $pdl_shape_str = join(',', ref($source_array) eq 'PDL' ? reverse @{ $pdl_shape } : @{ $pdl_shape }); @@ -222,7 +239,7 @@ method aspdl() my $pdl = PDL->new_from_specification($pdl_type, reverse @{ $self->shape }); my $perl_pack_type = DTYPE_MX_TO_PERL->{$dtype}; my $buf = pack("$perl_pack_type*", (0)x$self->size); - check_call(AI::MXNetCAPI::NDArraySyncCopyToCPU($self->handle, $buf, $self->size)); + check_call(AI::MXNetCAPI::NDArraySyncCopyToCPU($self->handle, $buf, $self->size)); ## special handling for float16 if($perl_pack_type eq 'S') { @@ -253,7 +270,7 @@ method asmpdl() my $pdl = PDL::Matrix->new_from_specification($pdl_type, @{ $self->shape }); my $perl_pack_type = DTYPE_MX_TO_PERL->{$dtype}; my $buf = pack("$perl_pack_type*", (0)x$self->size); - check_call(AI::MXNetCAPI::NDArraySyncCopyToCPU($self->handle, $buf, $self->size)); + check_call(AI::MXNetCAPI::NDArraySyncCopyToCPU($self->handle, $buf, $self->size)); ## special handling for float16 if($perl_pack_type eq 'S') { @@ -394,7 +411,7 @@ method moveaxis(Int $source, Int $dest) =head2 broadcast_to - Broadcasting the current NDArray into the given shape. + Broadcasting the current NDArray into the given shape. Parameters --------- @@ -404,7 +421,7 @@ method moveaxis(Int $source, Int $dest) method broadcast_to(Shape $shape) { my $cur_shape = $self->shape; - my $err_str = "operands could not be broadcast together with remapped shapes" + my $err_str = "operands could not be broadcast together with remapped shapes" ."[original->remapped]: [@$cur_shape] and requested shape [@$shape]"; if(@$shape < @$cur_shape) { @@ -494,7 +511,7 @@ method context() Returns ------- - a data type string ('float32', 'float64', 'float16', 'uint8', 'int32') + a data type string ('float32', 'float64', 'float16', 'uint8', 'int32') representing the data type of the ndarray. 'float32' is the default dtype for the ndarray class. =cut @@ -707,7 +724,7 @@ method stringify($other=, $reverse=) method iadd(AI::MXNet::NDArray|Num $other, $reverse=) { confess('trying to add to a readonly NDArray') unless $self->writable; - return ref $other + return ref $other ? __PACKAGE__->broadcast_add($self, $other, { out => $self }) : __PACKAGE__->_plus_scalar($self, $other, { out => $self }) } @@ -752,9 +769,9 @@ method multiply(AI::MXNet::NDArray|Num $other, $reverse=) method imultiply(AI::MXNet::NDArray|Num $other, $reverse=) { confess('trying to add to a readonly NDArray') unless $self->writable; - return ref $other - ? __PACKAGE__->broadcast_mul($self, $other, { out => $self }) - : __PACKAGE__->_mul_scalar($self, $other, { out => $self }) + return ref $other + ? __PACKAGE__->broadcast_mul($self, $other, { out => $self }) + : __PACKAGE__->_mul_scalar($self, $other, { out => $self }) } method divide(AI::MXNet::NDArray|Num $other, $reverse=) @@ -770,9 +787,9 @@ method divide(AI::MXNet::NDArray|Num $other, $reverse=) method idivide(AI::MXNet::NDArray|Num $other, $reverse=) { confess('trying to add to a readonly NDArray') unless $self->writable; - return ref $other - ? __PACKAGE__->broadcast_div($self, $other, { out => $self }) - : __PACKAGE__->_div_scalar($self, $other, { out => $self }) + return ref $other + ? __PACKAGE__->broadcast_div($self, $other, { out => $self }) + : __PACKAGE__->_div_scalar($self, $other, { out => $self }) } method power(AI::MXNet::NDArray|Num $other, $reverse=) @@ -1094,11 +1111,11 @@ method concatenate(ArrayRef[AI::MXNet::NDArray] $arrays, Index :$axis=0, :$alway $shape_axis += $arr->shape->[$axis]; my $arr_shape_rest1 = [@{ $arr->shape }[0..($axis-1)]]; my $arr_shape_rest2 = [@{ $arr->shape }[($axis+1)..(@{ $arr->shape }-1)]]; - confess("first array $arrays->[0] and $i array $arr do not match") + confess("first array $arrays->[0] and $i array $arr do not match") unless join(',',@$arr_shape_rest1) eq join(',',@$shape_rest1); - confess("first array $arrays->[0] and $i array $arr do not match") + confess("first array $arrays->[0] and $i array $arr do not match") unless join(',',@$arr_shape_rest2) eq join(',',@$shape_rest2); - confess("first array $arrays->[0] and $i array $arr dtypes do not match") + confess("first array $arrays->[0] and $i array $arr dtypes do not match") unless join(',',@$arr_shape_rest2) eq join(',',@$shape_rest2); $i++; } @@ -1118,8 +1135,8 @@ method concatenate(ArrayRef[AI::MXNet::NDArray] $arrays, Index :$axis=0, :$alway $begin->[$axis] = $idx; $end->[$axis] = $idx+$arr->shape->[$axis]; __PACKAGE__->_crop_assign( - $ret, $arr, - { + $ret, $arr, + { out => $ret, begin => $begin, end => $end diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/NDArray/Base.pm b/perl-package/AI-MXNet/lib/AI/MXNet/NDArray/Base.pm index 7fb6d0e61110..b51436157a82 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/NDArray/Base.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/NDArray/Base.pm @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + package AI::MXNet::NDArray::Base; use strict; use warnings; @@ -100,7 +117,7 @@ func _make_ndarray_function($handle, $func_name) } for my $key (keys %kwargs) { - $kwargs{ $key } = "(" .join(", ", @{ $kwargs{ $key } }) .")" + $kwargs{ $key } = "(" .join(", ", @{ $kwargs{ $key } }) .")" if ref $kwargs{ $key } eq 'ARRAY'; } my $out = check_call(AI::MXNetCAPI::ImperativeInvoke( diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/NDArray/Doc.pm b/perl-package/AI-MXNet/lib/AI/MXNet/NDArray/Doc.pm index a1a7812ca9a5..fc44812f2cff 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/NDArray/Doc.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/NDArray/Doc.pm @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + package AI::MXNet::NDArray::Doc; use strict; use warnings; diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/NDArray/Slice.pm b/perl-package/AI-MXNet/lib/AI/MXNet/NDArray/Slice.pm index a52f8eef7c1d..40312ebaa24f 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/NDArray/Slice.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/NDArray/Slice.pm @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + package AI::MXNet::NDArray::Slice; use strict; use warnings; @@ -13,7 +30,7 @@ use AI::MXNet::Function::Parameters; has parent => (is => 'ro', isa => 'AI::MXNet::NDArray', required => 1); has begin => (is => 'ro', isa => 'Shape', required => 1); has end => (is => 'ro', isa => 'Shape', required => 1); -use overload +use overload '.=' => \&set, '=' => sub { $_[0] }, '""' => \¬supported, @@ -37,10 +54,10 @@ method set(AcceptableInput $value, $reverse=) { confess("set value must be defined") unless defined $value; confess("${\ $self->parent } is not writable") unless $self->parent->writable; - my $shape = []; + my $shape = []; zip( sub { my ($begin, $end) = @_; push @$shape, ($end-$begin); }, - $self->begin, + $self->begin, $self->end ); if(ref $value) @@ -58,12 +75,12 @@ method set(AcceptableInput $value, $reverse=) $value = AI::MXNet::NDArray->array($value, ctx => $self->parent->context); } confess("value $value does not match slice dim sizes [@$shape]") - if @{$value->shape} != @$shape; + if @{$value->shape} != @$shape; zip( - sub { - my ($dsize, $vdsize) = @_; - confess("Slice [@$shape] != $value given as value") - if $dsize != $vdsize; + sub { + my ($dsize, $vdsize) = @_; + confess("Slice [@$shape] != $value given as value") + if $dsize != $vdsize; }, $shape, $value->shape diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Optimizer.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Optimizer.pm index 08b9565605eb..c6f682253833 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Optimizer.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Optimizer.pm @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + package AI::MXNet::Optimizer; use strict; use warnings; @@ -33,7 +50,7 @@ method register() { my $existing = $opt_registry{ $name }; warn( - "WARNING: New optimizer $self.$name" + "WARNING: New optimizer $self.$name" ."is overriding existing optimizer $existing.$name" ); } @@ -505,7 +522,7 @@ method update( if($self->clip_gradient) { $grad = AI::MXNet::NDArray->clip( - $grad, + $grad, -$self->clip_gradient, $self->clip_gradient ); @@ -566,7 +583,7 @@ method create_state(Index $index, AI::MXNet::NDArray $weight) } method update( - Index $index, + Index $index, AI::MXNet::NDArray $weight, AI::MXNet::NDArray $grad, AI::MXNet::NDArray|Undef $state @@ -678,7 +695,7 @@ method create_state(Index $index, AI::MXNet::NDArray $weight) } method update( - Index $index, + Index $index, AI::MXNet::NDArray $weight, AI::MXNet::NDArray $grad, ArrayRef[AI::MXNet::NDArray] $state @@ -748,7 +765,7 @@ has '+learning_rate' => (default => 0.05); method create_state(Index $index, AI::MXNet::NDArray $weight) { return AI::MXNet::NDArray->zeros( - $weight->shape, + $weight->shape, ctx => $weight->context ); # history } @@ -1025,7 +1042,7 @@ extends 'AI::MXNet::Optimizer'; method create_state(Index $index, AI::MXNet::NDArray $weight) { return AI::MXNet::NDArray->zeros( - $weight->shape, + $weight->shape, ctx => $weight->context ); } diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Profiler.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Profiler.pm index 6398fcb3e432..47d7a0ddf716 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Profiler.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Profiler.pm @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + package AI::MXNet::Profiler; use strict; use warnings; diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/RNN.pm b/perl-package/AI-MXNet/lib/AI/MXNet/RNN.pm index 13dc4f24de33..1ccab31fb7ac 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/RNN.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/RNN.pm @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + package AI::MXNet::RNN; use strict; use warnings; diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/RNN/Cell.pm b/perl-package/AI-MXNet/lib/AI/MXNet/RNN/Cell.pm index c7523aa86bbf..0221a90e7157 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/RNN/Cell.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/RNN/Cell.pm @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + package AI::MXNet::RNN::Params; use Mouse; use AI::MXNet::Function::Parameters; @@ -560,7 +577,7 @@ use Mouse; use AI::MXNet::Base; extends 'AI::MXNet::RNN::Cell'; -=head1 NAME +=head1 NAME AI::MXNet::RNN::LSTMCell =cut diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/RNN/IO.pm b/perl-package/AI-MXNet/lib/AI/MXNet/RNN/IO.pm index 731f7762b7a0..be3bdbd373cb 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/RNN/IO.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/RNN/IO.pm @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + package AI::MXNet::RNN::IO; use strict; use warnings; @@ -289,4 +306,4 @@ method next() ); } -1; \ No newline at end of file +1; diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Random.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Random.pm index dd17523e3cbf..9ca013c62348 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Random.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Random.pm @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + package AI::MXNet::Random; use strict; use warnings; @@ -59,4 +76,4 @@ for my $method ( } } -1; \ No newline at end of file +1; diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/RecordIO.pm b/perl-package/AI-MXNet/lib/AI/MXNet/RecordIO.pm index f0833bf77e5e..2027a901ec10 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/RecordIO.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/RecordIO.pm @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + package AI::MXNet::RecordIO; use strict; use warnings; @@ -24,7 +41,7 @@ use Mouse; has 'uri' => (is => 'ro', isa => 'Str', required => 1); has 'flag' => (is => 'ro', isa => enum([qw/r w/]), required => 1); has 'handle' => (is => 'rw', isa => 'RecordIOHandle'); -has [qw/writable +has [qw/writable is_open/] => (is => 'rw', isa => 'Bool'); sub BUILD @@ -336,4 +353,4 @@ method write_idx(Int $idx, Str $buf) push @{ $self->keys }, $idx; } -1; \ No newline at end of file +1; diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Rtc.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Rtc.pm index 92edcaf2b8c0..09dc66200322 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Rtc.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Rtc.pm @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + package AI::MXNet::Rtc; use strict; use warnings; @@ -124,4 +141,4 @@ method push( ); } -1; \ No newline at end of file +1; diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Symbol.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Symbol.pm index 8b14f4e2b1d3..a5298c7bc3af 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Symbol.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Symbol.pm @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + package AI::MXNet::Symbol; =head1 NAME @@ -340,7 +357,7 @@ method attr_dict() method _set_attr(Str @args) { - my %kwargs = @args; + my %kwargs = @args; while(my ($key, $val) = each(%kwargs)) { check_call( @@ -492,7 +509,7 @@ method list_inputs() method infer_type(Str|Undef @args) { - my ($positional_arguments, $kwargs, $kwargs_order) = _parse_arguments("Dtype", @args); + my ($positional_arguments, $kwargs, $kwargs_order) = _parse_arguments("Dtype", @args); my $sdata = []; my $keys = []; if(@$positional_arguments) @@ -710,7 +727,7 @@ method _get_ndarray_inputs( my ($arg_handles, $arg_arrays) = ([], []); if(ref $args eq 'ARRAY') { - confess("Length of $arg_key do not match number of arguments") + confess("Length of $arg_key do not match number of arguments") unless @$args == @$arg_names; @{ $arg_handles } = map { $_->handle } @{ $args }; $arg_arrays = $args; diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Symbol/AttrScope.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Symbol/AttrScope.pm index 017168439d7b..c728ed1b6ce8 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Symbol/AttrScope.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Symbol/AttrScope.pm @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + package AI::MXNet::Symbol::AttrScope; use strict; use warnings; diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Symbol/Base.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Symbol/Base.pm index 69ff952eca1a..4282f124a34b 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Symbol/Base.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Symbol/Base.pm @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + package AI::MXNet::Symbol::Base; use strict; use warnings; @@ -68,7 +85,7 @@ sub _compose # Create an atomic symbol function by handle and funciton name func _make_atomic_symbol_function($handle, $name) { - my ($real_name, $desc, $arg_names, + my ($real_name, $desc, $arg_names, $arg_types, $arg_descs, $key_var_num_args, $ret_type) = @{ check_call(AI::MXNetCAPI::SymbolGetAtomicSymbolInfo($handle)) }; $ret_type //= ''; @@ -76,7 +93,7 @@ func _make_atomic_symbol_function($handle, $name) my $doc_str = build_doc($func_name, $desc, $arg_names, - $arg_types, + $arg_types, $arg_descs, $key_var_num_args, $ret_type @@ -162,7 +179,7 @@ method _init_symbol_module() no strict 'refs'; { *{__PACKAGE__."::$name"} = $function; - } + } } } } diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Symbol/Doc.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Symbol/Doc.pm index 2485f21fe3a0..1d9a2c1288ea 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Symbol/Doc.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Symbol/Doc.pm @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + package AI::MXNet::Symbol::Doc; use strict; use warnings; diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Symbol/NameManager.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Symbol/NameManager.pm index 1e31730692a7..109949c79078 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Symbol/NameManager.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Symbol/NameManager.pm @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + package AI::MXNet::Symbol::NameManager; use strict; use warnings; diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/TestUtils.pm b/perl-package/AI-MXNet/lib/AI/MXNet/TestUtils.pm index 52050fa1ebdd..ea918c0cddf3 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/TestUtils.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/TestUtils.pm @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + package AI::MXNet::TestUtils; use strict; use warnings; @@ -399,4 +416,4 @@ func dies_like($code, $regexp) } } -1; \ No newline at end of file +1; diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Types.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Types.pm index 424591eb65a0..e48ae3c086ca 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Types.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Types.pm @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + package AI::MXNet::Types; use strict; use warnings; diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Util/Printable.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Util/Printable.pm index 4e8f8051ae10..1ae6c2d26c96 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Util/Printable.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Util/Printable.pm @@ -1,5 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + package AI::MXNet::Util::Printable; use strict; use warnings; use Data::Dumper qw(); -use overload '""' => sub { print Data::Dumper->new([shift])->Purity(1)->Deepcopy(1)->Terse(1)->Dump }; \ No newline at end of file +use overload '""' => sub { print Data::Dumper->new([shift])->Purity(1)->Deepcopy(1)->Terse(1)->Dump }; diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Visualization.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Visualization.pm index d6ea5aa97bf6..4cdc135c4206 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Visualization.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Visualization.pm @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + package AI::MXNet::Visualization; use strict; use warnings; @@ -37,7 +54,7 @@ use JSON::PP; my $softmax = mx->symbol->SoftmaxOutput(data => $fc2, name => 'softmax'); ## creates the image file working directory - mx->viz->plot_network($softmax, save_format => 'png')->render("network.png"); + mx->viz->plot_network($softmax, save_format => 'png')->render("network.png"); =head1 DESCRIPTION @@ -408,4 +425,4 @@ method render($output=) return $self->graph->$method($output); } -1; \ No newline at end of file +1; diff --git a/perl-package/AI-MXNetCAPI/lib/AI/MXNetCAPI.pm b/perl-package/AI-MXNetCAPI/lib/AI/MXNetCAPI.pm index 48ebe8090e4f..f09205733b1e 100644 --- a/perl-package/AI-MXNetCAPI/lib/AI/MXNetCAPI.pm +++ b/perl-package/AI-MXNetCAPI/lib/AI/MXNetCAPI.pm @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + package AI::MXNetCAPI; use base qw(DynaLoader); bootstrap AI::MXNetCAPI; diff --git a/perl-package/AI-NNVMCAPI/lib/AI/NNVMCAPI.pm b/perl-package/AI-NNVMCAPI/lib/AI/NNVMCAPI.pm index 62d4dd2b09ab..134d922b4d8d 100644 --- a/perl-package/AI-NNVMCAPI/lib/AI/NNVMCAPI.pm +++ b/perl-package/AI-NNVMCAPI/lib/AI/NNVMCAPI.pm @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + package AI::NNVMCAPI; use base qw(DynaLoader); bootstrap AI::NNVMCAPI; diff --git a/perl-package/test.sh b/perl-package/test.sh index 5aef8e6b82b1..c8509c141920 100755 --- a/perl-package/test.sh +++ b/perl-package/test.sh @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + MXNET_HOME=${PWD} export LD_LIBRARY_PATH=${MXNET_HOME}/lib export PERL5LIB=${MXNET_HOME}/perl5/lib/perl5 diff --git a/plugin/caffe/caffe_blob.cc b/plugin/caffe/caffe_blob.cc index c6d5156ffbc4..697efbfa99f2 100644 --- a/plugin/caffe/caffe_blob.cc +++ b/plugin/caffe/caffe_blob.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file caffe_blob.cc * \brief Implementations of SetDataGradToBlob given various device/dimension * \author Haoran Wang diff --git a/plugin/caffe/caffe_blob.h b/plugin/caffe/caffe_blob.h index 3037031ad991..666d269fdae1 100644 --- a/plugin/caffe/caffe_blob.h +++ b/plugin/caffe/caffe_blob.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file caffe_blob.h * \brief conversion between tensor and caffeBlob * \author Haoran Wang diff --git a/plugin/caffe/caffe_common.cc b/plugin/caffe/caffe_common.cc index 722b19138f79..53513a17d6c8 100644 --- a/plugin/caffe/caffe_common.cc +++ b/plugin/caffe/caffe_common.cc @@ -1,8 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file caffe_common.h * \brief Common functions for caffeOp and caffeLoss symbols - * \author Haoran Wang + * \author Haoran Wang */ #include #include diff --git a/plugin/caffe/caffe_common.h b/plugin/caffe/caffe_common.h index 6ee3c26202b9..8565d9e2e27c 100644 --- a/plugin/caffe/caffe_common.h +++ b/plugin/caffe/caffe_common.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file caffe_common.h * \brief Common functions for caffeOp and caffeLoss symbols * \author Haoran Wang diff --git a/plugin/caffe/caffe_data_iter.cc b/plugin/caffe/caffe_data_iter.cc index ecf776270a91..2682298b4b3d 100644 --- a/plugin/caffe/caffe_data_iter.cc +++ b/plugin/caffe/caffe_data_iter.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file caffe_data_iter.cc * \brief register mnist iterator */ diff --git a/plugin/caffe/caffe_fieldentry.h b/plugin/caffe/caffe_fieldentry.h index a020cf9d7e77..47d246f4439f 100644 --- a/plugin/caffe/caffe_fieldentry.h +++ b/plugin/caffe/caffe_fieldentry.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file caffe_fieldentry.h * \brief Implement FieldEntry * \author Haoran Wang diff --git a/plugin/caffe/caffe_loss-inl.h b/plugin/caffe/caffe_loss-inl.h index 038ee1458bcd..37bfcf06be4a 100644 --- a/plugin/caffe/caffe_loss-inl.h +++ b/plugin/caffe/caffe_loss-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file caffe_loss-inl.h * \brief Caffe Operator * \author Haoran Wang diff --git a/plugin/caffe/caffe_loss.cc b/plugin/caffe/caffe_loss.cc index a51f12602991..ce697d6c8ff9 100644 --- a/plugin/caffe/caffe_loss.cc +++ b/plugin/caffe/caffe_loss.cc @@ -1,8 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file caffe_loss.cc - * \brief caffe loss - * \author Haoran Wang + * \brief caffe loss + * \author Haoran Wang */ #include "./caffe_loss-inl.h" diff --git a/plugin/caffe/caffe_loss.cu b/plugin/caffe/caffe_loss.cu index 55489cafc316..2002cf2c661d 100644 --- a/plugin/caffe/caffe_loss.cu +++ b/plugin/caffe/caffe_loss.cu @@ -1,8 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file caffe_loss_gpu.cc - * \brief caffe loss - * \author Haoran Wang + * \brief caffe loss + * \author Haoran Wang */ #include "./caffe_loss-inl.h" diff --git a/plugin/caffe/caffe_op-inl.h b/plugin/caffe/caffe_op-inl.h index 1950865b76c3..43b9b5a091af 100644 --- a/plugin/caffe/caffe_op-inl.h +++ b/plugin/caffe/caffe_op-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file caffe_op-inl.h * \brief Caffe Operator * \author Haoran Wang diff --git a/plugin/caffe/caffe_op.cc b/plugin/caffe/caffe_op.cc index 90cb4da44b0f..5198ccaac7c9 100644 --- a/plugin/caffe/caffe_op.cc +++ b/plugin/caffe/caffe_op.cc @@ -1,8 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file caffe_op.cc * \brief caffe operator - * \author Haoran Wang + * \author Haoran Wang */ #include "./caffe_op-inl.h" namespace mxnet { diff --git a/plugin/caffe/caffe_op.cu b/plugin/caffe/caffe_op.cu index c52f2b69fb63..be6c20a1084f 100644 --- a/plugin/caffe/caffe_op.cu +++ b/plugin/caffe/caffe_op.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file caffe_operator_gpu.cc * \brief caffe operator * \author Haoran Wang diff --git a/plugin/caffe/caffe_stream.cc b/plugin/caffe/caffe_stream.cc index 99202bf9c09c..03badda65ca2 100644 --- a/plugin/caffe/caffe_stream.cc +++ b/plugin/caffe/caffe_stream.cc @@ -1,8 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file caffe_stream.cc * \brief define stream opertors >> and << - * \author Haoran Wang + * \author Haoran Wang */ #include"caffe_stream.h" diff --git a/plugin/caffe/caffe_stream.h b/plugin/caffe/caffe_stream.h index de9edb84feb4..b9a08d028f38 100644 --- a/plugin/caffe/caffe_stream.h +++ b/plugin/caffe/caffe_stream.h @@ -1,8 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file caffe_stream.h * \brief define stream opertors >> and << - * \author Haoran Wang + * \author Haoran Wang */ #ifndef PLUGIN_CAFFE_CAFFE_STREAM_H_ #define PLUGIN_CAFFE_CAFFE_STREAM_H_ diff --git a/plugin/opencv/__init__.py b/plugin/opencv/__init__.py index 072575177e41..bcf6d1ebc969 100644 --- a/plugin/opencv/__init__.py +++ b/plugin/opencv/__init__.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 # pylint: disable=wildcard-import diff --git a/plugin/opencv/cv_api.cc b/plugin/opencv/cv_api.cc index 78bec01548d4..b0bcbbce203e 100644 --- a/plugin/opencv/cv_api.cc +++ b/plugin/opencv/cv_api.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file cv_api.h * \brief C API for opencv * \author Junyuan Xie diff --git a/plugin/opencv/cv_api.h b/plugin/opencv/cv_api.h index fc224d0e1d05..e04357bf30b7 100644 --- a/plugin/opencv/cv_api.h +++ b/plugin/opencv/cv_api.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file cv_api.h * \brief C API for opencv * \author Junyuan Xie diff --git a/plugin/opencv/opencv.py b/plugin/opencv/opencv.py index 43b73b615cc8..52138af00660 100644 --- a/plugin/opencv/opencv.py +++ b/plugin/opencv/opencv.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 # pylint: disable=too-many-arguments,no-member,invalid-name diff --git a/plugin/sframe/iter_sframe.cc b/plugin/sframe/iter_sframe.cc index d91c0ded1d88..2a987e2b10f2 100644 --- a/plugin/sframe/iter_sframe.cc +++ b/plugin/sframe/iter_sframe.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file iter_sframe_image.cc * \brief * \author Bing Xu diff --git a/plugin/torch/torch_base.cc b/plugin/torch/torch_base.cc index af102d672f9e..89f832ccdfae 100644 --- a/plugin/torch/torch_base.cc +++ b/plugin/torch/torch_base.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file torch_base.cc * \brief torch_state * \author Junyuan Xie diff --git a/plugin/torch/torch_base.h b/plugin/torch/torch_base.h index acfefe7ac382..3aaaa2f13902 100644 --- a/plugin/torch/torch_base.h +++ b/plugin/torch/torch_base.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file torch_base.h * \brief Torch interface. * \author Junyuan Xie diff --git a/plugin/torch/torch_criterion-inl.h b/plugin/torch/torch_criterion-inl.h index 174ebf2dd9dd..7f592f156226 100644 --- a/plugin/torch/torch_criterion-inl.h +++ b/plugin/torch/torch_criterion-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file torch_module-inl.h * \brief torch module operator * \author Min Lin diff --git a/plugin/torch/torch_criterion.cc b/plugin/torch/torch_criterion.cc index a54be46a936d..bdfb2f42e61a 100644 --- a/plugin/torch/torch_criterion.cc +++ b/plugin/torch/torch_criterion.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file activation.cc * \brief activation op * \author Junyuan Xie diff --git a/plugin/torch/torch_criterion.cu b/plugin/torch/torch_criterion.cu index 57730a0bd88b..68c519c7c9f1 100644 --- a/plugin/torch/torch_criterion.cu +++ b/plugin/torch/torch_criterion.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file activation.cc * \brief activation op * \author Bing Xu diff --git a/plugin/torch/torch_function.cc b/plugin/torch/torch_function.cc index b47ab56f68d2..a1c5ff578da7 100644 --- a/plugin/torch/torch_function.cc +++ b/plugin/torch/torch_function.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file torch_base.cc * \brief torch_state * \author Junyuan Xie diff --git a/plugin/torch/torch_function.h b/plugin/torch/torch_function.h index 0151d5aa9925..8fb2ccfde454 100644 --- a/plugin/torch/torch_function.h +++ b/plugin/torch/torch_function.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file torch_function.h * \brief Torch interface. * \author Junyuan Xie diff --git a/plugin/torch/torch_module-inl.h b/plugin/torch/torch_module-inl.h index 31138fdcf16b..15b569fbbeef 100644 --- a/plugin/torch/torch_module-inl.h +++ b/plugin/torch/torch_module-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file torch_module-inl.h * \brief torch module operator * \author Min Lin diff --git a/plugin/torch/torch_module.cc b/plugin/torch/torch_module.cc index 81dc481c6632..658669fb419c 100644 --- a/plugin/torch/torch_module.cc +++ b/plugin/torch/torch_module.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file activation.cc * \brief activation op * \author Bing Xu diff --git a/plugin/torch/torch_module.cu b/plugin/torch/torch_module.cu index a298a23ae37e..caf9eb19911a 100644 --- a/plugin/torch/torch_module.cu +++ b/plugin/torch/torch_module.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file activation.cc * \brief activation op * \author Bing Xu diff --git a/plugin/warpctc/warpctc-inl.h b/plugin/warpctc/warpctc-inl.h index 328c08b2db02..d492656b2f15 100644 --- a/plugin/warpctc/warpctc-inl.h +++ b/plugin/warpctc/warpctc-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file warpctc-inl.h * \brief warpctc operator * \author Liang Xiang diff --git a/plugin/warpctc/warpctc.cc b/plugin/warpctc/warpctc.cc index db88a3316c7e..0ff61be758c7 100644 --- a/plugin/warpctc/warpctc.cc +++ b/plugin/warpctc/warpctc.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file warpctc.cc * \brief warpctc op * \author Liang Xiang diff --git a/plugin/warpctc/warpctc.cu b/plugin/warpctc/warpctc.cu index 186c4d0c18f4..7562a12a3c9d 100644 --- a/plugin/warpctc/warpctc.cu +++ b/plugin/warpctc/warpctc.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file warpctc.cc * \brief warpctc op * \author Liang Xiang diff --git a/prepare_mkl.sh b/prepare_mkl.sh index a320c44320f0..9769731f5396 100755 --- a/prepare_mkl.sh +++ b/prepare_mkl.sh @@ -1,4 +1,22 @@ #!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # set -ex # # All modification made by Intel Corporation: © 2016 Intel Corporation diff --git a/python/mxnet/__init__.py b/python/mxnet/__init__.py index d878f9bb0594..3c3ce76a9284 100644 --- a/python/mxnet/__init__.py +++ b/python/mxnet/__init__.py @@ -1,4 +1,22 @@ #!/usr/bin/env python + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 """MXNet: a concise, fast and flexible framework for deep learning.""" from __future__ import absolute_import diff --git a/python/mxnet/_ctypes/__init__.py b/python/mxnet/_ctypes/__init__.py index 2708cc5c1367..a9433ed06670 100644 --- a/python/mxnet/_ctypes/__init__.py +++ b/python/mxnet/_ctypes/__init__.py @@ -1 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + "ctypes module" diff --git a/python/mxnet/_ctypes/ndarray.py b/python/mxnet/_ctypes/ndarray.py index 396c57a41dfb..5a50f80498ec 100644 --- a/python/mxnet/_ctypes/ndarray.py +++ b/python/mxnet/_ctypes/ndarray.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 # pylint: disable=invalid-name, protected-access, too-many-arguments # pylint: disable=global-statement, unused-import diff --git a/python/mxnet/_ctypes/symbol.py b/python/mxnet/_ctypes/symbol.py index 5cbff551cf55..3ec2ddcdc548 100644 --- a/python/mxnet/_ctypes/symbol.py +++ b/python/mxnet/_ctypes/symbol.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 # pylint: disable=invalid-name, protected-access, too-many-arguments, global-statement """Symbolic configuration API.""" diff --git a/python/mxnet/_cy2/__init__.py b/python/mxnet/_cy2/__init__.py index 910cbe2e586b..1961cd9ff613 100644 --- a/python/mxnet/_cy2/__init__.py +++ b/python/mxnet/_cy2/__init__.py @@ -1 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Namespace for cython generated modules for python2""" diff --git a/python/mxnet/_cy3/__init__.py b/python/mxnet/_cy3/__init__.py index e89f266a0535..44dcca5ac003 100644 --- a/python/mxnet/_cy3/__init__.py +++ b/python/mxnet/_cy3/__init__.py @@ -1 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Namespace for cython generated modules for python3""" diff --git a/python/mxnet/_ndarray_internal.py b/python/mxnet/_ndarray_internal.py index 52ec16df4a8a..8f151f1b5b64 100644 --- a/python/mxnet/_ndarray_internal.py +++ b/python/mxnet/_ndarray_internal.py @@ -1 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """NDArray namespace used to register internal functions.""" diff --git a/python/mxnet/_symbol_internal.py b/python/mxnet/_symbol_internal.py index 58a8e4b2658a..cd6ae41c2a19 100644 --- a/python/mxnet/_symbol_internal.py +++ b/python/mxnet/_symbol_internal.py @@ -1 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Symbol namespace used to register internal functions.""" diff --git a/python/mxnet/attribute.py b/python/mxnet/attribute.py index b8604024efa4..15d38f81f2e3 100644 --- a/python/mxnet/attribute.py +++ b/python/mxnet/attribute.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 """Attribute scoping support for symbolic API.""" from __future__ import absolute_import diff --git a/python/mxnet/autograd.py b/python/mxnet/autograd.py index 2c3feabd525c..7340851cdef6 100644 --- a/python/mxnet/autograd.py +++ b/python/mxnet/autograd.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 """Autograd for NDArray.""" from __future__ import absolute_import diff --git a/python/mxnet/base.py b/python/mxnet/base.py index ddaeb6e77d54..7d5a5bf8f889 100644 --- a/python/mxnet/base.py +++ b/python/mxnet/base.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 # pylint: disable=invalid-name, no-member """ctypes library of mxnet and helper functions.""" diff --git a/python/mxnet/callback.py b/python/mxnet/callback.py index 12146009fe19..8c9f64a95144 100644 --- a/python/mxnet/callback.py +++ b/python/mxnet/callback.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 """Callback functions that can be used to track various status during epoch.""" from __future__ import absolute_import diff --git a/python/mxnet/context.py b/python/mxnet/context.py index 9822a6d86708..9798b480d235 100644 --- a/python/mxnet/context.py +++ b/python/mxnet/context.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 """Context management API of mxnet.""" from __future__ import absolute_import diff --git a/python/mxnet/contrib/__init__.py b/python/mxnet/contrib/__init__.py index c46fa2a24c7f..2730bc43863d 100644 --- a/python/mxnet/contrib/__init__.py +++ b/python/mxnet/contrib/__init__.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 """Experimental contributions""" diff --git a/python/mxnet/contrib/autograd.py b/python/mxnet/contrib/autograd.py index 9074e452c981..c7fb6e17803a 100644 --- a/python/mxnet/contrib/autograd.py +++ b/python/mxnet/contrib/autograd.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 """Autograd for NDArray.""" from __future__ import absolute_import diff --git a/python/mxnet/contrib/ndarray.py b/python/mxnet/contrib/ndarray.py index cf1815c99434..3c86fe7ba3fb 100644 --- a/python/mxnet/contrib/ndarray.py +++ b/python/mxnet/contrib/ndarray.py @@ -1,2 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 """NDArray namespace used to register contrib functions""" diff --git a/python/mxnet/contrib/symbol.py b/python/mxnet/contrib/symbol.py index 81c5ce889331..1d5334595f27 100644 --- a/python/mxnet/contrib/symbol.py +++ b/python/mxnet/contrib/symbol.py @@ -1,2 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 """Symbol namespace used to register contrib functions""" diff --git a/python/mxnet/contrib/tensorboard.py b/python/mxnet/contrib/tensorboard.py index 40e3104409a5..2bb766e7d69b 100644 --- a/python/mxnet/contrib/tensorboard.py +++ b/python/mxnet/contrib/tensorboard.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 """TensorBoard functions that can be used to log various status during epoch.""" from __future__ import absolute_import diff --git a/python/mxnet/executor.py b/python/mxnet/executor.py index d2b108cc04ed..baff834bb33a 100644 --- a/python/mxnet/executor.py +++ b/python/mxnet/executor.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 # pylint: disable=invalid-name, protected-access, too-many-locals, too-many-arguments """Symbolic Executor component of MXNet.""" diff --git a/python/mxnet/executor_manager.py b/python/mxnet/executor_manager.py index 0fb9eb3baade..33c6c9762717 100644 --- a/python/mxnet/executor_manager.py +++ b/python/mxnet/executor_manager.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 # pylint: disable=invalid-name, protected-access, too-many-locals, too-many-arguments, too-many-statements """Executor manager.""" diff --git a/python/mxnet/gluon/__init__.py b/python/mxnet/gluon/__init__.py index e3c341e97f02..089340efcd2c 100644 --- a/python/mxnet/gluon/__init__.py +++ b/python/mxnet/gluon/__init__.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 # pylint: disable=wildcard-import """Neural network module.""" diff --git a/python/mxnet/gluon/block.py b/python/mxnet/gluon/block.py index cfc5e5744338..74a9058e98e0 100644 --- a/python/mxnet/gluon/block.py +++ b/python/mxnet/gluon/block.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 # pylint: disable= arguments-differ """Base container class for all neural network models.""" diff --git a/python/mxnet/gluon/data/__init__.py b/python/mxnet/gluon/data/__init__.py index a0623257417c..23ae3e9b3be6 100644 --- a/python/mxnet/gluon/data/__init__.py +++ b/python/mxnet/gluon/data/__init__.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 # pylint: disable=wildcard-import """Dataset utilities.""" diff --git a/python/mxnet/gluon/data/dataloader.py b/python/mxnet/gluon/data/dataloader.py index b251deb9bb56..6497c7eb9be9 100644 --- a/python/mxnet/gluon/data/dataloader.py +++ b/python/mxnet/gluon/data/dataloader.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 # pylint: disable= """Dataset generator.""" diff --git a/python/mxnet/gluon/data/dataset.py b/python/mxnet/gluon/data/dataset.py index aefff0af16c9..f3dd691962bc 100644 --- a/python/mxnet/gluon/data/dataset.py +++ b/python/mxnet/gluon/data/dataset.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 # pylint: disable= """Dataset container.""" diff --git a/python/mxnet/gluon/data/sampler.py b/python/mxnet/gluon/data/sampler.py index f6cedf051727..80f115e0333f 100644 --- a/python/mxnet/gluon/data/sampler.py +++ b/python/mxnet/gluon/data/sampler.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 # pylint: disable= """Dataset sampler.""" diff --git a/python/mxnet/gluon/data/vision.py b/python/mxnet/gluon/data/vision.py index 36c4642e7665..4ddbbbdf48a2 100644 --- a/python/mxnet/gluon/data/vision.py +++ b/python/mxnet/gluon/data/vision.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 # pylint: disable= """Dataset container.""" diff --git a/python/mxnet/gluon/loss.py b/python/mxnet/gluon/loss.py index 38fdcb820919..2b31840ad959 100644 --- a/python/mxnet/gluon/loss.py +++ b/python/mxnet/gluon/loss.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 # pylint: disable=arguments-differ """ losses for training neural networks """ diff --git a/python/mxnet/gluon/model_zoo/__init__.py b/python/mxnet/gluon/model_zoo/__init__.py index aa5f148513bd..b8c32af38561 100644 --- a/python/mxnet/gluon/model_zoo/__init__.py +++ b/python/mxnet/gluon/model_zoo/__init__.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 """Predefined and pretrained models.""" diff --git a/python/mxnet/gluon/model_zoo/custom_layers.py b/python/mxnet/gluon/model_zoo/custom_layers.py index da1ca868dd76..cf91876888ee 100644 --- a/python/mxnet/gluon/model_zoo/custom_layers.py +++ b/python/mxnet/gluon/model_zoo/custom_layers.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 # pylint: disable= arguments-differ """Custom neural network layers in model_zoo.""" diff --git a/python/mxnet/gluon/model_zoo/model_store.py b/python/mxnet/gluon/model_zoo/model_store.py index 6a11626795f9..e3c48ba2235a 100644 --- a/python/mxnet/gluon/model_zoo/model_store.py +++ b/python/mxnet/gluon/model_zoo/model_store.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 """Model zoo for pre-trained models.""" from __future__ import print_function diff --git a/python/mxnet/gluon/model_zoo/vision/__init__.py b/python/mxnet/gluon/model_zoo/vision/__init__.py index 56e46f9a0c74..e4016db2ea20 100644 --- a/python/mxnet/gluon/model_zoo/vision/__init__.py +++ b/python/mxnet/gluon/model_zoo/vision/__init__.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 # pylint: disable=wildcard-import, arguments-differ r"""Module for pre-defined neural network models. diff --git a/python/mxnet/gluon/model_zoo/vision/alexnet.py b/python/mxnet/gluon/model_zoo/vision/alexnet.py index 86ff9324baca..4d5bc8c85b67 100644 --- a/python/mxnet/gluon/model_zoo/vision/alexnet.py +++ b/python/mxnet/gluon/model_zoo/vision/alexnet.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 # pylint: disable= arguments-differ """Alexnet, implemented in Gluon.""" diff --git a/python/mxnet/gluon/model_zoo/vision/densenet.py b/python/mxnet/gluon/model_zoo/vision/densenet.py index 9e1ff7799892..57dbe5d188a7 100644 --- a/python/mxnet/gluon/model_zoo/vision/densenet.py +++ b/python/mxnet/gluon/model_zoo/vision/densenet.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 # pylint: disable= arguments-differ """DenseNet, implemented in Gluon.""" diff --git a/python/mxnet/gluon/model_zoo/vision/inception.py b/python/mxnet/gluon/model_zoo/vision/inception.py index 8a28666d56cb..1afd3e346113 100644 --- a/python/mxnet/gluon/model_zoo/vision/inception.py +++ b/python/mxnet/gluon/model_zoo/vision/inception.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 # pylint: disable= arguments-differ """Inception, implemented in Gluon.""" diff --git a/python/mxnet/gluon/model_zoo/vision/resnet.py b/python/mxnet/gluon/model_zoo/vision/resnet.py index 48ba07941acb..78bc726f41d9 100644 --- a/python/mxnet/gluon/model_zoo/vision/resnet.py +++ b/python/mxnet/gluon/model_zoo/vision/resnet.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 # pylint: disable= arguments-differ """ResNets, implemented in Gluon.""" diff --git a/python/mxnet/gluon/model_zoo/vision/squeezenet.py b/python/mxnet/gluon/model_zoo/vision/squeezenet.py index bfcb5cbc9bae..1a14201c7998 100644 --- a/python/mxnet/gluon/model_zoo/vision/squeezenet.py +++ b/python/mxnet/gluon/model_zoo/vision/squeezenet.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 # pylint: disable= arguments-differ """SqueezeNet, implemented in Gluon.""" diff --git a/python/mxnet/gluon/model_zoo/vision/vgg.py b/python/mxnet/gluon/model_zoo/vision/vgg.py index 96a4fa1f3700..2f4daf9f6437 100644 --- a/python/mxnet/gluon/model_zoo/vision/vgg.py +++ b/python/mxnet/gluon/model_zoo/vision/vgg.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 # pylint: disable= arguments-differ """VGG, implemented in Gluon.""" diff --git a/python/mxnet/gluon/nn/__init__.py b/python/mxnet/gluon/nn/__init__.py index e4191b2a7dc2..0fc1ff12dd13 100644 --- a/python/mxnet/gluon/nn/__init__.py +++ b/python/mxnet/gluon/nn/__init__.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 # pylint: disable=wildcard-import """Neural network layers.""" diff --git a/python/mxnet/gluon/nn/basic_layers.py b/python/mxnet/gluon/nn/basic_layers.py index afe2df7b5f4c..63411caf2be7 100644 --- a/python/mxnet/gluon/nn/basic_layers.py +++ b/python/mxnet/gluon/nn/basic_layers.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 # pylint: disable= arguments-differ """Basic neural network layers.""" diff --git a/python/mxnet/gluon/nn/conv_layers.py b/python/mxnet/gluon/nn/conv_layers.py index 9094eddee3f3..e49340d27a4e 100644 --- a/python/mxnet/gluon/nn/conv_layers.py +++ b/python/mxnet/gluon/nn/conv_layers.py @@ -1,5 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 -# pylint: disable= arguments-differ +# pylint: disable= arguments-differ, too-many-lines """Convolutional neural network layers.""" from ..block import HybridBlock from ... import symbol diff --git a/python/mxnet/gluon/parameter.py b/python/mxnet/gluon/parameter.py index bdc967490dc5..69f60388fe25 100644 --- a/python/mxnet/gluon/parameter.py +++ b/python/mxnet/gluon/parameter.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 # pylint: disable= """Neural network parameter.""" diff --git a/python/mxnet/gluon/rnn/__init__.py b/python/mxnet/gluon/rnn/__init__.py index b4554ad884cb..24cce542274c 100644 --- a/python/mxnet/gluon/rnn/__init__.py +++ b/python/mxnet/gluon/rnn/__init__.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 # pylint: disable=wildcard-import """Recurrent neural network module.""" diff --git a/python/mxnet/gluon/rnn/rnn_cell.py b/python/mxnet/gluon/rnn/rnn_cell.py index 87c656c3020f..c9186fd3ce09 100644 --- a/python/mxnet/gluon/rnn/rnn_cell.py +++ b/python/mxnet/gluon/rnn/rnn_cell.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 # pylint: disable=no-member, invalid-name, protected-access, no-self-use # pylint: disable=too-many-branches, too-many-arguments, no-self-use diff --git a/python/mxnet/gluon/rnn/rnn_layer.py b/python/mxnet/gluon/rnn/rnn_layer.py index deb6898be55c..a9bcee52a6d3 100644 --- a/python/mxnet/gluon/rnn/rnn_layer.py +++ b/python/mxnet/gluon/rnn/rnn_layer.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 # pylint: disable=no-member, invalid-name, protected-access, no-self-use # pylint: disable=too-many-branches, too-many-arguments, no-self-use diff --git a/python/mxnet/gluon/trainer.py b/python/mxnet/gluon/trainer.py index e8aae71cfd99..bb2cc763b5ba 100644 --- a/python/mxnet/gluon/trainer.py +++ b/python/mxnet/gluon/trainer.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 # pylint: disable= """Parameter optimizer.""" diff --git a/python/mxnet/gluon/utils.py b/python/mxnet/gluon/utils.py index 505fbc55248c..7d9c378fe76d 100644 --- a/python/mxnet/gluon/utils.py +++ b/python/mxnet/gluon/utils.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 # pylint: disable= """Parallelization utility optimizer.""" diff --git a/python/mxnet/image/__init__.py b/python/mxnet/image/__init__.py index d484006ad7ce..9bb55fbfdddc 100644 --- a/python/mxnet/image/__init__.py +++ b/python/mxnet/image/__init__.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 # pylint: disable=wildcard-import """Image Iterators and image augmentation functions""" diff --git a/python/mxnet/image/detection.py b/python/mxnet/image/detection.py index 0a16ac36fc98..142ba25cc564 100644 --- a/python/mxnet/image/detection.py +++ b/python/mxnet/image/detection.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: disable=unused-import """Read images and perform augmentations for object detection.""" diff --git a/python/mxnet/image/image.py b/python/mxnet/image/image.py index cf17ecf13ffb..02cd3cd0d551 100644 --- a/python/mxnet/image/image.py +++ b/python/mxnet/image/image.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: disable=no-member, too-many-lines, redefined-builtin, protected-access, unused-import, invalid-name # pylint: disable=too-many-arguments, too-many-locals, no-name-in-module, too-many-branches, too-many-statements """Read individual image files and perform augmentations.""" diff --git a/python/mxnet/initializer.py b/python/mxnet/initializer.py index a5962b664e6b..78afa2dbd29a 100755 --- a/python/mxnet/initializer.py +++ b/python/mxnet/initializer.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Weight initializer.""" from __future__ import absolute_import, print_function diff --git a/python/mxnet/io.py b/python/mxnet/io.py index bb791cef035e..0404e34ea36c 100644 --- a/python/mxnet/io.py +++ b/python/mxnet/io.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Data iterators for common data formats.""" from __future__ import absolute_import from collections import OrderedDict, namedtuple diff --git a/python/mxnet/kvstore.py b/python/mxnet/kvstore.py index 10b83b04db97..fd0091182aea 100644 --- a/python/mxnet/kvstore.py +++ b/python/mxnet/kvstore.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 """ Key value store interface of MXNet for parameter synchronization.""" from __future__ import absolute_import diff --git a/python/mxnet/kvstore_server.py b/python/mxnet/kvstore_server.py index c6d0b073f824..1bb995a45ca8 100644 --- a/python/mxnet/kvstore_server.py +++ b/python/mxnet/kvstore_server.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 """A server node for the key value store.""" from __future__ import absolute_import diff --git a/python/mxnet/libinfo.py b/python/mxnet/libinfo.py index a24756632c10..d6521c558ac4 100644 --- a/python/mxnet/libinfo.py +++ b/python/mxnet/libinfo.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 """Information about mxnet.""" from __future__ import absolute_import diff --git a/python/mxnet/log.py b/python/mxnet/log.py index 46d97f6c390c..6dcaedbe6fe7 100644 --- a/python/mxnet/log.py +++ b/python/mxnet/log.py @@ -1,4 +1,22 @@ #!/usr/bin/env python + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # -*- coding: utf-8 -*- # pylint: disable= protected-access, invalid-name """Logging utilities.""" diff --git a/python/mxnet/lr_scheduler.py b/python/mxnet/lr_scheduler.py index 9998fc8769a1..e4af77aa8690 100644 --- a/python/mxnet/lr_scheduler.py +++ b/python/mxnet/lr_scheduler.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Scheduling learning rate.""" import logging diff --git a/python/mxnet/metric.py b/python/mxnet/metric.py index 2fe38ab751e0..1a8e67da5396 100644 --- a/python/mxnet/metric.py +++ b/python/mxnet/metric.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 # pylint: disable=no-member, too-many-lines diff --git a/python/mxnet/misc.py b/python/mxnet/misc.py index b158981ecf97..13b7dc23b5f6 100644 --- a/python/mxnet/misc.py +++ b/python/mxnet/misc.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: disable=invalid-name """Learning rate scheduler.""" diff --git a/python/mxnet/model.py b/python/mxnet/model.py index 5780ac1b9f9d..01b3fa50e18f 100644 --- a/python/mxnet/model.py +++ b/python/mxnet/model.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: disable=fixme, invalid-name, too-many-arguments, too-many-locals, too-many-lines # pylint: disable=too-many-branches, too-many-statements """MXNet model module""" diff --git a/python/mxnet/module/__init__.py b/python/mxnet/module/__init__.py index 916413782a17..32ecbb9c8be3 100644 --- a/python/mxnet/module/__init__.py +++ b/python/mxnet/module/__init__.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """A module is like a FeedForward model. But we would like to make it easier to compose, similar to Torch modules. """ diff --git a/python/mxnet/module/base_module.py b/python/mxnet/module/base_module.py index cacce25ff083..3123462f9c7c 100644 --- a/python/mxnet/module/base_module.py +++ b/python/mxnet/module/base_module.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: disable=fixme, too-many-arguments, too-many-locals, too-many-public-methods, too-many-branches """`BaseModule` defines an API for modules.""" diff --git a/python/mxnet/module/bucketing_module.py b/python/mxnet/module/bucketing_module.py index 7a1be96d093c..f3c7ecbddc05 100644 --- a/python/mxnet/module/bucketing_module.py +++ b/python/mxnet/module/bucketing_module.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: disable=too-many-instance-attributes, too-many-arguments, protected-access # pylint: disable=too-many-public-methods """A `BucketingModule` implement the `BaseModule` API, and allows multiple diff --git a/python/mxnet/module/executor_group.py b/python/mxnet/module/executor_group.py index 169e81ee326e..0f3c079f8fcb 100755 --- a/python/mxnet/module/executor_group.py +++ b/python/mxnet/module/executor_group.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: disable=too-many-instance-attributes,too-many-locals # pylint: disable=too-many-branches,too-many-statements,too-many-arguments """Executor group is a convenient tool for managing a group of executors.""" diff --git a/python/mxnet/module/module.py b/python/mxnet/module/module.py index b31ea0ffa319..058edd57eb3d 100644 --- a/python/mxnet/module/module.py +++ b/python/mxnet/module/module.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: disable=too-many-instance-attributes, too-many-arguments, protected-access, too-many-branches # pylint: disable=too-many-public-methods """A `Module` implement the `BaseModule` API by wrapping a `Symbol` and one or diff --git a/python/mxnet/module/python_module.py b/python/mxnet/module/python_module.py index af7c76e646a1..2d4343c80c72 100644 --- a/python/mxnet/module/python_module.py +++ b/python/mxnet/module/python_module.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: disable=too-many-instance-attributes, too-many-arguments """Provide some handy classes for user to implement a simple computation module in Python easily. diff --git a/python/mxnet/module/sequential_module.py b/python/mxnet/module/sequential_module.py index b2644c8e3864..642a398c08d4 100644 --- a/python/mxnet/module/sequential_module.py +++ b/python/mxnet/module/sequential_module.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: disable=too-many-arguments, too-many-locals, too-many-instance-attributes """`SequentialModule` is a container module that chains a number of modules together.""" diff --git a/python/mxnet/monitor.py b/python/mxnet/monitor.py index 15be41d585a8..e3185a1281af 100644 --- a/python/mxnet/monitor.py +++ b/python/mxnet/monitor.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 # pylint: disable=protected-access, logging-format-interpolation, invalid-name, no-member, too-many-branches """Monitor outputs, weights, and gradients for debugging.""" diff --git a/python/mxnet/name.py b/python/mxnet/name.py index 8003073f4d8a..966d38280ef7 100644 --- a/python/mxnet/name.py +++ b/python/mxnet/name.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 """Automatic naming support for symbolic API.""" from __future__ import absolute_import diff --git a/python/mxnet/ndarray.py b/python/mxnet/ndarray.py index d4a0cdbd8d51..42f0ff5e87cf 100644 --- a/python/mxnet/ndarray.py +++ b/python/mxnet/ndarray.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 # pylint: disable= too-many-lines, redefined-builtin, protected-access # pylint: disable=import-error, no-name-in-module, undefined-variable @@ -764,7 +781,7 @@ def shape(self): def size(self): """Number of elements in the array. - Equivalent to the product of the array’s dimensions. + Equivalent to the product of the array's dimensions. Examples -------- @@ -803,7 +820,7 @@ def context(self): @property def dtype(self): - """Data-type of the array’s elements. + """Data-type of the array's elements. Returns ------- diff --git a/python/mxnet/ndarray_doc.py b/python/mxnet/ndarray_doc.py index 9cc4545b9fe3..0c51036d8208 100644 --- a/python/mxnet/ndarray_doc.py +++ b/python/mxnet/ndarray_doc.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 # pylint: disable=unused-argument, too-many-arguments """Extra symbol documents""" diff --git a/python/mxnet/notebook/__init__.py b/python/mxnet/notebook/__init__.py index 71a30e3f037d..d605d7483330 100644 --- a/python/mxnet/notebook/__init__.py +++ b/python/mxnet/notebook/__init__.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: disable=invalid-name, missing-docstring, no-init, old-style-class, multiple-statements """MXNet notebook: an easy to use visualization platform""" diff --git a/python/mxnet/notebook/callback.py b/python/mxnet/notebook/callback.py index e9c6e97675aa..56321b715b40 100644 --- a/python/mxnet/notebook/callback.py +++ b/python/mxnet/notebook/callback.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: disable=fixme, invalid-name, missing-docstring, no-init, old-style-class, multiple-statements # pylint: disable=arguments-differ, too-many-arguments, no-member """Visualization callback function diff --git a/python/mxnet/operator.py b/python/mxnet/operator.py index 8274838a1f83..692c7fe827ee 100644 --- a/python/mxnet/operator.py +++ b/python/mxnet/operator.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 # pylint: disable=invalid-name, protected-access, too-many-arguments, no-self-use, too-many-locals, broad-except """numpy interface for operators.""" diff --git a/python/mxnet/optimizer.py b/python/mxnet/optimizer.py index 934566ec9d17..1ef9cc845036 100644 --- a/python/mxnet/optimizer.py +++ b/python/mxnet/optimizer.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Weight updating functions.""" import math import pickle diff --git a/python/mxnet/profiler.py b/python/mxnet/profiler.py index 0d4b8fb979ba..7356ed0fb8d0 100644 --- a/python/mxnet/profiler.py +++ b/python/mxnet/profiler.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 # pylint: disable=fixme, invalid-name, too-many-arguments, too-many-locals, too-many-lines # pylint: disable=too-many-branches, too-many-statements diff --git a/python/mxnet/random.py b/python/mxnet/random.py index 91c2f5035ffa..29b250d980ce 100644 --- a/python/mxnet/random.py +++ b/python/mxnet/random.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 # pylint: disable=no-member, protected-access, unused-import, no-name-in-module """Random number interface of MXNet.""" diff --git a/python/mxnet/recordio.py b/python/mxnet/recordio.py index 6661ac545b82..39f442b6aefc 100644 --- a/python/mxnet/recordio.py +++ b/python/mxnet/recordio.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Read and write for the RecordIO data format.""" from __future__ import absolute_import from collections import namedtuple diff --git a/python/mxnet/registry.py b/python/mxnet/registry.py index fdd095e1ebb5..4a4f22fa142b 100644 --- a/python/mxnet/registry.py +++ b/python/mxnet/registry.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 # pylint: disable=no-member diff --git a/python/mxnet/rnn/__init__.py b/python/mxnet/rnn/__init__.py index 99b0a2da0329..dbf382ecc411 100644 --- a/python/mxnet/rnn/__init__.py +++ b/python/mxnet/rnn/__init__.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 # pylint: disable=wildcard-import """Recurrent neural network module.""" diff --git a/python/mxnet/rnn/io.py b/python/mxnet/rnn/io.py index 8cfce9647374..ab51b09c5710 100644 --- a/python/mxnet/rnn/io.py +++ b/python/mxnet/rnn/io.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 # pylint: disable=too-many-arguments, too-many-locals """Definition of various recurrent neural network cells.""" diff --git a/python/mxnet/rnn/rnn.py b/python/mxnet/rnn/rnn.py index 10343c31bcb1..47307c55b042 100644 --- a/python/mxnet/rnn/rnn.py +++ b/python/mxnet/rnn/rnn.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 # pylint: disable=too-many-arguments, no-member """Functions for constructing recurrent neural networks.""" diff --git a/python/mxnet/rnn/rnn_cell.py b/python/mxnet/rnn/rnn_cell.py index c8213a20f9ef..1c3452041494 100644 --- a/python/mxnet/rnn/rnn_cell.py +++ b/python/mxnet/rnn/rnn_cell.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 # pylint: disable=no-member, invalid-name, protected-access, no-self-use # pylint: disable=too-many-branches, too-many-arguments, no-self-use diff --git a/python/mxnet/rtc.py b/python/mxnet/rtc.py index 759fc3d30042..9da38c6aaaf5 100644 --- a/python/mxnet/rtc.py +++ b/python/mxnet/rtc.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Interface to runtime cuda kernel compile module.""" from __future__ import absolute_import diff --git a/python/mxnet/symbol.py b/python/mxnet/symbol.py index a6c2b0ab689d..14cb3811deeb 100644 --- a/python/mxnet/symbol.py +++ b/python/mxnet/symbol.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 # pylint: disable=invalid-name, protected-access, too-many-arguments, too-many-lines # pylint: disable=import-error, no-name-in-module diff --git a/python/mxnet/symbol_doc.py b/python/mxnet/symbol_doc.py index dff5383e2682..3cb1997584d2 100644 --- a/python/mxnet/symbol_doc.py +++ b/python/mxnet/symbol_doc.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 # pylint: disable=unused-argument, too-many-arguments """Extra symbol documents diff --git a/python/mxnet/test_utils.py b/python/mxnet/test_utils.py index 83c773f53fe2..c5587f8d80a8 100644 --- a/python/mxnet/test_utils.py +++ b/python/mxnet/test_utils.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Tools for testing.""" # pylint: disable=too-many-lines from __future__ import absolute_import, print_function, division diff --git a/python/mxnet/torch.py b/python/mxnet/torch.py index 765c96bbfb60..b7fce6d5c8fd 100644 --- a/python/mxnet/torch.py +++ b/python/mxnet/torch.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 """Interface for NDArray functions executed by torch backend. Install Torch and compile with USE_TORCH=1 to use this module.""" diff --git a/python/mxnet/visualization.py b/python/mxnet/visualization.py index e67fee427be2..4dbf680c2e3a 100644 --- a/python/mxnet/visualization.py +++ b/python/mxnet/visualization.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # coding: utf-8 # pylint: disable=invalid-name, too-many-locals, fixme # pylint: disable=too-many-branches, too-many-statements diff --git a/python/setup.py b/python/setup.py index f5bd55de6cf6..14c8121d35ee 100644 --- a/python/setup.py +++ b/python/setup.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: disable=invalid-name, exec-used """Setup mxnet package.""" from __future__ import absolute_import diff --git a/scala-package/core/scripts/get_cifar_data.sh b/scala-package/core/scripts/get_cifar_data.sh index eba3a27805fa..9ec1c39a4f99 100755 --- a/scala-package/core/scripts/get_cifar_data.sh +++ b/scala-package/core/scripts/get_cifar_data.sh @@ -1,5 +1,23 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + set -e if [ ! -z "$MXNET_DATA_DIR" ]; then diff --git a/scala-package/core/scripts/get_mnist_data.sh b/scala-package/core/scripts/get_mnist_data.sh index a4cfe11e4b5a..97e151bf8333 100755 --- a/scala-package/core/scripts/get_mnist_data.sh +++ b/scala-package/core/scripts/get_mnist_data.sh @@ -1,5 +1,23 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + set -e if [ ! -z "$MXNET_DATA_DIR" ]; then diff --git a/scala-package/examples/scripts/customop/run_customop.sh b/scala-package/examples/scripts/customop/run_customop.sh index bd425edd5801..b11bb89a2d91 100644 --- a/scala-package/examples/scripts/customop/run_customop.sh +++ b/scala-package/examples/scripts/customop/run_customop.sh @@ -1,5 +1,23 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + MXNET_ROOT=$(cd "$(dirname $0)/../../../.."; pwd) CLASS_PATH=$MXNET_ROOT/scala-package/assembly/linux-x86_64-gpu/target/*:$MXNET_ROOT/scala-package/examples/target/*:$MXNET_ROOT/scala-package/examples/target/classes/lib/* diff --git a/scala-package/examples/scripts/customop/run_customopwithrtc.sh b/scala-package/examples/scripts/customop/run_customopwithrtc.sh index 6009193c8ebb..160525e4eb54 100644 --- a/scala-package/examples/scripts/customop/run_customopwithrtc.sh +++ b/scala-package/examples/scripts/customop/run_customopwithrtc.sh @@ -1,5 +1,23 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + MXNET_ROOT=$(cd "$(dirname $0)/../../../.."; pwd) CLASS_PATH=$MXNET_ROOT/scala-package/assembly/linux-x86_64-gpu/target/*:$MXNET_ROOT/scala-package/examples/target/*:$MXNET_ROOT/scala-package/examples/target/classes/lib/* diff --git a/scala-package/examples/scripts/module/mnist_mlp.sh b/scala-package/examples/scripts/module/mnist_mlp.sh index 0b450d7608a4..6bb9636e98da 100755 --- a/scala-package/examples/scripts/module/mnist_mlp.sh +++ b/scala-package/examples/scripts/module/mnist_mlp.sh @@ -1,4 +1,22 @@ #!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + ROOT_DIR=$(cd `dirname $0`/../../..; pwd) CLASSPATH=$ROOT_DIR/assembly/osx-x86_64-cpu/target/*:$ROOT_DIR/examples/target/*:$ROOT_DIR/examples/target/classes/lib/* diff --git a/scala-package/examples/scripts/module/run_sequential_module.sh b/scala-package/examples/scripts/module/run_sequential_module.sh index 15cc7dda8ba2..9d9edb719dd5 100644 --- a/scala-package/examples/scripts/module/run_sequential_module.sh +++ b/scala-package/examples/scripts/module/run_sequential_module.sh @@ -1,4 +1,22 @@ #!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + ROOT_DIR=$(cd `dirname $0`/../../..; pwd) CLASSPATH=$ROOT_DIR/assembly/linux-x86_64-cpu/target/*:$ROOT_DIR/examples/target/*:$ROOT_DIR/examples/target/classes/lib/* diff --git a/scala-package/examples/scripts/neuralstyle_end2end/run_test_end2end.sh b/scala-package/examples/scripts/neuralstyle_end2end/run_test_end2end.sh index 10bc2da4f9bf..1c683bfc9e34 100644 --- a/scala-package/examples/scripts/neuralstyle_end2end/run_test_end2end.sh +++ b/scala-package/examples/scripts/neuralstyle_end2end/run_test_end2end.sh @@ -1,9 +1,27 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + MXNET_ROOT=$(cd "$(dirname $0)/../../../.."; pwd) CLASS_PATH=$MXNET_ROOT/scala-package/assembly/linux-x86_64-gpu/target/*:$MXNET_ROOT/scala-package/examples/target/*:$MXNET_ROOT/scala-package/examples/target/classes/lib/* -INPUT_IMG=$1 +INPUT_IMG=$1 MODEL_DIR=$2 OUTPUT_DIR=$3 GPU=0 @@ -13,4 +31,4 @@ java -Xmx1024m -cp $CLASS_PATH \ --model-path $MODEL_DIR \ --input-image $INPUT_IMG \ --output-path $OUTPUT_DIR \ - --gpu $GPU \ No newline at end of file + --gpu $GPU diff --git a/scala-package/examples/scripts/neuralstyle_end2end/run_train_end2end.sh b/scala-package/examples/scripts/neuralstyle_end2end/run_train_end2end.sh index 3ede06a78b0b..fa08ff3713c8 100644 --- a/scala-package/examples/scripts/neuralstyle_end2end/run_train_end2end.sh +++ b/scala-package/examples/scripts/neuralstyle_end2end/run_train_end2end.sh @@ -1,12 +1,30 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + MXNET_ROOT=$(cd "$(dirname $0)/../../../.."; pwd) CLASS_PATH=$MXNET_ROOT/scala-package/assembly/linux-x86_64-gpu/target/*:$MXNET_ROOT/scala-package/examples/target/*:$MXNET_ROOT/scala-package/examples/target/classes/lib/* # more details please refer to # https://github.com/Ldpe2G/mxnet/blob/develop/example/neural-style/end_to_end/README.md -TRAIN_DATA_PATH=$1 -STYLE_IMG=$2 +TRAIN_DATA_PATH=$1 +STYLE_IMG=$2 VGG_MODEL_PATH=$3 SAVE_MODEL_DIR=$4 GPU=0 @@ -17,4 +35,4 @@ java -Xmx1024m -cp $CLASS_PATH \ --vgg--model-path $VGG_MODEL_PATH \ --save--model-path $SAVE_MODEL_DIR \ --style-image $STYLE_IMG \ - --gpu $GPU \ No newline at end of file + --gpu $GPU diff --git a/scala-package/examples/scripts/profiler/run_profiler_matmul.sh b/scala-package/examples/scripts/profiler/run_profiler_matmul.sh index b54a4226fb14..54aafafcfd61 100644 --- a/scala-package/examples/scripts/profiler/run_profiler_matmul.sh +++ b/scala-package/examples/scripts/profiler/run_profiler_matmul.sh @@ -1,5 +1,23 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + MXNET_ROOT=$(cd "$(dirname $0)/../../../.."; pwd) CLASS_PATH=$MXNET_ROOT/scala-package/assembly/linux-x86_64-gpu/target/*:$MXNET_ROOT/scala-package/examples/target/*:$MXNET_ROOT/scala-package/examples/target/classes/lib/* diff --git a/scala-package/examples/scripts/profiler/run_profiler_ndarray.sh b/scala-package/examples/scripts/profiler/run_profiler_ndarray.sh index 4a849c57b2ee..711fe5465404 100644 --- a/scala-package/examples/scripts/profiler/run_profiler_ndarray.sh +++ b/scala-package/examples/scripts/profiler/run_profiler_ndarray.sh @@ -1,5 +1,23 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + MXNET_ROOT=$(cd "$(dirname $0)/../../../.."; pwd) CLASS_PATH=$MXNET_ROOT/scala-package/assembly/linux-x86_64-gpu/target/*:$MXNET_ROOT/scala-package/examples/target/*:$MXNET_ROOT/scala-package/examples/target/classes/lib/* diff --git a/scala-package/examples/scripts/rnn/run_test_charrnn.sh b/scala-package/examples/scripts/rnn/run_test_charrnn.sh index 04eee3691435..a4ed91354e8c 100644 --- a/scala-package/examples/scripts/rnn/run_test_charrnn.sh +++ b/scala-package/examples/scripts/rnn/run_test_charrnn.sh @@ -1,5 +1,23 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + MXNET_ROOT=$(cd "$(dirname $0)/../../../.."; pwd) OS=$(uname) if [ "$OS" = "Darwin" ]; then diff --git a/scala-package/examples/scripts/rnn/run_train_charrnn.sh b/scala-package/examples/scripts/rnn/run_train_charrnn.sh index 07b7dda7d6e4..2e9a3a264c88 100755 --- a/scala-package/examples/scripts/rnn/run_train_charrnn.sh +++ b/scala-package/examples/scripts/rnn/run_train_charrnn.sh @@ -1,5 +1,23 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + MXNET_ROOT=$(cd "$(dirname $0)/../../../.."; pwd) OS=$(uname) if [ "$OS" = "Darwin" ]; then diff --git a/scala-package/examples/scripts/run_cnntextclassification.sh b/scala-package/examples/scripts/run_cnntextclassification.sh index a7cf7c0a6395..7939b0627422 100644 --- a/scala-package/examples/scripts/run_cnntextclassification.sh +++ b/scala-package/examples/scripts/run_cnntextclassification.sh @@ -1,16 +1,34 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + MXNET_ROOT=$(cd "$(dirname $0)/../../.."; pwd) CLASS_PATH=$MXNET_ROOT/scala-package/assembly/linux-x86_64-gpu/target/*:$MXNET_ROOT/scala-package/examples/target/*:$MXNET_ROOT/scala-package/examples/target/classes/lib/* # which gpu card to use, -1 means cpu GPU=$1 # the mr dataset path, you should put the pos and neg file in the same folder -MR_DATASET_PATH=$2 +MR_DATASET_PATH=$2 # the trained word2vec file path, binary or text format -W2V_FILE_PATH=$3 +W2V_FILE_PATH=$3 # whether the format of the word2vec file is binary,1 means binary, 0 means text -W2V_FORMAT_BIN=$4 +W2V_FORMAT_BIN=$4 BATCH_SIZE=$5 SAVE_MODEL_PATH=$6 diff --git a/scala-package/examples/scripts/run_gan_mnist.sh b/scala-package/examples/scripts/run_gan_mnist.sh index 2d3c545cf5d3..951241fb18b7 100644 --- a/scala-package/examples/scripts/run_gan_mnist.sh +++ b/scala-package/examples/scripts/run_gan_mnist.sh @@ -1,5 +1,23 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + MXNET_ROOT=$(cd "$(dirname $0)/../../.."; pwd) CLASS_PATH=$MXNET_ROOT/scala-package/assembly/linux-x86_64-gpu/target/*:$MXNET_ROOT/scala-package/examples/target/*:$MXNET_ROOT/scala-package/examples/target/classes/lib/* @@ -10,7 +28,7 @@ GPU=$1 # you can get the mnist data using the script core/scripts/get_mnist_data.sh MNIST_DATA_PATH=$2 -# the path to save the generated results +# the path to save the generated results OUTPUT_PATH=$3 java -Xmx4G -cp $CLASS_PATH \ diff --git a/scala-package/examples/scripts/run_multitask.sh b/scala-package/examples/scripts/run_multitask.sh index 1642cc8336f2..9e6a489e9fa7 100644 --- a/scala-package/examples/scripts/run_multitask.sh +++ b/scala-package/examples/scripts/run_multitask.sh @@ -1,5 +1,23 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + MXNET_ROOT=$(cd "$(dirname $0)/../../.."; pwd) CLASS_PATH=$MXNET_ROOT/scala-package/assembly/linux-x86_64-gpu/target/*:$MXNET_ROOT/scala-package/examples/target/*:$MXNET_ROOT/scala-package/examples/target/classes/lib/* diff --git a/scala-package/examples/scripts/run_neuralstyle.sh b/scala-package/examples/scripts/run_neuralstyle.sh index 5fbfc3227b7c..a9c2e5c1c1ea 100644 --- a/scala-package/examples/scripts/run_neuralstyle.sh +++ b/scala-package/examples/scripts/run_neuralstyle.sh @@ -1,9 +1,27 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + MXNET_ROOT=$(cd "$(dirname $0)/../../.."; pwd) CLASS_PATH=$MXNET_ROOT/scala-package/assembly/linux-x86_64-gpu/target/*:$MXNET_ROOT/scala-package/examples/target/*:$MXNET_ROOT/scala-package/examples/target/classes/lib/* -INPUT_IMG=$1 -STYLE_IMG=$2 +INPUT_IMG=$1 +STYLE_IMG=$2 MODEL_PATH=$MXNET_ROOT/example/neural-style/model/vgg19.params OUTPUT_DIR=$MXNET_ROOT/example/neural-style/output @@ -12,4 +30,4 @@ java -Xmx1024m -cp $CLASS_PATH \ --content-image $INPUT_IMG \ --style-image $STYLE_IMG \ --model-path $MODEL_PATH \ - --output-dir $OUTPUT_DIR + --output-dir $OUTPUT_DIR diff --git a/scala-package/examples/scripts/run_visualization.sh b/scala-package/examples/scripts/run_visualization.sh index 6f686adc06f5..a4b545e24484 100644 --- a/scala-package/examples/scripts/run_visualization.sh +++ b/scala-package/examples/scripts/run_visualization.sh @@ -1,5 +1,23 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + MXNET_ROOT=$(cd "$(dirname $0)/../../.."; pwd) CLASS_PATH=$MXNET_ROOT/scala-package/assembly/linux-x86_64-cpu/target/*:$MXNET_ROOT/scala-package/examples/target/*:$MXNET_ROOT/scala-package/examples/target/classes/lib/* @@ -15,4 +33,4 @@ NET=$2 java -Xmx1024m -cp $CLASS_PATH \ ml.dmlc.mxnetexamples.visualization.ExampleVis \ --out-dir $OUT_DIR \ - --net $NET + --net $NET diff --git a/scala-package/init-native/src/main/native/ml_dmlc_mxnet_init_native_c_api.cc b/scala-package/init-native/src/main/native/ml_dmlc_mxnet_init_native_c_api.cc index d6daa00eec9f..114510c66afb 100644 --- a/scala-package/init-native/src/main/native/ml_dmlc_mxnet_init_native_c_api.cc +++ b/scala-package/init-native/src/main/native/ml_dmlc_mxnet_init_native_c_api.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file ml_dmlc_mxnet_native_c_api.cc * \brief JNI function implementations */ diff --git a/scala-package/native/src/main/native/jni_helper_func.h b/scala-package/native/src/main/native/jni_helper_func.h index d1abd93d92e7..009bbec64e66 100644 --- a/scala-package/native/src/main/native/jni_helper_func.h +++ b/scala-package/native/src/main/native/jni_helper_func.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file jni_helper_func.h * \brief Helper functions for operating JVM objects */ diff --git a/scala-package/native/src/main/native/ml_dmlc_mxnet_native_c_api.cc b/scala-package/native/src/main/native/ml_dmlc_mxnet_native_c_api.cc index 07fd07596ee7..166f6b71eb9f 100644 --- a/scala-package/native/src/main/native/ml_dmlc_mxnet_native_c_api.cc +++ b/scala-package/native/src/main/native/ml_dmlc_mxnet_native_c_api.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file ml_dmlc_mxnet_native_c_api.cc * \brief JNI function implementations */ diff --git a/scala-package/spark/bin/run-mnist-example.sh b/scala-package/spark/bin/run-mnist-example.sh index dc2f3adbe7ac..cae19386a8ee 100755 --- a/scala-package/spark/bin/run-mnist-example.sh +++ b/scala-package/spark/bin/run-mnist-example.sh @@ -1,4 +1,22 @@ #!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + CURR_DIR=$(cd `dirname $0`; pwd) MODULE_DIR=$(cd $CURR_DIR/../; pwd) ROOT_DIR=$(cd $CURR_DIR/../../; pwd) diff --git a/setup-utils/install-mxnet-amz-linux.sh b/setup-utils/install-mxnet-amz-linux.sh index b8564a56ed3d..66788a984da6 100644 --- a/setup-utils/install-mxnet-amz-linux.sh +++ b/setup-utils/install-mxnet-amz-linux.sh @@ -1,4 +1,22 @@ #!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + ###################################################################### # This script installs MXNet for Python along with all required dependencies on a Amazon Linux Machine. ###################################################################### diff --git a/setup-utils/install-mxnet-fedora-python.sh b/setup-utils/install-mxnet-fedora-python.sh index 54b716b911db..86116665db88 100644 --- a/setup-utils/install-mxnet-fedora-python.sh +++ b/setup-utils/install-mxnet-fedora-python.sh @@ -1,4 +1,22 @@ #!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + ###################################################################### # This script installs MXNet for Python along with all required dependencies on a Fedora Machine. # Tested on Fedora 21.0 + distro. @@ -24,8 +42,8 @@ sudo yum install numpy echo "Installing Python setuptools..." sudo yum install -y python-setuptools python-pip -echo "Adding MXNet path to your ~/.bashrc file" -echo "export PYTHONPATH=$MXNET_HOME/python:$PYTHONPATH" >> ~/.bashrc +echo "Adding MXNet path to your ~/.bashrc file" +echo "export PYTHONPATH=$MXNET_HOME/python:$PYTHONPATH" >> ~/.bashrc source ~/.bashrc echo "Install Graphviz for plotting MXNet network graph..." diff --git a/setup-utils/install-mxnet-osx-python.sh b/setup-utils/install-mxnet-osx-python.sh index f9e4e775534e..8bfb7dade7b1 100755 --- a/setup-utils/install-mxnet-osx-python.sh +++ b/setup-utils/install-mxnet-osx-python.sh @@ -1,4 +1,22 @@ #!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # # This scripts installs the dependencies and compiles # MXNet source. @@ -12,7 +30,7 @@ export MXNET_GITPATH="https://github.com/dmlc/mxnet.git" if [ -z ${MXNET_TAG} ]; then # - # TODO: Change this to latest tag + # TODO: Change this to latest tag # to avoid updating this value for every release # export MXNET_TAG="v0.10.0" @@ -138,17 +156,17 @@ install_brew() { echo "END: Check/Install/Update Homebrew" echo $LINE echo " " - + echo "BEGIN: Install dependent brew packages for MXNet: ${BREW_PKGS}" - + chkret brew tap homebrew/science - + # install each individually to see progress for each for pkg in ${BREW_PKGS} do chkret brew_pkg_install ${pkg} done - + echo "END: Install dependent brew packages for MXNet: ${BREW_PKGS}" echo $LINE echo " " @@ -223,7 +241,7 @@ install_port () { # sudo mkdir -p /usr/local/opt/openblas/lib # sudo ln -s /opt/local/lib/libopenblas.a /usr/local/opt/openblas/lib/libopenblas.a #fi - + echo " " echo "END: Install dependent port packages for MXNet: ${PORT_PKGS}" echo $LINE @@ -265,9 +283,9 @@ install_mac_pkg_manager() { export PKG_MGR="" else export MAC_PKG_ASK=0 - + while true; do - echo "NOTE: Using the already installed package manager: $PKG_MGR" + echo "NOTE: Using the already installed package manager: $PKG_MGR" read -p "Do you want to continue? (y/n): " response echo " " case $response in @@ -368,7 +386,7 @@ compile_mxnet() { echo "BEGIN: Compile MXNet" cd ${MXNET_HOME} chkret cp make/osx.mk ./config.mk.tmp - + touch ./config.mk # rm any old setting of USE_BLAS, if present in config file egrep -v "^USE_BLAS" ./config.mk.tmp >> ./config.mk @@ -386,7 +404,7 @@ compile_mxnet() { echo "ADD_LDFLAGS += -L/usr/local/lib/graphviz/" >> ./config.mk fi echo " " - + echo "NOTE: The following compile-time configurations will be used." echo " If you want to change any of them, edit the following file" echo " in another terminal window and then press enter to continue." @@ -452,7 +470,7 @@ END echo " " echo $LINE echo " " - rm -f mxnet_test.log mxnet_test.expected + rm -f mxnet_test.log mxnet_test.expected exit 0 else echo " " diff --git a/setup-utils/install-mxnet-ubuntu-python.sh b/setup-utils/install-mxnet-ubuntu-python.sh index ba060745da8f..8aa0d0256a79 100644 --- a/setup-utils/install-mxnet-ubuntu-python.sh +++ b/setup-utils/install-mxnet-ubuntu-python.sh @@ -1,4 +1,22 @@ #!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + ###################################################################### # This script installs MXNet for Python along with all required dependencies on a Ubuntu Machine. # Tested on Ubuntu 14.0 + distro. diff --git a/setup-utils/install-mxnet-ubuntu-r.sh b/setup-utils/install-mxnet-ubuntu-r.sh index 8f4c07d0325d..ca46d7b37016 100644 --- a/setup-utils/install-mxnet-ubuntu-r.sh +++ b/setup-utils/install-mxnet-ubuntu-r.sh @@ -1,4 +1,22 @@ #!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + ###################################################################### # This script installs MXNet for R along with all required dependencies on a Ubuntu Machine. # Tested on Ubuntu 14.04+ distro. @@ -32,7 +50,7 @@ sudo apt-get -y install libcurl4-openssl-dev libssl-dev # Needed for R XML sudo apt-get install libxml2-dev -# Needed for R Cairo +# Needed for R Cairo sudo apt-get install libxt-dev sudo Rscript -e "install.packages('devtools', repo = 'https://cran.rstudio.com')" diff --git a/src/c_api/c_api.cc b/src/c_api/c_api.cc index 214e6ede5292..93458d21ac5a 100644 --- a/src/c_api/c_api.cc +++ b/src/c_api/c_api.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file c_api.cc * \brief C API of mxnet */ diff --git a/src/c_api/c_api_common.h b/src/c_api/c_api_common.h index d8857f80635d..846b53973b07 100644 --- a/src/c_api/c_api_common.h +++ b/src/c_api/c_api_common.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file c_api_error.h * \brief Error handling for C API. */ diff --git a/src/c_api/c_api_error.cc b/src/c_api/c_api_error.cc index 4ee6a35363c5..4d93b908fb31 100644 --- a/src/c_api/c_api_error.cc +++ b/src/c_api/c_api_error.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file c_api_error.cc * \brief C error handling */ diff --git a/src/c_api/c_api_executor.cc b/src/c_api/c_api_executor.cc index 3ba3154f2d97..a4c48e426879 100644 --- a/src/c_api/c_api_executor.cc +++ b/src/c_api/c_api_executor.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file c_api_executor.cc * \brief C API of mxnet */ diff --git a/src/c_api/c_api_ndarray.cc b/src/c_api/c_api_ndarray.cc index a37e3144c303..89605183e748 100644 --- a/src/c_api/c_api_ndarray.cc +++ b/src/c_api/c_api_ndarray.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file c_api_symbolic.cc * \brief C API of mxnet */ diff --git a/src/c_api/c_api_symbolic.cc b/src/c_api/c_api_symbolic.cc index d3603e94b2a1..e2c29b888ada 100644 --- a/src/c_api/c_api_symbolic.cc +++ b/src/c_api/c_api_symbolic.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file c_api_symbolic.cc * \brief C API of mxnet */ diff --git a/src/c_api/c_predict_api.cc b/src/c_api/c_predict_api.cc index 1dd784ba2249..5ca01492800e 100644 --- a/src/c_api/c_predict_api.cc +++ b/src/c_api/c_predict_api.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file c_predict_api.cc * \brief C predict API of mxnet */ diff --git a/src/common/cuda_utils.h b/src/common/cuda_utils.h index d0defc30ffa6..3c4d1a88de8e 100644 --- a/src/common/cuda_utils.h +++ b/src/common/cuda_utils.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file cuda_utils.h * \brief CUDA debugging utilities. */ diff --git a/src/common/lazy_alloc_array.h b/src/common/lazy_alloc_array.h index 61b81e5ce407..aa2cd4a139ee 100644 --- a/src/common/lazy_alloc_array.h +++ b/src/common/lazy_alloc_array.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file lazy_alloc_array.h * \brief An array that lazily allocate elements as * First time the cell get visited. diff --git a/src/common/mxrtc.cc b/src/common/mxrtc.cc index e808e11215bf..e72ac0bacdde 100644 --- a/src/common/mxrtc.cc +++ b/src/common/mxrtc.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file mxrtc.cc * \brief Wrapper for NVRTC * \author Junyuan Xie diff --git a/src/common/object_pool.h b/src/common/object_pool.h index 5e22d49a9e9b..6e11ce5ca785 100644 --- a/src/common/object_pool.h +++ b/src/common/object_pool.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors */ #ifndef MXNET_COMMON_OBJECT_POOL_H_ #define MXNET_COMMON_OBJECT_POOL_H_ diff --git a/src/common/utils.h b/src/common/utils.h index 5f50aab4781f..85e30970f1a0 100644 --- a/src/common/utils.h +++ b/src/common/utils.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file utils.h * \brief Basic utilility functions. */ diff --git a/src/engine/engine.cc b/src/engine/engine.cc index ae72861260e1..d6196085bee9 100644 --- a/src/engine/engine.cc +++ b/src/engine/engine.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file engine.cc * \brief Implementation of engine. */ diff --git a/src/engine/engine_impl.h b/src/engine/engine_impl.h index 9d3fc4cd09f7..cf727366f6d9 100644 --- a/src/engine/engine_impl.h +++ b/src/engine/engine_impl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file engine_impl.h * \brief Internal implementation header of engine components. */ diff --git a/src/engine/naive_engine.cc b/src/engine/naive_engine.cc index 11ff7c8138bf..85ec3ae672e2 100644 --- a/src/engine/naive_engine.cc +++ b/src/engine/naive_engine.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file naive_engine.cc * \brief Implementation of NaiveEngine */ diff --git a/src/engine/profiler.cc b/src/engine/profiler.cc index 44099c397783..99504f61ce17 100644 --- a/src/engine/profiler.cc +++ b/src/engine/profiler.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file profiler.cc * \brief implements profiler */ diff --git a/src/engine/profiler.h b/src/engine/profiler.h index f28d691e250d..b7f8e0e1f01a 100644 --- a/src/engine/profiler.h +++ b/src/engine/profiler.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file profiler.h * \brief implements profiler */ diff --git a/src/engine/stream_manager.h b/src/engine/stream_manager.h index 2d684bbb7b9a..1a66277bb4ec 100644 --- a/src/engine/stream_manager.h +++ b/src/engine/stream_manager.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors */ #ifndef MXNET_ENGINE_STREAM_MANAGER_H_ #define MXNET_ENGINE_STREAM_MANAGER_H_ diff --git a/src/engine/thread_pool.h b/src/engine/thread_pool.h index 060f4734a675..b6fe3c2d5d6a 100644 --- a/src/engine/thread_pool.h +++ b/src/engine/thread_pool.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors */ #ifndef MXNET_ENGINE_THREAD_POOL_H_ #define MXNET_ENGINE_THREAD_POOL_H_ diff --git a/src/engine/threaded_engine.cc b/src/engine/threaded_engine.cc index 3632a46ba80b..5f348fbb44a7 100644 --- a/src/engine/threaded_engine.cc +++ b/src/engine/threaded_engine.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file threaded_engine.cc * \brief implements base threaded engine. * \author Yutian Li diff --git a/src/engine/threaded_engine.h b/src/engine/threaded_engine.h index 4612cc6e02bf..9b7b74ddd631 100644 --- a/src/engine/threaded_engine.h +++ b/src/engine/threaded_engine.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file threaded_engine.h * \brief Implements base class of threaded engine * that tracks the dependency and pushes actions to execute. diff --git a/src/engine/threaded_engine_perdevice.cc b/src/engine/threaded_engine_perdevice.cc index 97356ae91e0d..66cfc9de1468 100644 --- a/src/engine/threaded_engine_perdevice.cc +++ b/src/engine/threaded_engine_perdevice.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file threaded_engine_perdevice.cc * \brief ThreadedEngine that uses fix amount of thread for each device. */ diff --git a/src/engine/threaded_engine_pooled.cc b/src/engine/threaded_engine_pooled.cc index d806c382390c..6db7c4bb7a92 100644 --- a/src/engine/threaded_engine_pooled.cc +++ b/src/engine/threaded_engine_pooled.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file threaded_engine_pooled.cc * \brief Pooled threaded engine * \author Yutian Li diff --git a/src/executor/attach_op_execs_pass.cc b/src/executor/attach_op_execs_pass.cc index c4b3a1895ad8..13b0018b6dae 100644 --- a/src/executor/attach_op_execs_pass.cc +++ b/src/executor/attach_op_execs_pass.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file attach_op_execs_pass.cc * \brief Operator executor to execute each operator. */ diff --git a/src/executor/attach_op_resource_pass.cc b/src/executor/attach_op_resource_pass.cc index 73c8f4807b77..ef26a3575c25 100644 --- a/src/executor/attach_op_resource_pass.cc +++ b/src/executor/attach_op_resource_pass.cc @@ -1,6 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file attach_op_resource_pass.cc * \brief Pass to attach resource to OpExecVector of the graph. */ diff --git a/src/executor/exec_pass.h b/src/executor/exec_pass.h index 76b02de736e9..0eda71d98214 100644 --- a/src/executor/exec_pass.h +++ b/src/executor/exec_pass.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file exec_pass.h * \brief All the execution related pass and data structures. */ diff --git a/src/executor/graph_executor.cc b/src/executor/graph_executor.cc index a17f44a7cff5..6dc8cf39970e 100644 --- a/src/executor/graph_executor.cc +++ b/src/executor/graph_executor.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file graph_executor.cc * \brief graph executor */ diff --git a/src/executor/graph_executor.h b/src/executor/graph_executor.h index 0efb8ae09f4a..dc50bef002ab 100644 --- a/src/executor/graph_executor.h +++ b/src/executor/graph_executor.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file graph_executor.h * \brief Executor to execute the computation graph. */ diff --git a/src/executor/inplace_addto_detect_pass.cc b/src/executor/inplace_addto_detect_pass.cc index 75a2608313aa..26a91e3f1b5e 100644 --- a/src/executor/inplace_addto_detect_pass.cc +++ b/src/executor/inplace_addto_detect_pass.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file inplace_addto_detect_pass.cc * \brief Detect whether inplace addto operation is possible for certain op. */ diff --git a/src/initialize.cc b/src/initialize.cc index c1e897f01a57..092dacfb26f2 100644 --- a/src/initialize.cc +++ b/src/initialize.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file initialize.cc * \brief initialize mxnet library */ diff --git a/src/io/image_aug_default.cc b/src/io/image_aug_default.cc index f7a79d19f81b..6db14bd583c8 100644 --- a/src/io/image_aug_default.cc +++ b/src/io/image_aug_default.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file image_aug_default.cc * \brief Default augmenter. */ diff --git a/src/io/image_augmenter.h b/src/io/image_augmenter.h index 6c0ad98bee7f..5b6c4e99e502 100644 --- a/src/io/image_augmenter.h +++ b/src/io/image_augmenter.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file image_augmenter.h * \brief Interface of opencv based image augmenter */ diff --git a/src/io/image_det_aug_default.cc b/src/io/image_det_aug_default.cc index cb7966dc7414..7d15facf7843 100644 --- a/src/io/image_det_aug_default.cc +++ b/src/io/image_det_aug_default.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file image_det_aug_default.cc * \brief Default augmenter. */ diff --git a/src/io/image_io.cc b/src/io/image_io.cc index 64fd2dde1908..f9d7f33a5a44 100644 --- a/src/io/image_io.cc +++ b/src/io/image_io.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file optimizer_op-inl.h * \brief Optimizer operators * \author Junyuan Xie diff --git a/src/io/image_iter_common.h b/src/io/image_iter_common.h index 59916c9b9997..f2f72dc928eb 100644 --- a/src/io/image_iter_common.h +++ b/src/io/image_iter_common.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * \file image_iter_common.h * \brief common types used by image data iterators */ diff --git a/src/io/image_recordio.h b/src/io/image_recordio.h index 10674ec20355..a931539aa296 100644 --- a/src/io/image_recordio.h +++ b/src/io/image_recordio.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file image_recordio.h * \brief image recordio struct */ @@ -24,7 +42,7 @@ struct ImageRecordIO { /*! * \brief label field that returns label of images * when image list was not presented, - * + * * NOTE: user do not need to repack recordio just to * change label field, just supply a list file that * maps image id to new labels @@ -58,9 +76,9 @@ struct ImageRecordIO { return header.image_id[0]; } /*! - * \brief load header from a record content + * \brief load header from a record content * \param buf the head of record - * \param size the size of the entire record + * \param size the size of the entire record */ inline void Load(void *buf, size_t size) { CHECK(size >= sizeof(header)); diff --git a/src/io/inst_vector.h b/src/io/inst_vector.h index d82bd48e2fa1..4bc2a6c758ba 100644 --- a/src/io/inst_vector.h +++ b/src/io/inst_vector.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file inst_vector.h * \brief holder of a sequence of DataInst in CPU * that are not necessarily of same shape diff --git a/src/io/io.cc b/src/io/io.cc index 822f66f47453..e7c92843b44e 100644 --- a/src/io/io.cc +++ b/src/io/io.cc @@ -1,4 +1,22 @@ -// Copyright (c) 2015 by Contributors +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + #include #include diff --git a/src/io/iter_batchloader.h b/src/io/iter_batchloader.h index a51e24503785..c5ec10618080 100644 --- a/src/io/iter_batchloader.h +++ b/src/io/iter_batchloader.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file iter_batchloader.h * \brief define a batch adapter to create tblob batch */ diff --git a/src/io/iter_csv.cc b/src/io/iter_csv.cc index 9dcbcb8a681d..a28b8d4d9d13 100644 --- a/src/io/iter_csv.cc +++ b/src/io/iter_csv.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file iter_csv.cc * \brief define a CSV Reader to read in arrays */ diff --git a/src/io/iter_image_det_recordio.cc b/src/io/iter_image_det_recordio.cc index 25e920d77c13..4e80d5d53172 100644 --- a/src/io/iter_image_det_recordio.cc +++ b/src/io/iter_image_det_recordio.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file iter_image_recordio-inl.hpp * \brief recordio data iterator */ diff --git a/src/io/iter_image_recordio.cc b/src/io/iter_image_recordio.cc index 49694d07f1bb..64f31a65fa51 100644 --- a/src/io/iter_image_recordio.cc +++ b/src/io/iter_image_recordio.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file iter_image_recordio-inl.hpp * \brief recordio data iterator */ diff --git a/src/io/iter_image_recordio_2.cc b/src/io/iter_image_recordio_2.cc index 9d4ebf4b2864..c4d1e8624bcc 100644 --- a/src/io/iter_image_recordio_2.cc +++ b/src/io/iter_image_recordio_2.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * \file iter_image_recordio_2.cc * \brief new version of recordio data iterator */ diff --git a/src/io/iter_mnist.cc b/src/io/iter_mnist.cc index 09799f2b1991..055af52aaebd 100644 --- a/src/io/iter_mnist.cc +++ b/src/io/iter_mnist.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file iter_mnist.cc * \brief register mnist iterator */ diff --git a/src/io/iter_normalize.h b/src/io/iter_normalize.h index 2cebaaa3a48e..409231b59bc8 100644 --- a/src/io/iter_normalize.h +++ b/src/io/iter_normalize.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file iter_normalize.h * \brief Iterator that subtracts mean and do a few augmentations. */ diff --git a/src/io/iter_prefetcher.h b/src/io/iter_prefetcher.h index 9050ef2d1b38..89960c71a12f 100644 --- a/src/io/iter_prefetcher.h +++ b/src/io/iter_prefetcher.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file iter_prefetcher.h * \brief define a prefetcher using threaditer to keep k batch fetched */ diff --git a/src/kvstore/comm.h b/src/kvstore/comm.h index 07f2d24bd223..ade9c95feda7 100644 --- a/src/kvstore/comm.h +++ b/src/kvstore/comm.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /** - * Copyright (c) 2015 by Contributors */ #ifndef MXNET_KVSTORE_COMM_H_ #define MXNET_KVSTORE_COMM_H_ diff --git a/src/kvstore/kvstore.cc b/src/kvstore/kvstore.cc index 78d4958096cc..a288676102cb 100644 --- a/src/kvstore/kvstore.cc +++ b/src/kvstore/kvstore.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file kvstore.cc * \brief implement kv_store */ diff --git a/src/kvstore/kvstore_dist.h b/src/kvstore/kvstore_dist.h index 5f5a0cc67a64..52c7c132cb5c 100644 --- a/src/kvstore/kvstore_dist.h +++ b/src/kvstore/kvstore_dist.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /** - * Copyright (c) 2015 by Contributors * @file kvstore_dist.h * @brief distributed implementation based on ps-lite */ diff --git a/src/kvstore/kvstore_dist_server.h b/src/kvstore/kvstore_dist_server.h index 02d4a38c2b10..4e9f887173c5 100644 --- a/src/kvstore/kvstore_dist_server.h +++ b/src/kvstore/kvstore_dist_server.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file mxnet_node.h * \brief implement mxnet nodes */ diff --git a/src/kvstore/kvstore_local.h b/src/kvstore/kvstore_local.h index dc5f7b786244..536a89b46e13 100644 --- a/src/kvstore/kvstore_local.h +++ b/src/kvstore/kvstore_local.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /** - * Copyright (c) 2015 by Contributors * @file kvstore_local.h * @brief local implementation */ diff --git a/src/ndarray/autograd.cc b/src/ndarray/autograd.cc index efb6bc9dbf8d..33d0d5d307ed 100644 --- a/src/ndarray/autograd.cc +++ b/src/ndarray/autograd.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * \file autograd.cc * \brief Implementation of AutogradRuntime module. */ diff --git a/src/ndarray/autograd.h b/src/ndarray/autograd.h index 474864009688..52e461d52c2d 100644 --- a/src/ndarray/autograd.h +++ b/src/ndarray/autograd.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * \file autograd.h * \brief AutogradRuntime can automatically compute gradients */ diff --git a/src/ndarray/ndarray.cc b/src/ndarray/ndarray.cc index 48499fa2cafd..8e71df729b73 100644 --- a/src/ndarray/ndarray.cc +++ b/src/ndarray/ndarray.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file ndarray.cc * \brief ndarry module of mxnet */ diff --git a/src/ndarray/ndarray_function-inl.h b/src/ndarray/ndarray_function-inl.h index 28524b73d0dd..2be55f50f934 100644 --- a/src/ndarray/ndarray_function-inl.h +++ b/src/ndarray/ndarray_function-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file ndarray_function-inl.h * \brief The real implementation of NDArray functions. */ diff --git a/src/ndarray/ndarray_function.cc b/src/ndarray/ndarray_function.cc index a5ba2660fd34..e4af86d2c824 100644 --- a/src/ndarray/ndarray_function.cc +++ b/src/ndarray/ndarray_function.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file ndarray_function_cpu.cc * \brief CPU Implementation of ndarray function. */ diff --git a/src/ndarray/ndarray_function.cu b/src/ndarray/ndarray_function.cu index 13d36a2c4293..30d532673cff 100644 --- a/src/ndarray/ndarray_function.cu +++ b/src/ndarray/ndarray_function.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file ndarray_function_cpu.cc * \brief GPU Implementation of ndarray function. */ diff --git a/src/ndarray/ndarray_function.h b/src/ndarray/ndarray_function.h index 479f6f99f07a..b1ed58db3e74 100644 --- a/src/ndarray/ndarray_function.h +++ b/src/ndarray/ndarray_function.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file ndarray_op.h * \brief the real execution functions of ndarray operations */ diff --git a/src/operator/activation-inl.h b/src/operator/activation-inl.h index ead79be95865..6e6222bb64dd 100644 --- a/src/operator/activation-inl.h +++ b/src/operator/activation-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file activation-inl.h * \brief Activation operator * \author Bing Xu diff --git a/src/operator/activation.cc b/src/operator/activation.cc index 7ef452f4ec62..a33c11ce546d 100644 --- a/src/operator/activation.cc +++ b/src/operator/activation.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file activation.cc * \brief activation op * \author Bing Xu diff --git a/src/operator/activation.cu b/src/operator/activation.cu index 9a55354de8b0..0ac51ad03109 100644 --- a/src/operator/activation.cu +++ b/src/operator/activation.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file activation.cu * \brief * \author Bing Xu diff --git a/src/operator/batch_norm-inl.h b/src/operator/batch_norm-inl.h index 2d9c96be0cb9..b53acb4d35cb 100644 --- a/src/operator/batch_norm-inl.h +++ b/src/operator/batch_norm-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * \file batch_norm-inl.h * \brief * \author Bing Xu, Chris Olivier diff --git a/src/operator/batch_norm.cc b/src/operator/batch_norm.cc index e56b30671e3e..86f47dd6163f 100644 --- a/src/operator/batch_norm.cc +++ b/src/operator/batch_norm.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file batch_norm.cc * \brief * \author Bing Xu, Chris Olivier diff --git a/src/operator/batch_norm.cu b/src/operator/batch_norm.cu index 9f7370f00faa..64f7d9373823 100644 --- a/src/operator/batch_norm.cu +++ b/src/operator/batch_norm.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * \file batch_norm.cu * \brief CUDA Batch Normalization code * \author Chris Olivier, Bing Xu diff --git a/src/operator/batch_norm_v1-inl.h b/src/operator/batch_norm_v1-inl.h index 19215c5400d5..092c4824f9e6 100644 --- a/src/operator/batch_norm_v1-inl.h +++ b/src/operator/batch_norm_v1-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file batch_norm-inl_v1.h * \brief * \author Bing Xu diff --git a/src/operator/batch_norm_v1.cc b/src/operator/batch_norm_v1.cc index 32c5034b0832..1abced8763c5 100644 --- a/src/operator/batch_norm_v1.cc +++ b/src/operator/batch_norm_v1.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file batch_norm_v1.cc * \brief * \author Bing Xu diff --git a/src/operator/batch_norm_v1.cu b/src/operator/batch_norm_v1.cu index 302dc47dbca4..8ed22a4dc6f1 100644 --- a/src/operator/batch_norm_v1.cu +++ b/src/operator/batch_norm_v1.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file batch_norm_v1.cu * \brief * \author Bing Xu diff --git a/src/operator/bilinear_sampler-inl.h b/src/operator/bilinear_sampler-inl.h index b4c9d991865f..2d68d7855b6d 100644 --- a/src/operator/bilinear_sampler-inl.h +++ b/src/operator/bilinear_sampler-inl.h @@ -1,219 +1,237 @@ -/*! - * Copyright (c) 2017 by Contributors - * \file bilinear_Sampler-inl.h - * \brief - * \author Xu Dong -*/ -#ifndef MXNET_OPERATOR_BILINEAR_SAMPLER_INL_H_ -#define MXNET_OPERATOR_BILINEAR_SAMPLER_INL_H_ - -#include -#include -#include -#include -#include -#include -#include -#include "./operator_common.h" - -namespace mxnet { -namespace op { - -namespace bs { -enum BilinearSamplerOpInputs {kData, kGrid}; -enum BilinearSamplerOpOutputs {kOut, kTmp}; -} - -struct BilinearSamplerParam : public dmlc::Parameter { - DMLC_DECLARE_PARAMETER(BilinearSamplerParam) { - } -}; - -template -class BilinearSamplerOp : public Operator { - public: - explicit BilinearSamplerOp(BilinearSamplerParam p) { - this->param_ = p; - } - - virtual void Forward(const OpContext &ctx, - const std::vector &in_data, - const std::vector &req, - const std::vector &out_data, - const std::vector &aux_args) { - using namespace mshadow; - using namespace mshadow::expr; - CHECK_EQ(req[bs::kOut], kWriteTo); - CHECK_EQ(in_data.size(), 2U); - Stream *s = ctx.get_stream(); - - Tensor data = in_data[bs::kData].get(s); - Tensor grid = in_data[bs::kGrid].get(s); - Tensor out = out_data[bs::kOut].get(s); - - BilinearSamplerForward(out, data, grid); - } - - virtual void Backward(const OpContext &ctx, - const std::vector &out_grad, - const std::vector &in_data, - const std::vector &out_data, - const std::vector &req, - const std::vector &in_grad, - const std::vector &aux_args) { - using namespace mshadow; - using namespace mshadow::expr; - CHECK_EQ(in_data.size(), 2U); - CHECK_NE(req[bs::kData], kWriteInplace); - CHECK_NE(req[bs::kGrid], kWriteInplace); - Stream *s = ctx.get_stream(); - - Tensor data = in_data[bs::kData].get(s); - Tensor grid = in_data[bs::kGrid].get(s); - Tensor gdata = in_grad[bs::kData].get(s); - Tensor ggrid = in_grad[bs::kGrid].get(s); - Tensor grad = out_grad[bs::kOut].get(s); - if (req[bs::kData] != kNullOp && req[bs::kGrid] != kNullOp) { - if (req[bs::kData] == kWriteTo) { - gdata = scalar(0.0f); - } - if (req[bs::kGrid] == kWriteTo) { - ggrid = scalar(0.0f); - } - BilinearSamplerBackward(gdata, ggrid, grad, data, grid); - } else if (req[bs::kData] == kNullOp && req[bs::kGrid] == kNullOp) { - return; - } else { - LOG(FATAL) << "Have not implemented the data req combinations! gdata_req=" - << req[bs::kData] << " ggrid_req=" << req[bs::kGrid]; - } - } - - private: - BilinearSamplerParam param_; -}; // class BilinearSamplerOp - -template -Operator* CreateOp(BilinearSamplerParam param, int dtype); - -#if DMLC_USE_CXX11 -class BilinearSamplerProp : public OperatorProperty { - public: - int NumVisibleOutputs() const override { - return 1; - } - - int NumOutputs() const override { - return 2; - } - - std::vector ListArguments() const override { - return {"data", "grid"}; - } - - std::vector ListOutputs() const override { - return {"output", "tmp"}; - } - - void Init(const std::vector >& kwargs) override { - param_.Init(kwargs); - } - - std::map GetParams() const override { - return param_.__DICT__(); - } - - bool InferShape(std::vector *in_shape, - std::vector *out_shape, - std::vector *aux_shape) const override { - using namespace mshadow; - CHECK_EQ(in_shape->size(), 2U) << "Input:[data, grid]"; - const TShape &dshape = (*in_shape)[bs::kData]; - const TShape &lshape = (*in_shape)[bs::kGrid]; - if (dshape.ndim() == 0) return false; - CHECK_EQ(dshape.ndim(), 4U) \ - << "input data should be 4D in batch-num_filter-y-x"; - if (lshape.ndim() == 0) return false; - CHECK_EQ(lshape.ndim(), 4U) \ - << "Sampler grid should be 4D in batch-2-y-x"; - CHECK_EQ(dshape[0], lshape[0]); - CHECK_EQ(lshape[1], 2U) << "incorrect grid shape[1], should be 2"; - // target height - CHECK_GT(lshape[2], 0U) \ - << "incorrect grid_shape: " << lshape[2]; - // target width - CHECK_GT(lshape[3], 0U) \ - << "incorrect grid_shape: " << lshape[3]; - out_shape->clear(); - // output_shape : (data.shape[0], data.shape[1], grid.shape[2], grid.shape[3]) - out_shape->push_back(dshape); - (*out_shape)[bs::kOut][2] = lshape[2]; - (*out_shape)[bs::kOut][3] = lshape[3]; - out_shape->push_back(Shape4(lshape[0], lshape[2], lshape[3], 2)); - return true; - } - - bool InferType(std::vector *in_type, - std::vector *out_type, - std::vector *aux_type) const override { - int dtype = -1; - for (size_t i = 0; i < in_type->size(); ++i) { - if (dtype == -1) { - dtype = in_type->at(i); - } else { - CHECK(in_type->at(i) == dtype || - in_type->at(i) == -1) << - "Non-uniform data type in BilinearSampler"; - } - } - if (dtype == -1) { - LOG(FATAL) << "Not enough information to infer type in BilinearSampler."; - return false; - } - size_t nin = this->ListArguments().size(); - in_type->clear(); - for (size_t i = 0; i < nin; ++i) in_type->push_back(dtype); - size_t naux = this->ListAuxiliaryStates().size(); - aux_type->clear(); - for (size_t i = 0; i < naux; ++i) aux_type->push_back(dtype); - size_t nout = this->ListOutputs().size(); - out_type->clear(); - for (size_t i = 0; i < nout; ++i) out_type->push_back(dtype); - return true; - } - - OperatorProperty* Copy() const override { - auto ptr = new BilinearSamplerProp(); - ptr->param_ = param_; - return ptr; - } - - std::string TypeString() const override { - return "BilinearSampler"; - } - - std::vector DeclareBackwardDependency( - const std::vector &out_grad, - const std::vector &in_data, - const std::vector &out_data) const override { - return {out_grad[bs::kOut], - in_data[bs::kData], - out_data[bs::kTmp], - in_data[bs::kGrid]}; - } - - Operator* CreateOperator(Context ctx) const override { - LOG(FATAL) << "Not Implemented."; - return NULL; - } - - Operator* CreateOperatorEx(Context ctx, std::vector *in_shape, - std::vector *in_type) const override; - - private: - BilinearSamplerParam param_; -}; // class BilinearSamplerProp -#endif // DMLC_USE_CXX11 -} // namespace op -} // namespace mxnet -#endif // MXNET_OPERATOR_BILINEAR_SAMPLER_INL_H_ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file bilinear_Sampler-inl.h + * \brief + * \author Xu Dong +*/ +#ifndef MXNET_OPERATOR_BILINEAR_SAMPLER_INL_H_ +#define MXNET_OPERATOR_BILINEAR_SAMPLER_INL_H_ + +#include +#include +#include +#include +#include +#include +#include +#include "./operator_common.h" + +namespace mxnet { +namespace op { + +namespace bs { +enum BilinearSamplerOpInputs {kData, kGrid}; +enum BilinearSamplerOpOutputs {kOut, kTmp}; +} + +struct BilinearSamplerParam : public dmlc::Parameter { + DMLC_DECLARE_PARAMETER(BilinearSamplerParam) { + } +}; + +template +class BilinearSamplerOp : public Operator { + public: + explicit BilinearSamplerOp(BilinearSamplerParam p) { + this->param_ = p; + } + + virtual void Forward(const OpContext &ctx, + const std::vector &in_data, + const std::vector &req, + const std::vector &out_data, + const std::vector &aux_args) { + using namespace mshadow; + using namespace mshadow::expr; + CHECK_EQ(req[bs::kOut], kWriteTo); + CHECK_EQ(in_data.size(), 2U); + Stream *s = ctx.get_stream(); + + Tensor data = in_data[bs::kData].get(s); + Tensor grid = in_data[bs::kGrid].get(s); + Tensor out = out_data[bs::kOut].get(s); + + BilinearSamplerForward(out, data, grid); + } + + virtual void Backward(const OpContext &ctx, + const std::vector &out_grad, + const std::vector &in_data, + const std::vector &out_data, + const std::vector &req, + const std::vector &in_grad, + const std::vector &aux_args) { + using namespace mshadow; + using namespace mshadow::expr; + CHECK_EQ(in_data.size(), 2U); + CHECK_NE(req[bs::kData], kWriteInplace); + CHECK_NE(req[bs::kGrid], kWriteInplace); + Stream *s = ctx.get_stream(); + + Tensor data = in_data[bs::kData].get(s); + Tensor grid = in_data[bs::kGrid].get(s); + Tensor gdata = in_grad[bs::kData].get(s); + Tensor ggrid = in_grad[bs::kGrid].get(s); + Tensor grad = out_grad[bs::kOut].get(s); + if (req[bs::kData] != kNullOp && req[bs::kGrid] != kNullOp) { + if (req[bs::kData] == kWriteTo) { + gdata = scalar(0.0f); + } + if (req[bs::kGrid] == kWriteTo) { + ggrid = scalar(0.0f); + } + BilinearSamplerBackward(gdata, ggrid, grad, data, grid); + } else if (req[bs::kData] == kNullOp && req[bs::kGrid] == kNullOp) { + return; + } else { + LOG(FATAL) << "Have not implemented the data req combinations! gdata_req=" + << req[bs::kData] << " ggrid_req=" << req[bs::kGrid]; + } + } + + private: + BilinearSamplerParam param_; +}; // class BilinearSamplerOp + +template +Operator* CreateOp(BilinearSamplerParam param, int dtype); + +#if DMLC_USE_CXX11 +class BilinearSamplerProp : public OperatorProperty { + public: + int NumVisibleOutputs() const override { + return 1; + } + + int NumOutputs() const override { + return 2; + } + + std::vector ListArguments() const override { + return {"data", "grid"}; + } + + std::vector ListOutputs() const override { + return {"output", "tmp"}; + } + + void Init(const std::vector >& kwargs) override { + param_.Init(kwargs); + } + + std::map GetParams() const override { + return param_.__DICT__(); + } + + bool InferShape(std::vector *in_shape, + std::vector *out_shape, + std::vector *aux_shape) const override { + using namespace mshadow; + CHECK_EQ(in_shape->size(), 2U) << "Input:[data, grid]"; + const TShape &dshape = (*in_shape)[bs::kData]; + const TShape &lshape = (*in_shape)[bs::kGrid]; + if (dshape.ndim() == 0) return false; + CHECK_EQ(dshape.ndim(), 4U) \ + << "input data should be 4D in batch-num_filter-y-x"; + if (lshape.ndim() == 0) return false; + CHECK_EQ(lshape.ndim(), 4U) \ + << "Sampler grid should be 4D in batch-2-y-x"; + CHECK_EQ(dshape[0], lshape[0]); + CHECK_EQ(lshape[1], 2U) << "incorrect grid shape[1], should be 2"; + // target height + CHECK_GT(lshape[2], 0U) \ + << "incorrect grid_shape: " << lshape[2]; + // target width + CHECK_GT(lshape[3], 0U) \ + << "incorrect grid_shape: " << lshape[3]; + out_shape->clear(); + // output_shape : (data.shape[0], data.shape[1], grid.shape[2], grid.shape[3]) + out_shape->push_back(dshape); + (*out_shape)[bs::kOut][2] = lshape[2]; + (*out_shape)[bs::kOut][3] = lshape[3]; + out_shape->push_back(Shape4(lshape[0], lshape[2], lshape[3], 2)); + return true; + } + + bool InferType(std::vector *in_type, + std::vector *out_type, + std::vector *aux_type) const override { + int dtype = -1; + for (size_t i = 0; i < in_type->size(); ++i) { + if (dtype == -1) { + dtype = in_type->at(i); + } else { + CHECK(in_type->at(i) == dtype || + in_type->at(i) == -1) << + "Non-uniform data type in BilinearSampler"; + } + } + if (dtype == -1) { + LOG(FATAL) << "Not enough information to infer type in BilinearSampler."; + return false; + } + size_t nin = this->ListArguments().size(); + in_type->clear(); + for (size_t i = 0; i < nin; ++i) in_type->push_back(dtype); + size_t naux = this->ListAuxiliaryStates().size(); + aux_type->clear(); + for (size_t i = 0; i < naux; ++i) aux_type->push_back(dtype); + size_t nout = this->ListOutputs().size(); + out_type->clear(); + for (size_t i = 0; i < nout; ++i) out_type->push_back(dtype); + return true; + } + + OperatorProperty* Copy() const override { + auto ptr = new BilinearSamplerProp(); + ptr->param_ = param_; + return ptr; + } + + std::string TypeString() const override { + return "BilinearSampler"; + } + + std::vector DeclareBackwardDependency( + const std::vector &out_grad, + const std::vector &in_data, + const std::vector &out_data) const override { + return {out_grad[bs::kOut], + in_data[bs::kData], + out_data[bs::kTmp], + in_data[bs::kGrid]}; + } + + Operator* CreateOperator(Context ctx) const override { + LOG(FATAL) << "Not Implemented."; + return NULL; + } + + Operator* CreateOperatorEx(Context ctx, std::vector *in_shape, + std::vector *in_type) const override; + + private: + BilinearSamplerParam param_; +}; // class BilinearSamplerProp +#endif // DMLC_USE_CXX11 +} // namespace op +} // namespace mxnet +#endif // MXNET_OPERATOR_BILINEAR_SAMPLER_INL_H_ diff --git a/src/operator/bilinear_sampler.cc b/src/operator/bilinear_sampler.cc index ca83a43343a6..d03f6798fde5 100644 --- a/src/operator/bilinear_sampler.cc +++ b/src/operator/bilinear_sampler.cc @@ -1,228 +1,246 @@ -/*! - * Copyright (c) 2017 by Contributors - * \file bilinear_sampler.cc - * \brief - * \author Xu Dong -*/ - -#include "./bilinear_sampler-inl.h" - -namespace mshadow { -template -bool between(DType value, int lowerBound, int upperBound) { - return (value >= lowerBound && value <= upperBound); -} -template -inline void BilinearSamplerForward(const Tensor &output, - const Tensor &input, - const Tensor &grid_src) { - DType *out = output.dptr_; - const DType *data = input.dptr_; - const DType *grid = grid_src.dptr_; - int o_n = output.size(0), o_c = output.size(1), o_h = output.size(2), o_w = output.size(3); - int i_c = input.size(1), i_h = input.size(2), i_w = input.size(3); - for (index_t n = 0; n < static_cast(o_n); ++n) { - for (index_t c = 0; c < static_cast(o_c); ++c) { - for (index_t h = 0; h < static_cast(o_h); ++h) { - for (index_t w = 0; w < static_cast(o_w); ++w) { - index_t out_index = n * o_c * o_h * o_w + c * o_h * o_w + h * o_w + w; - index_t grid_index = n * o_h * o_w * 2 + h * o_w + w; - DType y_real = (*(grid + grid_index + o_h * o_w) + 1) * (i_h - 1) / 2; - DType x_real = (*(grid + grid_index) + 1) * (i_w - 1) / 2; - int top_left_y = static_cast(floor(y_real)); - int top_left_x = static_cast(floor(x_real)); - DType top_left_y_w = 1.0 - (y_real - top_left_y); - DType top_left_x_w = 1.0 - (x_real - top_left_x); - int data_index = n * i_c * i_h * i_w + c * i_h * i_w + - top_left_y * i_w + top_left_x; - DType top_left_v = 0; - DType top_right_v = 0; - DType bottom_left_v = 0; - DType bottom_right_v = 0; - if (between(top_left_x, 0, i_w-1) && between(top_left_y, 0, i_h-1)) - top_left_v = *(data + data_index); - if (between(top_left_x + 1, 0, i_w-1) && between(top_left_y, 0, i_h-1)) - top_right_v = *(data + data_index + 1); - if (between(top_left_x, 0, i_w-1) && between(top_left_y + 1, 0, i_h-1)) - bottom_left_v = *(data + data_index + i_w); - if (between(top_left_x+1, 0, i_w-1) && between(top_left_y + 1, 0, i_h-1)) - bottom_right_v = *(data + data_index + i_w + 1); - *(out+out_index) = top_left_v * top_left_y_w * top_left_x_w + - top_right_v * top_left_y_w * (1.0 - top_left_x_w) + - bottom_left_v * (1.0 - top_left_y_w) * top_left_x_w + - bottom_right_v * (1.0 - top_left_y_w) * (1.0 - top_left_x_w); - } - } - } - } -} - -template -inline void BilinearSamplerBackward(const Tensor &gdata, - const Tensor &ggrid, - const Tensor &output_grad, - const Tensor &input_data, - const Tensor &grid) { - DType *g_input = gdata.dptr_; - DType *grad_grid = ggrid.dptr_; - const DType *grid_src = grid.dptr_; - const DType *grad = output_grad.dptr_; - const DType *data = input_data.dptr_; - int o_n = output_grad.size(0), o_c = output_grad.size(1), - o_h = output_grad.size(2), o_w = output_grad.size(3); - int i_c = input_data.size(1), i_h = input_data.size(2), i_w = input_data.size(3); - for (index_t n = 0; n < static_cast(o_n); ++n) { - for (index_t h = 0; h < static_cast(o_h); ++h) { - for (index_t w = 0; w < static_cast(o_w); ++w) { - DType top_left_y_gw = 0.0; - DType top_left_x_gw = 0.0; - index_t grid_src_index = n * o_h * o_w * 2 + h * o_w + w; - DType y_real = (*(grid_src + grid_src_index + o_h * o_w) + 1) * (i_h - 1) / 2; - DType x_real = (*(grid_src + grid_src_index) + 1) * (i_w - 1) / 2; - int top_left_y = static_cast(floor(y_real)); - int top_left_x = static_cast(floor(x_real)); - DType top_left_y_w = 1.0 - (y_real - top_left_y); - DType top_left_x_w = 1.0 - (x_real - top_left_x); - for (index_t c = 0; c < static_cast(o_c); ++c) { - index_t grad_index = n * o_c * o_h * o_w + c * o_h * o_w + h * o_w + w; - int data_index = n * i_c * i_h * i_w + c * i_h * i_w + top_left_y * i_w - + top_left_x; - // calc 4 vertex value in input data - DType top_left_v = 0; - DType top_right_v = 0; - DType bottom_left_v = 0; - DType bottom_right_v = 0; - // calc input grad - if (between(top_left_x, 0, i_w-1) && between(top_left_y, 0, i_h-1)) { - *(g_input + data_index) += *(grad + grad_index) * top_left_y_w * top_left_x_w; - top_left_v = *(data + data_index); - } - if (between(top_left_x+1, 0, i_w-1) && between(top_left_y, 0, i_h-1)) { - *(g_input + data_index + 1) += *(grad + grad_index) * top_left_y_w - * (1.0 - top_left_x_w); - top_right_v = *(data + data_index + 1); - } - if (between(top_left_x, 0, i_w-1) && between(top_left_y+1, 0, i_h-1)) { - *(g_input + data_index+ i_w) += *(grad + grad_index) * (1.0 - top_left_y_w) - * top_left_x_w; - bottom_left_v = *(data + data_index + i_w); - } - if (between(top_left_x+1, 0, i_w-1) && between(top_left_y+1, 0, i_h-1)) { - *(g_input + data_index+ i_w + 1) += *(grad + grad_index) * (1.0 - top_left_y_w) - * (1.0 - top_left_x_w); - bottom_right_v = *(data + data_index + i_w + 1); - } - // calc weight grad of top_left_w, then multiple -1 is the grad of grid_src - top_left_y_gw -= *(grad + grad_index) * (top_right_v - bottom_right_v + - (top_left_v - top_right_v - bottom_left_v + bottom_right_v) - * top_left_x_w); - top_left_x_gw -= *(grad + grad_index) * (bottom_left_v - bottom_right_v + - (top_left_v - top_right_v - bottom_left_v + bottom_right_v) - * top_left_y_w); - } - // calc grad of grid - *(grad_grid + grid_src_index + o_h * o_w) += top_left_y_gw * (i_h - 1) / 2; - *(grad_grid + grid_src_index) += top_left_x_gw * (i_w - 1) / 2; - } - } - } - } -} // namespace mshadow - -namespace mxnet { -namespace op { -template<> -Operator* CreateOp(BilinearSamplerParam param, int dtype) { - Operator *op = NULL; - MSHADOW_REAL_TYPE_SWITCH(dtype, DType, { - op = new BilinearSamplerOp(param); - }) - return op; -} - -Operator *BilinearSamplerProp::CreateOperatorEx(Context ctx, std::vector *in_shape, - std::vector *in_type) const { - DO_BIND_DISPATCH(CreateOp, param_, (*in_type)[0]); -} - -DMLC_REGISTER_PARAMETER(BilinearSamplerParam); - -MXNET_REGISTER_OP_PROPERTY(BilinearSampler, BilinearSamplerProp) -.add_argument("data", "NDArray-or-Symbol", "Input data to the BilinearsamplerOp.") -.add_argument("grid", "NDArray-or-Symbol", "Input grid to the BilinearsamplerOp." - "grid has two channels: x_src, y_src") -.add_arguments(BilinearSamplerParam::__FIELDS__()) -.describe(R"code(Applies bilinear sampling to input feature map. - -Bilinear Sampling is the key of [NIPS2015] \"Spatial Transformer Networks\". The usage of the operator is very similar to remap function in OpenCV, -except that the operator has the backward pass. - -Given :math:`data` and :math:`grid`, then the output is computed by - -.. math:: - x_{src} = grid[batch, 0, y_{dst}, x_{dst}] \\ - y_{src} = grid[batch, 1, y_{dst}, x_{dst}] \\ - output[batch, channel, y_{dst}, x_{dst}] = G(data[batch, channel, y_{src}, x_{src}) - -:math:`x_{dst}`, :math:`y_{dst}` enumerate all spatial locations in :math:`output`, and :math:`G()` denotes the bilinear interpolation kernel. -The out-boundary points will be padded with zeros.The shape of the output will be (data.shape[0], data.shape[1], grid.shape[2], grid.shape[3]). - -The operator assumes that :math:`data` has 'NCHW' layout and :math:`grid` has been normalized to [-1, 1]. - -BilinearSampler often cooperates with GridGenerator which generates sampling grids for BilinearSampler. -GridGenerator supports two kinds of transformation: ``affine`` and ``warp``. -If users want to design a CustomOp to manipulate :math:`grid`, please firstly refer to the code of GridGenerator. - -Example 1:: - - ## Zoom out data two times - data = array([[[[1, 4, 3, 6], - [1, 8, 8, 9], - [0, 4, 1, 5], - [1, 0, 1, 3]]]]) - - affine_matrix = array([[2, 0, 0], - [0, 2, 0]]) - - affine_matrix = reshape(affine_matrix, shape=(1, 6)) - - grid = GridGenerator(data=affine_matrix, transform_type='affine', target_shape=(4, 4)) - - out = BilinearSampler(data, grid) - - out - [[[[ 0, 0, 0, 0], - [ 0, 3.5, 6.5, 0], - [ 0, 1.25, 2.5, 0], - [ 0, 0, 0, 0]]] - - -Example 2:: - - ## shift data horizontally by -1 pixel - - data = array([[[[1, 4, 3, 6], - [1, 8, 8, 9], - [0, 4, 1, 5], - [1, 0, 1, 3]]]]) - - warp_maxtrix = array([[[[1, 1, 1, 1], - [1, 1, 1, 1], - [1, 1, 1, 1], - [1, 1, 1, 1]], - [[0, 0, 0, 0], - [0, 0, 0, 0], - [0, 0, 0, 0], - [0, 0, 0, 0]]]]) - - grid = GridGenerator(data=warp_matrix, transform_type='warp') - out = BilinearSampler(data, grid) - - out - [[[[ 4, 3, 6, 0], - [ 8, 8, 9, 0], - [ 4, 1, 5, 0], - [ 0, 1, 3, 0]]] -)code" ADD_FILELINE); -} // namespace op -} // namespace mxnet +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file bilinear_sampler.cc + * \brief + * \author Xu Dong +*/ + +#include "./bilinear_sampler-inl.h" + +namespace mshadow { +template +bool between(DType value, int lowerBound, int upperBound) { + return (value >= lowerBound && value <= upperBound); +} +template +inline void BilinearSamplerForward(const Tensor &output, + const Tensor &input, + const Tensor &grid_src) { + DType *out = output.dptr_; + const DType *data = input.dptr_; + const DType *grid = grid_src.dptr_; + int o_n = output.size(0), o_c = output.size(1), o_h = output.size(2), o_w = output.size(3); + int i_c = input.size(1), i_h = input.size(2), i_w = input.size(3); + for (index_t n = 0; n < static_cast(o_n); ++n) { + for (index_t c = 0; c < static_cast(o_c); ++c) { + for (index_t h = 0; h < static_cast(o_h); ++h) { + for (index_t w = 0; w < static_cast(o_w); ++w) { + index_t out_index = n * o_c * o_h * o_w + c * o_h * o_w + h * o_w + w; + index_t grid_index = n * o_h * o_w * 2 + h * o_w + w; + DType y_real = (*(grid + grid_index + o_h * o_w) + 1) * (i_h - 1) / 2; + DType x_real = (*(grid + grid_index) + 1) * (i_w - 1) / 2; + int top_left_y = static_cast(floor(y_real)); + int top_left_x = static_cast(floor(x_real)); + DType top_left_y_w = 1.0 - (y_real - top_left_y); + DType top_left_x_w = 1.0 - (x_real - top_left_x); + int data_index = n * i_c * i_h * i_w + c * i_h * i_w + + top_left_y * i_w + top_left_x; + DType top_left_v = 0; + DType top_right_v = 0; + DType bottom_left_v = 0; + DType bottom_right_v = 0; + if (between(top_left_x, 0, i_w-1) && between(top_left_y, 0, i_h-1)) + top_left_v = *(data + data_index); + if (between(top_left_x + 1, 0, i_w-1) && between(top_left_y, 0, i_h-1)) + top_right_v = *(data + data_index + 1); + if (between(top_left_x, 0, i_w-1) && between(top_left_y + 1, 0, i_h-1)) + bottom_left_v = *(data + data_index + i_w); + if (between(top_left_x+1, 0, i_w-1) && between(top_left_y + 1, 0, i_h-1)) + bottom_right_v = *(data + data_index + i_w + 1); + *(out+out_index) = top_left_v * top_left_y_w * top_left_x_w + + top_right_v * top_left_y_w * (1.0 - top_left_x_w) + + bottom_left_v * (1.0 - top_left_y_w) * top_left_x_w + + bottom_right_v * (1.0 - top_left_y_w) * (1.0 - top_left_x_w); + } + } + } + } +} + +template +inline void BilinearSamplerBackward(const Tensor &gdata, + const Tensor &ggrid, + const Tensor &output_grad, + const Tensor &input_data, + const Tensor &grid) { + DType *g_input = gdata.dptr_; + DType *grad_grid = ggrid.dptr_; + const DType *grid_src = grid.dptr_; + const DType *grad = output_grad.dptr_; + const DType *data = input_data.dptr_; + int o_n = output_grad.size(0), o_c = output_grad.size(1), + o_h = output_grad.size(2), o_w = output_grad.size(3); + int i_c = input_data.size(1), i_h = input_data.size(2), i_w = input_data.size(3); + for (index_t n = 0; n < static_cast(o_n); ++n) { + for (index_t h = 0; h < static_cast(o_h); ++h) { + for (index_t w = 0; w < static_cast(o_w); ++w) { + DType top_left_y_gw = 0.0; + DType top_left_x_gw = 0.0; + index_t grid_src_index = n * o_h * o_w * 2 + h * o_w + w; + DType y_real = (*(grid_src + grid_src_index + o_h * o_w) + 1) * (i_h - 1) / 2; + DType x_real = (*(grid_src + grid_src_index) + 1) * (i_w - 1) / 2; + int top_left_y = static_cast(floor(y_real)); + int top_left_x = static_cast(floor(x_real)); + DType top_left_y_w = 1.0 - (y_real - top_left_y); + DType top_left_x_w = 1.0 - (x_real - top_left_x); + for (index_t c = 0; c < static_cast(o_c); ++c) { + index_t grad_index = n * o_c * o_h * o_w + c * o_h * o_w + h * o_w + w; + int data_index = n * i_c * i_h * i_w + c * i_h * i_w + top_left_y * i_w + + top_left_x; + // calc 4 vertex value in input data + DType top_left_v = 0; + DType top_right_v = 0; + DType bottom_left_v = 0; + DType bottom_right_v = 0; + // calc input grad + if (between(top_left_x, 0, i_w-1) && between(top_left_y, 0, i_h-1)) { + *(g_input + data_index) += *(grad + grad_index) * top_left_y_w * top_left_x_w; + top_left_v = *(data + data_index); + } + if (between(top_left_x+1, 0, i_w-1) && between(top_left_y, 0, i_h-1)) { + *(g_input + data_index + 1) += *(grad + grad_index) * top_left_y_w + * (1.0 - top_left_x_w); + top_right_v = *(data + data_index + 1); + } + if (between(top_left_x, 0, i_w-1) && between(top_left_y+1, 0, i_h-1)) { + *(g_input + data_index+ i_w) += *(grad + grad_index) * (1.0 - top_left_y_w) + * top_left_x_w; + bottom_left_v = *(data + data_index + i_w); + } + if (between(top_left_x+1, 0, i_w-1) && between(top_left_y+1, 0, i_h-1)) { + *(g_input + data_index+ i_w + 1) += *(grad + grad_index) * (1.0 - top_left_y_w) + * (1.0 - top_left_x_w); + bottom_right_v = *(data + data_index + i_w + 1); + } + // calc weight grad of top_left_w, then multiple -1 is the grad of grid_src + top_left_y_gw -= *(grad + grad_index) * (top_right_v - bottom_right_v + + (top_left_v - top_right_v - bottom_left_v + bottom_right_v) + * top_left_x_w); + top_left_x_gw -= *(grad + grad_index) * (bottom_left_v - bottom_right_v + + (top_left_v - top_right_v - bottom_left_v + bottom_right_v) + * top_left_y_w); + } + // calc grad of grid + *(grad_grid + grid_src_index + o_h * o_w) += top_left_y_gw * (i_h - 1) / 2; + *(grad_grid + grid_src_index) += top_left_x_gw * (i_w - 1) / 2; + } + } + } + } +} // namespace mshadow + +namespace mxnet { +namespace op { +template<> +Operator* CreateOp(BilinearSamplerParam param, int dtype) { + Operator *op = NULL; + MSHADOW_REAL_TYPE_SWITCH(dtype, DType, { + op = new BilinearSamplerOp(param); + }) + return op; +} + +Operator *BilinearSamplerProp::CreateOperatorEx(Context ctx, std::vector *in_shape, + std::vector *in_type) const { + DO_BIND_DISPATCH(CreateOp, param_, (*in_type)[0]); +} + +DMLC_REGISTER_PARAMETER(BilinearSamplerParam); + +MXNET_REGISTER_OP_PROPERTY(BilinearSampler, BilinearSamplerProp) +.add_argument("data", "NDArray-or-Symbol", "Input data to the BilinearsamplerOp.") +.add_argument("grid", "NDArray-or-Symbol", "Input grid to the BilinearsamplerOp." + "grid has two channels: x_src, y_src") +.add_arguments(BilinearSamplerParam::__FIELDS__()) +.describe(R"code(Applies bilinear sampling to input feature map. + +Bilinear Sampling is the key of [NIPS2015] \"Spatial Transformer Networks\". The usage of the operator is very similar to remap function in OpenCV, +except that the operator has the backward pass. + +Given :math:`data` and :math:`grid`, then the output is computed by + +.. math:: + x_{src} = grid[batch, 0, y_{dst}, x_{dst}] \\ + y_{src} = grid[batch, 1, y_{dst}, x_{dst}] \\ + output[batch, channel, y_{dst}, x_{dst}] = G(data[batch, channel, y_{src}, x_{src}) + +:math:`x_{dst}`, :math:`y_{dst}` enumerate all spatial locations in :math:`output`, and :math:`G()` denotes the bilinear interpolation kernel. +The out-boundary points will be padded with zeros.The shape of the output will be (data.shape[0], data.shape[1], grid.shape[2], grid.shape[3]). + +The operator assumes that :math:`data` has 'NCHW' layout and :math:`grid` has been normalized to [-1, 1]. + +BilinearSampler often cooperates with GridGenerator which generates sampling grids for BilinearSampler. +GridGenerator supports two kinds of transformation: ``affine`` and ``warp``. +If users want to design a CustomOp to manipulate :math:`grid`, please firstly refer to the code of GridGenerator. + +Example 1:: + + ## Zoom out data two times + data = array([[[[1, 4, 3, 6], + [1, 8, 8, 9], + [0, 4, 1, 5], + [1, 0, 1, 3]]]]) + + affine_matrix = array([[2, 0, 0], + [0, 2, 0]]) + + affine_matrix = reshape(affine_matrix, shape=(1, 6)) + + grid = GridGenerator(data=affine_matrix, transform_type='affine', target_shape=(4, 4)) + + out = BilinearSampler(data, grid) + + out + [[[[ 0, 0, 0, 0], + [ 0, 3.5, 6.5, 0], + [ 0, 1.25, 2.5, 0], + [ 0, 0, 0, 0]]] + + +Example 2:: + + ## shift data horizontally by -1 pixel + + data = array([[[[1, 4, 3, 6], + [1, 8, 8, 9], + [0, 4, 1, 5], + [1, 0, 1, 3]]]]) + + warp_maxtrix = array([[[[1, 1, 1, 1], + [1, 1, 1, 1], + [1, 1, 1, 1], + [1, 1, 1, 1]], + [[0, 0, 0, 0], + [0, 0, 0, 0], + [0, 0, 0, 0], + [0, 0, 0, 0]]]]) + + grid = GridGenerator(data=warp_matrix, transform_type='warp') + out = BilinearSampler(data, grid) + + out + [[[[ 4, 3, 6, 0], + [ 8, 8, 9, 0], + [ 4, 1, 5, 0], + [ 0, 1, 3, 0]]] +)code" ADD_FILELINE); +} // namespace op +} // namespace mxnet diff --git a/src/operator/bilinear_sampler.cu b/src/operator/bilinear_sampler.cu index dc394f130b0a..14b5cd20a3d7 100644 --- a/src/operator/bilinear_sampler.cu +++ b/src/operator/bilinear_sampler.cu @@ -1,207 +1,225 @@ -/*! - * Copyright (c) 2017 by Contributors - * \file bilinear_sampler.cu - * \brief - * \author Xu Dong -*/ - -#include "./bilinear_sampler-inl.h" -#include -#include "../common/cuda_utils.h" -#if MXNET_USE_CUDNN == 1 && CUDNN_MAJOR >= 5 -#include "./cudnn_bilinear_sampler-inl.h" -#endif // MXNET_USE_CUDNN && CUDNN_MAJOR - -namespace mshadow { -namespace cuda { -template -__device__ bool between(DType value, int lowerBound, int upperBound) { - return (value >= lowerBound && value <= upperBound); -} -template -__global__ void BilinearSamplerForwardKernel(const int i_c, const int i_h, - const int i_w, const DType* data, - const DType* grid, const int o_n, - const int o_c, const int o_h, - const int o_w, DType* out) { - for (int index = (blockIdx.x + blockIdx.y * gridDim.x) * blockDim.x + threadIdx.x; - index < o_n * o_c * o_h * o_w; - index += blockDim.x * gridDim.x * gridDim.y) { - // (n, c, h, w) is the element in out - int w = index % o_w; - int h = (index / o_w) % o_h; - int c = (index / o_w / o_h) % o_c; - int n = index / o_w / o_h / o_c; - index_t out_index = n * o_c * o_h * o_w + c * o_h * o_w + h * o_w + w; - index_t grid_index = n * o_h * o_w * 2 + h * o_w + w; - DType y_real = (*(grid + grid_index + o_h * o_w) + 1) * (i_h - 1) / 2; - DType x_real = (*(grid + grid_index) + 1) * (i_w - 1) / 2; - int top_left_y = static_cast(floor(y_real)); - int top_left_x = static_cast(floor(x_real)); - DType top_left_y_w = 1.0 - (y_real - top_left_y); - DType top_left_x_w = 1.0 - (x_real - top_left_x); - int data_index = n * i_c * i_h * i_w + c * i_h * i_w + top_left_y * i_w + top_left_x; - DType top_left_v = 0; - DType top_right_v = 0; - DType bottom_left_v = 0; - DType bottom_right_v = 0; - if (between(top_left_x, 0, i_w-1) && between(top_left_y, 0, i_h-1)) - top_left_v = *(data + data_index); - if (between(top_left_x + 1, 0, i_w-1) && between(top_left_y, 0, i_h-1)) - top_right_v = *(data + data_index + 1); - if (between(top_left_x, 0, i_w-1) && between(top_left_y + 1, 0, i_h-1)) - bottom_left_v = *(data + data_index + i_w); - if (between(top_left_x+1, 0, i_w-1) && between(top_left_y + 1, 0, i_h-1)) - bottom_right_v = *(data + data_index + i_w + 1); - *(out+out_index) = top_left_v * top_left_y_w * top_left_x_w + - top_right_v * top_left_y_w * (1.0 - top_left_x_w) + - bottom_left_v * (1.0 - top_left_y_w) * top_left_x_w + - bottom_right_v * (1.0 - top_left_y_w) * (1.0 - top_left_x_w); - } -} - -template -__global__ void BilinearSamplerBackwardKernel(const int i_c, const int i_h, - const int i_w, const DType* grad, - const DType* data, const int o_n, - const int o_c, const int o_h, - const int o_w, DType* g_input, - const DType* grid_src, - DType* grad_grid) { - for (int index = (blockIdx.x + blockIdx.y * gridDim.x) * blockDim.x + threadIdx.x; - index < o_n * o_h * o_w; - index += blockDim.x * gridDim.x * gridDim.y) { - // (n, c, h, w) is the element in grad - int w = index % o_w; - int h = (index / o_w) % o_h; - int n = index / o_w / o_h; - DType top_left_y_gw = 0.0; - DType top_left_x_gw = 0.0; - index_t grid_src_index = n * o_h * o_w * 2 + h * o_w + w; - DType y_real = (*(grid_src + grid_src_index + o_h * o_w) + 1) * (i_h - 1) / 2; - DType x_real = (*(grid_src + grid_src_index) + 1) * (i_w - 1) / 2; - - int top_left_y = static_cast(floor(y_real)); - int top_left_x = static_cast(floor(x_real)); - DType top_left_y_w = 1.0 - (y_real - top_left_y); - DType top_left_x_w = 1.0 - (x_real - top_left_x); - for (index_t c = 0; c < o_c; ++c) { - index_t grad_index = n * o_c * o_h * o_w + c * o_h * o_w + h * o_w + w; - int data_index = n * i_c * i_h * i_w + c * i_h * i_w + top_left_y * i_w + top_left_x; - // calc 4 vertex value in input data - DType top_left_v = 0; - DType top_right_v = 0; - DType bottom_left_v = 0; - DType bottom_right_v = 0; - // calc input grad - if (between(top_left_x, 0, i_w-1) && between(top_left_y, 0, i_h-1)) { - atomicAdd(&g_input[data_index], *(grad + grad_index) * top_left_y_w * top_left_x_w); - top_left_v = *(data + data_index); - } - if (between(top_left_x+1, 0, i_w-1) && between(top_left_y, 0, i_h-1)) { - atomicAdd(&g_input[data_index + 1], *(grad + grad_index) * top_left_y_w - * (1.0 - top_left_x_w)); - top_right_v = *(data + data_index + 1); - } - if (between(top_left_x, 0, i_w-1) && between(top_left_y+1, 0, i_h-1)) { - atomicAdd(&g_input[data_index+ i_w], *(grad + grad_index) * (1.0 - top_left_y_w) - * top_left_x_w); - bottom_left_v = *(data + data_index + i_w); - } - if (between(top_left_x+1, 0, i_w-1) && between(top_left_y+1, 0, i_h-1)) { - atomicAdd(&g_input[data_index+ i_w + 1], *(grad + grad_index) * (1.0 - top_left_y_w) - * (1.0 - top_left_x_w)); - bottom_right_v = *(data + data_index + i_w + 1); - } - // calc weight grad of top_left_w, then multiple -1 is the grad of grid_src - top_left_y_gw -= *(grad + grad_index) * (top_right_v - bottom_right_v + - (top_left_v - top_right_v - bottom_left_v + bottom_right_v) - * top_left_x_w); - top_left_x_gw -= *(grad + grad_index) * (bottom_left_v - bottom_right_v + - (top_left_v - top_right_v - bottom_left_v + bottom_right_v) - * top_left_y_w); - } - // calc grad of grid - *(grad_grid + grid_src_index + o_h * o_w) += top_left_y_gw * (i_h - 1) / 2; - *(grad_grid + grid_src_index) += top_left_x_gw * (i_w - 1) / 2; - } -} -} // namespace cuda - -template -inline void BilinearSamplerForward(const Tensor &output, - const Tensor &input, - const Tensor &grid_src) { - DType *out = output.dptr_; - const DType *data = input.dptr_; - const DType *grid = grid_src.dptr_; - int o_n = output.size(0), o_c = output.size(1), o_h = output.size(2), o_w = output.size(3); - int i_c = input.size(1), i_h = input.size(2), i_w = input.size(3); - using namespace cuda; - const int max_block = (output.shape_.Size() + kMaxThreadsPerBlock - 1) / kMaxThreadsPerBlock; - const int grid_dim_x = (max_block > kMaxGridDim) ? kMaxGridDim : max_block; - const int grid_dim_y = - (max_block > kMaxGridDim) ? (max_block + kMaxGridDim - 1) / kMaxGridDim : 1; - dim3 num_blocks(grid_dim_x, grid_dim_y); - dim3 threads_per_block(kMaxThreadsPerBlock); - CheckLaunchParam(num_blocks, threads_per_block, "bilinear sampler forward"); - cudaStream_t stream = Stream::GetStream(output.stream_); - cuda::BilinearSamplerForwardKernel << > >( - i_c, i_h, i_w, data, grid, o_n, o_c, o_h, o_w, out); - // post kernel check - cudaError err = cudaPeekAtLastError(); - CHECK_EQ(err, cudaSuccess) << cudaGetErrorString(err); -} - -template -inline void BilinearSamplerBackward(const Tensor &input_grad, - const Tensor &ggrid, - const Tensor &output_grad, - const Tensor &input_data, - const Tensor &grid) { - DType *g_input = input_grad.dptr_; - DType *grad_grid = ggrid.dptr_; - const DType *grid_src = grid.dptr_; - const DType *grad = output_grad.dptr_; - const DType *data = input_data.dptr_; - int o_n = output_grad.size(0), o_c = output_grad.size(1), - o_h = output_grad.size(2), o_w = output_grad.size(3); - int i_c = input_data.size(1), i_h = input_data.size(2), i_w = input_data.size(3); - using namespace cuda; - const int max_block = (output_grad.shape_.Size() / o_c + kMaxThreadsPerBlock - 1) - / kMaxThreadsPerBlock; - const int grid_dim_x = (max_block > kMaxGridDim) ? kMaxGridDim : max_block; - const int grid_dim_y = - (max_block > kMaxGridDim) ? (max_block + kMaxGridDim - 1) / kMaxGridDim : 1; - dim3 num_blocks(grid_dim_x, grid_dim_y); - dim3 threads_per_block(kMaxThreadsPerBlock); - CheckLaunchParam(num_blocks, threads_per_block, "bilinear sampler backward"); - cudaStream_t stream = Stream::GetStream(input_grad.stream_); - cuda::BilinearSamplerBackwardKernel << > >( - i_c, i_h, i_w, grad, data, o_n, o_c, o_h, o_w, g_input, grid_src, grad_grid); - // post kernel check - cudaError err = cudaPeekAtLastError(); - CHECK_EQ(err, cudaSuccess) << cudaGetErrorString(err); -} - -} // namespace mshadow - -namespace mxnet { -namespace op { -template<> -Operator* CreateOp(BilinearSamplerParam param, int dtype) { - Operator *op = NULL; -#if MXNET_USE_CUDNN == 1 && CUDNN_MAJOR >= 5 - MSHADOW_REAL_TYPE_SWITCH(dtype, DType, { - op = new CuDNNBilinearSamplerOp(param); - }) -#else - MSHADOW_REAL_TYPE_SWITCH(dtype, DType, { - op = new BilinearSamplerOp(param); - }) -#endif // MXNET_USE_CUDNN && CUDNN_MAJOR - return op; -} - -} // namespace op -} // namespace mxnet +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file bilinear_sampler.cu + * \brief + * \author Xu Dong +*/ + +#include "./bilinear_sampler-inl.h" +#include +#include "../common/cuda_utils.h" +#if MXNET_USE_CUDNN == 1 && CUDNN_MAJOR >= 5 +#include "./cudnn_bilinear_sampler-inl.h" +#endif // MXNET_USE_CUDNN && CUDNN_MAJOR + +namespace mshadow { +namespace cuda { +template +__device__ bool between(DType value, int lowerBound, int upperBound) { + return (value >= lowerBound && value <= upperBound); +} +template +__global__ void BilinearSamplerForwardKernel(const int i_c, const int i_h, + const int i_w, const DType* data, + const DType* grid, const int o_n, + const int o_c, const int o_h, + const int o_w, DType* out) { + for (int index = (blockIdx.x + blockIdx.y * gridDim.x) * blockDim.x + threadIdx.x; + index < o_n * o_c * o_h * o_w; + index += blockDim.x * gridDim.x * gridDim.y) { + // (n, c, h, w) is the element in out + int w = index % o_w; + int h = (index / o_w) % o_h; + int c = (index / o_w / o_h) % o_c; + int n = index / o_w / o_h / o_c; + index_t out_index = n * o_c * o_h * o_w + c * o_h * o_w + h * o_w + w; + index_t grid_index = n * o_h * o_w * 2 + h * o_w + w; + DType y_real = (*(grid + grid_index + o_h * o_w) + 1) * (i_h - 1) / 2; + DType x_real = (*(grid + grid_index) + 1) * (i_w - 1) / 2; + int top_left_y = static_cast(floor(y_real)); + int top_left_x = static_cast(floor(x_real)); + DType top_left_y_w = 1.0 - (y_real - top_left_y); + DType top_left_x_w = 1.0 - (x_real - top_left_x); + int data_index = n * i_c * i_h * i_w + c * i_h * i_w + top_left_y * i_w + top_left_x; + DType top_left_v = 0; + DType top_right_v = 0; + DType bottom_left_v = 0; + DType bottom_right_v = 0; + if (between(top_left_x, 0, i_w-1) && between(top_left_y, 0, i_h-1)) + top_left_v = *(data + data_index); + if (between(top_left_x + 1, 0, i_w-1) && between(top_left_y, 0, i_h-1)) + top_right_v = *(data + data_index + 1); + if (between(top_left_x, 0, i_w-1) && between(top_left_y + 1, 0, i_h-1)) + bottom_left_v = *(data + data_index + i_w); + if (between(top_left_x+1, 0, i_w-1) && between(top_left_y + 1, 0, i_h-1)) + bottom_right_v = *(data + data_index + i_w + 1); + *(out+out_index) = top_left_v * top_left_y_w * top_left_x_w + + top_right_v * top_left_y_w * (1.0 - top_left_x_w) + + bottom_left_v * (1.0 - top_left_y_w) * top_left_x_w + + bottom_right_v * (1.0 - top_left_y_w) * (1.0 - top_left_x_w); + } +} + +template +__global__ void BilinearSamplerBackwardKernel(const int i_c, const int i_h, + const int i_w, const DType* grad, + const DType* data, const int o_n, + const int o_c, const int o_h, + const int o_w, DType* g_input, + const DType* grid_src, + DType* grad_grid) { + for (int index = (blockIdx.x + blockIdx.y * gridDim.x) * blockDim.x + threadIdx.x; + index < o_n * o_h * o_w; + index += blockDim.x * gridDim.x * gridDim.y) { + // (n, c, h, w) is the element in grad + int w = index % o_w; + int h = (index / o_w) % o_h; + int n = index / o_w / o_h; + DType top_left_y_gw = 0.0; + DType top_left_x_gw = 0.0; + index_t grid_src_index = n * o_h * o_w * 2 + h * o_w + w; + DType y_real = (*(grid_src + grid_src_index + o_h * o_w) + 1) * (i_h - 1) / 2; + DType x_real = (*(grid_src + grid_src_index) + 1) * (i_w - 1) / 2; + + int top_left_y = static_cast(floor(y_real)); + int top_left_x = static_cast(floor(x_real)); + DType top_left_y_w = 1.0 - (y_real - top_left_y); + DType top_left_x_w = 1.0 - (x_real - top_left_x); + for (index_t c = 0; c < o_c; ++c) { + index_t grad_index = n * o_c * o_h * o_w + c * o_h * o_w + h * o_w + w; + int data_index = n * i_c * i_h * i_w + c * i_h * i_w + top_left_y * i_w + top_left_x; + // calc 4 vertex value in input data + DType top_left_v = 0; + DType top_right_v = 0; + DType bottom_left_v = 0; + DType bottom_right_v = 0; + // calc input grad + if (between(top_left_x, 0, i_w-1) && between(top_left_y, 0, i_h-1)) { + atomicAdd(&g_input[data_index], *(grad + grad_index) * top_left_y_w * top_left_x_w); + top_left_v = *(data + data_index); + } + if (between(top_left_x+1, 0, i_w-1) && between(top_left_y, 0, i_h-1)) { + atomicAdd(&g_input[data_index + 1], *(grad + grad_index) * top_left_y_w + * (1.0 - top_left_x_w)); + top_right_v = *(data + data_index + 1); + } + if (between(top_left_x, 0, i_w-1) && between(top_left_y+1, 0, i_h-1)) { + atomicAdd(&g_input[data_index+ i_w], *(grad + grad_index) * (1.0 - top_left_y_w) + * top_left_x_w); + bottom_left_v = *(data + data_index + i_w); + } + if (between(top_left_x+1, 0, i_w-1) && between(top_left_y+1, 0, i_h-1)) { + atomicAdd(&g_input[data_index+ i_w + 1], *(grad + grad_index) * (1.0 - top_left_y_w) + * (1.0 - top_left_x_w)); + bottom_right_v = *(data + data_index + i_w + 1); + } + // calc weight grad of top_left_w, then multiple -1 is the grad of grid_src + top_left_y_gw -= *(grad + grad_index) * (top_right_v - bottom_right_v + + (top_left_v - top_right_v - bottom_left_v + bottom_right_v) + * top_left_x_w); + top_left_x_gw -= *(grad + grad_index) * (bottom_left_v - bottom_right_v + + (top_left_v - top_right_v - bottom_left_v + bottom_right_v) + * top_left_y_w); + } + // calc grad of grid + *(grad_grid + grid_src_index + o_h * o_w) += top_left_y_gw * (i_h - 1) / 2; + *(grad_grid + grid_src_index) += top_left_x_gw * (i_w - 1) / 2; + } +} +} // namespace cuda + +template +inline void BilinearSamplerForward(const Tensor &output, + const Tensor &input, + const Tensor &grid_src) { + DType *out = output.dptr_; + const DType *data = input.dptr_; + const DType *grid = grid_src.dptr_; + int o_n = output.size(0), o_c = output.size(1), o_h = output.size(2), o_w = output.size(3); + int i_c = input.size(1), i_h = input.size(2), i_w = input.size(3); + using namespace cuda; + const int max_block = (output.shape_.Size() + kMaxThreadsPerBlock - 1) / kMaxThreadsPerBlock; + const int grid_dim_x = (max_block > kMaxGridDim) ? kMaxGridDim : max_block; + const int grid_dim_y = + (max_block > kMaxGridDim) ? (max_block + kMaxGridDim - 1) / kMaxGridDim : 1; + dim3 num_blocks(grid_dim_x, grid_dim_y); + dim3 threads_per_block(kMaxThreadsPerBlock); + CheckLaunchParam(num_blocks, threads_per_block, "bilinear sampler forward"); + cudaStream_t stream = Stream::GetStream(output.stream_); + cuda::BilinearSamplerForwardKernel << > >( + i_c, i_h, i_w, data, grid, o_n, o_c, o_h, o_w, out); + // post kernel check + cudaError err = cudaPeekAtLastError(); + CHECK_EQ(err, cudaSuccess) << cudaGetErrorString(err); +} + +template +inline void BilinearSamplerBackward(const Tensor &input_grad, + const Tensor &ggrid, + const Tensor &output_grad, + const Tensor &input_data, + const Tensor &grid) { + DType *g_input = input_grad.dptr_; + DType *grad_grid = ggrid.dptr_; + const DType *grid_src = grid.dptr_; + const DType *grad = output_grad.dptr_; + const DType *data = input_data.dptr_; + int o_n = output_grad.size(0), o_c = output_grad.size(1), + o_h = output_grad.size(2), o_w = output_grad.size(3); + int i_c = input_data.size(1), i_h = input_data.size(2), i_w = input_data.size(3); + using namespace cuda; + const int max_block = (output_grad.shape_.Size() / o_c + kMaxThreadsPerBlock - 1) + / kMaxThreadsPerBlock; + const int grid_dim_x = (max_block > kMaxGridDim) ? kMaxGridDim : max_block; + const int grid_dim_y = + (max_block > kMaxGridDim) ? (max_block + kMaxGridDim - 1) / kMaxGridDim : 1; + dim3 num_blocks(grid_dim_x, grid_dim_y); + dim3 threads_per_block(kMaxThreadsPerBlock); + CheckLaunchParam(num_blocks, threads_per_block, "bilinear sampler backward"); + cudaStream_t stream = Stream::GetStream(input_grad.stream_); + cuda::BilinearSamplerBackwardKernel << > >( + i_c, i_h, i_w, grad, data, o_n, o_c, o_h, o_w, g_input, grid_src, grad_grid); + // post kernel check + cudaError err = cudaPeekAtLastError(); + CHECK_EQ(err, cudaSuccess) << cudaGetErrorString(err); +} + +} // namespace mshadow + +namespace mxnet { +namespace op { +template<> +Operator* CreateOp(BilinearSamplerParam param, int dtype) { + Operator *op = NULL; +#if MXNET_USE_CUDNN == 1 && CUDNN_MAJOR >= 5 + MSHADOW_REAL_TYPE_SWITCH(dtype, DType, { + op = new CuDNNBilinearSamplerOp(param); + }) +#else + MSHADOW_REAL_TYPE_SWITCH(dtype, DType, { + op = new BilinearSamplerOp(param); + }) +#endif // MXNET_USE_CUDNN && CUDNN_MAJOR + return op; +} + +} // namespace op +} // namespace mxnet diff --git a/src/operator/channel_op_common.h b/src/operator/channel_op_common.h index 9ae6a6602c2e..113da9b35825 100644 --- a/src/operator/channel_op_common.h +++ b/src/operator/channel_op_common.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file channel_op_common.h * \brief common function used for concat and split channel * \author Bing Xu diff --git a/src/operator/concat-inl.h b/src/operator/concat-inl.h index 09b0c4b21e89..ed553c8f99e7 100644 --- a/src/operator/concat-inl.h +++ b/src/operator/concat-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file concat-inl.h * \brief * \author Bing Xu diff --git a/src/operator/concat.cc b/src/operator/concat.cc index 49fa03e7e681..1bee4b45cd21 100644 --- a/src/operator/concat.cc +++ b/src/operator/concat.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file concat.cc * \brief * \author Bing Xu diff --git a/src/operator/concat.cu b/src/operator/concat.cu index a410e199637e..06828fcbcd7d 100644 --- a/src/operator/concat.cu +++ b/src/operator/concat.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file concat.cu * \brief * \author Bing Xu diff --git a/src/operator/contrib/count_sketch-inl.h b/src/operator/contrib/count_sketch-inl.h index 566327e3677c..5df00968e4e5 100644 --- a/src/operator/contrib/count_sketch-inl.h +++ b/src/operator/contrib/count_sketch-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file count_sketch-inl.h * \brief count_sketch operator and symbol * \author Chen Zhu diff --git a/src/operator/contrib/count_sketch.cc b/src/operator/contrib/count_sketch.cc index cf89c97bdc9b..6aba8f44b3ad 100644 --- a/src/operator/contrib/count_sketch.cc +++ b/src/operator/contrib/count_sketch.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file count_sketch.cc * \brief count_sketch op * \author Chen Zhu @@ -30,7 +48,7 @@ MXNET_REGISTER_OP_PROPERTY(_contrib_count_sketch, CountSketchProp) Assume input data has shape (N, d), sign hash table s has shape (N, d), index hash table h has shape (N, d) and mapping dimension out_dim = k, -each element in s is either +1 or -1, each element in h is random integer from 0 to k-1. +each element in s is either +1 or -1, each element in h is random integer from 0 to k-1. Then the operator computs: .. math:: diff --git a/src/operator/contrib/count_sketch.cu b/src/operator/contrib/count_sketch.cu index 7cf13e8a4993..0f3d295ae43f 100644 --- a/src/operator/contrib/count_sketch.cu +++ b/src/operator/contrib/count_sketch.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file count_sketch.cu * \brief count_sketch op * \author Chen Zhu, Yang Shi diff --git a/src/operator/contrib/ctc_loss-inl.h b/src/operator/contrib/ctc_loss-inl.h index 8431f65088f7..0d0c0bf4cd09 100644 --- a/src/operator/contrib/ctc_loss-inl.h +++ b/src/operator/contrib/ctc_loss-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file ctc_loss-inl.h * \brief * \author Sebastian Bodenstien diff --git a/src/operator/contrib/ctc_loss.cc b/src/operator/contrib/ctc_loss.cc index c3f3fe1621b4..3727cee10b1c 100644 --- a/src/operator/contrib/ctc_loss.cc +++ b/src/operator/contrib/ctc_loss.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file ctc_loss.cc * \brief * \author Sebastian Bodenstein @@ -58,24 +76,24 @@ The shapes of the inputs and outputs: - **label**: *(batch_size, label_sequence_length)* - **out**: *(batch_size)*. -``label`` is a tensor of integers between 1 and *alphabet_size*. If a -sequence of labels is shorter than *label_sequence_length*, use the special -padding character 0 at the end of the sequence to conform it to the correct -length. For example, if *label_sequence_length* = 4, and one has two sequences -of labels [2, 1] and [3, 2, 2], the resulting ```label``` tensor should be +``label`` is a tensor of integers between 1 and *alphabet_size*. If a +sequence of labels is shorter than *label_sequence_length*, use the special +padding character 0 at the end of the sequence to conform it to the correct +length. For example, if *label_sequence_length* = 4, and one has two sequences +of labels [2, 1] and [3, 2, 2], the resulting ```label``` tensor should be padded to be:: [[2, 1, 0, 0], [3, 2, 2, 0]] -The ``data`` tensor consists of sequences of activation vectors. The layer -applies a softmax to each vector, which then becomes a vector of probabilities -over the alphabet. Note that the 0th element of this vector is reserved for the +The ``data`` tensor consists of sequences of activation vectors. The layer +applies a softmax to each vector, which then becomes a vector of probabilities +over the alphabet. Note that the 0th element of this vector is reserved for the special blank character. ``out`` is a list of CTC loss values, one per example in the batch. -See *Connectionist Temporal Classification: Labelling Unsegmented -Sequence Data with Recurrent Neural Networks*, A. Graves *et al*. for more +See *Connectionist Temporal Classification: Labelling Unsegmented +Sequence Data with Recurrent Neural Networks*, A. Graves *et al*. for more information. )code" ADD_FILELINE) diff --git a/src/operator/contrib/ctc_loss.cu b/src/operator/contrib/ctc_loss.cu index ed80eb715516..4bdef752812b 100644 --- a/src/operator/contrib/ctc_loss.cu +++ b/src/operator/contrib/ctc_loss.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file ctc_loss.cu * \brief * \author Sebastian Bodenstein diff --git a/src/operator/contrib/deformable_convolution-inl.h b/src/operator/contrib/deformable_convolution-inl.h index d8397cfb715d..a8dc6b8f09ed 100644 --- a/src/operator/contrib/deformable_convolution-inl.h +++ b/src/operator/contrib/deformable_convolution-inl.h @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! * Copyright (c) 2017 Microsoft * Licensed under The Apache-2.0 License [see LICENSE for details] diff --git a/src/operator/contrib/deformable_convolution.cc b/src/operator/contrib/deformable_convolution.cc index 5af91a0aa407..352baa12fbc1 100644 --- a/src/operator/contrib/deformable_convolution.cc +++ b/src/operator/contrib/deformable_convolution.cc @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! * Copyright (c) 2017 Microsoft * Licensed under The Apache-2.0 License [see LICENSE for details] @@ -60,7 +79,7 @@ then we have:: If ``no_bias`` is set to be true, then the ``bias`` term is ignored. The default data ``layout`` is *NCHW*, namely *(batch_size, channle, height, -width)*. +width)*. If ``num_group`` is larger than 1, denoted by *g*, then split the input ``data`` evenly into *g* parts along the channel axis, and also evenly split ``weight`` diff --git a/src/operator/contrib/deformable_convolution.cu b/src/operator/contrib/deformable_convolution.cu index f690cc1ce24c..f2200a9978ca 100644 --- a/src/operator/contrib/deformable_convolution.cu +++ b/src/operator/contrib/deformable_convolution.cu @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! * Copyright (c) 2017 Microsoft * Licensed under The Apache-2.0 License [see LICENSE for details] diff --git a/src/operator/contrib/deformable_psroi_pooling-inl.h b/src/operator/contrib/deformable_psroi_pooling-inl.h index 16a98f76bcff..d391f045a1b5 100644 --- a/src/operator/contrib/deformable_psroi_pooling-inl.h +++ b/src/operator/contrib/deformable_psroi_pooling-inl.h @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! * Copyright (c) 2017 Microsoft * Licensed under The Apache-2.0 License [see LICENSE for details] diff --git a/src/operator/contrib/deformable_psroi_pooling.cc b/src/operator/contrib/deformable_psroi_pooling.cc index 290bad2a76cd..93bb64d2113c 100644 --- a/src/operator/contrib/deformable_psroi_pooling.cc +++ b/src/operator/contrib/deformable_psroi_pooling.cc @@ -1,8 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! * Copyright (c) 2017 Microsoft * Licensed under The Apache-2.0 License [see LICENSE for details] * \file deformable_psroi_pooling.cc - * \brief + * \brief * \author Yi Li, Guodong Zhang, Jifeng Dai */ #include "./deformable_psroi_pooling-inl.h" diff --git a/src/operator/contrib/deformable_psroi_pooling.cu b/src/operator/contrib/deformable_psroi_pooling.cu index f9eb01a26e38..71bbd4cd7f2a 100644 --- a/src/operator/contrib/deformable_psroi_pooling.cu +++ b/src/operator/contrib/deformable_psroi_pooling.cu @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! * Copyright (c) 2017 Microsoft * Licensed under The Apache-2.0 License [see LICENSE for details] diff --git a/src/operator/contrib/dequantize-inl.h b/src/operator/contrib/dequantize-inl.h index ecd0cb42aafc..61940c016b15 100644 --- a/src/operator/contrib/dequantize-inl.h +++ b/src/operator/contrib/dequantize-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * \file dequantize-inl.h * \brief Implementation of dequantize operation */ diff --git a/src/operator/contrib/dequantize.cc b/src/operator/contrib/dequantize.cc index 46e36fa3c891..422a9557dc1d 100644 --- a/src/operator/contrib/dequantize.cc +++ b/src/operator/contrib/dequantize.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * \file dequantize.cc * \brief */ diff --git a/src/operator/contrib/dequantize.cu b/src/operator/contrib/dequantize.cu index be09b797b1cd..7081c27c975b 100644 --- a/src/operator/contrib/dequantize.cu +++ b/src/operator/contrib/dequantize.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * \file dequantize.cu * \brief */ diff --git a/src/operator/contrib/fft-inl.h b/src/operator/contrib/fft-inl.h index 5996322c0cec..5092f586fdf7 100644 --- a/src/operator/contrib/fft-inl.h +++ b/src/operator/contrib/fft-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file fft-inl.h * \brief * \author Chen Zhu diff --git a/src/operator/contrib/fft.cc b/src/operator/contrib/fft.cc index e2094b3bf9a8..11f8425e07b1 100644 --- a/src/operator/contrib/fft.cc +++ b/src/operator/contrib/fft.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file fft-inl.h * \brief * \author Chen Zhu diff --git a/src/operator/contrib/fft.cu b/src/operator/contrib/fft.cu index 5dbd00c779fd..3017ce76756b 100644 --- a/src/operator/contrib/fft.cu +++ b/src/operator/contrib/fft.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * \file fft-inl.h * \brief * \author Chen Zhu diff --git a/src/operator/contrib/ifft-inl.h b/src/operator/contrib/ifft-inl.h index 98b601eeae0c..abd5bb22a389 100644 --- a/src/operator/contrib/ifft-inl.h +++ b/src/operator/contrib/ifft-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file Ifft-inl.h * \brief * \author Chen Zhu diff --git a/src/operator/contrib/ifft.cc b/src/operator/contrib/ifft.cc index b2afd46cb46d..0ea3a7ec112f 100644 --- a/src/operator/contrib/ifft.cc +++ b/src/operator/contrib/ifft.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file Ifft-inl.h * \brief * \author Chen Zhu diff --git a/src/operator/contrib/ifft.cu b/src/operator/contrib/ifft.cu index 93ec1e636a3b..79795d8561bf 100644 --- a/src/operator/contrib/ifft.cu +++ b/src/operator/contrib/ifft.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file Ifft-inl.h * \brief * \author Chen Zhu diff --git a/src/operator/contrib/krprod.h b/src/operator/contrib/krprod.h index a713f1e093a7..6ce94c648d79 100644 --- a/src/operator/contrib/krprod.h +++ b/src/operator/contrib/krprod.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * \file krprod.h * \brief Core function for Khatri-Rao product * \author Jencir Lee diff --git a/src/operator/contrib/multi_proposal-inl.h b/src/operator/contrib/multi_proposal-inl.h index 48f3535b5eab..7cd465e0b09e 100644 --- a/src/operator/contrib/multi_proposal-inl.h +++ b/src/operator/contrib/multi_proposal-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * Copyright (c) 2017 Microsoft * Licensed under The Apache-2.0 License [see LICENSE for details] * \file multi_proposal-inl.h diff --git a/src/operator/contrib/multi_proposal.cc b/src/operator/contrib/multi_proposal.cc index c8f75eaec547..cd00e877a11d 100644 --- a/src/operator/contrib/multi_proposal.cc +++ b/src/operator/contrib/multi_proposal.cc @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! * Copyright (c) 2017 Microsoft * Licensed under The Apache-2.0 License [see LICENSE for details] diff --git a/src/operator/contrib/multi_proposal.cu b/src/operator/contrib/multi_proposal.cu index 052d777d5fac..cb9996344e3e 100644 --- a/src/operator/contrib/multi_proposal.cu +++ b/src/operator/contrib/multi_proposal.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * Copyright (c) 2017 Microsoft * Licensed under The Apache-2.0 License [see LICENSE for details] * \file multi_proposal.cu diff --git a/src/operator/contrib/multibox_detection-inl.h b/src/operator/contrib/multibox_detection-inl.h index 3507281eba10..34099a3d6978 100644 --- a/src/operator/contrib/multibox_detection-inl.h +++ b/src/operator/contrib/multibox_detection-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file multibox_detection-inl.h * \brief post-process multibox detection predictions * \author Joshua Zhang diff --git a/src/operator/contrib/multibox_detection.cc b/src/operator/contrib/multibox_detection.cc index 2bf49f34d1ff..0f6982890f4f 100644 --- a/src/operator/contrib/multibox_detection.cc +++ b/src/operator/contrib/multibox_detection.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file multibox_detection.cc * \brief MultiBoxDetection op * \author Joshua Zhang diff --git a/src/operator/contrib/multibox_detection.cu b/src/operator/contrib/multibox_detection.cu index dab11ffbe701..56a1e88dfee0 100644 --- a/src/operator/contrib/multibox_detection.cu +++ b/src/operator/contrib/multibox_detection.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file multibox_detection.cu * \brief MultiBoxDetection op * \author Joshua Zhang diff --git a/src/operator/contrib/multibox_prior-inl.h b/src/operator/contrib/multibox_prior-inl.h index ee83fe462ce4..88ca3dc8de6f 100644 --- a/src/operator/contrib/multibox_prior-inl.h +++ b/src/operator/contrib/multibox_prior-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file multibox_prior-inl.h * \brief generate multibox prior boxes * \author Joshua Zhang diff --git a/src/operator/contrib/multibox_prior.cc b/src/operator/contrib/multibox_prior.cc index a9c747e7c6f5..af77fdaa8015 100644 --- a/src/operator/contrib/multibox_prior.cc +++ b/src/operator/contrib/multibox_prior.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file multibox_prior.cc * \brief generate multibox prior boxes cpu implementation * \author Joshua Zhang diff --git a/src/operator/contrib/multibox_prior.cu b/src/operator/contrib/multibox_prior.cu index a3f2cc22f552..b041b90d1d05 100644 --- a/src/operator/contrib/multibox_prior.cu +++ b/src/operator/contrib/multibox_prior.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file multibox_prior.cu * \brief generate multibox prior boxes cuda kernels * \author Joshua Zhang diff --git a/src/operator/contrib/multibox_target-inl.h b/src/operator/contrib/multibox_target-inl.h index 7185c9a1d2ff..f76df3504a28 100644 --- a/src/operator/contrib/multibox_target-inl.h +++ b/src/operator/contrib/multibox_target-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file multibox_target-inl.h * \brief * \author Joshua Zhang diff --git a/src/operator/contrib/multibox_target.cc b/src/operator/contrib/multibox_target.cc index 56c6ceefdc43..095613d4a938 100644 --- a/src/operator/contrib/multibox_target.cc +++ b/src/operator/contrib/multibox_target.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file multibox_target.cc * \brief MultiBoxTarget op * \author Joshua Zhang diff --git a/src/operator/contrib/multibox_target.cu b/src/operator/contrib/multibox_target.cu index adcfcf249eea..3d0da6ce6f5b 100644 --- a/src/operator/contrib/multibox_target.cu +++ b/src/operator/contrib/multibox_target.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file multibox_target.cu * \brief MultiBoxTarget op * \author Joshua Zhang diff --git a/src/operator/contrib/nn/deformable_im2col.cuh b/src/operator/contrib/nn/deformable_im2col.cuh index d9e7b970ca84..0238921933c5 100644 --- a/src/operator/contrib/nn/deformable_im2col.cuh +++ b/src/operator/contrib/nn/deformable_im2col.cuh @@ -1,34 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! ******************* BEGIN Caffe Copyright Notice and Disclaimer **************** * * COPYRIGHT - * + * * All contributions by the University of California: * Copyright (c) 2014-2017 The Regents of the University of California (Regents) * All rights reserved. - * + * * All other contributions: * Copyright (c) 2014-2017, the respective contributors * All rights reserved. - * + * * Caffe uses a shared copyright model: each contributor holds copyright over * their contributions to Caffe. The project versioning records all such * contribution and copyright details. If a contributor wants to further mark * their specific copyright on a particular contribution, they should indicate * their copyright solely in the commit message of the change when it is * committed. - * + * * LICENSE - * + * * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * + * modification, are permitted provided that the following conditions are met: + * * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. + * list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * + * and/or other materials provided with the distribution. + * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE @@ -39,9 +58,9 @@ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * + * * CONTRIBUTION AGREEMENT - * + * * By contributing to the BVLC/caffe repository through pull-request, comment, * or otherwise, the contributor releases their content to the * license and copyright terms herein. @@ -75,7 +94,7 @@ namespace mxnet { namespace op { template -__device__ DType deformable_im2col_bilinear(const DType* bottom_data, const int data_width, +__device__ DType deformable_im2col_bilinear(const DType* bottom_data, const int data_width, const int height, const int width, DType h, DType w) { int h_low = floor(h); @@ -114,7 +133,7 @@ __device__ DType deformable_im2col_bilinear(const DType* bottom_data, const int template -__device__ DType get_gradient_weight(DType argmax_h, DType argmax_w, +__device__ DType get_gradient_weight(DType argmax_h, DType argmax_w, const int h, const int w, const int height, const int width) { if (argmax_h < 0 || argmax_h > height || argmax_w < 0 || argmax_w > width) { @@ -282,9 +301,9 @@ __global__ void deformable_im2col_gpu_kernel(const int n, const DType* data_im, */ template inline void deformable_im2col(mshadow::Stream* s, - const DType* data_im, const DType* data_offset, + const DType* data_im, const DType* data_offset, const TShape& im_shape, const TShape& col_shape, const TShape& kernel_shape, - const TShape& pad, const TShape& stride, const TShape& dilation, + const TShape& pad, const TShape& stride, const TShape& dilation, const uint32_t deformable_group, DType* data_col) { // num_axes should be smaller than block size index_t num_spatial_axes = kernel_shape.ndim(); @@ -416,7 +435,7 @@ inline void deformable_col2im(mshadow::Stream* s, * \brief DO NOT call this directly. Use wrapper function deformable_col2im_coord() instead; */ template -__global__ void deformable_col2im_coord_gpu_kernel(const int n, const DType* data_col, +__global__ void deformable_col2im_coord_gpu_kernel(const int n, const DType* data_col, const DType* data_im, const DType* data_offset, const int channels, const int height, const int width, const int kernel_h, const int kernel_w, diff --git a/src/operator/contrib/nn/deformable_im2col.h b/src/operator/contrib/nn/deformable_im2col.h index 9d6180034c71..b477acb4c876 100644 --- a/src/operator/contrib/nn/deformable_im2col.h +++ b/src/operator/contrib/nn/deformable_im2col.h @@ -1,34 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! ******************* BEGIN Caffe Copyright Notice and Disclaimer **************** * * COPYRIGHT - * + * * All contributions by the University of California: * Copyright (c) 2014-2017 The Regents of the University of California (Regents) * All rights reserved. - * + * * All other contributions: * Copyright (c) 2014-2017, the respective contributors * All rights reserved. - * + * * Caffe uses a shared copyright model: each contributor holds copyright over * their contributions to Caffe. The project versioning records all such * contribution and copyright details. If a contributor wants to further mark * their specific copyright on a particular contribution, they should indicate * their copyright solely in the commit message of the change when it is * committed. - * + * * LICENSE - * + * * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * + * modification, are permitted provided that the following conditions are met: + * * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. + * list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * + * and/or other materials provided with the distribution. + * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE @@ -39,9 +58,9 @@ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * + * * CONTRIBUTION AGREEMENT - * + * * By contributing to the BVLC/caffe repository through pull-request, comment, * or otherwise, the contributor releases their content to the * license and copyright terms herein. @@ -70,7 +89,7 @@ namespace mxnet { namespace op { -/*!\brief +/*!\brief * cpu function of deformable_im2col algorithm * \param s device stream * \param data_im pointer of an image (C, H, W, ...) in the image batch diff --git a/src/operator/contrib/proposal-inl.h b/src/operator/contrib/proposal-inl.h index 686a8a354ff9..3d1851cedbac 100644 --- a/src/operator/contrib/proposal-inl.h +++ b/src/operator/contrib/proposal-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file proposal-inl.h * \brief Proposal Operator * \author Piotr Teterwak, Bing Xu, Jian Guo diff --git a/src/operator/contrib/proposal.cc b/src/operator/contrib/proposal.cc index fe4fe98a9f4b..ec539003b944 100644 --- a/src/operator/contrib/proposal.cc +++ b/src/operator/contrib/proposal.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file proposal.cc * \brief * \author Piotr Teterwak, Bing Xu, Jian Guo diff --git a/src/operator/contrib/proposal.cu b/src/operator/contrib/proposal.cu index ce1e9e5945d0..209ef79a2aaf 100644 --- a/src/operator/contrib/proposal.cu +++ b/src/operator/contrib/proposal.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file proposal.cu * \brief Proposal Operator * \author Shaoqing Ren, Jian Guo diff --git a/src/operator/contrib/psroi_pooling-inl.h b/src/operator/contrib/psroi_pooling-inl.h index 3a3a9c34927c..b4929725279d 100644 --- a/src/operator/contrib/psroi_pooling-inl.h +++ b/src/operator/contrib/psroi_pooling-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * Copyright (c) 2017 Microsoft * Licensed under The Apache-2.0 License [see LICENSE for details] * \file psroi_pooling-inl.h diff --git a/src/operator/contrib/psroi_pooling.cc b/src/operator/contrib/psroi_pooling.cc index ad25aec8eee8..dd3a9e08895d 100644 --- a/src/operator/contrib/psroi_pooling.cc +++ b/src/operator/contrib/psroi_pooling.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * Copyright (c) 2017 Microsoft * Licensed under The Apache-2.0 License [see LICENSE for details] * \file psroi_pooling.cc diff --git a/src/operator/contrib/psroi_pooling.cu b/src/operator/contrib/psroi_pooling.cu index 962c874c6d1a..6df64a1948d6 100644 --- a/src/operator/contrib/psroi_pooling.cu +++ b/src/operator/contrib/psroi_pooling.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * Copyright (c) 2017 Microsoft * Licensed under The Apache-2.0 License [see LICENSE for details] * \file psroi_pooling.cu diff --git a/src/operator/contrib/quantize-inl.h b/src/operator/contrib/quantize-inl.h index e005762cf0eb..1274a7ded58a 100644 --- a/src/operator/contrib/quantize-inl.h +++ b/src/operator/contrib/quantize-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * \file quantize-inl.h * \brief implementation of quantize operation */ diff --git a/src/operator/contrib/quantize.cc b/src/operator/contrib/quantize.cc index 86f35e117882..dbb8985c72f2 100644 --- a/src/operator/contrib/quantize.cc +++ b/src/operator/contrib/quantize.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * \file quantize.cc * \brief */ diff --git a/src/operator/contrib/quantize.cu b/src/operator/contrib/quantize.cu index c6d9035c9687..6c9db9aeecf4 100644 --- a/src/operator/contrib/quantize.cu +++ b/src/operator/contrib/quantize.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * \file quantize.cu * \brief */ diff --git a/src/operator/convolution-inl.h b/src/operator/convolution-inl.h index a97d53b322e8..0a2522cccb65 100644 --- a/src/operator/convolution-inl.h +++ b/src/operator/convolution-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * \file convolution-inl.h * \brief * \ref: https://github.com/Yangqing/caffe/wiki/Convolution-in-Caffe:-a-memo diff --git a/src/operator/convolution.cc b/src/operator/convolution.cc index fd604d90c546..35ab5f01afa1 100644 --- a/src/operator/convolution.cc +++ b/src/operator/convolution.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * \file convolution.cc * \brief * \author Bing Xu, Jun Wu diff --git a/src/operator/convolution.cu b/src/operator/convolution.cu index 34ae42f31af8..bf5f3053b2de 100644 --- a/src/operator/convolution.cu +++ b/src/operator/convolution.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * \file convolution.cu * \brief * \author Bing Xu, Jun Wu diff --git a/src/operator/convolution_v1-inl.h b/src/operator/convolution_v1-inl.h index ee8c8c0462b3..f39d8e0804bc 100644 --- a/src/operator/convolution_v1-inl.h +++ b/src/operator/convolution_v1-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file convolution_v1-inl.h * \brief * \author Bing Xu diff --git a/src/operator/convolution_v1.cc b/src/operator/convolution_v1.cc index a1d115fdae51..cb47ed11b5c9 100644 --- a/src/operator/convolution_v1.cc +++ b/src/operator/convolution_v1.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file convolution_v1.cc * \brief * \author Bing Xu diff --git a/src/operator/convolution_v1.cu b/src/operator/convolution_v1.cu index 83a0f1d0f7df..b20b4b249224 100644 --- a/src/operator/convolution_v1.cu +++ b/src/operator/convolution_v1.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file convolution_v1.cu * \brief * \author Bing Xu diff --git a/src/operator/correlation-inl.h b/src/operator/correlation-inl.h index 6ba209bfb28e..02507cb1d96c 100644 --- a/src/operator/correlation-inl.h +++ b/src/operator/correlation-inl.h @@ -1,236 +1,254 @@ -/*! - * Copyright (c) 2015 by Contributors - * \file correlation-inl.h - * \brief correlation operator and symbol - * \author Xu Dong -*/ -#ifndef MXNET_OPERATOR_CORRELATION_INL_H_ -#define MXNET_OPERATOR_CORRELATION_INL_H_ -#include -#include -#include -#include -#include -#include -#include -#include "./mshadow_op.h" -#include "./operator_common.h" -namespace mxnet { -namespace op { -// Declare enumeration of input order to make code more intuitive. -// These enums are only visible within this header -namespace Correlation { -enum CorrelationOpInputs{kData1, kData2}; -enum CorrelationOpOutputs{kOut, kTemp1, kTemp2}; -} // namespace Correlation -struct CorrelationParam : public dmlc::Parameter { - uint32_t max_displacement; - uint32_t kernel_size; - uint32_t pad_size; - uint32_t stride1; - uint32_t stride2; - bool is_multiply; - DMLC_DECLARE_PARAMETER(CorrelationParam) { - DMLC_DECLARE_FIELD(kernel_size).set_default(1) - .describe("kernel size for Correlation must be an odd number"); - DMLC_DECLARE_FIELD(max_displacement).set_default(1) - .describe("Max displacement of Correlation "); - DMLC_DECLARE_FIELD(stride1).set_default(1) - .describe("stride1 quantize data1 globally"); - DMLC_DECLARE_FIELD(stride2).set_default(1) - .describe("stride2 quantize data2 within the neighborhood centered around data1"); - DMLC_DECLARE_FIELD(pad_size).set_default(0) - .describe("pad for Correlation"); - DMLC_DECLARE_FIELD(is_multiply).set_default(true) - .describe("operation type is either multiplication or subduction"); - } -}; -template -class CorrelationOp : public Operator { - public: - explicit CorrelationOp(CorrelationParam param) { - this->param_ = param; - } - virtual void Forward(const OpContext &ctx, - const std::vector &in_data, - const std::vector &req, - const std::vector &out_data, - const std::vector &aux_args) { - using namespace mshadow; - CHECK_EQ(in_data.size(), 2U); - CHECK_EQ(out_data.size(), 3U); - Stream *s = ctx.get_stream(); - Tensor data1 = in_data[Correlation::kData1].get(s); - Tensor data2 = in_data[Correlation::kData2].get(s); - Tensor out = out_data[Correlation::kOut].get(s); - Tensor tmp1 = out_data[Correlation::kTemp1].get(s); - Tensor tmp2 = out_data[Correlation::kTemp2].get(s); - tmp1 = 0.0f; - tmp2 = 0.0f; - out = 0.0f; - CHECK_EQ(data1.CheckContiguous(), true); - CHECK_EQ(data2.CheckContiguous(), true); - CHECK_EQ(out.CheckContiguous(), true); - CHECK_EQ(tmp1.CheckContiguous(), true); - CHECK_EQ(tmp2.CheckContiguous(), true); - paddedbottomheight = data1.shape_[2] + 2 * param_.pad_size; - paddedbottomwidth = data1.shape_[3] + 2 * param_.pad_size; - kernel_radius_ = (param_.kernel_size - 1) / 2; - border_size_ = param_.max_displacement + kernel_radius_; - stride1 = param_.stride1; - stride2 = param_.stride2; - top_width_ = ceil(static_cast(paddedbottomwidth - border_size_ * 2)\ - / static_cast(stride1)); - top_height_ = ceil(static_cast(paddedbottomheight - border_size_ * 2)\ - / static_cast(stride1)); - neighborhood_grid_radius_ = param_.max_displacement / stride2; - neighborhood_grid_width_ = neighborhood_grid_radius_ * 2 + 1; - top_channels_ = neighborhood_grid_width_ * neighborhood_grid_width_; - num = data1.shape_[0]; - channels = data1.shape_[1]; - height = data1.shape_[2]; - width = data1.shape_[3]; - CorrelationForward(out, data1, data2, tmp1, tmp2, top_channels_, top_height_, top_width_, - param_.pad_size, param_.is_multiply, - param_.max_displacement, param_.kernel_size, - neighborhood_grid_radius_, neighborhood_grid_width_, - kernel_radius_, param_.stride1, param_.stride2); - } - virtual void Backward(const OpContext &ctx, - const std::vector &out_grad, - const std::vector &in_data, - const std::vector &out_data, - const std::vector &req, - const std::vector &in_grad, - const std::vector &aux_args) { - using namespace mshadow; - Stream *s = ctx.get_stream(); - Tensor grad_data1 = in_grad[Correlation::kData1].get(s); - Tensor grad_data2 = in_grad[Correlation::kData2].get(s); - Tensor out_g = out_grad[Correlation::kOut].get(s); - Tensor tmp1 = out_data[Correlation::kTemp1].get(s); - Tensor tmp2 = out_data[Correlation::kTemp2].get(s); - if (req[0] != kAddTo) grad_data1 = 0.0f; - if (req[1] != kAddTo) grad_data2 = 0.0f; - CHECK_EQ(grad_data1.CheckContiguous(), true); - CHECK_EQ(grad_data2.CheckContiguous(), true); - CHECK_EQ(out_g.CheckContiguous(), true); - CHECK_EQ(tmp1.CheckContiguous(), true); - CHECK_EQ(tmp2.CheckContiguous(), true); - CorrelationBackward(out_g, grad_data1, grad_data2, tmp1, tmp2, top_channels_, - top_height_, top_width_, param_.pad_size, param_.is_multiply, - param_.max_displacement, param_.kernel_size, neighborhood_grid_radius_, - neighborhood_grid_width_, kernel_radius_, param_.stride1, param_.stride2, - num, channels, height, width); - } - - private: - CorrelationParam param_; - int paddedbottomheight; - int paddedbottomwidth; - uint32_t kernel_radius_; - uint32_t border_size_; - uint32_t stride1; - uint32_t stride2; - uint32_t top_width_; - uint32_t top_height_; - uint32_t neighborhood_grid_radius_; - uint32_t neighborhood_grid_width_; - uint32_t top_channels_; - int num; - int channels; - int height; - int width; -}; // class CorrelationOp -// Decalre Factory function -template -Operator* CreateOp(CorrelationParam param); -#if DMLC_USE_CXX11 -class CorrelationProp : public OperatorProperty { - public: - std::vector ListArguments() const override { - return {"data1", "data2"}; - } - std::vector ListOutputs() const override { - return {"output", "tmp1", "tmp2"}; - } - int NumOutputs() const override { - return 3; - } - int NumVisibleOutputs() const override { - return 1; - } -void Init(const std::vector >& kwargs) override { - param_.Init(kwargs); - } - std::map GetParams() const override { - return param_.__DICT__(); - } - bool InferShape(std::vector *in_shape, - std::vector *out_shape, - std::vector *aux_shape) const override { - using namespace mshadow; - CHECK_EQ(in_shape->size(), 2U) << "Input:[data1, data2]"; - TShape dshape1 = in_shape->at(Correlation::kData1); - TShape dshape2 = in_shape->at(Correlation::kData2); - CHECK_EQ(dshape1.ndim(), 4U) << "data should be a 4D tensor"; - CHECK_EQ(dshape2.ndim(), 4U) << "data should be a 4D tensor"; - int paddedbottomheight; - int paddedbottomwidth; - uint32_t kernel_radius_; - uint32_t stride1; - uint32_t stride2; - uint32_t top_width_; - uint32_t top_height_; - uint32_t neighborhood_grid_radius_; - uint32_t neighborhood_grid_width_; - uint32_t top_channels_; - uint32_t border_size_; - paddedbottomheight = dshape1[2] + 2*param_.pad_size; - paddedbottomwidth = dshape1[3] + 2*param_.pad_size; - kernel_radius_ = (param_.kernel_size -1)/2; - border_size_ = param_.max_displacement + kernel_radius_; - stride1 = param_.stride1; - stride2 = param_.stride2; - top_width_ = ceil(static_cast(paddedbottomwidth - border_size_ * 2)\ - / static_cast(stride1)); - top_height_ = ceil(static_cast(paddedbottomheight - border_size_ * 2)\ - / static_cast(stride1)); - neighborhood_grid_radius_ = param_.max_displacement / stride2; - neighborhood_grid_width_ = neighborhood_grid_radius_ * 2 + 1; - top_channels_ = neighborhood_grid_width_ * neighborhood_grid_width_; - CHECK_GE(top_width_, 1U) << - "Correlation cannot be done with current settings.Neighborhood and kernel don't fit in blob"; - CHECK_GE(top_height_, 1U) << - "Correlation cannot be done with current settings.Neighborhood and kernel don't fit in blob"; - out_shape->clear(); - out_shape->push_back(Shape4(dshape1[0], top_channels_, top_height_, top_width_)); - out_shape->push_back(Shape4(dshape1[0], paddedbottomheight, paddedbottomwidth, dshape1[1])); - out_shape->push_back(Shape4(dshape1[0], paddedbottomheight, paddedbottomwidth, dshape1[1])); - return true; - } - OperatorProperty* Copy() const override { - CorrelationProp* Correlation_sym = new CorrelationProp(); - Correlation_sym->param_ = this->param_; - return Correlation_sym; - } - std::string TypeString() const override { - return "Correlation"; - } - // decalre dependency and inplace optimization options - std::vector DeclareBackwardDependency( - const std::vector &out_grad, - const std::vector &in_data, - const std::vector &out_data) const override { - return {out_grad[Correlation::kOut], - out_data[Correlation::kTemp1], out_data[Correlation::kTemp2]}; -} - Operator* CreateOperator(Context ctx) const override; - - private: - CorrelationParam param_; -}; // class CorrelationProp -#endif -} // namespace op -} // namespace mxnet -#endif // MXNET_OPERATOR_CORRELATION_INL_H_ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file correlation-inl.h + * \brief correlation operator and symbol + * \author Xu Dong +*/ +#ifndef MXNET_OPERATOR_CORRELATION_INL_H_ +#define MXNET_OPERATOR_CORRELATION_INL_H_ +#include +#include +#include +#include +#include +#include +#include +#include "./mshadow_op.h" +#include "./operator_common.h" +namespace mxnet { +namespace op { +// Declare enumeration of input order to make code more intuitive. +// These enums are only visible within this header +namespace Correlation { +enum CorrelationOpInputs{kData1, kData2}; +enum CorrelationOpOutputs{kOut, kTemp1, kTemp2}; +} // namespace Correlation +struct CorrelationParam : public dmlc::Parameter { + uint32_t max_displacement; + uint32_t kernel_size; + uint32_t pad_size; + uint32_t stride1; + uint32_t stride2; + bool is_multiply; + DMLC_DECLARE_PARAMETER(CorrelationParam) { + DMLC_DECLARE_FIELD(kernel_size).set_default(1) + .describe("kernel size for Correlation must be an odd number"); + DMLC_DECLARE_FIELD(max_displacement).set_default(1) + .describe("Max displacement of Correlation "); + DMLC_DECLARE_FIELD(stride1).set_default(1) + .describe("stride1 quantize data1 globally"); + DMLC_DECLARE_FIELD(stride2).set_default(1) + .describe("stride2 quantize data2 within the neighborhood centered around data1"); + DMLC_DECLARE_FIELD(pad_size).set_default(0) + .describe("pad for Correlation"); + DMLC_DECLARE_FIELD(is_multiply).set_default(true) + .describe("operation type is either multiplication or subduction"); + } +}; +template +class CorrelationOp : public Operator { + public: + explicit CorrelationOp(CorrelationParam param) { + this->param_ = param; + } + virtual void Forward(const OpContext &ctx, + const std::vector &in_data, + const std::vector &req, + const std::vector &out_data, + const std::vector &aux_args) { + using namespace mshadow; + CHECK_EQ(in_data.size(), 2U); + CHECK_EQ(out_data.size(), 3U); + Stream *s = ctx.get_stream(); + Tensor data1 = in_data[Correlation::kData1].get(s); + Tensor data2 = in_data[Correlation::kData2].get(s); + Tensor out = out_data[Correlation::kOut].get(s); + Tensor tmp1 = out_data[Correlation::kTemp1].get(s); + Tensor tmp2 = out_data[Correlation::kTemp2].get(s); + tmp1 = 0.0f; + tmp2 = 0.0f; + out = 0.0f; + CHECK_EQ(data1.CheckContiguous(), true); + CHECK_EQ(data2.CheckContiguous(), true); + CHECK_EQ(out.CheckContiguous(), true); + CHECK_EQ(tmp1.CheckContiguous(), true); + CHECK_EQ(tmp2.CheckContiguous(), true); + paddedbottomheight = data1.shape_[2] + 2 * param_.pad_size; + paddedbottomwidth = data1.shape_[3] + 2 * param_.pad_size; + kernel_radius_ = (param_.kernel_size - 1) / 2; + border_size_ = param_.max_displacement + kernel_radius_; + stride1 = param_.stride1; + stride2 = param_.stride2; + top_width_ = ceil(static_cast(paddedbottomwidth - border_size_ * 2)\ + / static_cast(stride1)); + top_height_ = ceil(static_cast(paddedbottomheight - border_size_ * 2)\ + / static_cast(stride1)); + neighborhood_grid_radius_ = param_.max_displacement / stride2; + neighborhood_grid_width_ = neighborhood_grid_radius_ * 2 + 1; + top_channels_ = neighborhood_grid_width_ * neighborhood_grid_width_; + num = data1.shape_[0]; + channels = data1.shape_[1]; + height = data1.shape_[2]; + width = data1.shape_[3]; + CorrelationForward(out, data1, data2, tmp1, tmp2, top_channels_, top_height_, top_width_, + param_.pad_size, param_.is_multiply, + param_.max_displacement, param_.kernel_size, + neighborhood_grid_radius_, neighborhood_grid_width_, + kernel_radius_, param_.stride1, param_.stride2); + } + virtual void Backward(const OpContext &ctx, + const std::vector &out_grad, + const std::vector &in_data, + const std::vector &out_data, + const std::vector &req, + const std::vector &in_grad, + const std::vector &aux_args) { + using namespace mshadow; + Stream *s = ctx.get_stream(); + Tensor grad_data1 = in_grad[Correlation::kData1].get(s); + Tensor grad_data2 = in_grad[Correlation::kData2].get(s); + Tensor out_g = out_grad[Correlation::kOut].get(s); + Tensor tmp1 = out_data[Correlation::kTemp1].get(s); + Tensor tmp2 = out_data[Correlation::kTemp2].get(s); + if (req[0] != kAddTo) grad_data1 = 0.0f; + if (req[1] != kAddTo) grad_data2 = 0.0f; + CHECK_EQ(grad_data1.CheckContiguous(), true); + CHECK_EQ(grad_data2.CheckContiguous(), true); + CHECK_EQ(out_g.CheckContiguous(), true); + CHECK_EQ(tmp1.CheckContiguous(), true); + CHECK_EQ(tmp2.CheckContiguous(), true); + CorrelationBackward(out_g, grad_data1, grad_data2, tmp1, tmp2, top_channels_, + top_height_, top_width_, param_.pad_size, param_.is_multiply, + param_.max_displacement, param_.kernel_size, neighborhood_grid_radius_, + neighborhood_grid_width_, kernel_radius_, param_.stride1, param_.stride2, + num, channels, height, width); + } + + private: + CorrelationParam param_; + int paddedbottomheight; + int paddedbottomwidth; + uint32_t kernel_radius_; + uint32_t border_size_; + uint32_t stride1; + uint32_t stride2; + uint32_t top_width_; + uint32_t top_height_; + uint32_t neighborhood_grid_radius_; + uint32_t neighborhood_grid_width_; + uint32_t top_channels_; + int num; + int channels; + int height; + int width; +}; // class CorrelationOp +// Decalre Factory function +template +Operator* CreateOp(CorrelationParam param); +#if DMLC_USE_CXX11 +class CorrelationProp : public OperatorProperty { + public: + std::vector ListArguments() const override { + return {"data1", "data2"}; + } + std::vector ListOutputs() const override { + return {"output", "tmp1", "tmp2"}; + } + int NumOutputs() const override { + return 3; + } + int NumVisibleOutputs() const override { + return 1; + } +void Init(const std::vector >& kwargs) override { + param_.Init(kwargs); + } + std::map GetParams() const override { + return param_.__DICT__(); + } + bool InferShape(std::vector *in_shape, + std::vector *out_shape, + std::vector *aux_shape) const override { + using namespace mshadow; + CHECK_EQ(in_shape->size(), 2U) << "Input:[data1, data2]"; + TShape dshape1 = in_shape->at(Correlation::kData1); + TShape dshape2 = in_shape->at(Correlation::kData2); + CHECK_EQ(dshape1.ndim(), 4U) << "data should be a 4D tensor"; + CHECK_EQ(dshape2.ndim(), 4U) << "data should be a 4D tensor"; + int paddedbottomheight; + int paddedbottomwidth; + uint32_t kernel_radius_; + uint32_t stride1; + uint32_t stride2; + uint32_t top_width_; + uint32_t top_height_; + uint32_t neighborhood_grid_radius_; + uint32_t neighborhood_grid_width_; + uint32_t top_channels_; + uint32_t border_size_; + paddedbottomheight = dshape1[2] + 2*param_.pad_size; + paddedbottomwidth = dshape1[3] + 2*param_.pad_size; + kernel_radius_ = (param_.kernel_size -1)/2; + border_size_ = param_.max_displacement + kernel_radius_; + stride1 = param_.stride1; + stride2 = param_.stride2; + top_width_ = ceil(static_cast(paddedbottomwidth - border_size_ * 2)\ + / static_cast(stride1)); + top_height_ = ceil(static_cast(paddedbottomheight - border_size_ * 2)\ + / static_cast(stride1)); + neighborhood_grid_radius_ = param_.max_displacement / stride2; + neighborhood_grid_width_ = neighborhood_grid_radius_ * 2 + 1; + top_channels_ = neighborhood_grid_width_ * neighborhood_grid_width_; + CHECK_GE(top_width_, 1U) << + "Correlation cannot be done with current settings.Neighborhood and kernel don't fit in blob"; + CHECK_GE(top_height_, 1U) << + "Correlation cannot be done with current settings.Neighborhood and kernel don't fit in blob"; + out_shape->clear(); + out_shape->push_back(Shape4(dshape1[0], top_channels_, top_height_, top_width_)); + out_shape->push_back(Shape4(dshape1[0], paddedbottomheight, paddedbottomwidth, dshape1[1])); + out_shape->push_back(Shape4(dshape1[0], paddedbottomheight, paddedbottomwidth, dshape1[1])); + return true; + } + OperatorProperty* Copy() const override { + CorrelationProp* Correlation_sym = new CorrelationProp(); + Correlation_sym->param_ = this->param_; + return Correlation_sym; + } + std::string TypeString() const override { + return "Correlation"; + } + // decalre dependency and inplace optimization options + std::vector DeclareBackwardDependency( + const std::vector &out_grad, + const std::vector &in_data, + const std::vector &out_data) const override { + return {out_grad[Correlation::kOut], + out_data[Correlation::kTemp1], out_data[Correlation::kTemp2]}; +} + Operator* CreateOperator(Context ctx) const override; + + private: + CorrelationParam param_; +}; // class CorrelationProp +#endif +} // namespace op +} // namespace mxnet +#endif // MXNET_OPERATOR_CORRELATION_INL_H_ diff --git a/src/operator/correlation.cc b/src/operator/correlation.cc index 18a3e1cb06b9..2522cd45c414 100644 --- a/src/operator/correlation.cc +++ b/src/operator/correlation.cc @@ -1,175 +1,193 @@ -/*! - * Copyright (c) 2015 by Contributors - * \file correlation.cc - * \brief correlation op - * \author Xu Dong -*/ -#include "./correlation-inl.h" -#include "./mshadow_op.h" - -namespace mshadow { -template -void AddPad(const Tensor &original, - const Tensor &out, - int pad_size) -{ for (index_t nbatch = 0 ; nbatch < original.size(0) ; nbatch++) - for (index_t channel = 0 ; channel < original.size(1) ; channel++) - for (index_t h = 0 ; h < original.size(2) ; h++) - for (index_t w = 0 ; w < original.size(3) ; w++) - out[nbatch][h+pad_size][w+pad_size][channel] = original[nbatch][channel][h][w]; -} -template -inline void CorrelationForward(const Tensor &out, - const Tensor &data1, - const Tensor &data2, - const Tensor &tmp1, - const Tensor &tmp2, - int top_channels_, int top_height_, int top_width_, - int pad_size_, bool is_multiply, - int max_displacement_, int kernel_size_, - int neighborhood_grid_radius_, int neighborhood_grid_width_, - int kernel_radius_, int stride1_, int stride2_) { - const index_t bnum = data1.size(0); - const int bchannels = data1.size(1); - const int sumelems = kernel_size_ * kernel_size_ * bchannels; - AddPad(data1, tmp1, pad_size_); - index_t top_channels_unsigned_ = static_cast(top_channels_); - AddPad(data2, tmp2, pad_size_); - for (index_t i = 0 ; i < static_cast(top_height_) ; i++) - for (index_t j = 0 ; j < static_cast(top_width_); j++) - for (index_t nbatch = 0 ; nbatch < bnum ; nbatch++) { - int x1 = j*stride1_+max_displacement_; - int y1 = i*stride1_+max_displacement_; - for (index_t top_channel = 0 ; top_channel < top_channels_unsigned_ ; top_channel++) { - int s2o = (top_channel % neighborhood_grid_width_ -\ - neighborhood_grid_radius_) * stride2_; - int s2p = (top_channel / neighborhood_grid_width_ -\ - neighborhood_grid_radius_) * stride2_; - int x2 = x1 + s2o; - int y2 = y1 + s2p; - for (index_t h = 0; h < static_cast(kernel_size_); h++) - for (index_t w = 0; w < static_cast(kernel_size_); w++) - for (index_t channel = 0; channel < static_cast(bchannels); channel++) { - if (is_multiply == true) - out[nbatch][top_channel][i][j] += \ - tmp1[nbatch][y1+h][x1+w][channel]*tmp2[nbatch][y2+h][x2+w][channel]; - else - out[nbatch][top_channel][i][j] += \ - fabsf(tmp1[nbatch][y1+h][x1+w][channel]-tmp2[nbatch][y2+h][x2+w][channel]); - } - out[nbatch][top_channel][i][j] /= sumelems; - } - } -} -template -inline void CorrelationBackward(const Tensor &out_grad, - const Tensor &in_grad1, - const Tensor &in_grad2, - const Tensor &tmp1, - const Tensor &tmp2, - int top_channels_, int top_height_, - int top_width_, int pad_size_, - bool is_multiply, int max_displacement_, - int kernel_size_, int neighborhood_grid_radius_, - int neighborhood_grid_width_, - int kernel_radius_, int stride1_, - int stride2_, int num, - int channels, int height, int width - ) { - const float sumelems = kernel_size_ * kernel_size_ * channels; - for (index_t i = 0 ; i < static_cast(top_height_) ; i++) - for (index_t j = 0 ; j < static_cast(top_width_); j++) - for (index_t nbatch = 0 ; nbatch < static_cast(num) ; nbatch++) { - int x1 = j*stride1_+max_displacement_; - int y1 = i*stride1_+max_displacement_; - for (int top_channel = 0 ; top_channel < top_channels_ ; top_channel++) { - int s2o = (top_channel % neighborhood_grid_width_ - \ - neighborhood_grid_radius_) * stride2_; - int s2p = (top_channel / neighborhood_grid_width_ - \ - neighborhood_grid_radius_) * stride2_; - int x2 = x1 + s2o; - int y2 = y1 + s2p; - for (int h = 0; h < kernel_size_; h++) - for (int w = 0; w < kernel_size_; w++) - for (int channel = 0 ; channel < channels; channel++) { - if (is_multiply == true) { - if ((y1 + h - pad_size_ >= 0) && (x1 + w - pad_size_ >= 0) && \ - (y1 + h < height +pad_size_) && (x1 + w < width + pad_size_)) { - in_grad1[nbatch][channel][y1+h-pad_size_][x1+w-pad_size_] += \ - out_grad[nbatch][top_channel][i][j] * \ - tmp2[nbatch][y2+h][x2+w][channel]/sumelems; - } - if ((y2 + h - pad_size_ >= 0) && (x2 + w -pad_size_ >=0) && \ - (y2 + h < height +pad_size_) && (x2 + w < width + pad_size_)) { - in_grad2[nbatch][channel][y2+h-pad_size_][x2+w-pad_size_] += \ - out_grad[nbatch][top_channel][i][j] * \ - tmp1[nbatch][y1+h][x1+w][channel]/sumelems; - } - } else { - if ((y1 + h - pad_size_ >= 0) && (x1 + w -pad_size_ >=0) && \ - (y1 + h < height + pad_size_) && (x1 + w < width + pad_size_)) { - Dtype sign = (tmp1[nbatch][y1+h][x1+w][channel] >= \ - tmp2[nbatch][y2+h][x2+w][channel])? Dtype(1.0) : Dtype(-1.0); - in_grad1[nbatch][channel][y1+h-pad_size_][x1+w-pad_size_] +=\ - out_grad[nbatch][top_channel][i][j]*sign/sumelems; - } - if ((y2 + h - pad_size_ >= 0) && (x2 + w - pad_size_ >=0) && \ - (y2 + h < height + pad_size_) && (x2 + w < width + pad_size_)) { - Dtype sign = (tmp1[nbatch][y1+h][x1+w][channel] >= \ - tmp2[nbatch][y2+h][x2+w][channel])? Dtype(-1.0) : Dtype(1.0); - in_grad2[nbatch][channel][y2+h-pad_size_][x2+w-pad_size_] +=\ - out_grad[nbatch][top_channel][i][j]*sign/sumelems; - } - } - } - } - } -} -} // namespace mshadow -namespace mxnet { -namespace op { -template<> -Operator *CreateOp(CorrelationParam param) { - return new CorrelationOp(param); -} -Operator* CorrelationProp::CreateOperator(Context ctx) const { - DO_BIND_DISPATCH(CreateOp, param_); -} -DMLC_REGISTER_PARAMETER(CorrelationParam); -MXNET_REGISTER_OP_PROPERTY(Correlation, CorrelationProp) -.add_argument("data1", "NDArray-or-Symbol", "Input data1 to the correlation.") -.add_argument("data2", "NDArray-or-Symbol", "Input data2 to the correlation.") -.add_arguments(CorrelationParam::__FIELDS__()) -.describe(R"code(Applies correlation to inputs. - -The correlation layer performs multiplicative patch comparisons between two feature maps. - -Given two multi-channel feature maps :math:`f_{1}, f_{2}`, with :math:`w`, :math:`h`, and :math:`c` being their width, height, and number of channels, -the correlation layer lets the network compare each patch from :math:`f_{1}` with each patch from :math:`f_{2}`. - -For now we consider only a single comparison of two patches. The 'correlation' of two patches centered at :math:`x_{1}` in the first map and -:math:`x_{2}` in the second map is then defined as: - -.. math:: - c(x_{1}, x_{2}) = \sum_{o \in [-k,k] \times [-k,k]} - -for a square patch of size :math:`K:=2k+1`. - -Note that the equation above is identical to one step of a convolution in neural networks, but instead of convolving data with a filter, it convolves data with other -data. For this reason, it has no training weights. - -Computing :math:`c(x_{1}, x_{2})` involves :math:`c * K^{2}` multiplications. Comparing all patch combinations involves :math:`w^{2}*h^{2}` such computations. - -Given a maximum displacement :math:`d`, for each location :math:`x_{1}` it computes correlations :math:`c(x_{1}, x_{2})` only in a neighborhood of size :math:`D:=2d+1`, -by limiting the range of :math:`x_{2}`. We use strides :math:`s_{1}, s_{2}`, to quantize :math:`x_{1}` globally and to quantize :math:`x_{2}` within the neighborhood -centered around :math:`x_{1}`. - -The final output is defined by the following expression: - -.. math:: - out[n, q, i, j] = c(x_{i, j}, x_{q}) - -where :math:`i` and :math:`j` enumerate spatial locations in :math:`f_{1}`, and :math:`q` denotes the :math:`q^{th}` neighborhood of :math:`x_{i,j}`. -)code" ADD_FILELINE); -} // namespace op -} // namespace mxnet +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file correlation.cc + * \brief correlation op + * \author Xu Dong +*/ +#include "./correlation-inl.h" +#include "./mshadow_op.h" + +namespace mshadow { +template +void AddPad(const Tensor &original, + const Tensor &out, + int pad_size) +{ for (index_t nbatch = 0 ; nbatch < original.size(0) ; nbatch++) + for (index_t channel = 0 ; channel < original.size(1) ; channel++) + for (index_t h = 0 ; h < original.size(2) ; h++) + for (index_t w = 0 ; w < original.size(3) ; w++) + out[nbatch][h+pad_size][w+pad_size][channel] = original[nbatch][channel][h][w]; +} +template +inline void CorrelationForward(const Tensor &out, + const Tensor &data1, + const Tensor &data2, + const Tensor &tmp1, + const Tensor &tmp2, + int top_channels_, int top_height_, int top_width_, + int pad_size_, bool is_multiply, + int max_displacement_, int kernel_size_, + int neighborhood_grid_radius_, int neighborhood_grid_width_, + int kernel_radius_, int stride1_, int stride2_) { + const index_t bnum = data1.size(0); + const int bchannels = data1.size(1); + const int sumelems = kernel_size_ * kernel_size_ * bchannels; + AddPad(data1, tmp1, pad_size_); + index_t top_channels_unsigned_ = static_cast(top_channels_); + AddPad(data2, tmp2, pad_size_); + for (index_t i = 0 ; i < static_cast(top_height_) ; i++) + for (index_t j = 0 ; j < static_cast(top_width_); j++) + for (index_t nbatch = 0 ; nbatch < bnum ; nbatch++) { + int x1 = j*stride1_+max_displacement_; + int y1 = i*stride1_+max_displacement_; + for (index_t top_channel = 0 ; top_channel < top_channels_unsigned_ ; top_channel++) { + int s2o = (top_channel % neighborhood_grid_width_ -\ + neighborhood_grid_radius_) * stride2_; + int s2p = (top_channel / neighborhood_grid_width_ -\ + neighborhood_grid_radius_) * stride2_; + int x2 = x1 + s2o; + int y2 = y1 + s2p; + for (index_t h = 0; h < static_cast(kernel_size_); h++) + for (index_t w = 0; w < static_cast(kernel_size_); w++) + for (index_t channel = 0; channel < static_cast(bchannels); channel++) { + if (is_multiply == true) + out[nbatch][top_channel][i][j] += \ + tmp1[nbatch][y1+h][x1+w][channel]*tmp2[nbatch][y2+h][x2+w][channel]; + else + out[nbatch][top_channel][i][j] += \ + fabsf(tmp1[nbatch][y1+h][x1+w][channel]-tmp2[nbatch][y2+h][x2+w][channel]); + } + out[nbatch][top_channel][i][j] /= sumelems; + } + } +} +template +inline void CorrelationBackward(const Tensor &out_grad, + const Tensor &in_grad1, + const Tensor &in_grad2, + const Tensor &tmp1, + const Tensor &tmp2, + int top_channels_, int top_height_, + int top_width_, int pad_size_, + bool is_multiply, int max_displacement_, + int kernel_size_, int neighborhood_grid_radius_, + int neighborhood_grid_width_, + int kernel_radius_, int stride1_, + int stride2_, int num, + int channels, int height, int width + ) { + const float sumelems = kernel_size_ * kernel_size_ * channels; + for (index_t i = 0 ; i < static_cast(top_height_) ; i++) + for (index_t j = 0 ; j < static_cast(top_width_); j++) + for (index_t nbatch = 0 ; nbatch < static_cast(num) ; nbatch++) { + int x1 = j*stride1_+max_displacement_; + int y1 = i*stride1_+max_displacement_; + for (int top_channel = 0 ; top_channel < top_channels_ ; top_channel++) { + int s2o = (top_channel % neighborhood_grid_width_ - \ + neighborhood_grid_radius_) * stride2_; + int s2p = (top_channel / neighborhood_grid_width_ - \ + neighborhood_grid_radius_) * stride2_; + int x2 = x1 + s2o; + int y2 = y1 + s2p; + for (int h = 0; h < kernel_size_; h++) + for (int w = 0; w < kernel_size_; w++) + for (int channel = 0 ; channel < channels; channel++) { + if (is_multiply == true) { + if ((y1 + h - pad_size_ >= 0) && (x1 + w - pad_size_ >= 0) && \ + (y1 + h < height +pad_size_) && (x1 + w < width + pad_size_)) { + in_grad1[nbatch][channel][y1+h-pad_size_][x1+w-pad_size_] += \ + out_grad[nbatch][top_channel][i][j] * \ + tmp2[nbatch][y2+h][x2+w][channel]/sumelems; + } + if ((y2 + h - pad_size_ >= 0) && (x2 + w -pad_size_ >=0) && \ + (y2 + h < height +pad_size_) && (x2 + w < width + pad_size_)) { + in_grad2[nbatch][channel][y2+h-pad_size_][x2+w-pad_size_] += \ + out_grad[nbatch][top_channel][i][j] * \ + tmp1[nbatch][y1+h][x1+w][channel]/sumelems; + } + } else { + if ((y1 + h - pad_size_ >= 0) && (x1 + w -pad_size_ >=0) && \ + (y1 + h < height + pad_size_) && (x1 + w < width + pad_size_)) { + Dtype sign = (tmp1[nbatch][y1+h][x1+w][channel] >= \ + tmp2[nbatch][y2+h][x2+w][channel])? Dtype(1.0) : Dtype(-1.0); + in_grad1[nbatch][channel][y1+h-pad_size_][x1+w-pad_size_] +=\ + out_grad[nbatch][top_channel][i][j]*sign/sumelems; + } + if ((y2 + h - pad_size_ >= 0) && (x2 + w - pad_size_ >=0) && \ + (y2 + h < height + pad_size_) && (x2 + w < width + pad_size_)) { + Dtype sign = (tmp1[nbatch][y1+h][x1+w][channel] >= \ + tmp2[nbatch][y2+h][x2+w][channel])? Dtype(-1.0) : Dtype(1.0); + in_grad2[nbatch][channel][y2+h-pad_size_][x2+w-pad_size_] +=\ + out_grad[nbatch][top_channel][i][j]*sign/sumelems; + } + } + } + } + } +} +} // namespace mshadow +namespace mxnet { +namespace op { +template<> +Operator *CreateOp(CorrelationParam param) { + return new CorrelationOp(param); +} +Operator* CorrelationProp::CreateOperator(Context ctx) const { + DO_BIND_DISPATCH(CreateOp, param_); +} +DMLC_REGISTER_PARAMETER(CorrelationParam); +MXNET_REGISTER_OP_PROPERTY(Correlation, CorrelationProp) +.add_argument("data1", "NDArray-or-Symbol", "Input data1 to the correlation.") +.add_argument("data2", "NDArray-or-Symbol", "Input data2 to the correlation.") +.add_arguments(CorrelationParam::__FIELDS__()) +.describe(R"code(Applies correlation to inputs. + +The correlation layer performs multiplicative patch comparisons between two feature maps. + +Given two multi-channel feature maps :math:`f_{1}, f_{2}`, with :math:`w`, :math:`h`, and :math:`c` being their width, height, and number of channels, +the correlation layer lets the network compare each patch from :math:`f_{1}` with each patch from :math:`f_{2}`. + +For now we consider only a single comparison of two patches. The 'correlation' of two patches centered at :math:`x_{1}` in the first map and +:math:`x_{2}` in the second map is then defined as: + +.. math:: + c(x_{1}, x_{2}) = \sum_{o \in [-k,k] \times [-k,k]} + +for a square patch of size :math:`K:=2k+1`. + +Note that the equation above is identical to one step of a convolution in neural networks, but instead of convolving data with a filter, it convolves data with other +data. For this reason, it has no training weights. + +Computing :math:`c(x_{1}, x_{2})` involves :math:`c * K^{2}` multiplications. Comparing all patch combinations involves :math:`w^{2}*h^{2}` such computations. + +Given a maximum displacement :math:`d`, for each location :math:`x_{1}` it computes correlations :math:`c(x_{1}, x_{2})` only in a neighborhood of size :math:`D:=2d+1`, +by limiting the range of :math:`x_{2}`. We use strides :math:`s_{1}, s_{2}`, to quantize :math:`x_{1}` globally and to quantize :math:`x_{2}` within the neighborhood +centered around :math:`x_{1}`. + +The final output is defined by the following expression: + +.. math:: + out[n, q, i, j] = c(x_{i, j}, x_{q}) + +where :math:`i` and :math:`j` enumerate spatial locations in :math:`f_{1}`, and :math:`q` denotes the :math:`q^{th}` neighborhood of :math:`x_{i,j}`. +)code" ADD_FILELINE); +} // namespace op +} // namespace mxnet diff --git a/src/operator/correlation.cu b/src/operator/correlation.cu index b26ae04f2d0b..149d73f2e83a 100644 --- a/src/operator/correlation.cu +++ b/src/operator/correlation.cu @@ -1,609 +1,628 @@ -/*! - * Copyright [2016] - * \file Correation.cu - * \brief Correlation operator - * \author Xu Dong -*/ -#include "./correlation-inl.h" -#include -#include -#include -#include - -#define ROUND_OFF 50000 -#define WARPS_PER_BLOCK 1 -#define THREADS_PER_WARP 32 -#define CORRELATION_CUDA_CHECK(condition) \ - /* Code block avoids redefinition of cudaError_t error */ \ - do { \ - cudaError_t error = condition; \ - CHECK_EQ(error, cudaSuccess) << " " << cudaGetErrorString(error); \ - } while (0) -#define CUDA_KERNEL_LOOP(i, n) \ -for (int i = blockIdx.x * blockDim.x + threadIdx.x; \ - i < (n); \ - i += blockDim.x * gridDim.x) -namespace mshadow { -namespace cuda { -// == Correlation Kernel -template -__global__ void CorrelateData(const int nthreads, int num, int topwidth, - int topheight, int topchannels, int topcount, - int max_displacement, int neighborhood_grid_radius, - int neighborhood_grid_width, int kernel_radius, int kernel_size, int stride1, int stride2, - int bottomwidth, int bottomheight, int bottomchannels, - const Dtype *bottom0, const Dtype *bottom1, Dtype *top) { - extern __shared__ char patch_data_char[]; - Dtype *patch_data = reinterpret_cast(patch_data_char); - // First (upper left) position of kernel upper-left corner - // in current center position of neighborhood in image 1 - int x1 = blockIdx.x * stride1 + max_displacement; - int y1 = blockIdx.y * stride1 + max_displacement; - int item = blockIdx.z; - int ch_off = threadIdx.x; - // Load 3D patch into shared shared memory - for (int j = 0; j < kernel_size; j++) { // HEIGHT - for (int i = 0; i < kernel_size; i++) { // WIDTH - int ji_off = ((j * kernel_size) + i) * bottomchannels; - for (int ch = ch_off; ch < bottomchannels; ch += (THREADS_PER_WARP * WARPS_PER_BLOCK)) { - // CHANNELS - int idx1 = ((item * bottomheight + y1+j) * bottomwidth + x1+i) * bottomchannels + ch; - int idxPatchData = ji_off + ch; - patch_data[idxPatchData] = bottom0[idx1]; - } - } - } - __syncthreads(); - __shared__ Dtype sum[THREADS_PER_WARP * WARPS_PER_BLOCK]; - // Compute correlation - for (int top_channel = 0; top_channel < topchannels; top_channel++) { - sum[ch_off] = 0; - int s2o = (top_channel % neighborhood_grid_width - neighborhood_grid_radius) * stride2; - int s2p = (top_channel / neighborhood_grid_width - neighborhood_grid_radius) * stride2; - for (int j = 0; j < kernel_size; j++) { // HEIGHT - for (int i = 0; i < kernel_size; i++) { // WIDTH - int ji_off = ((j * kernel_size) + i) * bottomchannels; - for (int ch = ch_off; ch < bottomchannels; ch += (THREADS_PER_WARP * WARPS_PER_BLOCK)) { - // CHANNELS - int x2 = x1 + s2o; - int y2 = y1 + s2p; - int idxPatchData = ji_off + ch; - int idx2 = ((item * bottomheight + y2 + j) * bottomwidth + x2 + i) * bottomchannels + ch; - sum[ch_off] += patch_data[idxPatchData] * bottom1[idx2]; - } - } - } - __syncthreads(); - if (ch_off == 0) { - Dtype total_sum = 0; - for (int idx = 0; idx < THREADS_PER_WARP * WARPS_PER_BLOCK; idx++) { - total_sum += sum[idx]; - } - const int sumelems = kernel_size * kernel_size * bottomchannels; - const int index = ((top_channel * topheight + blockIdx.y) * topwidth) + blockIdx.x; - top[index + item*topcount] = total_sum / static_cast(sumelems); - } // Aggregate result of different threads - } -} -// == Correlation Backward Pass Kernel (For data1) -template -__global__ void CorrelateDataBackward0(const int nthreads, int num, int item, - int topwidth, int topheight, int topchannels, - int max_displacement, int neighborhood_grid_radius, - int neighborhood_grid_width, int kernel_radius, int stride1, int stride2, - int bottomwidth, int bottomheight, int pbottomwidth, int pbottomheight, - int bottomchannels, int bottomcount, int pad_size, - Dtype *bottom0diff, const Dtype *bottom1, const Dtype *topdiff) { - CUDA_KERNEL_LOOP(index, nthreads) { - int n = index % bottomchannels; // channels - int l = (index / bottomchannels) % bottomwidth + pad_size; // w-pos - int m = (index / bottomchannels / bottomwidth) % bottomheight + pad_size; // h-pos - // Get X,Y ranges and clamp - // round_off is a trick to enable integer division with ceil, even for negative numbers - // We use a large offset, for the inner part not to become negative. - const int round_off = ROUND_OFF; - const int round_off_s1 = stride1 * round_off; - // We add round_off before_s1 the int division and subtract round_off after it, - // to ensure the formula matches ceil behavior: - int xmin = (l - 2*kernel_radius - max_displacement + round_off_s1 - 1)\ - / stride1 + 1 - round_off; // ceil (l - 2*kernel_radius - max_displacement) / stride1 - int ymin = (m - 2*kernel_radius - max_displacement + round_off_s1 - 1)\ - / stride1 + 1 - round_off; // ceil (l - 2*kernel_radius - max_displacement) / stride1 - // Same here: - int xmax = (l - max_displacement + round_off_s1) / stride1 - round_off; - // floor (l - max_displacement) / stride1 - int ymax = (m - max_displacement + round_off_s1) / stride1 - round_off; - // floor (m - max_displacement) / stride1 - Dtype sum = 0; - if (xmax >= 0 && ymax >= 0 && (xmin <= topwidth-1) && (ymin <= topheight-1)) { - xmin = max(0, xmin); - xmax = min(topwidth-1, xmax); - ymin = max(0, ymin); - ymax = min(topheight-1, ymax); - for (int p = -neighborhood_grid_radius; p <= neighborhood_grid_radius; p++) { - for (int o = -neighborhood_grid_radius; o <= neighborhood_grid_radius; o++) { - // Get bottom1 data: - int s2o = stride2 * o; - int s2p = stride2 * p; - int idxbot1 = ((item * pbottomheight + (m + s2p)) * pbottomwidth + (l + s2o))\ - * bottomchannels + n; - Dtype bot1tmp = bottom1[idxbot1]; // bottom1[l+s2o,m+s2p,n] - // Index offset for topdiff in following loops: - int op = (p+neighborhood_grid_radius) * neighborhood_grid_width\ - + (o + neighborhood_grid_radius); // index [o,p] - int idxopoffset = (item * topchannels + op); - for (int y = ymin; y <= ymax; y++) { - for (int x = xmin; x <= xmax; x++) { - int idxtopdiff = (idxopoffset * topheight + y) * topwidth + x; // topdiff[x,y,o,p] - sum += topdiff[idxtopdiff] * bot1tmp; - } - } - } - } - } - const int sumelems = (kernel_radius * 2 + 1) * (kernel_radius * 2+1) * bottomchannels; - const int bot0index = ((n * bottomheight) + (m-pad_size)) * bottomwidth + (l-pad_size); - bottom0diff[bot0index + item * bottomcount] = sum / static_cast(sumelems); - } -} -// == Correlation Backward Pass Kernel (For Blob 1) -template -__global__ void CorrelateDataBackward1(const int nthreads, - int num, int item, int topwidth, int topheight, int topchannels, - int max_displacement, int neighborhood_grid_radius, - int neighborhood_grid_width, int kernel_radius, int stride1, int stride2, - int bottomwidth, int bottomheight, int pbottomwidth, int pbottomheight, - int bottomchannels, int bottomcount, int pad_size, - const Dtype *bottom0, Dtype *bottom1diff, const Dtype *topdiff) { - CUDA_KERNEL_LOOP(index, nthreads) { - // int l = index % bottomwidth + pad_size; //w-pos - // int m = (index / bottomwidth) % bottomheight + pad_size; // h-pos - // int n = (index / bottomwidth / bottomheight) % bottomchannels; // channels - int n = index % bottomchannels; // channels - int l = (index / bottomchannels) % bottomwidth + pad_size; // w-pos - int m = (index / bottomchannels / bottomwidth) % bottomheight + pad_size; // h-pos - // round_off is a trick to enable integer division with ceil, even for negative numbers - // We use a large offset, for the inner part not to become negative. - const int round_off = ROUND_OFF; - const int round_off_s1 = stride1 * round_off; - Dtype sum = 0; - for (int p = -neighborhood_grid_radius; p <= neighborhood_grid_radius; p++) { - for (int o = -neighborhood_grid_radius; o <= neighborhood_grid_radius; o++) { - int s2o = stride2 * o; - int s2p = stride2 * p; - // Get X,Y ranges and clamp - // We add round_off before_s1 the int division and subtract round_off after it, - // to ensure the formula matches ceil behavior: - int xmin = (l - 2*kernel_radius - max_displacement - s2o + round_off_s1 - 1)\ - / stride1 + 1 - round_off; - // ceil (l - 2*kernel_radius - max_displacement - s2o) / stride1 - int ymin = (m - 2*kernel_radius - max_displacement - s2p + round_off_s1 - 1)\ - / stride1 + 1 - round_off; - // ceil (l - 2*kernel_radius - max_displacement - s2o) / stride1 - // Same here: - int xmax = (l - max_displacement - s2o + round_off_s1) / stride1 - round_off; - // floor (l - max_displacement - s2o) / stride1 - int ymax = (m - max_displacement - s2p + round_off_s1) / stride1 - round_off; - // floor (m - max_displacement - s2p) / stride1 - if (xmax >= 0 && ymax >= 0 && (xmin <= topwidth - 1) && (ymin <= topheight - 1)) { - xmin = max(0, xmin); - xmax = min(topwidth-1, xmax); - ymin = max(0, ymin); - ymax = min(topheight-1, ymax); - // Get bottom0 data: - int idxbot0 = ((item * pbottomheight + (m - s2p)) \ - * pbottomwidth + (l - s2o)) * bottomchannels + n; - Dtype bot0tmp = bottom0[idxbot0]; // bottom1[l+s2o,m+s2p,n] - // Index offset for topdiff in following loops: - int op = (p+neighborhood_grid_radius) * \ - neighborhood_grid_width + (o+neighborhood_grid_radius); // index [o,p] - int idxOpOffset = (item * topchannels + op); - for (int y = ymin; y <= ymax; y++) { - for (int x = xmin; x <= xmax; x++) { - int idxtopdiff = (idxOpOffset * topheight + y)\ - * topwidth + x; // topdiff[x,y,o,p] - sum += topdiff[idxtopdiff] * bot0tmp; - } - } - } - } - } - const int sumelems = (kernel_radius*2+1)*(kernel_radius*2+1)*bottomchannels; - const int bot1index = ((n * bottomheight) + (m - pad_size)) * bottomwidth + (l - pad_size); - bottom1diff[bot1index + item * bottomcount] = sum / static_cast(sumelems); - } -} -// == Correlation Kernel Subtraction -template -__global__ void CorrelateDataSubtract(const int nthreads, int num, int item, - int topwidth, int topheight, int topchannels, int topcount, - int max_displacement, int neighborhood_grid_radius, - int neighborhood_grid_width, int kernel_radius, int stride1, int stride2, - int bottomwidth, int bottomheight, int bottomchannels, - const Dtype *bottom0, const Dtype *bottom1, Dtype *top) { - CUDA_KERNEL_LOOP(index, nthreads) { - int x = index % topwidth; // w-pos - int y = (index / topwidth) % topheight; // h-pos - int c = (index / topwidth / topheight) % topchannels; // channels - // Offset of patch in image 2 - int s2o = (c % neighborhood_grid_width - neighborhood_grid_radius) * stride2; - int s2p = (c / neighborhood_grid_width - neighborhood_grid_radius) * stride2; - // First (upper left) position of kernel center in current neighborhood in image 1 - int x1 = x*stride1 + kernel_radius + max_displacement; - int y1 = y*stride1 + kernel_radius + max_displacement; - // Iterate through 3D patch - Dtype sum = 0; - for (int j = -kernel_radius; j <= kernel_radius; j++) { // HEIGHT - for (int i = -kernel_radius; i <= kernel_radius; i++) { // WIDTH - for (int l = 0; l < bottomchannels; l++) { // CHANNELS - // Calculate position in image 2 - int x2 = x1 + s2o; - int y2 = y1 + s2p; - // Indices in bottom data: (CH=l,W=x2,H=y2,N) - int idx1 = ((item * bottomheight + y1 + j) * bottomwidth + x1 + i) \ - * bottomchannels + l; - int idx2 = ((item * bottomheight + y2 + j) * bottomwidth + x2 + i) \ - * bottomchannels + l; - // Do the correlation: - sum += fabsf(bottom0[idx1] - bottom1[idx2]); - } - } - } - const int sumelems = (kernel_radius * 2 + 1) * (kernel_radius * 2 + 1) * bottomchannels; - top[index + item * topcount] = sum / static_cast(sumelems); - } -} -// == Correlation Backward Pass Kernel (For Blob 0) -template -__global__ void CorrelateDataBackward0Subtract(const int nthreads, int num, - int item, int topwidth, int topheight, int topchannels, - int max_displacement, int neighborhood_grid_radius, - int neighborhood_grid_width, int kernel_radius, - int stride1, int stride2, int bottomwidth, int bottomheight, - int pbottomwidth, int pbottomheight, - int bottomchannels, int bottomcount, int pad_size, - Dtype *bottom0diff, const Dtype *bottom0, const Dtype *bottom1, const Dtype *topdiff) { - CUDA_KERNEL_LOOP(index, nthreads) { - int n = index % bottomchannels; // channels - int l = (index / bottomchannels) % bottomwidth + pad_size; // w-pos - int m = (index / bottomchannels / bottomwidth) % bottomheight + pad_size; // h-pos - // Get X,Y ranges and clamp - // round_off is a trick to enable integer division with ceil, even for negative numbers - // We use a large offset, for the inner part not to become negative. - const int round_off = ROUND_OFF; - const int round_off_s1 = stride1 * round_off; - int idxbot0 = ((item * pbottomheight + m) * pbottomwidth + l)\ - * bottomchannels + n; - // We add round_off before_s1 the int division and subtract round_off after it, - // to ensure the formula matches ceil behavior: - int xmin = (l - 2*kernel_radius - max_displacement + round_off_s1 - 1)\ - / stride1 + 1 - round_off; // ceil (l - 2*kernel_radius - max_displacement) / stride1 - int ymin = (m - 2*kernel_radius - max_displacement + round_off_s1 - 1)\ - / stride1 + 1 - round_off; // ceil (l - 2*kernel_radius - max_displacement) / stride1 - // Same here: - int xmax = (l - max_displacement + round_off_s1) / stride1 - round_off; - // floor (l - max_displacement) / stride1 - int ymax = (m - max_displacement + round_off_s1) / stride1 - round_off; - // floor (m - max_displacement) / stride1 - Dtype sum = 0; - if (xmax >= 0 && ymax >= 0 && (xmin <= topwidth-1) && (ymin <= topheight-1)) { - xmin = max(0, xmin); - xmax = min(topwidth-1, xmax); - ymin = max(0, ymin); - ymax = min(topheight-1, ymax); - for (int p = -neighborhood_grid_radius; p <= neighborhood_grid_radius; p++) { - for (int o = -neighborhood_grid_radius; o <= neighborhood_grid_radius; o++) { - // Get bottom1 data: - int s2o = stride2 * o; - int s2p = stride2 * p; - int idxbot1 = ((item * pbottomheight + (m+s2p)) * pbottomwidth\ - + (l+s2o)) * bottomchannels + n; - Dtype bot0tmp = bottom0[idxbot0]; - Dtype bot1tmp = bottom1[idxbot1]; - Dtype sign = (bot0tmp >= bot1tmp) ? Dtype(1.0) : Dtype(-1.0); - // Index offset for topdiff in following loops: - int op = (p+neighborhood_grid_radius) * neighborhood_grid_width\ - + (o + neighborhood_grid_radius); // index [o,p] - int idxopoffset = (item * topchannels + op); - for (int y = ymin; y <= ymax; y++) { - for (int x = xmin; x <= xmax; x++) { - int idxtopdiff = (idxopoffset * topheight + y) * topwidth + x; // topdiff[x,y,o,p] - sum += topdiff[idxtopdiff] * sign; - } - } - } - } - } - const int sumelems = (kernel_radius * 2 + 1) * (kernel_radius * 2+1) * bottomchannels; - const int bot0index = ((n * bottomheight) + (m-pad_size)) * bottomwidth + (l-pad_size); - bottom0diff[bot0index + item * bottomcount] = sum / static_cast(sumelems); - } -} -// == Correlation Backward Pass Kernel (For Blob 1) -template -__global__ void CorrelateDataBackward1Subtract(const int nthreads, int num, - int item, int topwidth, int topheight, int topchannels, - int max_displacement, int neighborhood_grid_radius, - int neighborhood_grid_width, int kernel_radius, - int stride1, int stride2, int bottomwidth, int bottomheight, - int pbottomwidth, int pbottomheight, int bottomchannels, - int bottomcount, int pad_size, const Dtype *bottom0, - const Dtype *bottom1, Dtype *bottom1diff, const Dtype *topdiff) { - CUDA_KERNEL_LOOP(index, nthreads) { - // int l = index % bottomwidth + pad_size; //w-pos - // int m = (index / bottomwidth) % bottomheight + pad_size; // h-pos - // int n = (index / bottomwidth / bottomheight) % bottomchannels; // channels - int n = index % bottomchannels; // channels - int l = (index / bottomchannels) % bottomwidth + pad_size; // w-pos - int m = (index / bottomchannels / bottomwidth) % bottomheight + pad_size; // h-pos - // round_off is a trick to enable integer division with ceil, even for negative numbers - // We use a large offset, for the inner part not to become negative. - const int round_off = ROUND_OFF; - const int round_off_s1 = stride1 * round_off; - Dtype sum = 0; - int idxbot1 = ((item * pbottomheight + m) * pbottomwidth + l)\ - * bottomchannels + n; - for (int p = -neighborhood_grid_radius; p <= neighborhood_grid_radius; p++) { - for (int o = -neighborhood_grid_radius; o <= neighborhood_grid_radius; o++) { - int s2o = stride2 * o; - int s2p = stride2 * p; - // Get X,Y ranges and clamp - // We add round_off before_s1 the int division and subtract round_off after it, - // to ensure the formula matches ceil behavior: - int xmin = (l - 2*kernel_radius - max_displacement - s2o + round_off_s1 - 1)\ - / stride1 + 1 - round_off; - // ceil (l - 2*kernel_radius - max_displacement - s2o) / stride1 - int ymin = (m - 2*kernel_radius - max_displacement - s2p + round_off_s1 - 1)\ - / stride1 + 1 - round_off; - // ceil (l - 2*kernel_radius - max_displacement - s2o) / stride1 - // Same here: - int xmax = (l - max_displacement - s2o + round_off_s1) / stride1 - round_off; - // floor (l - max_displacement - s2o) / stride1 - int ymax = (m - max_displacement - s2p + round_off_s1) / stride1 - round_off; - // floor (m - max_displacement - s2p) / stride1 - if (xmax >= 0 && ymax >= 0 && (xmin <= topwidth - 1) && (ymin <= topheight - 1)) { - xmin = max(0, xmin); - xmax = min(topwidth-1, xmax); - ymin = max(0, ymin); - ymax = min(topheight-1, ymax); - // Get bottom0 data: - int idxbot0 = ((item * pbottomheight + (m - s2p)) * pbottomwidth + (l - s2o))\ - * bottomchannels + n; - // bottom0[l+s2o,m+s2p,n] - Dtype bot0tmp = bottom0[idxbot0]; - Dtype bot1tmp = bottom1[idxbot1]; - Dtype sign = (bot0tmp >= bot1tmp) ? Dtype(-1.0) : Dtype(1.0); - // Index offset for topdiff in following loops: - int op = (p+neighborhood_grid_radius) * \ - neighborhood_grid_width + (o+neighborhood_grid_radius); // index [o,p] - int idxOpOffset = (item * topchannels + op); - for (int y = ymin; y <= ymax; y++) { - for (int x = xmin; x <= xmax; x++) { - int idxtopdiff = (idxOpOffset * topheight + y)\ - * topwidth + x; // topdiff[x,y,o,p] - sum += topdiff[idxtopdiff] * sign; - } - } - } - } - } - const int sumelems = (kernel_radius*2+1)*(kernel_radius*2+1)*bottomchannels; - const int bot1index = ((n * bottomheight) + (m - pad_size)) * bottomwidth + (l - pad_size); - bottom1diff[bot1index + item * bottomcount] = sum / static_cast(sumelems); - } -} -// == Forward -// == Dimension rearrangement Kernel -template -__global__ void blob_rearrange_kernel2(const Dtype* in, Dtype* out, int num, -int channels, int width, int height, int widthheight, int padding, int pwidthheight) { - // change shape from [batchsize,channel,y,x] to [batchsize,y,x,channel] - int xy = blockIdx.x * blockDim.x + threadIdx.x; - if (xy >= widthheight ) - return; - int ch = blockIdx.y; - int n = blockIdx.z; - Dtype value = in[(n * channels + ch) * widthheight + xy]; - __syncthreads(); - int xpad = (xy % width + padding); - int ypad = (xy / width + padding); - int xypad = ypad * (width + 2 * padding) + xpad; - out[(n * pwidthheight + xypad) * channels + ch] = value; -} -template -void Forward_gpu( - const Tensor &out, - const Tensor &data1, - const Tensor &data2, - const Tensor &tmp1, - const Tensor &tmp2, - int top_channels_, int top_height_, int top_width_, int pad_size_, - bool is_multiply, int max_displacement_, int kernel_size_, - int neighborhood_grid_radius_, int neighborhood_grid_width_, - int kernel_radius_, int stride1_, int stride2_, cudaStream_t stream, - cudaStream_t stream_tmp1, cudaStream_t stream_tmp2) { - const Dtype *bottom_data1 = data1.dptr_; - const Dtype *bottom_data2 = data2.dptr_; - Dtype *rbot1 = tmp1.dptr_; - Dtype *rbot2 = tmp2.dptr_; - Dtype *top = out.dptr_; - const int bnum = data1.size(0); - const int bchannels = data1.size(1); - const int bheight = data1.size(2); - const int bwidth = data1.size(3); - const int bwidthheight = bwidth * bheight; - const int topcount = top_width_ * top_height_ * top_channels_; - dim3 threadsPerBlock(THREADS_PER_WARP * WARPS_PER_BLOCK); - int threads_per_block = 16; - dim3 totalBlocksRearr((bwidthheight - 1) / threads_per_block + 1, bchannels, bnum); - const int pwidthheight = (bwidth + 2 * pad_size_) * (bheight + 2 * pad_size_); - blob_rearrange_kernel2<<>> - (bottom_data1, rbot1, bnum, bchannels, bwidth, bheight, bwidthheight, pad_size_, pwidthheight); - blob_rearrange_kernel2<<>> - (bottom_data2, rbot2, bnum, bchannels, bwidth, bheight, bwidthheight, pad_size_, pwidthheight); - const int num = bnum; - const int channels = bchannels; - const int height = bheight + 2 * pad_size_; - const int width = bwidth + 2 * pad_size_; - const int shared_memory_per_block = (kernel_size_ * kernel_size_) * bchannels; - if (is_multiply == true) { - // CorrelationLayer - int topThreadCount = topcount; - dim3 totalBlocksCorr(top_width_, top_height_, num); - CorrelateData<<>>( - topThreadCount, - num, top_width_, top_height_, top_channels_, topcount, - max_displacement_, neighborhood_grid_radius_, - neighborhood_grid_width_, kernel_radius_, kernel_size_, - stride1_, stride2_, - width, height, channels, - rbot1, rbot2, top); - CORRELATION_CUDA_CHECK(cudaPeekAtLastError()); - } else { - // CorrelationLayer - for (int n = 0; n < num; n++) { - int topThreadCount = topcount; - const int gridSize = (topThreadCount + kMaxThreadsPerBlock - 1)\ - / kMaxThreadsPerBlock; - CorrelateDataSubtract<<>>( - topThreadCount, - num, n, top_width_, top_height_, top_channels_, topcount, - max_displacement_, neighborhood_grid_radius_, - neighborhood_grid_width_, kernel_radius_, - stride1_, stride2_, width, height, channels, rbot1, rbot2, top); - CORRELATION_CUDA_CHECK(cudaPeekAtLastError()); - } - } -} -template -void Backward_gpu( - const Tensor &out_grad, - const Tensor &in_grad1, - const Tensor &in_grad2, - const Tensor &tmp1, - const Tensor &tmp2, - int top_channels_, int top_height_, - int top_width_, int pad_size_, bool is_multiply, - int max_displacement_, int kernel_size_, - int neighborhood_grid_radius_, int neighborhood_grid_width_, - int kernel_radius_, int stride1_, int stride2_, - cudaStream_t stream0, cudaStream_t stream1, - int num, int channels, int height, int width) { - // Get top diff, compute bottom diff - const Dtype* top_diff = out_grad.dptr_; - Dtype* bottom0_diff = in_grad1.dptr_; - Dtype* bottom1_diff = in_grad2.dptr_; - const Dtype* rbot1 = tmp1.dptr_; - const Dtype* rbot2 = tmp2.dptr_; - const int paddedheight = height + 2 * pad_size_; - const int paddedwidth = width + 2 * pad_size_; - const int bottomcount = channels * height * width; - int botThreadCount = bottomcount; - const int gridSize = (botThreadCount + kMaxThreadsPerBlock - 1) / kMaxThreadsPerBlock; - // CorrelationLayerBackward - if (is_multiply == true) { - // == Run kernel Backward 0 - dim3 totalBlocksBackward0(width, height, channels * num); // First dim is fastest - const int buffer_size_backw0 = \ - (static_cast(ceil(static_cast(2 * kernel_radius_)\ - / static_cast(stride1_))) + 1) * top_channels_; - // == Run kernel Backward 0 - for (int n = 0; n < num; n++) { - CorrelateDataBackward0<<>>( - botThreadCount, - num, n, top_width_, top_height_, top_channels_, - max_displacement_, neighborhood_grid_radius_, neighborhood_grid_width_, kernel_radius_, - stride1_, stride2_, - width, height, paddedwidth, paddedheight, channels, bottomcount, pad_size_, - bottom0_diff, rbot2, top_diff); - CORRELATION_CUDA_CHECK(cudaPeekAtLastError()); - } - // == Run kernel Backward 1 - for (int n = 0; n < num; n++) { - CorrelateDataBackward1<<>>( - botThreadCount, - num, n, top_width_, top_height_, top_channels_, - max_displacement_, neighborhood_grid_radius_, neighborhood_grid_width_, kernel_radius_, - stride1_, stride2_, - width, height, paddedwidth, paddedheight, channels, bottomcount, pad_size_, - rbot1, bottom1_diff, top_diff); - CORRELATION_CUDA_CHECK(cudaPeekAtLastError()); - } - } else { - for (int n = 0; n < num; n++) { - // Bottom0: - CorrelateDataBackward0Subtract<<>>( - botThreadCount, - num, n, top_width_, top_height_, top_channels_, - max_displacement_, neighborhood_grid_radius_, neighborhood_grid_width_, kernel_radius_, - stride1_, stride2_, - width, height, paddedwidth, paddedheight, channels, bottomcount, pad_size_, - bottom0_diff, rbot1, rbot2, top_diff); - CORRELATION_CUDA_CHECK(cudaPeekAtLastError()); - } - for (int n = 0; n < num; n++) { - // Bottom1: - CorrelateDataBackward1Subtract<<>>( - botThreadCount, - num, n, top_width_, top_height_, top_channels_, - max_displacement_, neighborhood_grid_radius_, neighborhood_grid_width_, kernel_radius_, - stride1_, stride2_, - width, height, paddedwidth, paddedheight, channels, bottomcount, pad_size_, - rbot1, rbot2, bottom1_diff, top_diff); - CORRELATION_CUDA_CHECK(cudaPeekAtLastError()); - } - } -} -} // namespace cuda -template -inline void CorrelationForward(const Tensor &out, - const Tensor &data1, - const Tensor &data2, - const Tensor &tmp1, - const Tensor &tmp2, - int top_channels_, int top_height_, - int top_width_, int pad_size_, bool is_multiply, - int max_displacement_, int kernel_size_, - int neighborhood_grid_radius_, int neighborhood_grid_width_, - int kernel_radius_, int stride1_, int stride2_ - ) { - cudaStream_t stream = Stream::GetStream(out.stream_); - cudaStream_t stream_tmp1 = Stream::GetStream(tmp1.stream_); - cudaStream_t stream_tmp2 = Stream::GetStream(tmp2.stream_); - cuda::Forward_gpu(out, data1, data2, tmp1, tmp2, top_channels_, top_height_, - top_width_, pad_size_, is_multiply, max_displacement_, kernel_size_, - neighborhood_grid_radius_, neighborhood_grid_width_, kernel_radius_, - stride1_, stride2_, stream, stream_tmp1, stream_tmp2); -} - -template -inline void CorrelationBackward(const Tensor &out_grad, - const Tensor &in_grad1, - const Tensor &in_grad2, - const Tensor &tmp1, - const Tensor &tmp2, - int top_channels_, int top_height_, - int top_width_, int pad_size_, bool is_multiply, - int max_displacement_, int kernel_size_, - int neighborhood_grid_radius_, int neighborhood_grid_width_, - int kernel_radius_, int stride1_, - int stride2_, int num, int channels, int height, int width - ) { - cudaStream_t stream0 = Stream::GetStream(in_grad1.stream_); - cudaStream_t stream1 = Stream::GetStream(in_grad2.stream_); - cuda::Backward_gpu(out_grad, in_grad1, in_grad2, tmp1, tmp2, top_channels_, - top_height_, top_width_, pad_size_, is_multiply, - max_displacement_, kernel_size_, neighborhood_grid_radius_, - neighborhood_grid_width_, kernel_radius_, stride1_, stride2_, - stream0, stream1, num, channels, height, width); -} -} // namespace mshadow -namespace mxnet { -namespace op { -template<> -Operator* CreateOp(CorrelationParam param) { - return new CorrelationOp(param); -} -} // namespace op -} // namespace mxnet +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * Copyright [2016] + * \file Correation.cu + * \brief Correlation operator + * \author Xu Dong +*/ +#include "./correlation-inl.h" +#include +#include +#include +#include + +#define ROUND_OFF 50000 +#define WARPS_PER_BLOCK 1 +#define THREADS_PER_WARP 32 +#define CORRELATION_CUDA_CHECK(condition) \ + /* Code block avoids redefinition of cudaError_t error */ \ + do { \ + cudaError_t error = condition; \ + CHECK_EQ(error, cudaSuccess) << " " << cudaGetErrorString(error); \ + } while (0) +#define CUDA_KERNEL_LOOP(i, n) \ +for (int i = blockIdx.x * blockDim.x + threadIdx.x; \ + i < (n); \ + i += blockDim.x * gridDim.x) +namespace mshadow { +namespace cuda { +// == Correlation Kernel +template +__global__ void CorrelateData(const int nthreads, int num, int topwidth, + int topheight, int topchannels, int topcount, + int max_displacement, int neighborhood_grid_radius, + int neighborhood_grid_width, int kernel_radius, int kernel_size, int stride1, int stride2, + int bottomwidth, int bottomheight, int bottomchannels, + const Dtype *bottom0, const Dtype *bottom1, Dtype *top) { + extern __shared__ char patch_data_char[]; + Dtype *patch_data = reinterpret_cast(patch_data_char); + // First (upper left) position of kernel upper-left corner + // in current center position of neighborhood in image 1 + int x1 = blockIdx.x * stride1 + max_displacement; + int y1 = blockIdx.y * stride1 + max_displacement; + int item = blockIdx.z; + int ch_off = threadIdx.x; + // Load 3D patch into shared shared memory + for (int j = 0; j < kernel_size; j++) { // HEIGHT + for (int i = 0; i < kernel_size; i++) { // WIDTH + int ji_off = ((j * kernel_size) + i) * bottomchannels; + for (int ch = ch_off; ch < bottomchannels; ch += (THREADS_PER_WARP * WARPS_PER_BLOCK)) { + // CHANNELS + int idx1 = ((item * bottomheight + y1+j) * bottomwidth + x1+i) * bottomchannels + ch; + int idxPatchData = ji_off + ch; + patch_data[idxPatchData] = bottom0[idx1]; + } + } + } + __syncthreads(); + __shared__ Dtype sum[THREADS_PER_WARP * WARPS_PER_BLOCK]; + // Compute correlation + for (int top_channel = 0; top_channel < topchannels; top_channel++) { + sum[ch_off] = 0; + int s2o = (top_channel % neighborhood_grid_width - neighborhood_grid_radius) * stride2; + int s2p = (top_channel / neighborhood_grid_width - neighborhood_grid_radius) * stride2; + for (int j = 0; j < kernel_size; j++) { // HEIGHT + for (int i = 0; i < kernel_size; i++) { // WIDTH + int ji_off = ((j * kernel_size) + i) * bottomchannels; + for (int ch = ch_off; ch < bottomchannels; ch += (THREADS_PER_WARP * WARPS_PER_BLOCK)) { + // CHANNELS + int x2 = x1 + s2o; + int y2 = y1 + s2p; + int idxPatchData = ji_off + ch; + int idx2 = ((item * bottomheight + y2 + j) * bottomwidth + x2 + i) * bottomchannels + ch; + sum[ch_off] += patch_data[idxPatchData] * bottom1[idx2]; + } + } + } + __syncthreads(); + if (ch_off == 0) { + Dtype total_sum = 0; + for (int idx = 0; idx < THREADS_PER_WARP * WARPS_PER_BLOCK; idx++) { + total_sum += sum[idx]; + } + const int sumelems = kernel_size * kernel_size * bottomchannels; + const int index = ((top_channel * topheight + blockIdx.y) * topwidth) + blockIdx.x; + top[index + item*topcount] = total_sum / static_cast(sumelems); + } // Aggregate result of different threads + } +} +// == Correlation Backward Pass Kernel (For data1) +template +__global__ void CorrelateDataBackward0(const int nthreads, int num, int item, + int topwidth, int topheight, int topchannels, + int max_displacement, int neighborhood_grid_radius, + int neighborhood_grid_width, int kernel_radius, int stride1, int stride2, + int bottomwidth, int bottomheight, int pbottomwidth, int pbottomheight, + int bottomchannels, int bottomcount, int pad_size, + Dtype *bottom0diff, const Dtype *bottom1, const Dtype *topdiff) { + CUDA_KERNEL_LOOP(index, nthreads) { + int n = index % bottomchannels; // channels + int l = (index / bottomchannels) % bottomwidth + pad_size; // w-pos + int m = (index / bottomchannels / bottomwidth) % bottomheight + pad_size; // h-pos + // Get X,Y ranges and clamp + // round_off is a trick to enable integer division with ceil, even for negative numbers + // We use a large offset, for the inner part not to become negative. + const int round_off = ROUND_OFF; + const int round_off_s1 = stride1 * round_off; + // We add round_off before_s1 the int division and subtract round_off after it, + // to ensure the formula matches ceil behavior: + int xmin = (l - 2*kernel_radius - max_displacement + round_off_s1 - 1)\ + / stride1 + 1 - round_off; // ceil (l - 2*kernel_radius - max_displacement) / stride1 + int ymin = (m - 2*kernel_radius - max_displacement + round_off_s1 - 1)\ + / stride1 + 1 - round_off; // ceil (l - 2*kernel_radius - max_displacement) / stride1 + // Same here: + int xmax = (l - max_displacement + round_off_s1) / stride1 - round_off; + // floor (l - max_displacement) / stride1 + int ymax = (m - max_displacement + round_off_s1) / stride1 - round_off; + // floor (m - max_displacement) / stride1 + Dtype sum = 0; + if (xmax >= 0 && ymax >= 0 && (xmin <= topwidth-1) && (ymin <= topheight-1)) { + xmin = max(0, xmin); + xmax = min(topwidth-1, xmax); + ymin = max(0, ymin); + ymax = min(topheight-1, ymax); + for (int p = -neighborhood_grid_radius; p <= neighborhood_grid_radius; p++) { + for (int o = -neighborhood_grid_radius; o <= neighborhood_grid_radius; o++) { + // Get bottom1 data: + int s2o = stride2 * o; + int s2p = stride2 * p; + int idxbot1 = ((item * pbottomheight + (m + s2p)) * pbottomwidth + (l + s2o))\ + * bottomchannels + n; + Dtype bot1tmp = bottom1[idxbot1]; // bottom1[l+s2o,m+s2p,n] + // Index offset for topdiff in following loops: + int op = (p+neighborhood_grid_radius) * neighborhood_grid_width\ + + (o + neighborhood_grid_radius); // index [o,p] + int idxopoffset = (item * topchannels + op); + for (int y = ymin; y <= ymax; y++) { + for (int x = xmin; x <= xmax; x++) { + int idxtopdiff = (idxopoffset * topheight + y) * topwidth + x; // topdiff[x,y,o,p] + sum += topdiff[idxtopdiff] * bot1tmp; + } + } + } + } + } + const int sumelems = (kernel_radius * 2 + 1) * (kernel_radius * 2+1) * bottomchannels; + const int bot0index = ((n * bottomheight) + (m-pad_size)) * bottomwidth + (l-pad_size); + bottom0diff[bot0index + item * bottomcount] = sum / static_cast(sumelems); + } +} +// == Correlation Backward Pass Kernel (For Blob 1) +template +__global__ void CorrelateDataBackward1(const int nthreads, + int num, int item, int topwidth, int topheight, int topchannels, + int max_displacement, int neighborhood_grid_radius, + int neighborhood_grid_width, int kernel_radius, int stride1, int stride2, + int bottomwidth, int bottomheight, int pbottomwidth, int pbottomheight, + int bottomchannels, int bottomcount, int pad_size, + const Dtype *bottom0, Dtype *bottom1diff, const Dtype *topdiff) { + CUDA_KERNEL_LOOP(index, nthreads) { + // int l = index % bottomwidth + pad_size; //w-pos + // int m = (index / bottomwidth) % bottomheight + pad_size; // h-pos + // int n = (index / bottomwidth / bottomheight) % bottomchannels; // channels + int n = index % bottomchannels; // channels + int l = (index / bottomchannels) % bottomwidth + pad_size; // w-pos + int m = (index / bottomchannels / bottomwidth) % bottomheight + pad_size; // h-pos + // round_off is a trick to enable integer division with ceil, even for negative numbers + // We use a large offset, for the inner part not to become negative. + const int round_off = ROUND_OFF; + const int round_off_s1 = stride1 * round_off; + Dtype sum = 0; + for (int p = -neighborhood_grid_radius; p <= neighborhood_grid_radius; p++) { + for (int o = -neighborhood_grid_radius; o <= neighborhood_grid_radius; o++) { + int s2o = stride2 * o; + int s2p = stride2 * p; + // Get X,Y ranges and clamp + // We add round_off before_s1 the int division and subtract round_off after it, + // to ensure the formula matches ceil behavior: + int xmin = (l - 2*kernel_radius - max_displacement - s2o + round_off_s1 - 1)\ + / stride1 + 1 - round_off; + // ceil (l - 2*kernel_radius - max_displacement - s2o) / stride1 + int ymin = (m - 2*kernel_radius - max_displacement - s2p + round_off_s1 - 1)\ + / stride1 + 1 - round_off; + // ceil (l - 2*kernel_radius - max_displacement - s2o) / stride1 + // Same here: + int xmax = (l - max_displacement - s2o + round_off_s1) / stride1 - round_off; + // floor (l - max_displacement - s2o) / stride1 + int ymax = (m - max_displacement - s2p + round_off_s1) / stride1 - round_off; + // floor (m - max_displacement - s2p) / stride1 + if (xmax >= 0 && ymax >= 0 && (xmin <= topwidth - 1) && (ymin <= topheight - 1)) { + xmin = max(0, xmin); + xmax = min(topwidth-1, xmax); + ymin = max(0, ymin); + ymax = min(topheight-1, ymax); + // Get bottom0 data: + int idxbot0 = ((item * pbottomheight + (m - s2p)) \ + * pbottomwidth + (l - s2o)) * bottomchannels + n; + Dtype bot0tmp = bottom0[idxbot0]; // bottom1[l+s2o,m+s2p,n] + // Index offset for topdiff in following loops: + int op = (p+neighborhood_grid_radius) * \ + neighborhood_grid_width + (o+neighborhood_grid_radius); // index [o,p] + int idxOpOffset = (item * topchannels + op); + for (int y = ymin; y <= ymax; y++) { + for (int x = xmin; x <= xmax; x++) { + int idxtopdiff = (idxOpOffset * topheight + y)\ + * topwidth + x; // topdiff[x,y,o,p] + sum += topdiff[idxtopdiff] * bot0tmp; + } + } + } + } + } + const int sumelems = (kernel_radius*2+1)*(kernel_radius*2+1)*bottomchannels; + const int bot1index = ((n * bottomheight) + (m - pad_size)) * bottomwidth + (l - pad_size); + bottom1diff[bot1index + item * bottomcount] = sum / static_cast(sumelems); + } +} +// == Correlation Kernel Subtraction +template +__global__ void CorrelateDataSubtract(const int nthreads, int num, int item, + int topwidth, int topheight, int topchannels, int topcount, + int max_displacement, int neighborhood_grid_radius, + int neighborhood_grid_width, int kernel_radius, int stride1, int stride2, + int bottomwidth, int bottomheight, int bottomchannels, + const Dtype *bottom0, const Dtype *bottom1, Dtype *top) { + CUDA_KERNEL_LOOP(index, nthreads) { + int x = index % topwidth; // w-pos + int y = (index / topwidth) % topheight; // h-pos + int c = (index / topwidth / topheight) % topchannels; // channels + // Offset of patch in image 2 + int s2o = (c % neighborhood_grid_width - neighborhood_grid_radius) * stride2; + int s2p = (c / neighborhood_grid_width - neighborhood_grid_radius) * stride2; + // First (upper left) position of kernel center in current neighborhood in image 1 + int x1 = x*stride1 + kernel_radius + max_displacement; + int y1 = y*stride1 + kernel_radius + max_displacement; + // Iterate through 3D patch + Dtype sum = 0; + for (int j = -kernel_radius; j <= kernel_radius; j++) { // HEIGHT + for (int i = -kernel_radius; i <= kernel_radius; i++) { // WIDTH + for (int l = 0; l < bottomchannels; l++) { // CHANNELS + // Calculate position in image 2 + int x2 = x1 + s2o; + int y2 = y1 + s2p; + // Indices in bottom data: (CH=l,W=x2,H=y2,N) + int idx1 = ((item * bottomheight + y1 + j) * bottomwidth + x1 + i) \ + * bottomchannels + l; + int idx2 = ((item * bottomheight + y2 + j) * bottomwidth + x2 + i) \ + * bottomchannels + l; + // Do the correlation: + sum += fabsf(bottom0[idx1] - bottom1[idx2]); + } + } + } + const int sumelems = (kernel_radius * 2 + 1) * (kernel_radius * 2 + 1) * bottomchannels; + top[index + item * topcount] = sum / static_cast(sumelems); + } +} +// == Correlation Backward Pass Kernel (For Blob 0) +template +__global__ void CorrelateDataBackward0Subtract(const int nthreads, int num, + int item, int topwidth, int topheight, int topchannels, + int max_displacement, int neighborhood_grid_radius, + int neighborhood_grid_width, int kernel_radius, + int stride1, int stride2, int bottomwidth, int bottomheight, + int pbottomwidth, int pbottomheight, + int bottomchannels, int bottomcount, int pad_size, + Dtype *bottom0diff, const Dtype *bottom0, const Dtype *bottom1, const Dtype *topdiff) { + CUDA_KERNEL_LOOP(index, nthreads) { + int n = index % bottomchannels; // channels + int l = (index / bottomchannels) % bottomwidth + pad_size; // w-pos + int m = (index / bottomchannels / bottomwidth) % bottomheight + pad_size; // h-pos + // Get X,Y ranges and clamp + // round_off is a trick to enable integer division with ceil, even for negative numbers + // We use a large offset, for the inner part not to become negative. + const int round_off = ROUND_OFF; + const int round_off_s1 = stride1 * round_off; + int idxbot0 = ((item * pbottomheight + m) * pbottomwidth + l)\ + * bottomchannels + n; + // We add round_off before_s1 the int division and subtract round_off after it, + // to ensure the formula matches ceil behavior: + int xmin = (l - 2*kernel_radius - max_displacement + round_off_s1 - 1)\ + / stride1 + 1 - round_off; // ceil (l - 2*kernel_radius - max_displacement) / stride1 + int ymin = (m - 2*kernel_radius - max_displacement + round_off_s1 - 1)\ + / stride1 + 1 - round_off; // ceil (l - 2*kernel_radius - max_displacement) / stride1 + // Same here: + int xmax = (l - max_displacement + round_off_s1) / stride1 - round_off; + // floor (l - max_displacement) / stride1 + int ymax = (m - max_displacement + round_off_s1) / stride1 - round_off; + // floor (m - max_displacement) / stride1 + Dtype sum = 0; + if (xmax >= 0 && ymax >= 0 && (xmin <= topwidth-1) && (ymin <= topheight-1)) { + xmin = max(0, xmin); + xmax = min(topwidth-1, xmax); + ymin = max(0, ymin); + ymax = min(topheight-1, ymax); + for (int p = -neighborhood_grid_radius; p <= neighborhood_grid_radius; p++) { + for (int o = -neighborhood_grid_radius; o <= neighborhood_grid_radius; o++) { + // Get bottom1 data: + int s2o = stride2 * o; + int s2p = stride2 * p; + int idxbot1 = ((item * pbottomheight + (m+s2p)) * pbottomwidth\ + + (l+s2o)) * bottomchannels + n; + Dtype bot0tmp = bottom0[idxbot0]; + Dtype bot1tmp = bottom1[idxbot1]; + Dtype sign = (bot0tmp >= bot1tmp) ? Dtype(1.0) : Dtype(-1.0); + // Index offset for topdiff in following loops: + int op = (p+neighborhood_grid_radius) * neighborhood_grid_width\ + + (o + neighborhood_grid_radius); // index [o,p] + int idxopoffset = (item * topchannels + op); + for (int y = ymin; y <= ymax; y++) { + for (int x = xmin; x <= xmax; x++) { + int idxtopdiff = (idxopoffset * topheight + y) * topwidth + x; // topdiff[x,y,o,p] + sum += topdiff[idxtopdiff] * sign; + } + } + } + } + } + const int sumelems = (kernel_radius * 2 + 1) * (kernel_radius * 2+1) * bottomchannels; + const int bot0index = ((n * bottomheight) + (m-pad_size)) * bottomwidth + (l-pad_size); + bottom0diff[bot0index + item * bottomcount] = sum / static_cast(sumelems); + } +} +// == Correlation Backward Pass Kernel (For Blob 1) +template +__global__ void CorrelateDataBackward1Subtract(const int nthreads, int num, + int item, int topwidth, int topheight, int topchannels, + int max_displacement, int neighborhood_grid_radius, + int neighborhood_grid_width, int kernel_radius, + int stride1, int stride2, int bottomwidth, int bottomheight, + int pbottomwidth, int pbottomheight, int bottomchannels, + int bottomcount, int pad_size, const Dtype *bottom0, + const Dtype *bottom1, Dtype *bottom1diff, const Dtype *topdiff) { + CUDA_KERNEL_LOOP(index, nthreads) { + // int l = index % bottomwidth + pad_size; //w-pos + // int m = (index / bottomwidth) % bottomheight + pad_size; // h-pos + // int n = (index / bottomwidth / bottomheight) % bottomchannels; // channels + int n = index % bottomchannels; // channels + int l = (index / bottomchannels) % bottomwidth + pad_size; // w-pos + int m = (index / bottomchannels / bottomwidth) % bottomheight + pad_size; // h-pos + // round_off is a trick to enable integer division with ceil, even for negative numbers + // We use a large offset, for the inner part not to become negative. + const int round_off = ROUND_OFF; + const int round_off_s1 = stride1 * round_off; + Dtype sum = 0; + int idxbot1 = ((item * pbottomheight + m) * pbottomwidth + l)\ + * bottomchannels + n; + for (int p = -neighborhood_grid_radius; p <= neighborhood_grid_radius; p++) { + for (int o = -neighborhood_grid_radius; o <= neighborhood_grid_radius; o++) { + int s2o = stride2 * o; + int s2p = stride2 * p; + // Get X,Y ranges and clamp + // We add round_off before_s1 the int division and subtract round_off after it, + // to ensure the formula matches ceil behavior: + int xmin = (l - 2*kernel_radius - max_displacement - s2o + round_off_s1 - 1)\ + / stride1 + 1 - round_off; + // ceil (l - 2*kernel_radius - max_displacement - s2o) / stride1 + int ymin = (m - 2*kernel_radius - max_displacement - s2p + round_off_s1 - 1)\ + / stride1 + 1 - round_off; + // ceil (l - 2*kernel_radius - max_displacement - s2o) / stride1 + // Same here: + int xmax = (l - max_displacement - s2o + round_off_s1) / stride1 - round_off; + // floor (l - max_displacement - s2o) / stride1 + int ymax = (m - max_displacement - s2p + round_off_s1) / stride1 - round_off; + // floor (m - max_displacement - s2p) / stride1 + if (xmax >= 0 && ymax >= 0 && (xmin <= topwidth - 1) && (ymin <= topheight - 1)) { + xmin = max(0, xmin); + xmax = min(topwidth-1, xmax); + ymin = max(0, ymin); + ymax = min(topheight-1, ymax); + // Get bottom0 data: + int idxbot0 = ((item * pbottomheight + (m - s2p)) * pbottomwidth + (l - s2o))\ + * bottomchannels + n; + // bottom0[l+s2o,m+s2p,n] + Dtype bot0tmp = bottom0[idxbot0]; + Dtype bot1tmp = bottom1[idxbot1]; + Dtype sign = (bot0tmp >= bot1tmp) ? Dtype(-1.0) : Dtype(1.0); + // Index offset for topdiff in following loops: + int op = (p+neighborhood_grid_radius) * \ + neighborhood_grid_width + (o+neighborhood_grid_radius); // index [o,p] + int idxOpOffset = (item * topchannels + op); + for (int y = ymin; y <= ymax; y++) { + for (int x = xmin; x <= xmax; x++) { + int idxtopdiff = (idxOpOffset * topheight + y)\ + * topwidth + x; // topdiff[x,y,o,p] + sum += topdiff[idxtopdiff] * sign; + } + } + } + } + } + const int sumelems = (kernel_radius*2+1)*(kernel_radius*2+1)*bottomchannels; + const int bot1index = ((n * bottomheight) + (m - pad_size)) * bottomwidth + (l - pad_size); + bottom1diff[bot1index + item * bottomcount] = sum / static_cast(sumelems); + } +} +// == Forward +// == Dimension rearrangement Kernel +template +__global__ void blob_rearrange_kernel2(const Dtype* in, Dtype* out, int num, +int channels, int width, int height, int widthheight, int padding, int pwidthheight) { + // change shape from [batchsize,channel,y,x] to [batchsize,y,x,channel] + int xy = blockIdx.x * blockDim.x + threadIdx.x; + if (xy >= widthheight ) + return; + int ch = blockIdx.y; + int n = blockIdx.z; + Dtype value = in[(n * channels + ch) * widthheight + xy]; + __syncthreads(); + int xpad = (xy % width + padding); + int ypad = (xy / width + padding); + int xypad = ypad * (width + 2 * padding) + xpad; + out[(n * pwidthheight + xypad) * channels + ch] = value; +} +template +void Forward_gpu( + const Tensor &out, + const Tensor &data1, + const Tensor &data2, + const Tensor &tmp1, + const Tensor &tmp2, + int top_channels_, int top_height_, int top_width_, int pad_size_, + bool is_multiply, int max_displacement_, int kernel_size_, + int neighborhood_grid_radius_, int neighborhood_grid_width_, + int kernel_radius_, int stride1_, int stride2_, cudaStream_t stream, + cudaStream_t stream_tmp1, cudaStream_t stream_tmp2) { + const Dtype *bottom_data1 = data1.dptr_; + const Dtype *bottom_data2 = data2.dptr_; + Dtype *rbot1 = tmp1.dptr_; + Dtype *rbot2 = tmp2.dptr_; + Dtype *top = out.dptr_; + const int bnum = data1.size(0); + const int bchannels = data1.size(1); + const int bheight = data1.size(2); + const int bwidth = data1.size(3); + const int bwidthheight = bwidth * bheight; + const int topcount = top_width_ * top_height_ * top_channels_; + dim3 threadsPerBlock(THREADS_PER_WARP * WARPS_PER_BLOCK); + int threads_per_block = 16; + dim3 totalBlocksRearr((bwidthheight - 1) / threads_per_block + 1, bchannels, bnum); + const int pwidthheight = (bwidth + 2 * pad_size_) * (bheight + 2 * pad_size_); + blob_rearrange_kernel2<<>> + (bottom_data1, rbot1, bnum, bchannels, bwidth, bheight, bwidthheight, pad_size_, pwidthheight); + blob_rearrange_kernel2<<>> + (bottom_data2, rbot2, bnum, bchannels, bwidth, bheight, bwidthheight, pad_size_, pwidthheight); + const int num = bnum; + const int channels = bchannels; + const int height = bheight + 2 * pad_size_; + const int width = bwidth + 2 * pad_size_; + const int shared_memory_per_block = (kernel_size_ * kernel_size_) * bchannels; + if (is_multiply == true) { + // CorrelationLayer + int topThreadCount = topcount; + dim3 totalBlocksCorr(top_width_, top_height_, num); + CorrelateData<<>>( + topThreadCount, + num, top_width_, top_height_, top_channels_, topcount, + max_displacement_, neighborhood_grid_radius_, + neighborhood_grid_width_, kernel_radius_, kernel_size_, + stride1_, stride2_, + width, height, channels, + rbot1, rbot2, top); + CORRELATION_CUDA_CHECK(cudaPeekAtLastError()); + } else { + // CorrelationLayer + for (int n = 0; n < num; n++) { + int topThreadCount = topcount; + const int gridSize = (topThreadCount + kMaxThreadsPerBlock - 1)\ + / kMaxThreadsPerBlock; + CorrelateDataSubtract<<>>( + topThreadCount, + num, n, top_width_, top_height_, top_channels_, topcount, + max_displacement_, neighborhood_grid_radius_, + neighborhood_grid_width_, kernel_radius_, + stride1_, stride2_, width, height, channels, rbot1, rbot2, top); + CORRELATION_CUDA_CHECK(cudaPeekAtLastError()); + } + } +} +template +void Backward_gpu( + const Tensor &out_grad, + const Tensor &in_grad1, + const Tensor &in_grad2, + const Tensor &tmp1, + const Tensor &tmp2, + int top_channels_, int top_height_, + int top_width_, int pad_size_, bool is_multiply, + int max_displacement_, int kernel_size_, + int neighborhood_grid_radius_, int neighborhood_grid_width_, + int kernel_radius_, int stride1_, int stride2_, + cudaStream_t stream0, cudaStream_t stream1, + int num, int channels, int height, int width) { + // Get top diff, compute bottom diff + const Dtype* top_diff = out_grad.dptr_; + Dtype* bottom0_diff = in_grad1.dptr_; + Dtype* bottom1_diff = in_grad2.dptr_; + const Dtype* rbot1 = tmp1.dptr_; + const Dtype* rbot2 = tmp2.dptr_; + const int paddedheight = height + 2 * pad_size_; + const int paddedwidth = width + 2 * pad_size_; + const int bottomcount = channels * height * width; + int botThreadCount = bottomcount; + const int gridSize = (botThreadCount + kMaxThreadsPerBlock - 1) / kMaxThreadsPerBlock; + // CorrelationLayerBackward + if (is_multiply == true) { + // == Run kernel Backward 0 + dim3 totalBlocksBackward0(width, height, channels * num); // First dim is fastest + const int buffer_size_backw0 = \ + (static_cast(ceil(static_cast(2 * kernel_radius_)\ + / static_cast(stride1_))) + 1) * top_channels_; + // == Run kernel Backward 0 + for (int n = 0; n < num; n++) { + CorrelateDataBackward0<<>>( + botThreadCount, + num, n, top_width_, top_height_, top_channels_, + max_displacement_, neighborhood_grid_radius_, neighborhood_grid_width_, kernel_radius_, + stride1_, stride2_, + width, height, paddedwidth, paddedheight, channels, bottomcount, pad_size_, + bottom0_diff, rbot2, top_diff); + CORRELATION_CUDA_CHECK(cudaPeekAtLastError()); + } + // == Run kernel Backward 1 + for (int n = 0; n < num; n++) { + CorrelateDataBackward1<<>>( + botThreadCount, + num, n, top_width_, top_height_, top_channels_, + max_displacement_, neighborhood_grid_radius_, neighborhood_grid_width_, kernel_radius_, + stride1_, stride2_, + width, height, paddedwidth, paddedheight, channels, bottomcount, pad_size_, + rbot1, bottom1_diff, top_diff); + CORRELATION_CUDA_CHECK(cudaPeekAtLastError()); + } + } else { + for (int n = 0; n < num; n++) { + // Bottom0: + CorrelateDataBackward0Subtract<<>>( + botThreadCount, + num, n, top_width_, top_height_, top_channels_, + max_displacement_, neighborhood_grid_radius_, neighborhood_grid_width_, kernel_radius_, + stride1_, stride2_, + width, height, paddedwidth, paddedheight, channels, bottomcount, pad_size_, + bottom0_diff, rbot1, rbot2, top_diff); + CORRELATION_CUDA_CHECK(cudaPeekAtLastError()); + } + for (int n = 0; n < num; n++) { + // Bottom1: + CorrelateDataBackward1Subtract<<>>( + botThreadCount, + num, n, top_width_, top_height_, top_channels_, + max_displacement_, neighborhood_grid_radius_, neighborhood_grid_width_, kernel_radius_, + stride1_, stride2_, + width, height, paddedwidth, paddedheight, channels, bottomcount, pad_size_, + rbot1, rbot2, bottom1_diff, top_diff); + CORRELATION_CUDA_CHECK(cudaPeekAtLastError()); + } + } +} +} // namespace cuda +template +inline void CorrelationForward(const Tensor &out, + const Tensor &data1, + const Tensor &data2, + const Tensor &tmp1, + const Tensor &tmp2, + int top_channels_, int top_height_, + int top_width_, int pad_size_, bool is_multiply, + int max_displacement_, int kernel_size_, + int neighborhood_grid_radius_, int neighborhood_grid_width_, + int kernel_radius_, int stride1_, int stride2_ + ) { + cudaStream_t stream = Stream::GetStream(out.stream_); + cudaStream_t stream_tmp1 = Stream::GetStream(tmp1.stream_); + cudaStream_t stream_tmp2 = Stream::GetStream(tmp2.stream_); + cuda::Forward_gpu(out, data1, data2, tmp1, tmp2, top_channels_, top_height_, + top_width_, pad_size_, is_multiply, max_displacement_, kernel_size_, + neighborhood_grid_radius_, neighborhood_grid_width_, kernel_radius_, + stride1_, stride2_, stream, stream_tmp1, stream_tmp2); +} + +template +inline void CorrelationBackward(const Tensor &out_grad, + const Tensor &in_grad1, + const Tensor &in_grad2, + const Tensor &tmp1, + const Tensor &tmp2, + int top_channels_, int top_height_, + int top_width_, int pad_size_, bool is_multiply, + int max_displacement_, int kernel_size_, + int neighborhood_grid_radius_, int neighborhood_grid_width_, + int kernel_radius_, int stride1_, + int stride2_, int num, int channels, int height, int width + ) { + cudaStream_t stream0 = Stream::GetStream(in_grad1.stream_); + cudaStream_t stream1 = Stream::GetStream(in_grad2.stream_); + cuda::Backward_gpu(out_grad, in_grad1, in_grad2, tmp1, tmp2, top_channels_, + top_height_, top_width_, pad_size_, is_multiply, + max_displacement_, kernel_size_, neighborhood_grid_radius_, + neighborhood_grid_width_, kernel_radius_, stride1_, stride2_, + stream0, stream1, num, channels, height, width); +} +} // namespace mshadow +namespace mxnet { +namespace op { +template<> +Operator* CreateOp(CorrelationParam param) { + return new CorrelationOp(param); +} +} // namespace op +} // namespace mxnet diff --git a/src/operator/crop-inl.h b/src/operator/crop-inl.h index 5b5adbf15874..5a8709633f21 100644 --- a/src/operator/crop-inl.h +++ b/src/operator/crop-inl.h @@ -1,214 +1,232 @@ -/*! - * Copyright (c) 2015 by Contributors - * \file crop-inl.h - * \brief - * \author Wei Wu -*/ -#ifndef MXNET_OPERATOR_CROP_INL_H_ -#define MXNET_OPERATOR_CROP_INL_H_ -#include -#include -#include -#include -#include -#include -#include -#include -#include "./operator_common.h" - -namespace mxnet { -namespace op { - -namespace crop_enum { -enum CropOpInputs {kData, kCropLike}; -enum CropOpOutputs {kOut}; -} // namespace crop_enum - -struct CropParam : public dmlc::Parameter { - int num_args; - TShape offset; - TShape h_w; - bool center_crop; - DMLC_DECLARE_PARAMETER(CropParam) { - DMLC_DECLARE_FIELD(num_args).set_range(1, 3) - .describe("Number of inputs for crop, if equals one, then we will use the h_w" - "for crop height and width, else if equals two, then we will use the height" - "and width of the second input symbol, we name crop_like here"); - int shape[] = {0, 0}; - DMLC_DECLARE_FIELD(offset).set_default(TShape(shape, shape + 2)) - .describe("crop offset coordinate: (y, x)"); - DMLC_DECLARE_FIELD(h_w).set_default(TShape(shape, shape + 2)) - .describe("crop height and width: (h, w)"); - DMLC_DECLARE_FIELD(center_crop).set_default(false) - .describe("If set to true, then it will use be the center_crop," - "or it will crop using the shape of crop_like"); - } -}; // struct CropParam - -template -class CropOp : public Operator { - public: - explicit CropOp(CropParam param) { - this->param_ = param; - } - - virtual void Forward(const OpContext &ctx, - const std::vector &in_data, - const std::vector &req, - const std::vector &out_data, - const std::vector &aux_args) { - using namespace mshadow; - using namespace mshadow::expr; - CHECK_EQ(static_cast(in_data.size()), param_.num_args); - CHECK_EQ(out_data.size(), 1U); - CHECK_EQ(req[crop_enum::kOut], kWriteTo); - Stream *s = ctx.get_stream(); - Tensor data = in_data[crop_enum::kData].get(s); - Tensor out = out_data[crop_enum::kOut].get(s); - offset_hw_ = InferCropOfferset(data.shape_, out.shape_); - out = crop(data, Shape2(out.size(2), out.size(3)), offset_hw_[0], offset_hw_[1]); - } - - // because the crop_like input is only used with it's shape, so we should be - // careful setting its backwrd grad value to zeros, so that it will not hurt - // the connection of crop_like. - virtual void Backward(const OpContext &ctx, - const std::vector &out_grad, - const std::vector &in_data, - const std::vector &out_data, - const std::vector &req, - const std::vector &in_grad, - const std::vector &aux_states) { - using namespace mshadow; - using namespace mshadow::expr; - CHECK_EQ(in_grad.size(), static_cast(param_.num_args)) << in_grad.size(); - CHECK_EQ(out_grad.size(), 1U) << out_grad.size(); - Stream *s = ctx.get_stream(); - Tensor grad = out_grad[crop_enum::kOut].get(s); - Tensor gdata = in_grad[crop_enum::kData].get(s); - if (param_.num_args > 1) { - // here backward grad is set to zero for crop_like - // however, this should only be done when num_args > 1, i.e., crop_like exists - Tensor gcrop_like = in_grad[crop_enum::kCropLike].get(s); - gcrop_like = (real_t)0.0f; - } - offset_hw_ = InferCropOfferset(gdata.shape_, grad.shape_); - gdata = (real_t)0.0f; - slice<3>(slice<2>(gdata, offset_hw_[0], offset_hw_[0]+grad.size(2)), - offset_hw_[1], offset_hw_[1]+grad.size(3)) = grad; - } - - private: - CropParam param_; - std::vector offset_hw_; - std::vector InferCropOfferset(const mshadow::Shape<4> &data_shape, - const mshadow::Shape<4> &out_shape) { - std::vector offset_hw; - CHECK_GE(data_shape[2], out_shape[2]) << - "data_shape'height should be larger than that of out_shape"; - CHECK_GE(data_shape[3], out_shape[3]) << - "data_shape'weight should be larger than that of out_shape"; - if (param_.center_crop) { - offset_hw.push_back(static_cast((data_shape[2]-out_shape[2])/2)); - offset_hw.push_back(static_cast((data_shape[3]-out_shape[3])/2)); - } else { - CHECK_GE(static_cast(param_.offset[0]), 0) << - "offset[0] should be larger than 0"; - CHECK_LE(param_.offset[0], data_shape[2]-out_shape[2]) << - "offset[0] should be less than the residual space of height"; - CHECK_GE(static_cast(param_.offset[1]), 0) << - "offset[1] should be larger than 0"; - CHECK_LE(param_.offset[1], data_shape[3]-out_shape[3]) << - "offset[1] should be less than the residual space of width"; - offset_hw.push_back(static_cast(param_.offset[0])); - offset_hw.push_back(static_cast(param_.offset[1])); - } - return offset_hw; - } -}; // class CropOp - -template -Operator *CreateOp(CropParam param); - -#if DMLC_USE_CXX11 -class CropProp : public OperatorProperty { - public: - void Init(const std::vector >& kwargs) override { - param_.Init(kwargs); - } - - std::map GetParams() const override { - return param_.__DICT__(); - } - - std::vector ListArguments() const override { - // return {"data", "crop_like"}; - std::vector ret; - for (int i = 0; i < param_.num_args; ++i) { - ret.push_back(std::string("arg") + std::to_string(i)); - } - return ret; - } - - bool InferShape(std::vector *in_shape, - std::vector *out_shape, - std::vector *aux_shape) const override { - using namespace mshadow; - CHECK_EQ(in_shape->size(), static_cast(param_.num_args)); - TShape data_shape = in_shape->at(crop_enum::kData); - if (data_shape.ndim() == 0) return false; - CHECK_EQ(data_shape.ndim(), 4U) << \ - "Input data should be 4D in batch-num_filter-y-x"; - std::vector crop_shape; - if (param_.num_args == 1) { - CHECK_GE(static_cast(param_.h_w[0]), 1) << - "the crop height(h_w[0]) should be larger than 1"; - CHECK_LE(static_cast(param_.h_w[0]), static_cast(data_shape[2])) << - "the crop height(h_w[0]) should be less than the input data's height"; - CHECK_GE(static_cast(param_.h_w[1]), 1) << - "the crop width(h_w[1]) should be larger than 1"; - CHECK_LE(static_cast(param_.h_w[1]), static_cast(data_shape[3])) << - "the crop width(h_w[1]) should be less than the input data's width"; - crop_shape.push_back(param_.h_w[0]); - crop_shape.push_back(param_.h_w[1]); - } else if (param_.num_args == 2) { - TShape crop_like_shape = in_shape->at(crop_enum::kCropLike); - crop_shape.push_back(crop_like_shape[2]); - crop_shape.push_back(crop_like_shape[3]); - } - if (crop_shape.size() == 0) return false; - CHECK_EQ(crop_shape.size(), 2U) << \ - "Input crop_like should be 2D in height-width"; - out_shape->clear(); - data_shape[2] = crop_shape[0]; - data_shape[3] = crop_shape[1]; - out_shape->push_back(data_shape); - return true; - } - - OperatorProperty* Copy() const override { - auto ptr = new CropProp(); - ptr->param_ = param_; - return ptr; - } - - std::string TypeString() const override { - return "Crop"; - } - - std::vector DeclareBackwardDependency( - const std::vector &out_grad, - const std::vector &in_data, - const std::vector &out_data) const override { - return out_grad; - } - - Operator* CreateOperator(Context ctx) const override; - - private: - CropParam param_; -}; // class CropProp -#endif // DMLC_USE_CXX11 -} // namespace op -} // namespace mxnet -#endif // MXNET_OPERATOR_CROP_INL_H_ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file crop-inl.h + * \brief + * \author Wei Wu +*/ +#ifndef MXNET_OPERATOR_CROP_INL_H_ +#define MXNET_OPERATOR_CROP_INL_H_ +#include +#include +#include +#include +#include +#include +#include +#include +#include "./operator_common.h" + +namespace mxnet { +namespace op { + +namespace crop_enum { +enum CropOpInputs {kData, kCropLike}; +enum CropOpOutputs {kOut}; +} // namespace crop_enum + +struct CropParam : public dmlc::Parameter { + int num_args; + TShape offset; + TShape h_w; + bool center_crop; + DMLC_DECLARE_PARAMETER(CropParam) { + DMLC_DECLARE_FIELD(num_args).set_range(1, 3) + .describe("Number of inputs for crop, if equals one, then we will use the h_w" + "for crop height and width, else if equals two, then we will use the height" + "and width of the second input symbol, we name crop_like here"); + int shape[] = {0, 0}; + DMLC_DECLARE_FIELD(offset).set_default(TShape(shape, shape + 2)) + .describe("crop offset coordinate: (y, x)"); + DMLC_DECLARE_FIELD(h_w).set_default(TShape(shape, shape + 2)) + .describe("crop height and width: (h, w)"); + DMLC_DECLARE_FIELD(center_crop).set_default(false) + .describe("If set to true, then it will use be the center_crop," + "or it will crop using the shape of crop_like"); + } +}; // struct CropParam + +template +class CropOp : public Operator { + public: + explicit CropOp(CropParam param) { + this->param_ = param; + } + + virtual void Forward(const OpContext &ctx, + const std::vector &in_data, + const std::vector &req, + const std::vector &out_data, + const std::vector &aux_args) { + using namespace mshadow; + using namespace mshadow::expr; + CHECK_EQ(static_cast(in_data.size()), param_.num_args); + CHECK_EQ(out_data.size(), 1U); + CHECK_EQ(req[crop_enum::kOut], kWriteTo); + Stream *s = ctx.get_stream(); + Tensor data = in_data[crop_enum::kData].get(s); + Tensor out = out_data[crop_enum::kOut].get(s); + offset_hw_ = InferCropOfferset(data.shape_, out.shape_); + out = crop(data, Shape2(out.size(2), out.size(3)), offset_hw_[0], offset_hw_[1]); + } + + // because the crop_like input is only used with it's shape, so we should be + // careful setting its backwrd grad value to zeros, so that it will not hurt + // the connection of crop_like. + virtual void Backward(const OpContext &ctx, + const std::vector &out_grad, + const std::vector &in_data, + const std::vector &out_data, + const std::vector &req, + const std::vector &in_grad, + const std::vector &aux_states) { + using namespace mshadow; + using namespace mshadow::expr; + CHECK_EQ(in_grad.size(), static_cast(param_.num_args)) << in_grad.size(); + CHECK_EQ(out_grad.size(), 1U) << out_grad.size(); + Stream *s = ctx.get_stream(); + Tensor grad = out_grad[crop_enum::kOut].get(s); + Tensor gdata = in_grad[crop_enum::kData].get(s); + if (param_.num_args > 1) { + // here backward grad is set to zero for crop_like + // however, this should only be done when num_args > 1, i.e., crop_like exists + Tensor gcrop_like = in_grad[crop_enum::kCropLike].get(s); + gcrop_like = (real_t)0.0f; + } + offset_hw_ = InferCropOfferset(gdata.shape_, grad.shape_); + gdata = (real_t)0.0f; + slice<3>(slice<2>(gdata, offset_hw_[0], offset_hw_[0]+grad.size(2)), + offset_hw_[1], offset_hw_[1]+grad.size(3)) = grad; + } + + private: + CropParam param_; + std::vector offset_hw_; + std::vector InferCropOfferset(const mshadow::Shape<4> &data_shape, + const mshadow::Shape<4> &out_shape) { + std::vector offset_hw; + CHECK_GE(data_shape[2], out_shape[2]) << + "data_shape'height should be larger than that of out_shape"; + CHECK_GE(data_shape[3], out_shape[3]) << + "data_shape'weight should be larger than that of out_shape"; + if (param_.center_crop) { + offset_hw.push_back(static_cast((data_shape[2]-out_shape[2])/2)); + offset_hw.push_back(static_cast((data_shape[3]-out_shape[3])/2)); + } else { + CHECK_GE(static_cast(param_.offset[0]), 0) << + "offset[0] should be larger than 0"; + CHECK_LE(param_.offset[0], data_shape[2]-out_shape[2]) << + "offset[0] should be less than the residual space of height"; + CHECK_GE(static_cast(param_.offset[1]), 0) << + "offset[1] should be larger than 0"; + CHECK_LE(param_.offset[1], data_shape[3]-out_shape[3]) << + "offset[1] should be less than the residual space of width"; + offset_hw.push_back(static_cast(param_.offset[0])); + offset_hw.push_back(static_cast(param_.offset[1])); + } + return offset_hw; + } +}; // class CropOp + +template +Operator *CreateOp(CropParam param); + +#if DMLC_USE_CXX11 +class CropProp : public OperatorProperty { + public: + void Init(const std::vector >& kwargs) override { + param_.Init(kwargs); + } + + std::map GetParams() const override { + return param_.__DICT__(); + } + + std::vector ListArguments() const override { + // return {"data", "crop_like"}; + std::vector ret; + for (int i = 0; i < param_.num_args; ++i) { + ret.push_back(std::string("arg") + std::to_string(i)); + } + return ret; + } + + bool InferShape(std::vector *in_shape, + std::vector *out_shape, + std::vector *aux_shape) const override { + using namespace mshadow; + CHECK_EQ(in_shape->size(), static_cast(param_.num_args)); + TShape data_shape = in_shape->at(crop_enum::kData); + if (data_shape.ndim() == 0) return false; + CHECK_EQ(data_shape.ndim(), 4U) << \ + "Input data should be 4D in batch-num_filter-y-x"; + std::vector crop_shape; + if (param_.num_args == 1) { + CHECK_GE(static_cast(param_.h_w[0]), 1) << + "the crop height(h_w[0]) should be larger than 1"; + CHECK_LE(static_cast(param_.h_w[0]), static_cast(data_shape[2])) << + "the crop height(h_w[0]) should be less than the input data's height"; + CHECK_GE(static_cast(param_.h_w[1]), 1) << + "the crop width(h_w[1]) should be larger than 1"; + CHECK_LE(static_cast(param_.h_w[1]), static_cast(data_shape[3])) << + "the crop width(h_w[1]) should be less than the input data's width"; + crop_shape.push_back(param_.h_w[0]); + crop_shape.push_back(param_.h_w[1]); + } else if (param_.num_args == 2) { + TShape crop_like_shape = in_shape->at(crop_enum::kCropLike); + crop_shape.push_back(crop_like_shape[2]); + crop_shape.push_back(crop_like_shape[3]); + } + if (crop_shape.size() == 0) return false; + CHECK_EQ(crop_shape.size(), 2U) << \ + "Input crop_like should be 2D in height-width"; + out_shape->clear(); + data_shape[2] = crop_shape[0]; + data_shape[3] = crop_shape[1]; + out_shape->push_back(data_shape); + return true; + } + + OperatorProperty* Copy() const override { + auto ptr = new CropProp(); + ptr->param_ = param_; + return ptr; + } + + std::string TypeString() const override { + return "Crop"; + } + + std::vector DeclareBackwardDependency( + const std::vector &out_grad, + const std::vector &in_data, + const std::vector &out_data) const override { + return out_grad; + } + + Operator* CreateOperator(Context ctx) const override; + + private: + CropParam param_; +}; // class CropProp +#endif // DMLC_USE_CXX11 +} // namespace op +} // namespace mxnet +#endif // MXNET_OPERATOR_CROP_INL_H_ diff --git a/src/operator/crop.cc b/src/operator/crop.cc index f1233ba8a135..8465819903ce 100644 --- a/src/operator/crop.cc +++ b/src/operator/crop.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file crop.cc * \brief * \author Wei Wu diff --git a/src/operator/crop.cu b/src/operator/crop.cu index 64f8cb219f30..0b51b1449581 100644 --- a/src/operator/crop.cu +++ b/src/operator/crop.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file concat.cu * \brief * \author Wei Wu diff --git a/src/operator/cross_device_copy.cc b/src/operator/cross_device_copy.cc index a9a5f475f0bc..b32a68d3038c 100644 --- a/src/operator/cross_device_copy.cc +++ b/src/operator/cross_device_copy.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file cross_device_copy.cc * \brief Special operator that copys NDArray */ diff --git a/src/operator/cudnn_activation-inl.h b/src/operator/cudnn_activation-inl.h index 68f68b6225be..317ef47c126a 100644 --- a/src/operator/cudnn_activation-inl.h +++ b/src/operator/cudnn_activation-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file cudnn_activation-inl.h * \brief * \author Bing Xu diff --git a/src/operator/cudnn_algoreg-inl.h b/src/operator/cudnn_algoreg-inl.h index 0d1c3948186c..1078d658597e 100644 --- a/src/operator/cudnn_algoreg-inl.h +++ b/src/operator/cudnn_algoreg-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file cudnn_algoreg-inl.h * \brief * \author Bing Xu diff --git a/src/operator/cudnn_algoreg.cc b/src/operator/cudnn_algoreg.cc index 103c4819d951..5aa8688c8148 100644 --- a/src/operator/cudnn_algoreg.cc +++ b/src/operator/cudnn_algoreg.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file cudnn_algoreg.cc * \brief * \author Junyuan Xie diff --git a/src/operator/cudnn_batch_norm-inl.h b/src/operator/cudnn_batch_norm-inl.h index 6005b0f58b12..b0c5f43157d0 100644 --- a/src/operator/cudnn_batch_norm-inl.h +++ b/src/operator/cudnn_batch_norm-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file cudnn_batch_norm-inl.h * \brief * \author Junyuan Xie diff --git a/src/operator/cudnn_batch_norm.cc b/src/operator/cudnn_batch_norm.cc index 424299d93260..28c592b78ccf 100644 --- a/src/operator/cudnn_batch_norm.cc +++ b/src/operator/cudnn_batch_norm.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file cudnn_batch_norm.cc * \brief * \author Junyuan Xie diff --git a/src/operator/cudnn_batch_norm.cu b/src/operator/cudnn_batch_norm.cu index 3ab43cabd6cd..c16fc0cac25b 100644 --- a/src/operator/cudnn_batch_norm.cu +++ b/src/operator/cudnn_batch_norm.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file cudnn_batch_norm.cu * \brief * \author Junyuan Xie diff --git a/src/operator/cudnn_bilinear_sampler-inl.h b/src/operator/cudnn_bilinear_sampler-inl.h index 8b012b71723b..57592dabd891 100644 --- a/src/operator/cudnn_bilinear_sampler-inl.h +++ b/src/operator/cudnn_bilinear_sampler-inl.h @@ -1,167 +1,185 @@ -/*! - * Copyright (c) 2016 by Contributors - * \file cudnn_bilinear_sampler-inl.h - * \brief - * \author Xu Dong -*/ -#ifndef MXNET_OPERATOR_CUDNN_BILINEAR_SAMPLER_INL_H_ -#define MXNET_OPERATOR_CUDNN_BILINEAR_SAMPLER_INL_H_ - -#include -#include -#include "./bilinear_sampler-inl.h" -namespace mxnet { -namespace op { -#if defined(__CUDACC__) && MXNET_USE_CUDNN == 1 && CUDNN_MAJOR >= 5 -template -class CuDNNBilinearSamplerOp : public Operator { - public: - explicit CuDNNBilinearSamplerOp(BilinearSamplerParam param) { - this->param_ = param; - init_cudnn_ = false; - dtype_ = mshadow::DataType::kCudnnFlag; - sampler_ = CUDNN_SAMPLER_BILINEAR; - } - - ~CuDNNBilinearSamplerOp() { - if (init_cudnn_) { - CUDNN_CALL(cudnnDestroySpatialTransformerDescriptor(st_desc_)); - CUDNN_CALL(cudnnDestroyTensorDescriptor(in_desc_)); - CUDNN_CALL(cudnnDestroyTensorDescriptor(out_desc_)); - } - } - - virtual void Forward(const OpContext &ctx, - const std::vector &in_data, - const std::vector &req, - const std::vector &out_data, - const std::vector &aux_args) { - using namespace mshadow; - CHECK_EQ(req[bs::kOut], kWriteTo); - CHECK_EQ(in_data.size(), 2U); - CHECK_EQ(out_data.size(), 2U); - Stream *s = ctx.get_stream(); - - Tensor data = in_data[bs::kData].get(s); - Tensor grid = in_data[bs::kGrid].get(s); - Tensor grid_tmp = out_data[bs::kTmp].get(s); - Tensor out = out_data[bs::kOut].get(s); - // grid_tmp : (batch, h, w, 2) - grid_tmp = transpose(grid, Shape4(0, 2, 3, 1)); - if (!init_cudnn_) { - Init(s, in_data, out_data); - } - CHECK_EQ(data.CheckContiguous(), true); - CHECK_EQ(out.CheckContiguous(), true); - CHECK_EQ(grid_tmp.CheckContiguous(), true); - typename DataType::ScaleType alpha = 1.0f; - typename DataType::ScaleType beta = 0.0f; - CUDNN_CALL(cudnnSpatialTfSamplerForward(s->dnn_handle_, - st_desc_, - &alpha, - in_desc_, - data.dptr_, - grid_tmp.dptr_, - &beta, - out_desc_, - out.dptr_)); - } - - virtual void Backward(const OpContext &ctx, - const std::vector &out_grad, - const std::vector &in_data, - const std::vector &out_data, - const std::vector &req, - const std::vector &in_grad, - const std::vector &aux_args) { - using namespace mshadow; - CHECK_NE(req[bs::kData], kWriteInplace); - CHECK_NE(req[bs::kGrid], kWriteInplace); - CHECK_EQ(in_data.size(), 2U); - CHECK_EQ(out_data.size(), 2U); - CHECK_EQ(out_grad.size(), 1U); - Stream *s = ctx.get_stream(); - Tensor data = in_data[bs::kData].get(s); - Tensor grid_tmp = out_data[bs::kTmp].get(s); - Tensor gdata = in_grad[bs::kData].get(s); - Tensor ggrid = in_grad[bs::kGrid].get(s); - Tensor grad = out_grad[bs::kOut].get(s); - - typename DataType::ScaleType alpha = (req[bs::kData] == kNullOp) ? 0.0f : 1.0f; - typename DataType::ScaleType beta = (req[bs::kData] == kAddTo) ? 1.0f : 0.0f; - typename DataType::ScaleType alpha_dgrid = 1.0f; - typename DataType::ScaleType beta_dgrid = 0.0f; - CUDNN_CALL(cudnnSpatialTfSamplerBackward(s->dnn_handle_, - st_desc_, - &alpha, - in_desc_, - data.dptr_, - &beta, - in_desc_/*reuse in_desc_*/, - gdata.dptr_/*output*/, - &alpha_dgrid, - out_desc_/*reuse out_desc_*/, - grad.dptr_, - grid_tmp.dptr_, - &beta_dgrid, - grid_tmp.dptr_)); - Assign(ggrid, req[bs::kGrid], transpose(grid_tmp, Shape4(0, 3, 1, 2))); - } - - private: - inline void Init(mshadow::Stream *s, - const std::vector &in_data, - const std::vector &out_data) { - using namespace mshadow; - #if CUDNN_MAJOR >= 5 - format_ = CUDNN_TENSOR_NCHW; - #endif - CHECK_EQ(in_data.size(), 2U); - CHECK_EQ(out_data.size(), 2U); - if (!init_cudnn_) { - init_cudnn_ = true; - Tensor data = in_data[bs::kData].get(s); - Tensor out = out_data[bs::kOut].get(s); - CUDNN_CALL(cudnnCreateSpatialTransformerDescriptor(&st_desc_)); - CUDNN_CALL(cudnnCreateTensorDescriptor(&in_desc_)); - CUDNN_CALL(cudnnCreateTensorDescriptor(&out_desc_)); - CUDNN_CALL(cudnnSetTensor4dDescriptor(in_desc_, - format_, - dtype_, - data.size(0), - data.size(1), - data.size(2), - data.size(3))); - CUDNN_CALL(cudnnSetTensor4dDescriptor(out_desc_, - format_, - dtype_, - out.size(0), - out.size(1), - out.size(2), - out.size(3))); - int dim[] = {static_cast(out.size(0)), static_cast(out.size(1)), - static_cast(out.size(2)), static_cast(out.size(3))}; - CUDNN_CALL(cudnnSetSpatialTransformerNdDescriptor(st_desc_, - sampler_, - dtype_, - 4, - dim)); - } - } - - bool init_cudnn_; - cudnnDataType_t dtype_; - cudnnSpatialTransformerDescriptor_t st_desc_; - cudnnTensorDescriptor_t in_desc_; - cudnnTensorDescriptor_t out_desc_; - cudnnSamplerType_t sampler_; - #if CUDNN_MAJOR >= 5 - cudnnTensorFormat_t format_; - #endif - BilinearSamplerParam param_; -}; -#endif // __CUDACC__ && CUDNN -} // namespace op -} // namespace mxnet - -#endif // MXNET_OPERATOR_CUDNN_BILINEAR_SAMPLER_INL_H_ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file cudnn_bilinear_sampler-inl.h + * \brief + * \author Xu Dong +*/ +#ifndef MXNET_OPERATOR_CUDNN_BILINEAR_SAMPLER_INL_H_ +#define MXNET_OPERATOR_CUDNN_BILINEAR_SAMPLER_INL_H_ + +#include +#include +#include "./bilinear_sampler-inl.h" +namespace mxnet { +namespace op { +#if defined(__CUDACC__) && MXNET_USE_CUDNN == 1 && CUDNN_MAJOR >= 5 +template +class CuDNNBilinearSamplerOp : public Operator { + public: + explicit CuDNNBilinearSamplerOp(BilinearSamplerParam param) { + this->param_ = param; + init_cudnn_ = false; + dtype_ = mshadow::DataType::kCudnnFlag; + sampler_ = CUDNN_SAMPLER_BILINEAR; + } + + ~CuDNNBilinearSamplerOp() { + if (init_cudnn_) { + CUDNN_CALL(cudnnDestroySpatialTransformerDescriptor(st_desc_)); + CUDNN_CALL(cudnnDestroyTensorDescriptor(in_desc_)); + CUDNN_CALL(cudnnDestroyTensorDescriptor(out_desc_)); + } + } + + virtual void Forward(const OpContext &ctx, + const std::vector &in_data, + const std::vector &req, + const std::vector &out_data, + const std::vector &aux_args) { + using namespace mshadow; + CHECK_EQ(req[bs::kOut], kWriteTo); + CHECK_EQ(in_data.size(), 2U); + CHECK_EQ(out_data.size(), 2U); + Stream *s = ctx.get_stream(); + + Tensor data = in_data[bs::kData].get(s); + Tensor grid = in_data[bs::kGrid].get(s); + Tensor grid_tmp = out_data[bs::kTmp].get(s); + Tensor out = out_data[bs::kOut].get(s); + // grid_tmp : (batch, h, w, 2) + grid_tmp = transpose(grid, Shape4(0, 2, 3, 1)); + if (!init_cudnn_) { + Init(s, in_data, out_data); + } + CHECK_EQ(data.CheckContiguous(), true); + CHECK_EQ(out.CheckContiguous(), true); + CHECK_EQ(grid_tmp.CheckContiguous(), true); + typename DataType::ScaleType alpha = 1.0f; + typename DataType::ScaleType beta = 0.0f; + CUDNN_CALL(cudnnSpatialTfSamplerForward(s->dnn_handle_, + st_desc_, + &alpha, + in_desc_, + data.dptr_, + grid_tmp.dptr_, + &beta, + out_desc_, + out.dptr_)); + } + + virtual void Backward(const OpContext &ctx, + const std::vector &out_grad, + const std::vector &in_data, + const std::vector &out_data, + const std::vector &req, + const std::vector &in_grad, + const std::vector &aux_args) { + using namespace mshadow; + CHECK_NE(req[bs::kData], kWriteInplace); + CHECK_NE(req[bs::kGrid], kWriteInplace); + CHECK_EQ(in_data.size(), 2U); + CHECK_EQ(out_data.size(), 2U); + CHECK_EQ(out_grad.size(), 1U); + Stream *s = ctx.get_stream(); + Tensor data = in_data[bs::kData].get(s); + Tensor grid_tmp = out_data[bs::kTmp].get(s); + Tensor gdata = in_grad[bs::kData].get(s); + Tensor ggrid = in_grad[bs::kGrid].get(s); + Tensor grad = out_grad[bs::kOut].get(s); + + typename DataType::ScaleType alpha = (req[bs::kData] == kNullOp) ? 0.0f : 1.0f; + typename DataType::ScaleType beta = (req[bs::kData] == kAddTo) ? 1.0f : 0.0f; + typename DataType::ScaleType alpha_dgrid = 1.0f; + typename DataType::ScaleType beta_dgrid = 0.0f; + CUDNN_CALL(cudnnSpatialTfSamplerBackward(s->dnn_handle_, + st_desc_, + &alpha, + in_desc_, + data.dptr_, + &beta, + in_desc_/*reuse in_desc_*/, + gdata.dptr_/*output*/, + &alpha_dgrid, + out_desc_/*reuse out_desc_*/, + grad.dptr_, + grid_tmp.dptr_, + &beta_dgrid, + grid_tmp.dptr_)); + Assign(ggrid, req[bs::kGrid], transpose(grid_tmp, Shape4(0, 3, 1, 2))); + } + + private: + inline void Init(mshadow::Stream *s, + const std::vector &in_data, + const std::vector &out_data) { + using namespace mshadow; + #if CUDNN_MAJOR >= 5 + format_ = CUDNN_TENSOR_NCHW; + #endif + CHECK_EQ(in_data.size(), 2U); + CHECK_EQ(out_data.size(), 2U); + if (!init_cudnn_) { + init_cudnn_ = true; + Tensor data = in_data[bs::kData].get(s); + Tensor out = out_data[bs::kOut].get(s); + CUDNN_CALL(cudnnCreateSpatialTransformerDescriptor(&st_desc_)); + CUDNN_CALL(cudnnCreateTensorDescriptor(&in_desc_)); + CUDNN_CALL(cudnnCreateTensorDescriptor(&out_desc_)); + CUDNN_CALL(cudnnSetTensor4dDescriptor(in_desc_, + format_, + dtype_, + data.size(0), + data.size(1), + data.size(2), + data.size(3))); + CUDNN_CALL(cudnnSetTensor4dDescriptor(out_desc_, + format_, + dtype_, + out.size(0), + out.size(1), + out.size(2), + out.size(3))); + int dim[] = {static_cast(out.size(0)), static_cast(out.size(1)), + static_cast(out.size(2)), static_cast(out.size(3))}; + CUDNN_CALL(cudnnSetSpatialTransformerNdDescriptor(st_desc_, + sampler_, + dtype_, + 4, + dim)); + } + } + + bool init_cudnn_; + cudnnDataType_t dtype_; + cudnnSpatialTransformerDescriptor_t st_desc_; + cudnnTensorDescriptor_t in_desc_; + cudnnTensorDescriptor_t out_desc_; + cudnnSamplerType_t sampler_; + #if CUDNN_MAJOR >= 5 + cudnnTensorFormat_t format_; + #endif + BilinearSamplerParam param_; +}; +#endif // __CUDACC__ && CUDNN +} // namespace op +} // namespace mxnet + +#endif // MXNET_OPERATOR_CUDNN_BILINEAR_SAMPLER_INL_H_ diff --git a/src/operator/cudnn_convolution-inl.h b/src/operator/cudnn_convolution-inl.h index 06887a94aa70..e966b56d2a20 100644 --- a/src/operator/cudnn_convolution-inl.h +++ b/src/operator/cudnn_convolution-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file cudnn_convolution-inl.h * \brief * \author Bing Xu diff --git a/src/operator/cudnn_deconvolution-inl.h b/src/operator/cudnn_deconvolution-inl.h index 2e2ae3a8cb8f..8c8f0551dde3 100644 --- a/src/operator/cudnn_deconvolution-inl.h +++ b/src/operator/cudnn_deconvolution-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * \file cudnn_deconvolution-inl.h * \brief * \author Wei Wu, Leonard Lausen diff --git a/src/operator/cudnn_lrn-inl.h b/src/operator/cudnn_lrn-inl.h index d65a678bc07d..241ec704a904 100644 --- a/src/operator/cudnn_lrn-inl.h +++ b/src/operator/cudnn_lrn-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file cudnn_lrn-inl.h * \brief * \author Bing Xu diff --git a/src/operator/cudnn_pooling-inl.h b/src/operator/cudnn_pooling-inl.h index 3c9344ec5aeb..5b03fe5ee6f3 100644 --- a/src/operator/cudnn_pooling-inl.h +++ b/src/operator/cudnn_pooling-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file cudnn_pooling-inl.h * \brief * \author Bing Xu diff --git a/src/operator/cudnn_rnn-inl.h b/src/operator/cudnn_rnn-inl.h index 17acf4a5b35f..1122aff033f2 100644 --- a/src/operator/cudnn_rnn-inl.h +++ b/src/operator/cudnn_rnn-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file cudnn_rnn-inl.h * \brief * \author Sebastian Bodenstein diff --git a/src/operator/cudnn_softmax_activation-inl.h b/src/operator/cudnn_softmax_activation-inl.h index 86c27317f923..c604a8f3f4c1 100644 --- a/src/operator/cudnn_softmax_activation-inl.h +++ b/src/operator/cudnn_softmax_activation-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file cudnn_activation-inl.h * \brief * \author Bing Xu diff --git a/src/operator/cudnn_spatial_transformer-inl.h b/src/operator/cudnn_spatial_transformer-inl.h index b25e8cebc077..fc767841447b 100644 --- a/src/operator/cudnn_spatial_transformer-inl.h +++ b/src/operator/cudnn_spatial_transformer-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file cudnn_spatial_transformer-inl.h * \brief * \author Wei Wu diff --git a/src/operator/custom/custom-inl.h b/src/operator/custom/custom-inl.h index 3c688feb05a1..4b2d620be1d6 100644 --- a/src/operator/custom/custom-inl.h +++ b/src/operator/custom/custom-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file native_op-inl.h * \brief * \author Junyuan Xie diff --git a/src/operator/custom/custom.cc b/src/operator/custom/custom.cc index 5a40be92b68e..7b257ba843c3 100644 --- a/src/operator/custom/custom.cc +++ b/src/operator/custom/custom.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file custom.cc * \brief * \author Junyuan Xie diff --git a/src/operator/custom/native_op-inl.h b/src/operator/custom/native_op-inl.h index 780b0ae41f67..ebce18611b56 100644 --- a/src/operator/custom/native_op-inl.h +++ b/src/operator/custom/native_op-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file native_op-inl.h * \brief * \author Junyuan Xie diff --git a/src/operator/custom/native_op.cc b/src/operator/custom/native_op.cc index 2ccd286e8cd3..5dd35049d5bd 100644 --- a/src/operator/custom/native_op.cc +++ b/src/operator/custom/native_op.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file native_op.cc * \brief * \author Junyuan Xie diff --git a/src/operator/custom/native_op.cu b/src/operator/custom/native_op.cu index 807592626e8b..ad8d65e3c2eb 100644 --- a/src/operator/custom/native_op.cu +++ b/src/operator/custom/native_op.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file native_op.cu * \brief * \author Junyuan Xie diff --git a/src/operator/custom/ndarray_op-inl.h b/src/operator/custom/ndarray_op-inl.h index fa4208f1da89..b3a4662b669e 100644 --- a/src/operator/custom/ndarray_op-inl.h +++ b/src/operator/custom/ndarray_op-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file native_op-inl.h * \brief * \author Junyuan Xie diff --git a/src/operator/custom/ndarray_op.cc b/src/operator/custom/ndarray_op.cc index 9815f888a98b..48426baea866 100644 --- a/src/operator/custom/ndarray_op.cc +++ b/src/operator/custom/ndarray_op.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file ndarray_op.cc * \brief * \author Junyuan Xie diff --git a/src/operator/deconvolution-inl.h b/src/operator/deconvolution-inl.h index 909a6fd5fed6..43530138b8ea 100644 --- a/src/operator/deconvolution-inl.h +++ b/src/operator/deconvolution-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file deconvolution-inl.h * \brief * \author Wei Wu diff --git a/src/operator/deconvolution.cc b/src/operator/deconvolution.cc index 397bd0065f80..6a59ff6588ff 100644 --- a/src/operator/deconvolution.cc +++ b/src/operator/deconvolution.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file deconvolution.cc * \brief * \author Wei Wu diff --git a/src/operator/deconvolution.cu b/src/operator/deconvolution.cu index 60cf0ad5a21a..b9dd1c156187 100644 --- a/src/operator/deconvolution.cu +++ b/src/operator/deconvolution.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file deconvolution.cu * \brief * \author Wei Wu diff --git a/src/operator/dropout-inl.h b/src/operator/dropout-inl.h index 57d78146a68d..b2fb7823bedc 100644 --- a/src/operator/dropout-inl.h +++ b/src/operator/dropout-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file dropout-inl.h * \brief * \author Bing Xu diff --git a/src/operator/dropout.cc b/src/operator/dropout.cc index e206214e9b64..af65578ec6f8 100644 --- a/src/operator/dropout.cc +++ b/src/operator/dropout.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file dropout.cc * \brief * \author Bing Xu diff --git a/src/operator/dropout.cu b/src/operator/dropout.cu index ea9eb7dfa200..5265d8013ff7 100644 --- a/src/operator/dropout.cu +++ b/src/operator/dropout.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file dropout.cc * \brief * \author Bing Xu diff --git a/src/operator/elemwise_op_common.h b/src/operator/elemwise_op_common.h index 228303c85a82..9b398f947e30 100644 --- a/src/operator/elemwise_op_common.h +++ b/src/operator/elemwise_op_common.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! -* Copyright (c) 2016 by Contributors * \file elemwise_op_common.h * \brief common function used for broadcasting and reducing * \author Xingjian Shi diff --git a/src/operator/fully_connected-inl.h b/src/operator/fully_connected-inl.h index 94616bc2e7d7..e2fab9f1f7dd 100644 --- a/src/operator/fully_connected-inl.h +++ b/src/operator/fully_connected-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file fully_connect_op-inl.h * \brief fully connect operator and symbol */ diff --git a/src/operator/fully_connected.cc b/src/operator/fully_connected.cc index 56cf4f6dbdde..5dbaf8c82005 100644 --- a/src/operator/fully_connected.cc +++ b/src/operator/fully_connected.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file fully_connected.cc * \brief fully connect operator */ diff --git a/src/operator/fully_connected.cu b/src/operator/fully_connected.cu index 7b834a3b08ef..28a0307b70bd 100644 --- a/src/operator/fully_connected.cu +++ b/src/operator/fully_connected.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file fully_connected.cu * \brief fully connect operator */ diff --git a/src/operator/grid_generator-inl.h b/src/operator/grid_generator-inl.h index 04fe7ec1c5fd..65fb8ccf2e07 100644 --- a/src/operator/grid_generator-inl.h +++ b/src/operator/grid_generator-inl.h @@ -1,318 +1,336 @@ -/*! - * Copyright (c) 2017 by Contributors - * \file grid_generator-inl.h - * \brief - * The operator generate sampling grid - * \author Xu Dong -*/ -#ifndef MXNET_OPERATOR_GRID_GENERATOR_INL_H_ -#define MXNET_OPERATOR_GRID_GENERATOR_INL_H_ - -#include -#include -#include -#include -#include -#include -#include -#include "./mshadow_op.h" -#include "./operator_common.h" - -namespace mxnet { -namespace op { - -namespace grid { -enum GridGeneratorOpInputs {kData}; -enum GridGeneratorOpOutputs {kOut, kGridDst}; -enum GridGeneratorOpResource {kTempSpace}; -enum GridGeneratorTransformType {kAffine, kWarp}; -} - -struct GridGeneratorParam : public dmlc::Parameter { - int transform_type; - TShape target_shape; - DMLC_DECLARE_PARAMETER(GridGeneratorParam) { - int shape[] = {0, 0}; - DMLC_DECLARE_FIELD(transform_type) - .add_enum("affine", grid::kAffine) - .add_enum("warp", grid::kWarp) - .describe("The type of transformation. For `affine`, input data should be an affine matrix " - "of size (batch, 6). For `warp`, input data should be an optical flow of size " - "(batch, 2, h, w)."); - DMLC_DECLARE_FIELD(target_shape).set_default(TShape(shape, shape + 2)) - .describe("Specifies the output shape (H, W). This is required if transformation type is " - "`affine`. If transformation type is `warp`, this parameter is ignored."); - } -}; - -template -class GridGeneratorOp : public Operator { - public: - explicit GridGeneratorOp(GridGeneratorParam p) { - this->param_ = p; - } - - virtual void Forward(const OpContext &ctx, - const std::vector &in_data, - const std::vector &req, - const std::vector &out_data, - const std::vector &aux_args) { - using namespace mshadow; - using namespace mshadow::expr; - CHECK_EQ(req[grid::kOut], kWriteTo); - CHECK_EQ(in_data.size(), 1U); - CHECK_EQ(out_data.size(), 2U); - Stream *s = ctx.get_stream(); - switch (param_.transform_type) { - case grid::kAffine: { - // if transform_type is affine, data is affine matrix, input shape : (batch, 2, 3) - Tensor out = out_data[grid::kOut]. - get_with_shape(Shape2(out_data[grid::kOut].shape_[0] * 2, - out_data[grid::kOut].shape_[2] * out_data[grid::kOut].shape_[3]), s); - Tensor grid_dst = out_data[grid::kGridDst].get(s); - Shape<2> data_shape = Shape2(out_data[grid::kOut].shape_[0] * 2, 3); - Tensor data = in_data[grid::kData] - .get_with_shape(data_shape, s); - // x, y, 1 - grid_dst[0] = range(0, grid_dst.shape_[1]); - grid_dst[0] = grid_dst[0] - tcast(tcast(grid_dst[0] / - scalar(param_.target_shape[1]))) * scalar(param_.target_shape[1]); - grid_dst[0] = scalar(-1.0) + grid_dst[0] * - scalar(2.0 / (param_.target_shape[1] - 1)); - grid_dst[1] = range(0, grid_dst.shape_[1]); - grid_dst[1] = scalar(-1.0) + tcast(tcast(grid_dst[1] / - scalar(param_.target_shape[1]))) * scalar(2.0/(param_.target_shape[0] - 1)); - grid_dst[2] = scalar(1.0); - Assign(out, req[grid::kOut], dot(data, grid_dst)); - break; - } - // Warping transformation - case grid::kWarp: { - // if transform_type is warp, data is optical flow, input shape : (batch, 2, height, width) - // grid_src = grid_dst + optical flow - Tensor data = in_data[grid::kData].get(s); - Tensor out = out_data[grid::kOut].get(s); - // grid_dst : (2, H, W) - Tensor grid_dst = out_data[grid::kGridDst].get(s); - Tensor workspace = ctx.requested[grid::kTempSpace] - .get_space_typed(Shape2(2, 1), s); - grid_dst[0] = repmat(range(0, data.size(3)), data.size(2)); - grid_dst[1] = reshape(range(0, data.size(2), 1, data.size(3)), - Shape2(data.size(2), data.size(3))); - workspace[0] = scalar((DType(data.size(3)) - 1.0) / 2.0); - workspace[1] = scalar((DType(data.size(2)) - 1.0) / 2.0); - Assign(out, req[grid::kOut], - (data + broadcast_with_axis(grid_dst, -1, data.shape_[0])) / - broadcast_to(reshape(workspace, Shape4(1, 2, 1, 1)), - TShape(data.shape_)) - scalar(1)); - break; - } - } - } - - virtual void Backward(const OpContext &ctx, - const std::vector &out_grad, - const std::vector &in_data, - const std::vector &out_data, - const std::vector &req, - const std::vector &in_grad, - const std::vector &aux_args) { - using namespace mshadow; - using namespace mshadow::expr; - CHECK_EQ(in_data.size(), 1U); - CHECK_EQ(out_data.size(), 2U); - Stream *s = ctx.get_stream(); - switch (param_.transform_type) { - case grid::kAffine: { - Tensor grid_dst = out_data[grid::kGridDst].get(s); - Shape<2> data_shape = Shape2(in_grad[grid::kData].shape_[0] * 2, 3); - Tensor gdata = in_grad[grid::kData] - .get_with_shape(data_shape, s); - Shape<2> grad_shape = Shape2(out_grad[grid::kOut].shape_[0] * 2, - param_.target_shape[0] * param_.target_shape[1]); - Tensor grad = out_grad[grid::kOut] - .get_with_shape(grad_shape, s); - // grad : (batch * 2, H * W) grid_dst.T : (H * W, 3) - Assign(gdata, req[grid::kData] , dot(grad, grid_dst.T())); - break; - } - case grid::kWarp: { - Tensor grad = out_grad[grid::kOut].get(s); - Tensor gdata = in_grad[grid::kData].get(s); - Tensor workspace = ctx.requested[grid::kTempSpace] - .get_space_typed(Shape2(2, 1), s); - workspace[0] = scalar((DType(gdata.size(3)) - 1.0) / 2.0); - workspace[1] = scalar((DType(gdata.size(2)) - 1.0) / 2.0); - Assign(gdata, req[grid::kData], - grad / broadcast_to(reshape(workspace, Shape4(1, 2, 1, 1)), - TShape(gdata.shape_))); - break; - } - } - } - - private: - GridGeneratorParam param_; -}; // class GridGeneratorOp - -template -Operator* CreateOp(GridGeneratorParam param, int dtype); - -#if DMLC_USE_CXX11 -class GridGeneratorProp : public OperatorProperty { - public: - int NumVisibleOutputs() const override { - return 1; - } - - int NumOutputs() const override { - return 2; - } - - std::vector ListArguments() const override { - return {"data"}; - } - - std::vector ListOutputs() const override { - return {"output", "grid_dst"}; - } - - void Init(const std::vector >& kwargs) override { - param_.Init(kwargs); - } - - std::map GetParams() const override { - return param_.__DICT__(); - } - - bool InferShape(std::vector *in_shape, - std::vector *out_shape, - std::vector *aux_shape) const override { - using namespace mshadow; - CHECK_EQ(in_shape->size(), 1U) << "Input:[data]"; - const TShape &lshape = (*in_shape)[grid::kData]; - if (lshape.ndim() == 0) return false; - out_shape->clear(); - switch (param_.transform_type) { - case grid::kAffine: { - CHECK_EQ(lshape.ndim(), 2U) \ - << "if transform_type is affine, data is affine matrix" - "affine matrix should be 2D in batch-num_hidden"; - CHECK_EQ(lshape[1], 6U) << "incorrect data shape[1], should be 6"; - CHECK_GT(param_.target_shape[0], 0U) \ - << "incorrect target_shape: " << param_.target_shape[0]; - CHECK_GT(param_.target_shape[1], 0U) \ - << "incorrect target_shape: " << param_.target_shape[1]; - out_shape->push_back(Shape4(lshape[0], 2, param_.target_shape[0], param_.target_shape[1])); - out_shape->push_back(Shape2(3, param_.target_shape[0] * param_.target_shape[1])); - break; - } - case grid::kWarp: { - CHECK_EQ(lshape.ndim(), 4U) \ - << "if transform_type is warp, data is optical flow" - "optical flow should be 4D in batch-num_hidden-y-x"; - CHECK_EQ(lshape[1], 2U) << "incorrect data shape[1], should be 2"; - out_shape->push_back(lshape); - out_shape->push_back(Shape3(2, lshape[2], lshape[3])); - break; - } - } - return true; - } - - bool InferType(std::vector *in_type, - std::vector *out_type, - std::vector *aux_type) const override { - int dtype = -1; - for (size_t i = 0; i < in_type->size(); ++i) { - if (dtype == -1) { - dtype = in_type->at(i); - } else { - CHECK(in_type->at(i) == dtype || - in_type->at(i) == -1) << - "Non-uniform data type in GridGenerator"; - } - } - if (dtype == -1) { - LOG(FATAL) << "Not enough information to infer type in GridGenerator."; - return false; - } - size_t nin = this->ListArguments().size(); - in_type->clear(); - for (size_t i = 0; i < nin; ++i) in_type->push_back(dtype); - size_t naux = this->ListAuxiliaryStates().size(); - aux_type->clear(); - for (size_t i = 0; i < naux; ++i) aux_type->push_back(dtype); - size_t nout = this->ListOutputs().size(); - out_type->clear(); - for (size_t i = 0; i < nout; ++i) out_type->push_back(dtype); - return true; - } - - OperatorProperty* Copy() const override { - auto ptr = new GridGeneratorProp(); - ptr->param_ = param_; - return ptr; - } - - std::string TypeString() const override { - return "GridGenerator"; - } - - std::vector DeclareBackwardDependency( - const std::vector &out_grad, - const std::vector &in_data, - const std::vector &out_data) const override { - switch (param_.transform_type) { - case grid::kAffine: { - return {out_grad[grid::kOut], - out_data[grid::kGridDst]}; - } - case grid::kWarp: { - return {out_grad[grid::kOut]}; - } - } - return {}; - } - - std::vector ForwardResource( - const std::vector &in_shape) const override { - switch (param_.transform_type) { - case grid::kAffine: { - return{}; - } - case grid::kWarp: { - return{ ResourceRequest::kTempSpace }; - } - } - return{}; - } - - std::vector BackwardResource( - const std::vector &in_shape) const override { - switch (param_.transform_type) { - case grid::kAffine: { - return {}; - } - case grid::kWarp: { - return {ResourceRequest::kTempSpace}; - } - } - return {}; - } - - Operator* CreateOperator(Context ctx) const override { - LOG(FATAL) << "Not Implemented."; - return NULL; - } - - Operator* CreateOperatorEx(Context ctx, std::vector *in_shape, - std::vector *in_type) const override; - - private: - GridGeneratorParam param_; -}; // class GridGeneratorProp -#endif // DMLC_USE_CXX11 -} // namespace op -} // namespace mxnet -#endif // MXNET_OPERATOR_GRID_GENERATOR_INL_H_ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file grid_generator-inl.h + * \brief + * The operator generate sampling grid + * \author Xu Dong +*/ +#ifndef MXNET_OPERATOR_GRID_GENERATOR_INL_H_ +#define MXNET_OPERATOR_GRID_GENERATOR_INL_H_ + +#include +#include +#include +#include +#include +#include +#include +#include "./mshadow_op.h" +#include "./operator_common.h" + +namespace mxnet { +namespace op { + +namespace grid { +enum GridGeneratorOpInputs {kData}; +enum GridGeneratorOpOutputs {kOut, kGridDst}; +enum GridGeneratorOpResource {kTempSpace}; +enum GridGeneratorTransformType {kAffine, kWarp}; +} + +struct GridGeneratorParam : public dmlc::Parameter { + int transform_type; + TShape target_shape; + DMLC_DECLARE_PARAMETER(GridGeneratorParam) { + int shape[] = {0, 0}; + DMLC_DECLARE_FIELD(transform_type) + .add_enum("affine", grid::kAffine) + .add_enum("warp", grid::kWarp) + .describe("The type of transformation. For `affine`, input data should be an affine matrix " + "of size (batch, 6). For `warp`, input data should be an optical flow of size " + "(batch, 2, h, w)."); + DMLC_DECLARE_FIELD(target_shape).set_default(TShape(shape, shape + 2)) + .describe("Specifies the output shape (H, W). This is required if transformation type is " + "`affine`. If transformation type is `warp`, this parameter is ignored."); + } +}; + +template +class GridGeneratorOp : public Operator { + public: + explicit GridGeneratorOp(GridGeneratorParam p) { + this->param_ = p; + } + + virtual void Forward(const OpContext &ctx, + const std::vector &in_data, + const std::vector &req, + const std::vector &out_data, + const std::vector &aux_args) { + using namespace mshadow; + using namespace mshadow::expr; + CHECK_EQ(req[grid::kOut], kWriteTo); + CHECK_EQ(in_data.size(), 1U); + CHECK_EQ(out_data.size(), 2U); + Stream *s = ctx.get_stream(); + switch (param_.transform_type) { + case grid::kAffine: { + // if transform_type is affine, data is affine matrix, input shape : (batch, 2, 3) + Tensor out = out_data[grid::kOut]. + get_with_shape(Shape2(out_data[grid::kOut].shape_[0] * 2, + out_data[grid::kOut].shape_[2] * out_data[grid::kOut].shape_[3]), s); + Tensor grid_dst = out_data[grid::kGridDst].get(s); + Shape<2> data_shape = Shape2(out_data[grid::kOut].shape_[0] * 2, 3); + Tensor data = in_data[grid::kData] + .get_with_shape(data_shape, s); + // x, y, 1 + grid_dst[0] = range(0, grid_dst.shape_[1]); + grid_dst[0] = grid_dst[0] - tcast(tcast(grid_dst[0] / + scalar(param_.target_shape[1]))) * scalar(param_.target_shape[1]); + grid_dst[0] = scalar(-1.0) + grid_dst[0] * + scalar(2.0 / (param_.target_shape[1] - 1)); + grid_dst[1] = range(0, grid_dst.shape_[1]); + grid_dst[1] = scalar(-1.0) + tcast(tcast(grid_dst[1] / + scalar(param_.target_shape[1]))) * scalar(2.0/(param_.target_shape[0] - 1)); + grid_dst[2] = scalar(1.0); + Assign(out, req[grid::kOut], dot(data, grid_dst)); + break; + } + // Warping transformation + case grid::kWarp: { + // if transform_type is warp, data is optical flow, input shape : (batch, 2, height, width) + // grid_src = grid_dst + optical flow + Tensor data = in_data[grid::kData].get(s); + Tensor out = out_data[grid::kOut].get(s); + // grid_dst : (2, H, W) + Tensor grid_dst = out_data[grid::kGridDst].get(s); + Tensor workspace = ctx.requested[grid::kTempSpace] + .get_space_typed(Shape2(2, 1), s); + grid_dst[0] = repmat(range(0, data.size(3)), data.size(2)); + grid_dst[1] = reshape(range(0, data.size(2), 1, data.size(3)), + Shape2(data.size(2), data.size(3))); + workspace[0] = scalar((DType(data.size(3)) - 1.0) / 2.0); + workspace[1] = scalar((DType(data.size(2)) - 1.0) / 2.0); + Assign(out, req[grid::kOut], + (data + broadcast_with_axis(grid_dst, -1, data.shape_[0])) / + broadcast_to(reshape(workspace, Shape4(1, 2, 1, 1)), + TShape(data.shape_)) - scalar(1)); + break; + } + } + } + + virtual void Backward(const OpContext &ctx, + const std::vector &out_grad, + const std::vector &in_data, + const std::vector &out_data, + const std::vector &req, + const std::vector &in_grad, + const std::vector &aux_args) { + using namespace mshadow; + using namespace mshadow::expr; + CHECK_EQ(in_data.size(), 1U); + CHECK_EQ(out_data.size(), 2U); + Stream *s = ctx.get_stream(); + switch (param_.transform_type) { + case grid::kAffine: { + Tensor grid_dst = out_data[grid::kGridDst].get(s); + Shape<2> data_shape = Shape2(in_grad[grid::kData].shape_[0] * 2, 3); + Tensor gdata = in_grad[grid::kData] + .get_with_shape(data_shape, s); + Shape<2> grad_shape = Shape2(out_grad[grid::kOut].shape_[0] * 2, + param_.target_shape[0] * param_.target_shape[1]); + Tensor grad = out_grad[grid::kOut] + .get_with_shape(grad_shape, s); + // grad : (batch * 2, H * W) grid_dst.T : (H * W, 3) + Assign(gdata, req[grid::kData] , dot(grad, grid_dst.T())); + break; + } + case grid::kWarp: { + Tensor grad = out_grad[grid::kOut].get(s); + Tensor gdata = in_grad[grid::kData].get(s); + Tensor workspace = ctx.requested[grid::kTempSpace] + .get_space_typed(Shape2(2, 1), s); + workspace[0] = scalar((DType(gdata.size(3)) - 1.0) / 2.0); + workspace[1] = scalar((DType(gdata.size(2)) - 1.0) / 2.0); + Assign(gdata, req[grid::kData], + grad / broadcast_to(reshape(workspace, Shape4(1, 2, 1, 1)), + TShape(gdata.shape_))); + break; + } + } + } + + private: + GridGeneratorParam param_; +}; // class GridGeneratorOp + +template +Operator* CreateOp(GridGeneratorParam param, int dtype); + +#if DMLC_USE_CXX11 +class GridGeneratorProp : public OperatorProperty { + public: + int NumVisibleOutputs() const override { + return 1; + } + + int NumOutputs() const override { + return 2; + } + + std::vector ListArguments() const override { + return {"data"}; + } + + std::vector ListOutputs() const override { + return {"output", "grid_dst"}; + } + + void Init(const std::vector >& kwargs) override { + param_.Init(kwargs); + } + + std::map GetParams() const override { + return param_.__DICT__(); + } + + bool InferShape(std::vector *in_shape, + std::vector *out_shape, + std::vector *aux_shape) const override { + using namespace mshadow; + CHECK_EQ(in_shape->size(), 1U) << "Input:[data]"; + const TShape &lshape = (*in_shape)[grid::kData]; + if (lshape.ndim() == 0) return false; + out_shape->clear(); + switch (param_.transform_type) { + case grid::kAffine: { + CHECK_EQ(lshape.ndim(), 2U) \ + << "if transform_type is affine, data is affine matrix" + "affine matrix should be 2D in batch-num_hidden"; + CHECK_EQ(lshape[1], 6U) << "incorrect data shape[1], should be 6"; + CHECK_GT(param_.target_shape[0], 0U) \ + << "incorrect target_shape: " << param_.target_shape[0]; + CHECK_GT(param_.target_shape[1], 0U) \ + << "incorrect target_shape: " << param_.target_shape[1]; + out_shape->push_back(Shape4(lshape[0], 2, param_.target_shape[0], param_.target_shape[1])); + out_shape->push_back(Shape2(3, param_.target_shape[0] * param_.target_shape[1])); + break; + } + case grid::kWarp: { + CHECK_EQ(lshape.ndim(), 4U) \ + << "if transform_type is warp, data is optical flow" + "optical flow should be 4D in batch-num_hidden-y-x"; + CHECK_EQ(lshape[1], 2U) << "incorrect data shape[1], should be 2"; + out_shape->push_back(lshape); + out_shape->push_back(Shape3(2, lshape[2], lshape[3])); + break; + } + } + return true; + } + + bool InferType(std::vector *in_type, + std::vector *out_type, + std::vector *aux_type) const override { + int dtype = -1; + for (size_t i = 0; i < in_type->size(); ++i) { + if (dtype == -1) { + dtype = in_type->at(i); + } else { + CHECK(in_type->at(i) == dtype || + in_type->at(i) == -1) << + "Non-uniform data type in GridGenerator"; + } + } + if (dtype == -1) { + LOG(FATAL) << "Not enough information to infer type in GridGenerator."; + return false; + } + size_t nin = this->ListArguments().size(); + in_type->clear(); + for (size_t i = 0; i < nin; ++i) in_type->push_back(dtype); + size_t naux = this->ListAuxiliaryStates().size(); + aux_type->clear(); + for (size_t i = 0; i < naux; ++i) aux_type->push_back(dtype); + size_t nout = this->ListOutputs().size(); + out_type->clear(); + for (size_t i = 0; i < nout; ++i) out_type->push_back(dtype); + return true; + } + + OperatorProperty* Copy() const override { + auto ptr = new GridGeneratorProp(); + ptr->param_ = param_; + return ptr; + } + + std::string TypeString() const override { + return "GridGenerator"; + } + + std::vector DeclareBackwardDependency( + const std::vector &out_grad, + const std::vector &in_data, + const std::vector &out_data) const override { + switch (param_.transform_type) { + case grid::kAffine: { + return {out_grad[grid::kOut], + out_data[grid::kGridDst]}; + } + case grid::kWarp: { + return {out_grad[grid::kOut]}; + } + } + return {}; + } + + std::vector ForwardResource( + const std::vector &in_shape) const override { + switch (param_.transform_type) { + case grid::kAffine: { + return{}; + } + case grid::kWarp: { + return{ ResourceRequest::kTempSpace }; + } + } + return{}; + } + + std::vector BackwardResource( + const std::vector &in_shape) const override { + switch (param_.transform_type) { + case grid::kAffine: { + return {}; + } + case grid::kWarp: { + return {ResourceRequest::kTempSpace}; + } + } + return {}; + } + + Operator* CreateOperator(Context ctx) const override { + LOG(FATAL) << "Not Implemented."; + return NULL; + } + + Operator* CreateOperatorEx(Context ctx, std::vector *in_shape, + std::vector *in_type) const override; + + private: + GridGeneratorParam param_; +}; // class GridGeneratorProp +#endif // DMLC_USE_CXX11 +} // namespace op +} // namespace mxnet +#endif // MXNET_OPERATOR_GRID_GENERATOR_INL_H_ diff --git a/src/operator/grid_generator.cc b/src/operator/grid_generator.cc index 62ff75a88359..411f856be08b 100644 --- a/src/operator/grid_generator.cc +++ b/src/operator/grid_generator.cc @@ -1,36 +1,54 @@ -/*! - * Copyright (c) 2017 by Contributors - * \file grid_generator.cc - * \brief - * \author Xu Dong -*/ - -#include "./grid_generator-inl.h" - -namespace mxnet { -namespace op { -template<> -Operator* CreateOp(GridGeneratorParam param, int dtype) { - Operator *op = NULL; - if (dtype == mshadow::kFloat32) { - op = new GridGeneratorOp(param); - } else { - LOG(FATAL) << "Other DTypes are not supported!"; - } - return op; -} - -Operator *GridGeneratorProp::CreateOperatorEx(Context ctx, std::vector *in_shape, - std::vector *in_type) const { - DO_BIND_DISPATCH(CreateOp, param_, (*in_type)[0]); -} - -DMLC_REGISTER_PARAMETER(GridGeneratorParam); - -MXNET_REGISTER_OP_PROPERTY(GridGenerator, GridGeneratorProp) -.add_argument("data", "NDArray-or-Symbol", "Input data to the function.") -.add_arguments(GridGeneratorParam::__FIELDS__()) -.describe("Generates 2D sampling grid for bilinear sampling."); - -} // namespace op -} // namespace mxnet +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file grid_generator.cc + * \brief + * \author Xu Dong +*/ + +#include "./grid_generator-inl.h" + +namespace mxnet { +namespace op { +template<> +Operator* CreateOp(GridGeneratorParam param, int dtype) { + Operator *op = NULL; + if (dtype == mshadow::kFloat32) { + op = new GridGeneratorOp(param); + } else { + LOG(FATAL) << "Other DTypes are not supported!"; + } + return op; +} + +Operator *GridGeneratorProp::CreateOperatorEx(Context ctx, std::vector *in_shape, + std::vector *in_type) const { + DO_BIND_DISPATCH(CreateOp, param_, (*in_type)[0]); +} + +DMLC_REGISTER_PARAMETER(GridGeneratorParam); + +MXNET_REGISTER_OP_PROPERTY(GridGenerator, GridGeneratorProp) +.add_argument("data", "NDArray-or-Symbol", "Input data to the function.") +.add_arguments(GridGeneratorParam::__FIELDS__()) +.describe("Generates 2D sampling grid for bilinear sampling."); + +} // namespace op +} // namespace mxnet diff --git a/src/operator/grid_generator.cu b/src/operator/grid_generator.cu index 991948cd1581..7c0a80258d36 100644 --- a/src/operator/grid_generator.cu +++ b/src/operator/grid_generator.cu @@ -1,21 +1,39 @@ -/*! - * Copyright (c) 2017 by Contributors - * \file grid_generator.cu - * \brief - * \author Xu Dong -*/ - -#include "./grid_generator-inl.h" - -namespace mxnet { -namespace op { -template<> -Operator* CreateOp(GridGeneratorParam param, int dtype) { - Operator *op = NULL; - MSHADOW_REAL_TYPE_SWITCH(dtype, DType, { - op = new GridGeneratorOp(param); - }) - return op; -} -} // namespace op -} // namespace mxnet +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file grid_generator.cu + * \brief + * \author Xu Dong +*/ + +#include "./grid_generator-inl.h" + +namespace mxnet { +namespace op { +template<> +Operator* CreateOp(GridGeneratorParam param, int dtype) { + Operator *op = NULL; + MSHADOW_REAL_TYPE_SWITCH(dtype, DType, { + op = new GridGeneratorOp(param); + }) + return op; +} +} // namespace op +} // namespace mxnet diff --git a/src/operator/identity_attach_KL_sparse_reg-inl.h b/src/operator/identity_attach_KL_sparse_reg-inl.h index 413bac90c0ac..2307914f62a5 100644 --- a/src/operator/identity_attach_KL_sparse_reg-inl.h +++ b/src/operator/identity_attach_KL_sparse_reg-inl.h @@ -1,177 +1,195 @@ -/*! - * Copyright (c) 2015 by Contributors - * \file sparse_reg-inl.h - * \brief -*/ -#ifndef MXNET_OPERATOR_IDENTITY_ATTACH_KL_SPARSE_REG_INL_H_ -#define MXNET_OPERATOR_IDENTITY_ATTACH_KL_SPARSE_REG_INL_H_ -#include -#include -#include -#include -#include -#include -#include -#include "./mshadow_op.h" -#include "./operator_common.h" - -namespace mxnet { -namespace op { - -namespace sparsereg { -enum IdentityAttachKLSparseRegOpInputs {kData}; -enum IdentityAttachKLSparseRegOpOutputs {kOut}; -enum IdentityAttachKLSparseRegOpAuxiliary {kMovingAvg}; -enum IdentityAttachKLSparseRegBackResource {kTempSpace}; -} // namespace sparsereg - -struct IdentityAttachKLSparseRegParam : public dmlc::Parameter { - float penalty; - float sparseness_target; - float momentum; - DMLC_DECLARE_PARAMETER(IdentityAttachKLSparseRegParam) { - DMLC_DECLARE_FIELD(sparseness_target).set_default(0.1) - .set_range(0, 1) - .describe("The sparseness target"); - DMLC_DECLARE_FIELD(penalty).set_default(0.001) - .describe("The tradeoff parameter for the sparseness penalty"); - DMLC_DECLARE_FIELD(momentum).set_default(0.9) - .set_range(0, 1) - .describe("The momentum for running average"); - } -}; // struct IdentityAttachKLSparseRegParam - -// This op regularizes the output of a sigmoid activation function. -// In forward, it simply copies the input. -// In backward, it attaches sparseness penalty to the gradient. -// The regularization is based on the KL divergence of mean activation and target. -// More details: P11 of https://www.cs.toronto.edu/~hinton/absps/guideTR.pdf -// Please make sure that it is only paired with sigmoid activation, otherwise NaN may occur. -template -class IdentityAttachKLSparseRegOp : public Operator { - public: - explicit IdentityAttachKLSparseRegOp(IdentityAttachKLSparseRegParam param) { - this->param_ = param; - } - virtual void Forward(const OpContext &ctx, - const std::vector &in_data, - const std::vector &req, - const std::vector &out_data, - const std::vector &aux_args) { - using namespace mshadow; - using namespace mshadow::expr; - CHECK_EQ(in_data.size(), 1U); - CHECK_EQ(out_data.size(), 1U); - Stream *s = ctx.get_stream(); - Tensor data = in_data[sparsereg::kData].FlatTo2D(s); - Tensor out = out_data[sparsereg::kOut].FlatTo2D(s); - Assign(out, req[sparsereg::kData], F(data)); - } - - virtual void Backward(const OpContext &ctx, - const std::vector &out_grad, - const std::vector &in_data, - const std::vector &out_data, - const std::vector &req, - const std::vector &in_grad, - const std::vector &aux_args) { - using namespace mshadow; - using namespace mshadow::expr; - Stream *s = ctx.get_stream(); - Tensor grad_in = in_grad[sparsereg::kData].FlatTo2D(s); - Tensor data_in = in_data[sparsereg::kData].FlatTo2D(s); - Tensor grad_out = out_grad[sparsereg::kOut].FlatTo2D(s); - Tensor moving_avg = aux_args[sparsereg::kMovingAvg].get(s); - Tensor avg = ctx.requested[sparsereg::kTempSpace].get_space( - mshadow::Shape1(moving_avg.shape_[0]), s); - avg = sumall_except_dim<1>(data_in); - avg /= data_in.shape_[0]; - moving_avg = param_.momentum * moving_avg + (1 - param_.momentum) * avg; - Assign(grad_in, req[sparsereg::kData], grad_out + param_.penalty * - (-param_.sparseness_target / broadcast<1>(moving_avg, data_in.shape_) + - ((1 - param_.sparseness_target) / (1 - broadcast<1>(moving_avg, data_in.shape_))))); - } - - private: - IdentityAttachKLSparseRegParam param_; -}; // class IdentityAttachKLSparseRegOp - -template -Operator *CreateOp(IdentityAttachKLSparseRegParam param); - -#if DMLC_USE_CXX11 -class IdentityAttachKLSparseRegProp : public OperatorProperty { - public: - void Init(const std::vector >& kwargs) override { - param_.Init(kwargs); - } - - std::map GetParams() const override { - return param_.__DICT__(); - } - - bool InferShape(std::vector *in_shape, - std::vector *out_shape, - std::vector *aux_shape) const override { - using namespace mshadow; - CHECK_EQ(in_shape->size(), 1U); - const TShape &dshape = in_shape->at(sparsereg::kData); - if (dshape.ndim() == 0) return false; - out_shape->clear(); - out_shape->push_back(dshape); - aux_shape->clear(); - aux_shape->push_back(Shape1(dshape[1])); - return true; - } - - OperatorProperty* Copy() const override { - auto ptr = new IdentityAttachKLSparseRegProp(); - ptr->param_ = param_; - return ptr; - } - - std::string TypeString() const override { - return "IdentityAttachKLSparseReg"; - } - - std::vector DeclareBackwardDependency( - const std::vector &out_grad, - const std::vector &in_data, - const std::vector &out_data) const override { - return {out_grad[sparsereg::kOut], in_data[sparsereg::kData]}; - } - - std::vector > ForwardInplaceOption( - const std::vector &in_data, - const std::vector &out_data) const override { - return {{in_data[sparsereg::kData], out_data[sparsereg::kOut]}}; - } - - std::vector > BackwardInplaceOption( - const std::vector &out_grad, - const std::vector &in_data, - const std::vector &out_data, - const std::vector &in_grad) const override { - return { {out_grad[sparsereg::kOut], in_grad[sparsereg::kData]} }; - } - - std::vector ListAuxiliaryStates() const override { - return {"moving_avg"}; - } - - std::vector BackwardResource( - const std::vector &in_shape) const override { - return {ResourceRequest::kTempSpace}; - } - - Operator* CreateOperator(Context ctx) const override; - - private: - IdentityAttachKLSparseRegParam param_; -}; // class IdentityAttachKLSparseRegProperty - -#endif // DMLC_USE_CXX11 -} // namespace op -} // namespace mxnet - -#endif // MXNET_OPERATOR_IDENTITY_ATTACH_KL_SPARSE_REG_INL_H_ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file sparse_reg-inl.h + * \brief +*/ +#ifndef MXNET_OPERATOR_IDENTITY_ATTACH_KL_SPARSE_REG_INL_H_ +#define MXNET_OPERATOR_IDENTITY_ATTACH_KL_SPARSE_REG_INL_H_ +#include +#include +#include +#include +#include +#include +#include +#include "./mshadow_op.h" +#include "./operator_common.h" + +namespace mxnet { +namespace op { + +namespace sparsereg { +enum IdentityAttachKLSparseRegOpInputs {kData}; +enum IdentityAttachKLSparseRegOpOutputs {kOut}; +enum IdentityAttachKLSparseRegOpAuxiliary {kMovingAvg}; +enum IdentityAttachKLSparseRegBackResource {kTempSpace}; +} // namespace sparsereg + +struct IdentityAttachKLSparseRegParam : public dmlc::Parameter { + float penalty; + float sparseness_target; + float momentum; + DMLC_DECLARE_PARAMETER(IdentityAttachKLSparseRegParam) { + DMLC_DECLARE_FIELD(sparseness_target).set_default(0.1) + .set_range(0, 1) + .describe("The sparseness target"); + DMLC_DECLARE_FIELD(penalty).set_default(0.001) + .describe("The tradeoff parameter for the sparseness penalty"); + DMLC_DECLARE_FIELD(momentum).set_default(0.9) + .set_range(0, 1) + .describe("The momentum for running average"); + } +}; // struct IdentityAttachKLSparseRegParam + +// This op regularizes the output of a sigmoid activation function. +// In forward, it simply copies the input. +// In backward, it attaches sparseness penalty to the gradient. +// The regularization is based on the KL divergence of mean activation and target. +// More details: P11 of https://www.cs.toronto.edu/~hinton/absps/guideTR.pdf +// Please make sure that it is only paired with sigmoid activation, otherwise NaN may occur. +template +class IdentityAttachKLSparseRegOp : public Operator { + public: + explicit IdentityAttachKLSparseRegOp(IdentityAttachKLSparseRegParam param) { + this->param_ = param; + } + virtual void Forward(const OpContext &ctx, + const std::vector &in_data, + const std::vector &req, + const std::vector &out_data, + const std::vector &aux_args) { + using namespace mshadow; + using namespace mshadow::expr; + CHECK_EQ(in_data.size(), 1U); + CHECK_EQ(out_data.size(), 1U); + Stream *s = ctx.get_stream(); + Tensor data = in_data[sparsereg::kData].FlatTo2D(s); + Tensor out = out_data[sparsereg::kOut].FlatTo2D(s); + Assign(out, req[sparsereg::kData], F(data)); + } + + virtual void Backward(const OpContext &ctx, + const std::vector &out_grad, + const std::vector &in_data, + const std::vector &out_data, + const std::vector &req, + const std::vector &in_grad, + const std::vector &aux_args) { + using namespace mshadow; + using namespace mshadow::expr; + Stream *s = ctx.get_stream(); + Tensor grad_in = in_grad[sparsereg::kData].FlatTo2D(s); + Tensor data_in = in_data[sparsereg::kData].FlatTo2D(s); + Tensor grad_out = out_grad[sparsereg::kOut].FlatTo2D(s); + Tensor moving_avg = aux_args[sparsereg::kMovingAvg].get(s); + Tensor avg = ctx.requested[sparsereg::kTempSpace].get_space( + mshadow::Shape1(moving_avg.shape_[0]), s); + avg = sumall_except_dim<1>(data_in); + avg /= data_in.shape_[0]; + moving_avg = param_.momentum * moving_avg + (1 - param_.momentum) * avg; + Assign(grad_in, req[sparsereg::kData], grad_out + param_.penalty * + (-param_.sparseness_target / broadcast<1>(moving_avg, data_in.shape_) + + ((1 - param_.sparseness_target) / (1 - broadcast<1>(moving_avg, data_in.shape_))))); + } + + private: + IdentityAttachKLSparseRegParam param_; +}; // class IdentityAttachKLSparseRegOp + +template +Operator *CreateOp(IdentityAttachKLSparseRegParam param); + +#if DMLC_USE_CXX11 +class IdentityAttachKLSparseRegProp : public OperatorProperty { + public: + void Init(const std::vector >& kwargs) override { + param_.Init(kwargs); + } + + std::map GetParams() const override { + return param_.__DICT__(); + } + + bool InferShape(std::vector *in_shape, + std::vector *out_shape, + std::vector *aux_shape) const override { + using namespace mshadow; + CHECK_EQ(in_shape->size(), 1U); + const TShape &dshape = in_shape->at(sparsereg::kData); + if (dshape.ndim() == 0) return false; + out_shape->clear(); + out_shape->push_back(dshape); + aux_shape->clear(); + aux_shape->push_back(Shape1(dshape[1])); + return true; + } + + OperatorProperty* Copy() const override { + auto ptr = new IdentityAttachKLSparseRegProp(); + ptr->param_ = param_; + return ptr; + } + + std::string TypeString() const override { + return "IdentityAttachKLSparseReg"; + } + + std::vector DeclareBackwardDependency( + const std::vector &out_grad, + const std::vector &in_data, + const std::vector &out_data) const override { + return {out_grad[sparsereg::kOut], in_data[sparsereg::kData]}; + } + + std::vector > ForwardInplaceOption( + const std::vector &in_data, + const std::vector &out_data) const override { + return {{in_data[sparsereg::kData], out_data[sparsereg::kOut]}}; + } + + std::vector > BackwardInplaceOption( + const std::vector &out_grad, + const std::vector &in_data, + const std::vector &out_data, + const std::vector &in_grad) const override { + return { {out_grad[sparsereg::kOut], in_grad[sparsereg::kData]} }; + } + + std::vector ListAuxiliaryStates() const override { + return {"moving_avg"}; + } + + std::vector BackwardResource( + const std::vector &in_shape) const override { + return {ResourceRequest::kTempSpace}; + } + + Operator* CreateOperator(Context ctx) const override; + + private: + IdentityAttachKLSparseRegParam param_; +}; // class IdentityAttachKLSparseRegProperty + +#endif // DMLC_USE_CXX11 +} // namespace op +} // namespace mxnet + +#endif // MXNET_OPERATOR_IDENTITY_ATTACH_KL_SPARSE_REG_INL_H_ diff --git a/src/operator/identity_attach_KL_sparse_reg.cc b/src/operator/identity_attach_KL_sparse_reg.cc index 51e67721032c..5e776774e00b 100644 --- a/src/operator/identity_attach_KL_sparse_reg.cc +++ b/src/operator/identity_attach_KL_sparse_reg.cc @@ -1,37 +1,55 @@ -/*! - * Copyright (c) 2015 by Contributors - * \file identity_attach_KL_sparse_reg.cc - * \brief\ -*/ -#include "./identity_attach_KL_sparse_reg-inl.h" -#include - -namespace mxnet { -namespace op { -template<> -Operator *CreateOp(IdentityAttachKLSparseRegParam param) { - return new IdentityAttachKLSparseRegOp(param); -} - -Operator *IdentityAttachKLSparseRegProp::CreateOperator(Context ctx) const { - DO_BIND_DISPATCH(CreateOp, param_); -} - -DMLC_REGISTER_PARAMETER(IdentityAttachKLSparseRegParam); - -MXNET_REGISTER_OP_PROPERTY(IdentityAttachKLSparseReg, IdentityAttachKLSparseRegProp) -.describe("Apply a sparse regularization to the output a sigmoid activation function.") -.add_argument("data", "NDArray-or-Symbol", "Input data.") -.add_arguments(IdentityAttachKLSparseRegParam::__FIELDS__()); - -NNVM_REGISTER_OP(IdentityAttachKLSparseReg) -.set_attr("FSetInputVarAttrOnCompose", - [](const nnvm::NodeAttrs& attrs, nnvm::NodePtr var, const int index) { - if (var->attrs.dict.find("__init__") != var->attrs.dict.end()) return; - if (index == 1) { - var->attrs.dict["__init__"] = "[\"zero\", {}]"; - } - }); -} // namespace op -} // namespace mxnet - +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file identity_attach_KL_sparse_reg.cc + * \brief\ +*/ +#include "./identity_attach_KL_sparse_reg-inl.h" +#include + +namespace mxnet { +namespace op { +template<> +Operator *CreateOp(IdentityAttachKLSparseRegParam param) { + return new IdentityAttachKLSparseRegOp(param); +} + +Operator *IdentityAttachKLSparseRegProp::CreateOperator(Context ctx) const { + DO_BIND_DISPATCH(CreateOp, param_); +} + +DMLC_REGISTER_PARAMETER(IdentityAttachKLSparseRegParam); + +MXNET_REGISTER_OP_PROPERTY(IdentityAttachKLSparseReg, IdentityAttachKLSparseRegProp) +.describe("Apply a sparse regularization to the output a sigmoid activation function.") +.add_argument("data", "NDArray-or-Symbol", "Input data.") +.add_arguments(IdentityAttachKLSparseRegParam::__FIELDS__()); + +NNVM_REGISTER_OP(IdentityAttachKLSparseReg) +.set_attr("FSetInputVarAttrOnCompose", + [](const nnvm::NodeAttrs& attrs, nnvm::NodePtr var, const int index) { + if (var->attrs.dict.find("__init__") != var->attrs.dict.end()) return; + if (index == 1) { + var->attrs.dict["__init__"] = "[\"zero\", {}]"; + } + }); +} // namespace op +} // namespace mxnet + diff --git a/src/operator/identity_attach_KL_sparse_reg.cu b/src/operator/identity_attach_KL_sparse_reg.cu index 6188fb9d954f..0a11fb167399 100644 --- a/src/operator/identity_attach_KL_sparse_reg.cu +++ b/src/operator/identity_attach_KL_sparse_reg.cu @@ -1,16 +1,34 @@ -/*! - * Copyright (c) 2015 by Contributors - * \file identity_attach_KL_sparse_reg.cu - * \brief -*/ -#include "./identity_attach_KL_sparse_reg-inl.h" - -namespace mxnet { -namespace op { -template<> -Operator *CreateOp(IdentityAttachKLSparseRegParam param) { - return new IdentityAttachKLSparseRegOp(param); -} - -} // namespace op -} // namespace mxnet +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file identity_attach_KL_sparse_reg.cu + * \brief +*/ +#include "./identity_attach_KL_sparse_reg-inl.h" + +namespace mxnet { +namespace op { +template<> +Operator *CreateOp(IdentityAttachKLSparseRegParam param) { + return new IdentityAttachKLSparseRegOp(param); +} + +} // namespace op +} // namespace mxnet diff --git a/src/operator/instance_norm-inl.h b/src/operator/instance_norm-inl.h index 4a4f515ed601..6e78f7628a11 100644 --- a/src/operator/instance_norm-inl.h +++ b/src/operator/instance_norm-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file instance_norm-inl.h * \brief Reproducing paper Instance Normalization: The Missing Ingredient for * Fast Stylization, D. Ulyanov, A. Vedaldi, V. Lempitsky, 2016 diff --git a/src/operator/instance_norm.cc b/src/operator/instance_norm.cc index cc2bd6b93e8b..0666b4bd0303 100644 --- a/src/operator/instance_norm.cc +++ b/src/operator/instance_norm.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file instance_norm.cc * \brief * \author Sebastian Bodenstein diff --git a/src/operator/instance_norm.cu b/src/operator/instance_norm.cu index 096008463c4a..9f8cbea797ed 100644 --- a/src/operator/instance_norm.cu +++ b/src/operator/instance_norm.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file instance_norm.cu * \brief * \author Sebastian Bodenstein diff --git a/src/operator/l2_normalization-inl.h b/src/operator/l2_normalization-inl.h index a49c8362645d..c1f17acbbce1 100644 --- a/src/operator/l2_normalization-inl.h +++ b/src/operator/l2_normalization-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file l2_normalization_op-inl.h * \brief instance l2 Normalization op */ diff --git a/src/operator/l2_normalization.cc b/src/operator/l2_normalization.cc index 8a4112d8db35..6995a0d1e440 100644 --- a/src/operator/l2_normalization.cc +++ b/src/operator/l2_normalization.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file l2_normalization.cc * \brief l2 normalization operator */ diff --git a/src/operator/l2_normalization.cu b/src/operator/l2_normalization.cu index d7bab3586a27..ae76278559a8 100644 --- a/src/operator/l2_normalization.cu +++ b/src/operator/l2_normalization.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file l2_normalization.cu * \brief l2 normalization operator */ diff --git a/src/operator/leaky_relu-inl.h b/src/operator/leaky_relu-inl.h index b0a5c0e53300..828930a0e405 100644 --- a/src/operator/leaky_relu-inl.h +++ b/src/operator/leaky_relu-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file leaky_relu-inl.h * \brief leaky relu family operator * \author Bing Xu diff --git a/src/operator/leaky_relu.cc b/src/operator/leaky_relu.cc index 25fe17edb5ee..da58bd94bd57 100644 --- a/src/operator/leaky_relu.cc +++ b/src/operator/leaky_relu.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file leaky_relu.cc * \brief * \author Bing Xu @@ -24,7 +42,7 @@ DMLC_REGISTER_PARAMETER(LeakyReLUParam); MXNET_REGISTER_OP_PROPERTY(LeakyReLU, LeakyReLUProp) .describe(R"code(Applies Leaky rectified linear unit activation element-wise to the input. -Leaky ReLUs attempt to fix the "dying ReLU" problem by allowing a small `slope` +Leaky ReLUs attempt to fix the "dying ReLU" problem by allowing a small `slope` when the input is negative and has a slope of one when input is positive. The following modified ReLU Activation functions are supported: diff --git a/src/operator/leaky_relu.cu b/src/operator/leaky_relu.cu index c9af119a96ed..b9b3a7b73f9c 100644 --- a/src/operator/leaky_relu.cu +++ b/src/operator/leaky_relu.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file leaky_relu.cc * \brief * \author Bing Xu diff --git a/src/operator/loss_binary_op-inl.h b/src/operator/loss_binary_op-inl.h index a61cee7e3d3c..8add82725292 100644 --- a/src/operator/loss_binary_op-inl.h +++ b/src/operator/loss_binary_op-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file loss_binary_op-inl.h * \brief Loss functions */ diff --git a/src/operator/loss_binary_op.cc b/src/operator/loss_binary_op.cc index 43bf6943e0c5..d0a77946ffba 100644 --- a/src/operator/loss_binary_op.cc +++ b/src/operator/loss_binary_op.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file loss_binary_op.cc * \brief loss function that takes a data and label */ diff --git a/src/operator/loss_binary_op.cu b/src/operator/loss_binary_op.cu index 66700e7918b8..8694b9f2844f 100644 --- a/src/operator/loss_binary_op.cu +++ b/src/operator/loss_binary_op.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file loss_binary_op.cu * \brief loss function that takes a data and label */ diff --git a/src/operator/lrn-inl.h b/src/operator/lrn-inl.h index 66be9ac7f4ed..a320a26bed30 100644 --- a/src/operator/lrn-inl.h +++ b/src/operator/lrn-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file lrn-inl.h * \brief * \author Bing Xu diff --git a/src/operator/lrn.cc b/src/operator/lrn.cc index ac4a309cbe05..46f4fca486b5 100644 --- a/src/operator/lrn.cc +++ b/src/operator/lrn.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file lrn.cc * \brief * \author Bing Xu diff --git a/src/operator/lrn.cu b/src/operator/lrn.cu index 681de80508c7..702f4b2fa92a 100644 --- a/src/operator/lrn.cu +++ b/src/operator/lrn.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file lrn.cu * \brief * \author Bing Xu diff --git a/src/operator/make_loss-inl.h b/src/operator/make_loss-inl.h index 65af62732373..3f4a99373ca3 100644 --- a/src/operator/make_loss-inl.h +++ b/src/operator/make_loss-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file make_loss-inl.h * \brief special layer for propagating loss */ diff --git a/src/operator/make_loss.cc b/src/operator/make_loss.cc index 1be93def5e87..748357d243f5 100644 --- a/src/operator/make_loss.cc +++ b/src/operator/make_loss.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file make_loss.cc * \brief special layer for propagating loss */ diff --git a/src/operator/make_loss.cu b/src/operator/make_loss.cu index 5f5fad6955fe..7f508500f58e 100644 --- a/src/operator/make_loss.cu +++ b/src/operator/make_loss.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file make_loss.cu * \brief special layer for propagating loss */ diff --git a/src/operator/mshadow_op.h b/src/operator/mshadow_op.h index 8d867043dccd..f7815d2f8d4c 100644 --- a/src/operator/mshadow_op.h +++ b/src/operator/mshadow_op.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file mshadow_op.h * \brief * \author Bing Xu diff --git a/src/operator/mxnet_op.h b/src/operator/mxnet_op.h index 9b5dcfe3d3b1..0af7d026d9d5 100644 --- a/src/operator/mxnet_op.h +++ b/src/operator/mxnet_op.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * \file mxnet_op.h * \brief * \author Junyuan Xie diff --git a/src/operator/nn/im2col.cuh b/src/operator/nn/im2col.cuh index 786fd22f8c9b..edd5b0dcfb2f 100644 --- a/src/operator/nn/im2col.cuh +++ b/src/operator/nn/im2col.cuh @@ -1,34 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! ******************* BEGIN Caffe Copyright Notice and Disclaimer **************** * * COPYRIGHT - * + * * All contributions by the University of California: * Copyright (c) 2014-2017 The Regents of the University of California (Regents) * All rights reserved. - * + * * All other contributions: * Copyright (c) 2014-2017, the respective contributors * All rights reserved. - * + * * Caffe uses a shared copyright model: each contributor holds copyright over * their contributions to Caffe. The project versioning records all such * contribution and copyright details. If a contributor wants to further mark * their specific copyright on a particular contribution, they should indicate * their copyright solely in the commit message of the change when it is * committed. - * + * * LICENSE - * + * * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * + * modification, are permitted provided that the following conditions are met: + * * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. + * list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * + * and/or other materials provided with the distribution. + * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE @@ -39,9 +58,9 @@ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * + * * CONTRIBUTION AGREEMENT - * + * * By contributing to the BVLC/caffe repository through pull-request, comment, * or otherwise, the contributor releases their content to the * license and copyright terms herein. @@ -304,7 +323,7 @@ inline void im2col(mshadow::Stream* s, <<::GetStream(s)>>>( num_kernels, data_im, im_shape[2], im_shape[3], kernel_shape[0], kernel_shape[1], - pad[0], pad[1], stride[0], stride[1], dilation[0], dilation[1], + pad[0], pad[1], stride[0], stride[1], dilation[0], dilation[1], col_shape[1], col_shape[2], data_col); break; case 3: diff --git a/src/operator/nn/im2col.h b/src/operator/nn/im2col.h index ce4d9e31db9c..621b2451a19e 100644 --- a/src/operator/nn/im2col.h +++ b/src/operator/nn/im2col.h @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! ******************* BEGIN Caffe Copyright Notice and Disclaimer **************** * @@ -48,7 +67,6 @@ * ***************** END Caffe Copyright Notice and Disclaimer ******************** * - * Copyright (c) 2017 by Contributors * \file im2col.h * \brief Function definitions of converting an image to * column matrix based on kernel, padding, and dilation. diff --git a/src/operator/nn/pool.cuh b/src/operator/nn/pool.cuh index 54fd3461d80f..0e9cff0c51e4 100644 --- a/src/operator/nn/pool.cuh +++ b/src/operator/nn/pool.cuh @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! ******************* BEGIN Caffe Copyright Notice and Disclaimer **************** * diff --git a/src/operator/nn/pool.h b/src/operator/nn/pool.h index 79accb5d521f..3bac86560407 100644 --- a/src/operator/nn/pool.h +++ b/src/operator/nn/pool.h @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! ******************* BEGIN Caffe Copyright Notice and Disclaimer **************** * @@ -48,7 +67,6 @@ * ***************** END Caffe Copyright Notice and Disclaimer ******************** * - * Copyright (c) 2017 by Contributors * \file pool.h * \brief Function definitions of pooling 1/2/3-D images. * We adopted looping 2-D image pixels from Caffe and extended it to 1-D and 3-D cases. diff --git a/src/operator/nn/softmax-inl.h b/src/operator/nn/softmax-inl.h index 749ad7374b96..e1150b14f69d 100644 --- a/src/operator/nn/softmax-inl.h +++ b/src/operator/nn/softmax-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * \file softmax-inl.h * \brief */ diff --git a/src/operator/nn/softmax.cc b/src/operator/nn/softmax.cc index f1f2078ddce1..58c1a051248e 100644 --- a/src/operator/nn/softmax.cc +++ b/src/operator/nn/softmax.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * \file softmax.cc * \brief CPU Implementation of softmax */ diff --git a/src/operator/nn/softmax.cu b/src/operator/nn/softmax.cu index 570f5bf15c88..d5a843ddc07b 100644 --- a/src/operator/nn/softmax.cu +++ b/src/operator/nn/softmax.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * \file softmax.cc * \brief CPU Implementation of softmax */ diff --git a/src/operator/nnpack/nnpack_convolution-inl.h b/src/operator/nnpack/nnpack_convolution-inl.h index 03f4a0bcfdb4..4a1342688969 100644 --- a/src/operator/nnpack/nnpack_convolution-inl.h +++ b/src/operator/nnpack/nnpack_convolution-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file nnpack_convolution-inl.h * \brief * \author Carwin diff --git a/src/operator/nnpack/nnpack_fully_connected-inl.h b/src/operator/nnpack/nnpack_fully_connected-inl.h index 2d87db1e1aec..f85ddd89c702 100644 --- a/src/operator/nnpack/nnpack_fully_connected-inl.h +++ b/src/operator/nnpack/nnpack_fully_connected-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file nnpack_fully_connected-inl.h * \brief * \author Wei Wu diff --git a/src/operator/nnpack/nnpack_pooling-inl.h b/src/operator/nnpack/nnpack_pooling-inl.h index 0df070de812c..968ead16204d 100644 --- a/src/operator/nnpack/nnpack_pooling-inl.h +++ b/src/operator/nnpack/nnpack_pooling-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file nnpack_pooling-inl.h * \brief * \author Wei Wu diff --git a/src/operator/nnpack/nnpack_util.cc b/src/operator/nnpack/nnpack_util.cc index 8004bb1063dc..b873b591fa57 100644 --- a/src/operator/nnpack/nnpack_util.cc +++ b/src/operator/nnpack/nnpack_util.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file nnpack_util.cc * \brief * \author Wei Wu diff --git a/src/operator/nnpack/nnpack_util.h b/src/operator/nnpack/nnpack_util.h index 280c6ffce875..cde1880257a3 100644 --- a/src/operator/nnpack/nnpack_util.h +++ b/src/operator/nnpack/nnpack_util.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file nnpack_util.h * \brief * \author Carwin diff --git a/src/operator/operator.cc b/src/operator/operator.cc index e476c583e668..9117c1c1288a 100644 --- a/src/operator/operator.cc +++ b/src/operator/operator.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file operator.cc * \brief operator module of mxnet */ diff --git a/src/operator/operator_common.h b/src/operator/operator_common.h index a43d092bceb6..2d46bd3230ce 100644 --- a/src/operator/operator_common.h +++ b/src/operator/operator_common.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file operator_common.h * \brief common internal header of most operators * this header includes utility functions operator can use diff --git a/src/operator/operator_util.cc b/src/operator/operator_util.cc index 84a19d6b4b15..25fa209a026c 100644 --- a/src/operator/operator_util.cc +++ b/src/operator/operator_util.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file operator_util.cc * Implementation of operator util. */ diff --git a/src/operator/optimizer_op-inl.h b/src/operator/optimizer_op-inl.h index 9f4959350362..70759b15251a 100644 --- a/src/operator/optimizer_op-inl.h +++ b/src/operator/optimizer_op-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file optimizer_op-inl.h * \brief Optimizer operators * \author Junyuan Xie diff --git a/src/operator/optimizer_op.cc b/src/operator/optimizer_op.cc index 3fdb9c2498fb..b26c333edaef 100644 --- a/src/operator/optimizer_op.cc +++ b/src/operator/optimizer_op.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file optimizer_op.cc * \brief Optimizer operators * \author Junyuan Xie diff --git a/src/operator/optimizer_op.cu b/src/operator/optimizer_op.cu index a30584dd183f..0e74e303dbc9 100644 --- a/src/operator/optimizer_op.cu +++ b/src/operator/optimizer_op.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file optimizer_op.cu * \brief Optimizer operators * \author Junyuan Xie diff --git a/src/operator/pad-inl.h b/src/operator/pad-inl.h index e6e6b7b30327..80f9e0bf92ac 100644 --- a/src/operator/pad-inl.h +++ b/src/operator/pad-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file pad-inl.h * \brief * \author Sebastian Bodenstien diff --git a/src/operator/pad.cc b/src/operator/pad.cc index 77177b5758ec..468629a43672 100644 --- a/src/operator/pad.cc +++ b/src/operator/pad.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file pad.cc * \brief * \author Sebastian Bodenstein diff --git a/src/operator/pad.cu b/src/operator/pad.cu index bf7265cfa342..98220b6c39ef 100644 --- a/src/operator/pad.cu +++ b/src/operator/pad.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file pad.cu * \brief * \author Sebastian Bodenstein diff --git a/src/operator/pooling-inl.h b/src/operator/pooling-inl.h index 8156c3796539..fbc6981a7591 100644 --- a/src/operator/pooling-inl.h +++ b/src/operator/pooling-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * \file pooling-inl.h * \brief * \author Bing Xu, Jun Wu diff --git a/src/operator/pooling.cc b/src/operator/pooling.cc index 72b17038af9f..51dce873cd04 100644 --- a/src/operator/pooling.cc +++ b/src/operator/pooling.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * \file pooling.cc * \brief * \author Bing Xu, Jun Wu diff --git a/src/operator/pooling.cu b/src/operator/pooling.cu index bc7716b946af..950f09956258 100644 --- a/src/operator/pooling.cu +++ b/src/operator/pooling.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * \file pooling.cu * \brief * \author Bing Xu, Jun Wu diff --git a/src/operator/pooling_v1-inl.h b/src/operator/pooling_v1-inl.h index 0b9f7adce62f..e541298ed2ab 100644 --- a/src/operator/pooling_v1-inl.h +++ b/src/operator/pooling_v1-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file pooling_v1-inl.h * \brief * \author Bing Xu diff --git a/src/operator/pooling_v1.cc b/src/operator/pooling_v1.cc index 20acff76a5fb..40de7457520f 100644 --- a/src/operator/pooling_v1.cc +++ b/src/operator/pooling_v1.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file pooling_v1.cc * \brief * \author Bing Xu diff --git a/src/operator/pooling_v1.cu b/src/operator/pooling_v1.cu index 99aebbc6446c..4db22c18420d 100644 --- a/src/operator/pooling_v1.cu +++ b/src/operator/pooling_v1.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file pooling_v1.cu * \brief * \author Bing Xu diff --git a/src/operator/random/multisample_op.cc b/src/operator/random/multisample_op.cc index 303d1d2f0086..f1264e5dc3cc 100644 --- a/src/operator/random/multisample_op.cc +++ b/src/operator/random/multisample_op.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * \file multisample_op.cc * \brief CPU-implementation of multi-sampling operators */ diff --git a/src/operator/random/multisample_op.h b/src/operator/random/multisample_op.h index 0b5b4cee6217..748b3ba0ccad 100644 --- a/src/operator/random/multisample_op.h +++ b/src/operator/random/multisample_op.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * \file sampling_op.h * \brief Function definitions of operators for sampling from multiple distributions */ diff --git a/src/operator/random/sample_multinomial_op.cc b/src/operator/random/sample_multinomial_op.cc index 9e6dbe99c045..b358b3b2b4f2 100644 --- a/src/operator/random/sample_multinomial_op.cc +++ b/src/operator/random/sample_multinomial_op.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * \file sample_multinomial_op.h * \brief Operator for sampling from multinomial distributions */ diff --git a/src/operator/random/sample_multinomial_op.cu b/src/operator/random/sample_multinomial_op.cu index 434202d5e09b..c2bc99b7323e 100644 --- a/src/operator/random/sample_multinomial_op.cu +++ b/src/operator/random/sample_multinomial_op.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * \file sample_multinomial_op.h * \brief Operator for sampling from multinomial distributions */ diff --git a/src/operator/random/sample_multinomial_op.h b/src/operator/random/sample_multinomial_op.h index ab73ebf0543e..2b016329f390 100644 --- a/src/operator/random/sample_multinomial_op.h +++ b/src/operator/random/sample_multinomial_op.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * \file sample_multinomial_op.h * \brief Operator for sampling from multinomial distributions */ diff --git a/src/operator/random/sample_op.cc b/src/operator/random/sample_op.cc index 1b3c293548e1..8d87d2b99d14 100644 --- a/src/operator/random/sample_op.cc +++ b/src/operator/random/sample_op.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file sample_op.cc * \brief CPU Implementation of sample op */ @@ -106,7 +124,7 @@ MXNET_OPERATOR_REGISTER_SAMPLE(random_negative_binomial, SampleNegBinomialParam) .add_alias("_sample_negbinomial") .describe(R"code(Draw random samples from a negative binomial distribution. -Samples are distributed according to a negative binomial distribution parametrized by +Samples are distributed according to a negative binomial distribution parametrized by *k* (limit of unsuccessful experiments) and *p* (failure probability in each experiment). Samples will always be returned as a floating point data type. @@ -121,8 +139,8 @@ MXNET_OPERATOR_REGISTER_SAMPLE(random_generalized_negative_binomial, SampleGenNe .add_alias("_sample_gennegbinomial") .describe(R"code(Draw random samples from a generalized negative binomial distribution. -Samples are distributed according to a generalized negative binomial distribution parametrized by -*mu* (mean) and *alpha* (dispersion). *alpha* is defined as *1/k* where *k* is the failure limit of the +Samples are distributed according to a generalized negative binomial distribution parametrized by +*mu* (mean) and *alpha* (dispersion). *alpha* is defined as *1/k* where *k* is the failure limit of the number of unsuccessful experiments (generalized to real numbers). Samples will always be returned as a floating point data type. diff --git a/src/operator/random/sample_op.cu b/src/operator/random/sample_op.cu index 62c8a73249cb..0d4b2e5a8270 100644 --- a/src/operator/random/sample_op.cu +++ b/src/operator/random/sample_op.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file sample_op.cu * \brief GPU Implementation of sample op */ diff --git a/src/operator/random/sample_op.h b/src/operator/random/sample_op.h index 79655b3a3ba1..a1a6a2345b1b 100644 --- a/src/operator/random/sample_op.h +++ b/src/operator/random/sample_op.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file sample_op.h * \brief Elementary sampling operators */ diff --git a/src/operator/regression_output-inl.h b/src/operator/regression_output-inl.h index 7f8b2948ebfa..0de312cff8d6 100644 --- a/src/operator/regression_output-inl.h +++ b/src/operator/regression_output-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file regression_ouput-inl.h * \brief Regression output operator. */ diff --git a/src/operator/regression_output.cc b/src/operator/regression_output.cc index fc71a993d43a..d19f336d2aa2 100644 --- a/src/operator/regression_output.cc +++ b/src/operator/regression_output.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file regression_output.cc * \brief regression output operator */ diff --git a/src/operator/regression_output.cu b/src/operator/regression_output.cu index 18e7a1f4184c..64dcef3df6f0 100644 --- a/src/operator/regression_output.cu +++ b/src/operator/regression_output.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file regression_output.cu * \brief regression output operator */ diff --git a/src/operator/rnn-inl.h b/src/operator/rnn-inl.h index bc5f3d7da6ee..4f09ebe9c3ea 100644 --- a/src/operator/rnn-inl.h +++ b/src/operator/rnn-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file rnn-inl.h * \brief * \author Sebastian Bodenstein diff --git a/src/operator/rnn.cc b/src/operator/rnn.cc index f19c3bbad04b..4c7954f3e5a6 100644 --- a/src/operator/rnn.cc +++ b/src/operator/rnn.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file rnn.cc * \brief * \author Sebastian Bodenstein diff --git a/src/operator/rnn.cu b/src/operator/rnn.cu index 4e3998eac269..0daee32abe5b 100644 --- a/src/operator/rnn.cu +++ b/src/operator/rnn.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file rnn.cu * \brief * \author Sebastian Bodenstein diff --git a/src/operator/roi_pooling-inl.h b/src/operator/roi_pooling-inl.h index 7eb14b7219c7..05a0ae41ab94 100644 --- a/src/operator/roi_pooling-inl.h +++ b/src/operator/roi_pooling-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file roi_pooling-inl.h * \brief roi pooling operator and symbol * \author Kye-Hyeon Kim, Jian Guo diff --git a/src/operator/roi_pooling.cc b/src/operator/roi_pooling.cc index 0faca1e463bc..9af00bb450d4 100644 --- a/src/operator/roi_pooling.cc +++ b/src/operator/roi_pooling.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file roi_pooling.cc * \brief roi pooling operator * \author Ross Girshick, Kye-Hyeon Kim, Jian Guo diff --git a/src/operator/roi_pooling.cu b/src/operator/roi_pooling.cu index 3d744680d009..80d38e476f80 100644 --- a/src/operator/roi_pooling.cu +++ b/src/operator/roi_pooling.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file roi_pooling.cu * \brief roi pooling operator * \author Ross Girshick, Kye-Hyeon Kim, Jian Guo diff --git a/src/operator/sequence_last-inl.h b/src/operator/sequence_last-inl.h index c2acbf164197..0551ee933f0a 100644 --- a/src/operator/sequence_last-inl.h +++ b/src/operator/sequence_last-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file sequence_last-inl.h * \brief * \author Sebastian Bodenstien diff --git a/src/operator/sequence_last.cc b/src/operator/sequence_last.cc index 8a50ff73ec64..6c04bdd34d86 100644 --- a/src/operator/sequence_last.cc +++ b/src/operator/sequence_last.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file sequence_last.cc * \brief * \author Sebastian Bodenstein diff --git a/src/operator/sequence_last.cu b/src/operator/sequence_last.cu index 329c2c77f6b4..9215b2478c1d 100644 --- a/src/operator/sequence_last.cu +++ b/src/operator/sequence_last.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file sequence_last.cu * \brief * \author Sebastian Bodenstein diff --git a/src/operator/sequence_mask-inl.h b/src/operator/sequence_mask-inl.h index 69c98746553b..dec1f2a2b7ed 100644 --- a/src/operator/sequence_mask-inl.h +++ b/src/operator/sequence_mask-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file wl_sequence_mask-inl.h * \brief * \author Sebastian Bodenstien diff --git a/src/operator/sequence_mask.cc b/src/operator/sequence_mask.cc index 0ac782e51c3c..ed90f3ee4a4f 100644 --- a/src/operator/sequence_mask.cc +++ b/src/operator/sequence_mask.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file sequence_mask.cc * \brief * \author Sebastian Bodenstein diff --git a/src/operator/sequence_mask.cu b/src/operator/sequence_mask.cu index 41c08942cdff..d370ff3d13ec 100644 --- a/src/operator/sequence_mask.cu +++ b/src/operator/sequence_mask.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file sequence_mask.cu * \brief * \author Sebastian Bodenstein diff --git a/src/operator/sequence_op_common.h b/src/operator/sequence_op_common.h index a2924921218f..9e5843161087 100644 --- a/src/operator/sequence_op_common.h +++ b/src/operator/sequence_op_common.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file sequence_op_common.h * \brief common function used for sequence layers * \author Sebastian Bodenstein diff --git a/src/operator/sequence_reverse-inl.h b/src/operator/sequence_reverse-inl.h index 0ead25ceba72..0a43138a085c 100644 --- a/src/operator/sequence_reverse-inl.h +++ b/src/operator/sequence_reverse-inl.h @@ -1,5 +1,23 @@ /* - * Copyright (c) 2016 by Contributors + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* * \file sequence_reverse-inl.h * \brief * \author Sebastian Bodenstien diff --git a/src/operator/sequence_reverse.cc b/src/operator/sequence_reverse.cc index 01dcb6810e62..61821d3945f7 100644 --- a/src/operator/sequence_reverse.cc +++ b/src/operator/sequence_reverse.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file sequence_reverse.cc * \brief * \author Sebastian Bodenstein diff --git a/src/operator/sequence_reverse.cu b/src/operator/sequence_reverse.cu index cdd8f348950c..c6cc3f66d0fe 100644 --- a/src/operator/sequence_reverse.cu +++ b/src/operator/sequence_reverse.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file sequence_reverse.cu * \brief * \author Sebastian Bodenstein diff --git a/src/operator/slice_channel-inl.h b/src/operator/slice_channel-inl.h index 69d1f2ad5449..a48c52f0b70e 100644 --- a/src/operator/slice_channel-inl.h +++ b/src/operator/slice_channel-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file slice_channel-inl.h * \brief * \author Bing Xu diff --git a/src/operator/slice_channel.cc b/src/operator/slice_channel.cc index 689f0109214f..7293ba6afcf3 100644 --- a/src/operator/slice_channel.cc +++ b/src/operator/slice_channel.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file slice_channel.cc * \brief * \author Bing Xu @@ -31,7 +49,7 @@ MXNET_REGISTER_OP_PROPERTY(SliceChannel, SliceChannelProp) .. note:: ``SliceChannel`` is deprecated. Use ``split`` instead. -**Note** that `num_outputs` should evenly divide the length of the axis +**Note** that `num_outputs` should evenly divide the length of the axis along which to split the array. Example:: diff --git a/src/operator/slice_channel.cu b/src/operator/slice_channel.cu index 6afd45003ed3..eb1c9c8b6e93 100644 --- a/src/operator/slice_channel.cu +++ b/src/operator/slice_channel.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file slice_channel.cc * \brief * \author Bing Xu diff --git a/src/operator/softmax_activation-inl.h b/src/operator/softmax_activation-inl.h index 0f4bf5f290af..b1b76930b483 100644 --- a/src/operator/softmax_activation-inl.h +++ b/src/operator/softmax_activation-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file softmax_activation-inl.h * \brief SoftmaxActivation operator * \author Junyuan Xie diff --git a/src/operator/softmax_activation.cc b/src/operator/softmax_activation.cc index 827ea03a4fd4..115b0a730cde 100644 --- a/src/operator/softmax_activation.cc +++ b/src/operator/softmax_activation.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file activation.cc * \brief softmax_activation op * \author Junyuan Xie diff --git a/src/operator/softmax_activation.cu b/src/operator/softmax_activation.cu index b2d903a98ae6..5bebed2846b8 100644 --- a/src/operator/softmax_activation.cu +++ b/src/operator/softmax_activation.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file softmax_activation.cu * \brief * \author Junyuan Xie diff --git a/src/operator/softmax_output-inl.h b/src/operator/softmax_output-inl.h index 3bed9387c1db..d14193bdc9fc 100644 --- a/src/operator/softmax_output-inl.h +++ b/src/operator/softmax_output-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file softmax_output-inl.h * \brief * \author Bing Xu diff --git a/src/operator/softmax_output.cc b/src/operator/softmax_output.cc index 08580e9328cd..52bb2a400755 100644 --- a/src/operator/softmax_output.cc +++ b/src/operator/softmax_output.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file softmax_output.cc * \brief * \author Bing Xu diff --git a/src/operator/softmax_output.cu b/src/operator/softmax_output.cu index 7d9324bd2632..8de5df6655f7 100644 --- a/src/operator/softmax_output.cu +++ b/src/operator/softmax_output.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file softmax_output.cu * \brief * \author Bing Xu diff --git a/src/operator/spatial_transformer-inl.h b/src/operator/spatial_transformer-inl.h index 8c02d35ebc44..77967579340f 100644 --- a/src/operator/spatial_transformer-inl.h +++ b/src/operator/spatial_transformer-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file spatial_transformer-inl.h * \brief * Reproducing paper: aderberg M, Simonyan K, Zisserman A. "Spatial transformer networks" diff --git a/src/operator/spatial_transformer.cc b/src/operator/spatial_transformer.cc index 409339b3c445..0d8ee2917637 100644 --- a/src/operator/spatial_transformer.cc +++ b/src/operator/spatial_transformer.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file spatial_transformer.cc * \brief * \author Wei Wu diff --git a/src/operator/spatial_transformer.cu b/src/operator/spatial_transformer.cu index 4942f6573dac..b3d635c5e8ab 100644 --- a/src/operator/spatial_transformer.cu +++ b/src/operator/spatial_transformer.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file spatial_transformer.cu * \brief * \author Wei Wu diff --git a/src/operator/special_functions-inl.h b/src/operator/special_functions-inl.h index 743391e0fce0..b9460a3e7f0f 100644 --- a/src/operator/special_functions-inl.h +++ b/src/operator/special_functions-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file special_functions-inl.h * \brief * \author Valentin Flunkert diff --git a/src/operator/svm_output-inl.h b/src/operator/svm_output-inl.h index 7f460ace8967..f02546144107 100644 --- a/src/operator/svm_output-inl.h +++ b/src/operator/svm_output-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file svm_output-inl.h * \brief * \author Jonas Amaro diff --git a/src/operator/svm_output.cc b/src/operator/svm_output.cc index 5f1f77ad9fc1..766968dfaf0f 100644 --- a/src/operator/svm_output.cc +++ b/src/operator/svm_output.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file svm_output.cc * \brief * \author Jonas Amaro diff --git a/src/operator/svm_output.cu b/src/operator/svm_output.cu index d4b959683287..250df9147f87 100644 --- a/src/operator/svm_output.cu +++ b/src/operator/svm_output.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file svm_output.cu * \brief * \author Jonas Amaro diff --git a/src/operator/swapaxis-inl.h b/src/operator/swapaxis-inl.h index 9595f6e93884..89c724556b8b 100644 --- a/src/operator/swapaxis-inl.h +++ b/src/operator/swapaxis-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file swapaxis-inl.h * \brief * \author Ming Zhang diff --git a/src/operator/swapaxis.cc b/src/operator/swapaxis.cc index 097f9837025f..a6c3e8bff0c7 100644 --- a/src/operator/swapaxis.cc +++ b/src/operator/swapaxis.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file swapaxis.cc * \brief * \author Ming Zhang diff --git a/src/operator/swapaxis.cu b/src/operator/swapaxis.cu index 93f78c2e733d..e9b105d71ea4 100644 --- a/src/operator/swapaxis.cu +++ b/src/operator/swapaxis.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file swapaxis.cu * \brief * \author Ming Zhang diff --git a/src/operator/tensor/broadcast_reduce-inl.cuh b/src/operator/tensor/broadcast_reduce-inl.cuh index 2ba0eb5cec17..b1a259f9b791 100644 --- a/src/operator/tensor/broadcast_reduce-inl.cuh +++ b/src/operator/tensor/broadcast_reduce-inl.cuh @@ -1,605 +1,624 @@ -/*! - * Copyright (c) 2015-2017 by Contributors - * \file broadcast_reduce-inl.cuh - * \brief CUDA implementations for binary broadcast and reduce - * \author Antti-Pekka Hynninen -*/ -#ifndef MXNET_OPERATOR_TENSOR_BROADCAST_REDUCE_INL_CUH_ -#define MXNET_OPERATOR_TENSOR_BROADCAST_REDUCE_INL_CUH_ - -using namespace mshadow::cuda; - -template -__launch_bounds__(kMaxThreadsPerBlock) -__global__ void binary_broadcast_kernel(const int N, const bool addto, - const DType* __restrict lhs, - const DType* __restrict rhs, DType *out, - const Shape lstride, const Shape rstride, - const Shape oshape) { - for (int idx = blockIdx.x * blockDim.x * unroll + threadIdx.x; idx < N; - idx += blockDim.x * gridDim.x * unroll) - { - int j[unroll]; - int k[unroll]; - DType val[unroll]; - #pragma unroll - for (int i=0;i < unroll;i++) { - unravel_dot(idx + i*blockDim.x, oshape, lstride, rstride, &j[i], &k[i]); - val[i] = OP::Map(lhs[j[i]], rhs[k[i]]); - } - #pragma unroll - for (int i=0;i < unroll;i++) { - if (idx + i*blockDim.x < N) assign(&out[idx + i*blockDim.x], addto, val[i]); - } - - } -} - -template -void BinaryBroadcastComputeImpl(Stream *s, const OpReqType req, - const TBlob& lhs, const TBlob& rhs, const TBlob& out) { - if (req == kNullOp) return; - cudaStream_t stream = Stream::GetStream(s); - int N = out.shape_.Size(); - const int warpSize = 32; - const int unroll = 2; - int nthread = std::min(kMaxThreadsPerBlock, ((N + warpSize - 1)/warpSize)*warpSize ); - int ngrid = std::min(kBaseGridNum, (N + nthread*unroll - 1) / (nthread*unroll)); - Shape lstride = calc_stride(lhs.shape_.get()); - Shape rstride = calc_stride(rhs.shape_.get()); - binary_broadcast_kernel<<>>( - N, req == kAddTo, lhs.dptr(), rhs.dptr(), out.dptr(), lstride, rstride, - out.shape_.get()); -} - -const int nthread_reduce = kMaxThreadsPerBlock; -template -__launch_bounds__(nthread_reduce) -__global__ void reduce_kernel(const int N, const int M, const bool addto, - const DType* __restrict big, DType *small, - const Shape big_shape0, const Shape small_shape, - const Shape big_shape, const Shape big_stride, - const int Mnext, const bool do_transpose) { - extern __shared__ char shTileChar[]; - DType* shTile = (DType*)(shTileChar); - const int tid = threadIdx.x + threadIdx.y*blockDim.x; - const int bx = (do_transpose) ? blockDim.y : blockDim.x; - const int by = (do_transpose) ? blockDim.x : blockDim.y; - const int tidx = (do_transpose) ? tid / by : threadIdx.x; - const int tidy = (do_transpose) ? tid % by : threadIdx.y; - for (int m0 = blockIdx.y; m0 < Mnext; m0 += gridDim.y) { - // This TB handles M range [Mstart, ...., Mend - 1] - const int Mstart = (int)((uint64_t)M*(uint64_t)m0/(uint64_t)Mnext); - const int Mend = (int)((uint64_t)M*(uint64_t)(m0 + 1)/(uint64_t)Mnext); - for (int idx0 = blockIdx.x*bx; idx0 < N; idx0 += bx*gridDim.x) { - int idx = idx0 + tidx; - Shape coord = unravel(idx, small_shape); - int idx_big0 = ravel(coord, big_shape0); - - DType val; - Reducer::SetInitValue(val); - if (idx < N) { - for (int k = tidy + Mstart; k < Mend; k += by*unroll) { - int idx_big[unroll]; - #pragma unroll - for (int u=0;u < unroll;u++) { - idx_big[u] = idx_big0 + unravel_dot(k + u*by, big_shape, big_stride); - } - DType tmp[unroll]; - #pragma unroll - for (int u=0;u < unroll;u++) { - if (k + u*by < Mend) { - tmp[u] = OP::Map(big[idx_big[u]]); - } - } - #pragma unroll - for (int u=0;u < unroll;u++) { - if (k + u*by < Mend) Reducer::Reduce(val, tmp[u]); - } - } - } - - // Shared memory block bx * by. Reduction is along by. Final result is in tidy=0 - if (by > 1) { - // Fix bx to avoid bank conflicts. Assumes warpSize number of banks - const int fbx = (do_transpose && ((bx & (warpSize - 1)) == 0)) ? (bx + 1) : bx; - const int it0 = tidx + tidy*fbx; - shTile[it0] = val; - __syncthreads(); - for (int t=1;t < by;t <<= 1) { - DType tmp; - Reducer::SetInitValue(tmp); - if (tidy + t < by) tmp = shTile[it0 + t*fbx]; - __syncthreads(); - Reducer::Reduce(shTile[it0], tmp); - __syncthreads(); - } - if (idx < N && tidy == 0) { - assign(&small[idx + m0*N], addto, shTile[tidx]); - } - } else { - if (idx < N) { - assign(&small[idx + m0*N], addto, val); - } - } - } - } - -} - -template -__launch_bounds__(nthread_reduce) -__global__ void reduce_kernel(const int N, const int M, const bool addto, - const DType* __restrict big, const DType* __restrict lhs, - const DType* __restrict rhs, DType *small, - const Shape big_shape0, const Shape lhs_shape0, - const Shape rhs_shape0, const Shape small_shape, - const Shape big_shape, const Shape lhs_shape, - const Shape rhs_shape, const Shape big_stride, - const Shape lhs_stride, const Shape rhs_stride, - const int Mnext, const bool do_transpose) { - extern __shared__ char shTileChar[]; - DType* shTile = (DType*)(shTileChar); - const int tid = threadIdx.x + threadIdx.y*blockDim.x; - const int bx = (do_transpose) ? blockDim.y : blockDim.x; - const int by = (do_transpose) ? blockDim.x : blockDim.y; - const int tidx = (do_transpose) ? tid / by : threadIdx.x; - const int tidy = (do_transpose) ? tid % by : threadIdx.y; - for (int m0 = blockIdx.y; m0 < Mnext; m0 += gridDim.y) { - // This TB handles M range [Mstart, ...., Mend - 1] - const int Mstart = (int)((uint64_t)M*(uint64_t)m0/(uint64_t)Mnext); - const int Mend = (int)((uint64_t)M*(uint64_t)(m0 + 1)/(uint64_t)Mnext); - for (int idx0 = blockIdx.x*bx; idx0 < N; idx0 += bx*gridDim.x) { - int idx = idx0 + tidx; - Shape coord = unravel(idx, small_shape); - int idx_big0 = ravel(coord, big_shape0); - int idx_lhs0 = ravel(coord, lhs_shape0); - int idx_rhs0 = ravel(coord, rhs_shape0); - - DType val; - Reducer::SetInitValue(val); - if (idx < N) { - for (int k = tidy + Mstart; k < Mend; k += by*unroll) { - int idx_big[unroll]; - int idx_lhs[unroll]; - int idx_rhs[unroll]; - #pragma unroll - for (int u=0;u < unroll;u++) { - idx_big[u] = idx_big0 + unravel_dot(k + u*by, big_shape, big_stride); - idx_lhs[u] = idx_lhs0 + unravel_dot(k + u*by, lhs_shape, lhs_stride); - idx_rhs[u] = idx_rhs0 + unravel_dot(k + u*by, rhs_shape, rhs_stride); - } - DType tmp[unroll]; - #pragma unroll - for (int u=0;u < unroll;u++) { - if (k + u*by < Mend) { - tmp[u] = OP1::Map(big[idx_big[u]], OP2::Map(lhs[idx_lhs[u]], rhs[idx_rhs[u]])); - } - } - #pragma unroll - for (int u=0;u < unroll;u++) { - if (k + u*by < Mend) Reducer::Reduce(val, tmp[u]); - } - } - } - - // Shared memory block bx * by. Reduction is along by. Final result is in tidy=0 - if (by > 1) { - // Fix bx to avoid bank conflicts. Assumes warpSize number of banks - const int fbx = (do_transpose && ((bx & (warpSize - 1)) == 0)) ? (bx + 1) : bx; - const int it0 = tidx + tidy*fbx; - shTile[it0] = val; - __syncthreads(); - for (int t=1;t < by;t <<= 1) { - DType tmp; - Reducer::SetInitValue(tmp); - if (tidy + t < by) tmp = shTile[it0 + t*fbx]; - __syncthreads(); - Reducer::Reduce(shTile[it0], tmp); - __syncthreads(); - } - if (idx < N && tidy == 0) { - assign(&small[idx + m0*N], addto, shTile[tidx]); - } - } else { - if (idx < N) { - assign(&small[idx + m0*N], addto, val); - } - } - } - } - -} - -// Simple reduction of lines when M is small -template -__launch_bounds__(kMaxThreadsPerBlock) -__global__ void reduce_lines_kernel(const int N, const int M, const bool addto, - const int small_in_stride, const DType* __restrict small_in, DType *small_out) { - for (int idx = threadIdx.x + blockIdx.x*blockDim.x; idx < N; idx += blockDim.x*gridDim.x) { - - DType val; - Reducer::SetInitValue(val); - for (int k = 0; k < M; k++) { - Reducer::Reduce(val, small_in[idx + k*small_in_stride]); - } - - if (idx < N) { - assign(&small_out[idx], addto, val); - } - - } -} - -template -__global__ void reduce_kernel_M1(const int N, const bool addto, - const DType* __restrict big, DType *small, const Shape bshape, - const Shape sshape) { - for (int idx = threadIdx.x + blockIdx.x*blockDim.x; idx < N; idx += blockDim.x*gridDim.x) { - Shape coord = unravel(idx, sshape); - int j = ravel(coord, bshape); - assign(&small[idx], addto, OP::Map(big[j])); - } -} - -template -__global__ void reduce_kernel_M1(const int N, const bool addto, - const DType* __restrict big, - const DType* __restrict lhs, - const DType* __restrict rhs, - DType *small, - const Shape big_shape, - const Shape lhs_shape, - const Shape rhs_shape, - const Shape small_shape) { - for (int idx = threadIdx.x + blockIdx.x*blockDim.x; idx < N; idx += blockDim.x*gridDim.x) { - Shape coord = unravel(idx, small_shape); - int idx_big = ravel(coord, big_shape); - int idx_lhs = ravel(coord, lhs_shape); - int idx_rhs = ravel(coord, rhs_shape); - DType val = OP1::Map(big[idx_big], OP2::Map(lhs[idx_lhs], rhs[idx_rhs])); - assign(&small[idx], addto, val); - } -} - -// Returns the stride with which the fastest dimension is moving. -// Used to detect memory access scatter. -template -MSHADOW_XINLINE int fastest_stride(const Shape& small, const Shape& big, - const Shape& big_stride) { - for (int i = ndim-1; i >= 0; --i) { - if (big[i] != 1) { - return (small[i] == big[i]) ? 1 : big_stride[i]; - } - } - return 1; -} - -// Returns a/b integer division rounded up -template -Type ceil_idiv(const Type a, const Type b) { - return (a + b - 1)/b; -} - -// Configuration for ReduceImpl() -template -struct ReduceImplConfig { - static const int warpSize = 32; - static const int unroll_reduce = 2; - static const int maxLoopPerTB = 64; - int N; - int M; - int Mnext; - struct { - dim3 blockDim; - dim3 gridDim; - int shMemSize; - bool do_transpose; - } kernel_1; - struct { - int blockSize; - int gridSize; - } kernel_2; - size_t workspace_size; - - Shape rshape, rstride; - Shape lhs_shape, lhs_stride; - Shape rhs_shape, rhs_stride; -}; - -static inline uint64_t calc_num_load(const int X, const int Y, const int* strides) { - const int warpSize = ReduceImplConfig<1>::warpSize; - // Number of full warps - uint64_t num_full_warp = X / warpSize; - // Length of the partial warp i.e. number of threads that are performing loads - uint64_t len_part_warp = X % warpSize; - - uint64_t num_load_full = (std::min(warpSize, strides[0]) + - std::min(warpSize, strides[1]) + - std::min(warpSize, strides[2]))*num_full_warp; - - uint64_t num_load_part = - (std::min(len_part_warp, ceil_idiv(len_part_warp*strides[0], warpSize)) + - std::min(len_part_warp, ceil_idiv(len_part_warp*strides[1], warpSize)) + - std::min(len_part_warp, ceil_idiv(len_part_warp*strides[2], warpSize)))* - (len_part_warp != 0); - - uint64_t num_load = (num_load_full + num_load_part)*(uint64_t)Y; - return num_load; -} - -template -ReduceImplConfig ConfigureReduceImpl(const TBlob& small, const TBlob& big, const TBlob* lhs, - const TBlob* rhs) { - - ReduceImplConfig config; - - diff(small.shape_.get(), big.shape_.get(), &config.rshape, &config.rstride); - config.N = small.shape_.Size(); - config.M = config.rshape.Size(); - - bool multiOp = false; - if (lhs != NULL) { - CHECK_NOTNULL(rhs); - diff(small.shape_.get(), lhs->shape_.get(), &config.lhs_shape, - &config.lhs_stride); - diff(small.shape_.get(), rhs->shape_.get(), &config.rhs_shape, - &config.rhs_stride); - multiOp = true; - } - - config.workspace_size = 0; - - if (config.M == 1) { - config.kernel_1.blockDim.x = kMaxThreadsPerBlock; - config.kernel_1.gridDim.x = std::min((unsigned int)kBaseGridNum, - (config.N + config.kernel_1.blockDim.x - 1)/config.kernel_1.blockDim.x); - } else { - - int reduce_strides[3]; - reduce_strides[0] = fastest_stride(small.shape_.get(), big.shape_.get(), - big.shape_.get()); - reduce_strides[1] = (multiOp) ? fastest_stride(small.shape_.get(), - lhs->shape_.get(), lhs->shape_.get()) : 1; - reduce_strides[2] = (multiOp) ? fastest_stride(small.shape_.get(), - rhs->shape_.get(), rhs->shape_.get()) : 1; - - int reduce_strides_transp[3]; - reduce_strides_transp[0] = fastest_stride(small.shape_.get(), config.rshape, - config.rstride); - reduce_strides_transp[1] = (multiOp) ? - fastest_stride(small.shape_.get(), config.lhs_shape, config.lhs_stride) : 1; - reduce_strides_transp[2] = (multiOp) ? - fastest_stride(small.shape_.get(), config.rhs_shape, config.rhs_stride) : 1; - - uint64_t num_load = calc_num_load(config.N, config.M, reduce_strides); - uint64_t num_load_transp = calc_num_load(config.M, config.N, reduce_strides_transp); - - config.Mnext = 1; - config.kernel_1.do_transpose = (num_load > num_load_transp); - - config.kernel_1.blockDim.x = 0; - config.kernel_1.blockDim.y = 0; - - if (config.kernel_1.do_transpose) { - // Fastest thread ID goes through M - // Loop over N has step size config.kernel_1.blockDim.y - if (config.N < 8) { - config.kernel_1.blockDim.y = 1; - } else if (config.N < 256) { - config.kernel_1.blockDim.y = 4; - } else { - if (config.M < 8) { - config.kernel_1.blockDim.x = 1; - } else if (config.M < 256) { - config.kernel_1.blockDim.x = 4; - } else { - config.kernel_1.blockDim.x = config.warpSize; - } - } - } else { - // Fastest thread ID goes through N - // Loop over M has step size config.kernel_1.blockDim.y - if (config.M < 8) { - config.kernel_1.blockDim.y = 1; - } else if (config.M < 256) { - config.kernel_1.blockDim.y = 4; - } else { - if (config.N < 8) { - config.kernel_1.blockDim.x = 1; - } else if (config.N < 256) { - config.kernel_1.blockDim.x = 4; - } else { - config.kernel_1.blockDim.x = config.warpSize; - } - } - } - - if (config.kernel_1.blockDim.x == 0 && config.kernel_1.blockDim.y == 0) { - LOG(FATAL) << "Unable to set blockDim"; - } else if (config.kernel_1.blockDim.x == 0) { - config.kernel_1.blockDim.x = nthread_reduce / config.kernel_1.blockDim.y; - } else if (config.kernel_1.blockDim.y == 0) { - config.kernel_1.blockDim.y = nthread_reduce / config.kernel_1.blockDim.x; - } - - if (config.kernel_1.do_transpose) { - // Fastest thread ID goes through M - config.kernel_1.gridDim.x = std::min((unsigned int)kBaseGridNum, - ceil_idiv(config.N, config.kernel_1.blockDim.y)); - config.kernel_1.gridDim.y = std::min(kBaseGridNum, config.Mnext); - int by = config.kernel_1.blockDim.y; - if (config.kernel_1.blockDim.y % config.warpSize == 0) { - // Fix shared memory bank conflict - by++; - } - config.kernel_1.shMemSize = (config.kernel_1.blockDim.x > 1) ? - config.kernel_1.blockDim.x*by*sizeof(DType) : 0; - // Maximum number of times we want TB to loop in M - // Max size of M-block each TB can handle - int maxMblock = config.kernel_1.blockDim.x*config.maxLoopPerTB; - config.Mnext = (config.M + maxMblock - 1) / maxMblock; - } else { - // Fastest thread ID goes through N - config.kernel_1.gridDim.x = std::min((unsigned int)kBaseGridNum, - ceil_idiv(config.N, config.kernel_1.blockDim.x)); - config.kernel_1.gridDim.y = std::min(kBaseGridNum, config.Mnext); - config.kernel_1.shMemSize = (config.kernel_1.blockDim.y > 1) ? - config.kernel_1.blockDim.x*config.kernel_1.blockDim.y*sizeof(DType) : 0; - // Maximum number of times we want TB to loop in M - // Max size of M-block each TB can handle - int maxMblock = config.kernel_1.blockDim.y*config.maxLoopPerTB; - config.Mnext = (config.M + maxMblock - 1) / maxMblock; - } - - if (config.Mnext > 1) { - // small_dptr[] is N*Mnext*sizeof(DType) bytes - config.workspace_size += config.N*config.Mnext*sizeof(DType); - // Set gridDim.y to Mnext - config.kernel_1.gridDim.y = std::min(kBaseGridNum, config.Mnext); - } - - if (config.Mnext > 1) { - config.kernel_2.blockSize = kMaxThreadsPerBlock; - config.kernel_2.gridSize = std::min((int)kBaseGridNum, - (config.N + config.kernel_2.blockSize - 1)/config.kernel_2.blockSize ); - } - - } - - return config; -} - -#define KERNEL_UNROLL_SWITCH(do_unroll, unrollAmount, unrollVar, ...) \ - if (do_unroll) { \ - const int unrollVar = unrollAmount; \ - {__VA_ARGS__} \ - } else { \ - const int unrollVar = 1; \ - {__VA_ARGS__} \ - } - -template -void ReduceImpl(cudaStream_t stream, const TBlob& small, const OpReqType req, - const TBlob& big, const Tensor& workspace, - const ReduceImplConfig& config) { - if (config.M == 1) { - reduce_kernel_M1 - <<< config.kernel_1.gridDim, config.kernel_1.blockDim, 0, stream >>>( - config.N, req == kAddTo, big.dptr(), small.dptr(), big.shape_.get(), - small.shape_.get()); - } else { - - DType* small_dptr = small.dptr(); - bool addto = (req == kAddTo); - if (config.Mnext > 1) { - // small_dptr[] is N*Mnext*sizeof(DType) bytes - small_dptr = reinterpret_cast(workspace.dptr_); - addto = false; - // Check that the workspace is contigiuous - CHECK_EQ(workspace.CheckContiguous(), true); - // Check that we have enough storage - CHECK_GE(workspace.size(0), config.workspace_size); - } - - const int by = (config.kernel_1.do_transpose) ? - config.kernel_1.blockDim.x : config.kernel_1.blockDim.y; - const bool do_unroll = ( config.M / (by*config.Mnext) >= config.unroll_reduce ); - KERNEL_UNROLL_SWITCH(do_unroll, ReduceImplConfig::unroll_reduce, UNROLL, { - reduce_kernel - <<< config.kernel_1.gridDim, config.kernel_1.blockDim, config.kernel_1.shMemSize, stream>>>( - config.N, config.M, addto, big.dptr(), small_dptr, big.shape_.get(), - small.shape_.get(), config.rshape, config.rstride, config.Mnext, - config.kernel_1.do_transpose); - }); - - if (config.Mnext > 1) { - reduce_lines_kernel - <<< config.kernel_2.gridSize, config.kernel_2.blockSize, 0, stream >>> - (config.N, config.Mnext, req == kAddTo, config.N, small_dptr, small.dptr()); - } - } -} - -template -void ReduceImpl(cudaStream_t stream, const TBlob& small, const TBlob& lhs, const TBlob& rhs, - const OpReqType req, const TBlob& big, const Tensor& workspace, - const ReduceImplConfig& config) { - if (config.M == 1) { - reduce_kernel_M1 - <<< config.kernel_1.gridDim, config.kernel_1.blockDim, 0, stream >>>( - config.N, req == kAddTo, big.dptr(), lhs.dptr(), rhs.dptr(), - small.dptr(), big.shape_.get(), lhs.shape_.get(), - rhs.shape_.get(), small.shape_.get()); - } else { - DType* small_dptr = small.dptr(); - bool addto = (req == kAddTo); - if (config.Mnext > 1) { - // small_dptr[] is N*Mnext*sizeof(DType) bytes - small_dptr = reinterpret_cast(workspace.dptr_); - addto = false; - // Check that the workspace is contigiuous - CHECK_EQ(workspace.CheckContiguous(), true); - // Check that we have enough storage - CHECK_GE(workspace.size(0), config.workspace_size); - } - - const int by = (config.kernel_1.do_transpose) ? - config.kernel_1.blockDim.x : config.kernel_1.blockDim.y; - const bool do_unroll = ( config.M / (by*config.Mnext) >= config.unroll_reduce ); - KERNEL_UNROLL_SWITCH(do_unroll, ReduceImplConfig::unroll_reduce, UNROLL, { - reduce_kernel - <<< config.kernel_1.gridDim, config.kernel_1.blockDim, config.kernel_1.shMemSize, stream>>>( - config.N, config.M, addto, big.dptr(), lhs.dptr(), rhs.dptr(), - small_dptr, big.shape_.get(), lhs.shape_.get(), - rhs.shape_.get(), small.shape_.get(), config.rshape, config.lhs_shape, - config.rhs_shape, config.rstride, config.lhs_stride, config.rhs_stride, config.Mnext, - config.kernel_1.do_transpose); - }); - - if (config.Mnext > 1) { - reduce_lines_kernel - <<< config.kernel_2.gridSize, config.kernel_2.blockSize, 0, stream >>> - (config.N, config.Mnext, req == kAddTo, config.N, small_dptr, small.dptr()); - } - } -} - -#undef KERNEL_UNROLL_SWITCH - -template -void Reduce(Stream *s, const TBlob& small, const OpReqType req, - const Tensor& workspace, const TBlob& big) { - if (req == kNullOp) return; - cudaStream_t stream = Stream::GetStream(s); - ReduceImplConfig config = ConfigureReduceImpl(small, big, NULL, NULL); - ReduceImpl(stream, small, req, big, workspace, config); -} - -template -void Reduce(Stream *s, const TBlob& small, const OpReqType req, - const Tensor& workspace, const TBlob& big, - const TBlob& lhs, const TBlob& rhs) { - if (req == kNullOp) return; - cudaStream_t stream = Stream::GetStream(s); - ReduceImplConfig config = ConfigureReduceImpl(small, big, &lhs, &rhs); - ReduceImpl(stream, small, lhs, rhs, req, big, workspace, config); -} - -template -size_t ReduceWorkspaceSize(Stream *s, const TBlob& small, const OpReqType req, - const TBlob& big) { - if (req == kNullOp) return 0; - ReduceImplConfig config = ConfigureReduceImpl(small, big, NULL, NULL); - return config.workspace_size; -} - -template -size_t ReduceWorkspaceSize(Stream *s, const TBlob& small, const OpReqType req, - const TBlob& big, const TBlob& lhs, const TBlob& rhs) { - if (req == kNullOp) return 0; - ReduceImplConfig config = ConfigureReduceImpl(small, big, &lhs, &rhs); - return config.workspace_size; -} - -#endif //MXNET_OPERATOR_TENSOR_BROADCAST_REDUCE_INL_CUH_ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * Copyright (c) 2015-2017 by Contributors + * \file broadcast_reduce-inl.cuh + * \brief CUDA implementations for binary broadcast and reduce + * \author Antti-Pekka Hynninen +*/ +#ifndef MXNET_OPERATOR_TENSOR_BROADCAST_REDUCE_INL_CUH_ +#define MXNET_OPERATOR_TENSOR_BROADCAST_REDUCE_INL_CUH_ + +using namespace mshadow::cuda; + +template +__launch_bounds__(kMaxThreadsPerBlock) +__global__ void binary_broadcast_kernel(const int N, const bool addto, + const DType* __restrict lhs, + const DType* __restrict rhs, DType *out, + const Shape lstride, const Shape rstride, + const Shape oshape) { + for (int idx = blockIdx.x * blockDim.x * unroll + threadIdx.x; idx < N; + idx += blockDim.x * gridDim.x * unroll) + { + int j[unroll]; + int k[unroll]; + DType val[unroll]; + #pragma unroll + for (int i=0;i < unroll;i++) { + unravel_dot(idx + i*blockDim.x, oshape, lstride, rstride, &j[i], &k[i]); + val[i] = OP::Map(lhs[j[i]], rhs[k[i]]); + } + #pragma unroll + for (int i=0;i < unroll;i++) { + if (idx + i*blockDim.x < N) assign(&out[idx + i*blockDim.x], addto, val[i]); + } + + } +} + +template +void BinaryBroadcastComputeImpl(Stream *s, const OpReqType req, + const TBlob& lhs, const TBlob& rhs, const TBlob& out) { + if (req == kNullOp) return; + cudaStream_t stream = Stream::GetStream(s); + int N = out.shape_.Size(); + const int warpSize = 32; + const int unroll = 2; + int nthread = std::min(kMaxThreadsPerBlock, ((N + warpSize - 1)/warpSize)*warpSize ); + int ngrid = std::min(kBaseGridNum, (N + nthread*unroll - 1) / (nthread*unroll)); + Shape lstride = calc_stride(lhs.shape_.get()); + Shape rstride = calc_stride(rhs.shape_.get()); + binary_broadcast_kernel<<>>( + N, req == kAddTo, lhs.dptr(), rhs.dptr(), out.dptr(), lstride, rstride, + out.shape_.get()); +} + +const int nthread_reduce = kMaxThreadsPerBlock; +template +__launch_bounds__(nthread_reduce) +__global__ void reduce_kernel(const int N, const int M, const bool addto, + const DType* __restrict big, DType *small, + const Shape big_shape0, const Shape small_shape, + const Shape big_shape, const Shape big_stride, + const int Mnext, const bool do_transpose) { + extern __shared__ char shTileChar[]; + DType* shTile = (DType*)(shTileChar); + const int tid = threadIdx.x + threadIdx.y*blockDim.x; + const int bx = (do_transpose) ? blockDim.y : blockDim.x; + const int by = (do_transpose) ? blockDim.x : blockDim.y; + const int tidx = (do_transpose) ? tid / by : threadIdx.x; + const int tidy = (do_transpose) ? tid % by : threadIdx.y; + for (int m0 = blockIdx.y; m0 < Mnext; m0 += gridDim.y) { + // This TB handles M range [Mstart, ...., Mend - 1] + const int Mstart = (int)((uint64_t)M*(uint64_t)m0/(uint64_t)Mnext); + const int Mend = (int)((uint64_t)M*(uint64_t)(m0 + 1)/(uint64_t)Mnext); + for (int idx0 = blockIdx.x*bx; idx0 < N; idx0 += bx*gridDim.x) { + int idx = idx0 + tidx; + Shape coord = unravel(idx, small_shape); + int idx_big0 = ravel(coord, big_shape0); + + DType val; + Reducer::SetInitValue(val); + if (idx < N) { + for (int k = tidy + Mstart; k < Mend; k += by*unroll) { + int idx_big[unroll]; + #pragma unroll + for (int u=0;u < unroll;u++) { + idx_big[u] = idx_big0 + unravel_dot(k + u*by, big_shape, big_stride); + } + DType tmp[unroll]; + #pragma unroll + for (int u=0;u < unroll;u++) { + if (k + u*by < Mend) { + tmp[u] = OP::Map(big[idx_big[u]]); + } + } + #pragma unroll + for (int u=0;u < unroll;u++) { + if (k + u*by < Mend) Reducer::Reduce(val, tmp[u]); + } + } + } + + // Shared memory block bx * by. Reduction is along by. Final result is in tidy=0 + if (by > 1) { + // Fix bx to avoid bank conflicts. Assumes warpSize number of banks + const int fbx = (do_transpose && ((bx & (warpSize - 1)) == 0)) ? (bx + 1) : bx; + const int it0 = tidx + tidy*fbx; + shTile[it0] = val; + __syncthreads(); + for (int t=1;t < by;t <<= 1) { + DType tmp; + Reducer::SetInitValue(tmp); + if (tidy + t < by) tmp = shTile[it0 + t*fbx]; + __syncthreads(); + Reducer::Reduce(shTile[it0], tmp); + __syncthreads(); + } + if (idx < N && tidy == 0) { + assign(&small[idx + m0*N], addto, shTile[tidx]); + } + } else { + if (idx < N) { + assign(&small[idx + m0*N], addto, val); + } + } + } + } + +} + +template +__launch_bounds__(nthread_reduce) +__global__ void reduce_kernel(const int N, const int M, const bool addto, + const DType* __restrict big, const DType* __restrict lhs, + const DType* __restrict rhs, DType *small, + const Shape big_shape0, const Shape lhs_shape0, + const Shape rhs_shape0, const Shape small_shape, + const Shape big_shape, const Shape lhs_shape, + const Shape rhs_shape, const Shape big_stride, + const Shape lhs_stride, const Shape rhs_stride, + const int Mnext, const bool do_transpose) { + extern __shared__ char shTileChar[]; + DType* shTile = (DType*)(shTileChar); + const int tid = threadIdx.x + threadIdx.y*blockDim.x; + const int bx = (do_transpose) ? blockDim.y : blockDim.x; + const int by = (do_transpose) ? blockDim.x : blockDim.y; + const int tidx = (do_transpose) ? tid / by : threadIdx.x; + const int tidy = (do_transpose) ? tid % by : threadIdx.y; + for (int m0 = blockIdx.y; m0 < Mnext; m0 += gridDim.y) { + // This TB handles M range [Mstart, ...., Mend - 1] + const int Mstart = (int)((uint64_t)M*(uint64_t)m0/(uint64_t)Mnext); + const int Mend = (int)((uint64_t)M*(uint64_t)(m0 + 1)/(uint64_t)Mnext); + for (int idx0 = blockIdx.x*bx; idx0 < N; idx0 += bx*gridDim.x) { + int idx = idx0 + tidx; + Shape coord = unravel(idx, small_shape); + int idx_big0 = ravel(coord, big_shape0); + int idx_lhs0 = ravel(coord, lhs_shape0); + int idx_rhs0 = ravel(coord, rhs_shape0); + + DType val; + Reducer::SetInitValue(val); + if (idx < N) { + for (int k = tidy + Mstart; k < Mend; k += by*unroll) { + int idx_big[unroll]; + int idx_lhs[unroll]; + int idx_rhs[unroll]; + #pragma unroll + for (int u=0;u < unroll;u++) { + idx_big[u] = idx_big0 + unravel_dot(k + u*by, big_shape, big_stride); + idx_lhs[u] = idx_lhs0 + unravel_dot(k + u*by, lhs_shape, lhs_stride); + idx_rhs[u] = idx_rhs0 + unravel_dot(k + u*by, rhs_shape, rhs_stride); + } + DType tmp[unroll]; + #pragma unroll + for (int u=0;u < unroll;u++) { + if (k + u*by < Mend) { + tmp[u] = OP1::Map(big[idx_big[u]], OP2::Map(lhs[idx_lhs[u]], rhs[idx_rhs[u]])); + } + } + #pragma unroll + for (int u=0;u < unroll;u++) { + if (k + u*by < Mend) Reducer::Reduce(val, tmp[u]); + } + } + } + + // Shared memory block bx * by. Reduction is along by. Final result is in tidy=0 + if (by > 1) { + // Fix bx to avoid bank conflicts. Assumes warpSize number of banks + const int fbx = (do_transpose && ((bx & (warpSize - 1)) == 0)) ? (bx + 1) : bx; + const int it0 = tidx + tidy*fbx; + shTile[it0] = val; + __syncthreads(); + for (int t=1;t < by;t <<= 1) { + DType tmp; + Reducer::SetInitValue(tmp); + if (tidy + t < by) tmp = shTile[it0 + t*fbx]; + __syncthreads(); + Reducer::Reduce(shTile[it0], tmp); + __syncthreads(); + } + if (idx < N && tidy == 0) { + assign(&small[idx + m0*N], addto, shTile[tidx]); + } + } else { + if (idx < N) { + assign(&small[idx + m0*N], addto, val); + } + } + } + } + +} + +// Simple reduction of lines when M is small +template +__launch_bounds__(kMaxThreadsPerBlock) +__global__ void reduce_lines_kernel(const int N, const int M, const bool addto, + const int small_in_stride, const DType* __restrict small_in, DType *small_out) { + for (int idx = threadIdx.x + blockIdx.x*blockDim.x; idx < N; idx += blockDim.x*gridDim.x) { + + DType val; + Reducer::SetInitValue(val); + for (int k = 0; k < M; k++) { + Reducer::Reduce(val, small_in[idx + k*small_in_stride]); + } + + if (idx < N) { + assign(&small_out[idx], addto, val); + } + + } +} + +template +__global__ void reduce_kernel_M1(const int N, const bool addto, + const DType* __restrict big, DType *small, const Shape bshape, + const Shape sshape) { + for (int idx = threadIdx.x + blockIdx.x*blockDim.x; idx < N; idx += blockDim.x*gridDim.x) { + Shape coord = unravel(idx, sshape); + int j = ravel(coord, bshape); + assign(&small[idx], addto, OP::Map(big[j])); + } +} + +template +__global__ void reduce_kernel_M1(const int N, const bool addto, + const DType* __restrict big, + const DType* __restrict lhs, + const DType* __restrict rhs, + DType *small, + const Shape big_shape, + const Shape lhs_shape, + const Shape rhs_shape, + const Shape small_shape) { + for (int idx = threadIdx.x + blockIdx.x*blockDim.x; idx < N; idx += blockDim.x*gridDim.x) { + Shape coord = unravel(idx, small_shape); + int idx_big = ravel(coord, big_shape); + int idx_lhs = ravel(coord, lhs_shape); + int idx_rhs = ravel(coord, rhs_shape); + DType val = OP1::Map(big[idx_big], OP2::Map(lhs[idx_lhs], rhs[idx_rhs])); + assign(&small[idx], addto, val); + } +} + +// Returns the stride with which the fastest dimension is moving. +// Used to detect memory access scatter. +template +MSHADOW_XINLINE int fastest_stride(const Shape& small, const Shape& big, + const Shape& big_stride) { + for (int i = ndim-1; i >= 0; --i) { + if (big[i] != 1) { + return (small[i] == big[i]) ? 1 : big_stride[i]; + } + } + return 1; +} + +// Returns a/b integer division rounded up +template +Type ceil_idiv(const Type a, const Type b) { + return (a + b - 1)/b; +} + +// Configuration for ReduceImpl() +template +struct ReduceImplConfig { + static const int warpSize = 32; + static const int unroll_reduce = 2; + static const int maxLoopPerTB = 64; + int N; + int M; + int Mnext; + struct { + dim3 blockDim; + dim3 gridDim; + int shMemSize; + bool do_transpose; + } kernel_1; + struct { + int blockSize; + int gridSize; + } kernel_2; + size_t workspace_size; + + Shape rshape, rstride; + Shape lhs_shape, lhs_stride; + Shape rhs_shape, rhs_stride; +}; + +static inline uint64_t calc_num_load(const int X, const int Y, const int* strides) { + const int warpSize = ReduceImplConfig<1>::warpSize; + // Number of full warps + uint64_t num_full_warp = X / warpSize; + // Length of the partial warp i.e. number of threads that are performing loads + uint64_t len_part_warp = X % warpSize; + + uint64_t num_load_full = (std::min(warpSize, strides[0]) + + std::min(warpSize, strides[1]) + + std::min(warpSize, strides[2]))*num_full_warp; + + uint64_t num_load_part = + (std::min(len_part_warp, ceil_idiv(len_part_warp*strides[0], warpSize)) + + std::min(len_part_warp, ceil_idiv(len_part_warp*strides[1], warpSize)) + + std::min(len_part_warp, ceil_idiv(len_part_warp*strides[2], warpSize)))* + (len_part_warp != 0); + + uint64_t num_load = (num_load_full + num_load_part)*(uint64_t)Y; + return num_load; +} + +template +ReduceImplConfig ConfigureReduceImpl(const TBlob& small, const TBlob& big, const TBlob* lhs, + const TBlob* rhs) { + + ReduceImplConfig config; + + diff(small.shape_.get(), big.shape_.get(), &config.rshape, &config.rstride); + config.N = small.shape_.Size(); + config.M = config.rshape.Size(); + + bool multiOp = false; + if (lhs != NULL) { + CHECK_NOTNULL(rhs); + diff(small.shape_.get(), lhs->shape_.get(), &config.lhs_shape, + &config.lhs_stride); + diff(small.shape_.get(), rhs->shape_.get(), &config.rhs_shape, + &config.rhs_stride); + multiOp = true; + } + + config.workspace_size = 0; + + if (config.M == 1) { + config.kernel_1.blockDim.x = kMaxThreadsPerBlock; + config.kernel_1.gridDim.x = std::min((unsigned int)kBaseGridNum, + (config.N + config.kernel_1.blockDim.x - 1)/config.kernel_1.blockDim.x); + } else { + + int reduce_strides[3]; + reduce_strides[0] = fastest_stride(small.shape_.get(), big.shape_.get(), + big.shape_.get()); + reduce_strides[1] = (multiOp) ? fastest_stride(small.shape_.get(), + lhs->shape_.get(), lhs->shape_.get()) : 1; + reduce_strides[2] = (multiOp) ? fastest_stride(small.shape_.get(), + rhs->shape_.get(), rhs->shape_.get()) : 1; + + int reduce_strides_transp[3]; + reduce_strides_transp[0] = fastest_stride(small.shape_.get(), config.rshape, + config.rstride); + reduce_strides_transp[1] = (multiOp) ? + fastest_stride(small.shape_.get(), config.lhs_shape, config.lhs_stride) : 1; + reduce_strides_transp[2] = (multiOp) ? + fastest_stride(small.shape_.get(), config.rhs_shape, config.rhs_stride) : 1; + + uint64_t num_load = calc_num_load(config.N, config.M, reduce_strides); + uint64_t num_load_transp = calc_num_load(config.M, config.N, reduce_strides_transp); + + config.Mnext = 1; + config.kernel_1.do_transpose = (num_load > num_load_transp); + + config.kernel_1.blockDim.x = 0; + config.kernel_1.blockDim.y = 0; + + if (config.kernel_1.do_transpose) { + // Fastest thread ID goes through M + // Loop over N has step size config.kernel_1.blockDim.y + if (config.N < 8) { + config.kernel_1.blockDim.y = 1; + } else if (config.N < 256) { + config.kernel_1.blockDim.y = 4; + } else { + if (config.M < 8) { + config.kernel_1.blockDim.x = 1; + } else if (config.M < 256) { + config.kernel_1.blockDim.x = 4; + } else { + config.kernel_1.blockDim.x = config.warpSize; + } + } + } else { + // Fastest thread ID goes through N + // Loop over M has step size config.kernel_1.blockDim.y + if (config.M < 8) { + config.kernel_1.blockDim.y = 1; + } else if (config.M < 256) { + config.kernel_1.blockDim.y = 4; + } else { + if (config.N < 8) { + config.kernel_1.blockDim.x = 1; + } else if (config.N < 256) { + config.kernel_1.blockDim.x = 4; + } else { + config.kernel_1.blockDim.x = config.warpSize; + } + } + } + + if (config.kernel_1.blockDim.x == 0 && config.kernel_1.blockDim.y == 0) { + LOG(FATAL) << "Unable to set blockDim"; + } else if (config.kernel_1.blockDim.x == 0) { + config.kernel_1.blockDim.x = nthread_reduce / config.kernel_1.blockDim.y; + } else if (config.kernel_1.blockDim.y == 0) { + config.kernel_1.blockDim.y = nthread_reduce / config.kernel_1.blockDim.x; + } + + if (config.kernel_1.do_transpose) { + // Fastest thread ID goes through M + config.kernel_1.gridDim.x = std::min((unsigned int)kBaseGridNum, + ceil_idiv(config.N, config.kernel_1.blockDim.y)); + config.kernel_1.gridDim.y = std::min(kBaseGridNum, config.Mnext); + int by = config.kernel_1.blockDim.y; + if (config.kernel_1.blockDim.y % config.warpSize == 0) { + // Fix shared memory bank conflict + by++; + } + config.kernel_1.shMemSize = (config.kernel_1.blockDim.x > 1) ? + config.kernel_1.blockDim.x*by*sizeof(DType) : 0; + // Maximum number of times we want TB to loop in M + // Max size of M-block each TB can handle + int maxMblock = config.kernel_1.blockDim.x*config.maxLoopPerTB; + config.Mnext = (config.M + maxMblock - 1) / maxMblock; + } else { + // Fastest thread ID goes through N + config.kernel_1.gridDim.x = std::min((unsigned int)kBaseGridNum, + ceil_idiv(config.N, config.kernel_1.blockDim.x)); + config.kernel_1.gridDim.y = std::min(kBaseGridNum, config.Mnext); + config.kernel_1.shMemSize = (config.kernel_1.blockDim.y > 1) ? + config.kernel_1.blockDim.x*config.kernel_1.blockDim.y*sizeof(DType) : 0; + // Maximum number of times we want TB to loop in M + // Max size of M-block each TB can handle + int maxMblock = config.kernel_1.blockDim.y*config.maxLoopPerTB; + config.Mnext = (config.M + maxMblock - 1) / maxMblock; + } + + if (config.Mnext > 1) { + // small_dptr[] is N*Mnext*sizeof(DType) bytes + config.workspace_size += config.N*config.Mnext*sizeof(DType); + // Set gridDim.y to Mnext + config.kernel_1.gridDim.y = std::min(kBaseGridNum, config.Mnext); + } + + if (config.Mnext > 1) { + config.kernel_2.blockSize = kMaxThreadsPerBlock; + config.kernel_2.gridSize = std::min((int)kBaseGridNum, + (config.N + config.kernel_2.blockSize - 1)/config.kernel_2.blockSize ); + } + + } + + return config; +} + +#define KERNEL_UNROLL_SWITCH(do_unroll, unrollAmount, unrollVar, ...) \ + if (do_unroll) { \ + const int unrollVar = unrollAmount; \ + {__VA_ARGS__} \ + } else { \ + const int unrollVar = 1; \ + {__VA_ARGS__} \ + } + +template +void ReduceImpl(cudaStream_t stream, const TBlob& small, const OpReqType req, + const TBlob& big, const Tensor& workspace, + const ReduceImplConfig& config) { + if (config.M == 1) { + reduce_kernel_M1 + <<< config.kernel_1.gridDim, config.kernel_1.blockDim, 0, stream >>>( + config.N, req == kAddTo, big.dptr(), small.dptr(), big.shape_.get(), + small.shape_.get()); + } else { + + DType* small_dptr = small.dptr(); + bool addto = (req == kAddTo); + if (config.Mnext > 1) { + // small_dptr[] is N*Mnext*sizeof(DType) bytes + small_dptr = reinterpret_cast(workspace.dptr_); + addto = false; + // Check that the workspace is contigiuous + CHECK_EQ(workspace.CheckContiguous(), true); + // Check that we have enough storage + CHECK_GE(workspace.size(0), config.workspace_size); + } + + const int by = (config.kernel_1.do_transpose) ? + config.kernel_1.blockDim.x : config.kernel_1.blockDim.y; + const bool do_unroll = ( config.M / (by*config.Mnext) >= config.unroll_reduce ); + KERNEL_UNROLL_SWITCH(do_unroll, ReduceImplConfig::unroll_reduce, UNROLL, { + reduce_kernel + <<< config.kernel_1.gridDim, config.kernel_1.blockDim, config.kernel_1.shMemSize, stream>>>( + config.N, config.M, addto, big.dptr(), small_dptr, big.shape_.get(), + small.shape_.get(), config.rshape, config.rstride, config.Mnext, + config.kernel_1.do_transpose); + }); + + if (config.Mnext > 1) { + reduce_lines_kernel + <<< config.kernel_2.gridSize, config.kernel_2.blockSize, 0, stream >>> + (config.N, config.Mnext, req == kAddTo, config.N, small_dptr, small.dptr()); + } + } +} + +template +void ReduceImpl(cudaStream_t stream, const TBlob& small, const TBlob& lhs, const TBlob& rhs, + const OpReqType req, const TBlob& big, const Tensor& workspace, + const ReduceImplConfig& config) { + if (config.M == 1) { + reduce_kernel_M1 + <<< config.kernel_1.gridDim, config.kernel_1.blockDim, 0, stream >>>( + config.N, req == kAddTo, big.dptr(), lhs.dptr(), rhs.dptr(), + small.dptr(), big.shape_.get(), lhs.shape_.get(), + rhs.shape_.get(), small.shape_.get()); + } else { + DType* small_dptr = small.dptr(); + bool addto = (req == kAddTo); + if (config.Mnext > 1) { + // small_dptr[] is N*Mnext*sizeof(DType) bytes + small_dptr = reinterpret_cast(workspace.dptr_); + addto = false; + // Check that the workspace is contigiuous + CHECK_EQ(workspace.CheckContiguous(), true); + // Check that we have enough storage + CHECK_GE(workspace.size(0), config.workspace_size); + } + + const int by = (config.kernel_1.do_transpose) ? + config.kernel_1.blockDim.x : config.kernel_1.blockDim.y; + const bool do_unroll = ( config.M / (by*config.Mnext) >= config.unroll_reduce ); + KERNEL_UNROLL_SWITCH(do_unroll, ReduceImplConfig::unroll_reduce, UNROLL, { + reduce_kernel + <<< config.kernel_1.gridDim, config.kernel_1.blockDim, config.kernel_1.shMemSize, stream>>>( + config.N, config.M, addto, big.dptr(), lhs.dptr(), rhs.dptr(), + small_dptr, big.shape_.get(), lhs.shape_.get(), + rhs.shape_.get(), small.shape_.get(), config.rshape, config.lhs_shape, + config.rhs_shape, config.rstride, config.lhs_stride, config.rhs_stride, config.Mnext, + config.kernel_1.do_transpose); + }); + + if (config.Mnext > 1) { + reduce_lines_kernel + <<< config.kernel_2.gridSize, config.kernel_2.blockSize, 0, stream >>> + (config.N, config.Mnext, req == kAddTo, config.N, small_dptr, small.dptr()); + } + } +} + +#undef KERNEL_UNROLL_SWITCH + +template +void Reduce(Stream *s, const TBlob& small, const OpReqType req, + const Tensor& workspace, const TBlob& big) { + if (req == kNullOp) return; + cudaStream_t stream = Stream::GetStream(s); + ReduceImplConfig config = ConfigureReduceImpl(small, big, NULL, NULL); + ReduceImpl(stream, small, req, big, workspace, config); +} + +template +void Reduce(Stream *s, const TBlob& small, const OpReqType req, + const Tensor& workspace, const TBlob& big, + const TBlob& lhs, const TBlob& rhs) { + if (req == kNullOp) return; + cudaStream_t stream = Stream::GetStream(s); + ReduceImplConfig config = ConfigureReduceImpl(small, big, &lhs, &rhs); + ReduceImpl(stream, small, lhs, rhs, req, big, workspace, config); +} + +template +size_t ReduceWorkspaceSize(Stream *s, const TBlob& small, const OpReqType req, + const TBlob& big) { + if (req == kNullOp) return 0; + ReduceImplConfig config = ConfigureReduceImpl(small, big, NULL, NULL); + return config.workspace_size; +} + +template +size_t ReduceWorkspaceSize(Stream *s, const TBlob& small, const OpReqType req, + const TBlob& big, const TBlob& lhs, const TBlob& rhs) { + if (req == kNullOp) return 0; + ReduceImplConfig config = ConfigureReduceImpl(small, big, &lhs, &rhs); + return config.workspace_size; +} + +#endif //MXNET_OPERATOR_TENSOR_BROADCAST_REDUCE_INL_CUH_ diff --git a/src/operator/tensor/broadcast_reduce-inl.h b/src/operator/tensor/broadcast_reduce-inl.h index 118b50578385..744308d9a486 100644 --- a/src/operator/tensor/broadcast_reduce-inl.h +++ b/src/operator/tensor/broadcast_reduce-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015-2017 by Contributors * \file broadcast_reduce_kernel.h * \brief Function definition of elementwise unary operators */ diff --git a/src/operator/tensor/broadcast_reduce_op.h b/src/operator/tensor/broadcast_reduce_op.h index 1de33bc86f89..aa678fd7dd82 100644 --- a/src/operator/tensor/broadcast_reduce_op.h +++ b/src/operator/tensor/broadcast_reduce_op.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file elementwise_unary_op-inl.h * \brief Function definition of elementwise unary operators */ diff --git a/src/operator/tensor/broadcast_reduce_op_index.cc b/src/operator/tensor/broadcast_reduce_op_index.cc index 5944e2374ea3..6887955880bc 100644 --- a/src/operator/tensor/broadcast_reduce_op_index.cc +++ b/src/operator/tensor/broadcast_reduce_op_index.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file broadcast_reduce_op.cc * \brief CPU Implementation of broadcast and reduce functions. */ diff --git a/src/operator/tensor/broadcast_reduce_op_index.cu b/src/operator/tensor/broadcast_reduce_op_index.cu index e07b3a2b66d7..defa35ea6227 100644 --- a/src/operator/tensor/broadcast_reduce_op_index.cu +++ b/src/operator/tensor/broadcast_reduce_op_index.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file broadcast_reduce_op.cu * \brief GPU Implementation of broadcast and reduce functions. */ diff --git a/src/operator/tensor/broadcast_reduce_op_value.cc b/src/operator/tensor/broadcast_reduce_op_value.cc index fdbaf7610e9b..551ee8be89d5 100644 --- a/src/operator/tensor/broadcast_reduce_op_value.cc +++ b/src/operator/tensor/broadcast_reduce_op_value.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file broadcast_reduce_op.cc * \brief CPU Implementation of broadcast and reduce functions. */ diff --git a/src/operator/tensor/broadcast_reduce_op_value.cu b/src/operator/tensor/broadcast_reduce_op_value.cu index e4b90d580043..2c216e78982d 100644 --- a/src/operator/tensor/broadcast_reduce_op_value.cu +++ b/src/operator/tensor/broadcast_reduce_op_value.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file broadcast_reduce_op.cu * \brief GPU Implementation of broadcast and reduce functions. */ diff --git a/src/operator/tensor/control_flow_op.cc b/src/operator/tensor/control_flow_op.cc index 5dd525ea391e..bf08fe7e9d94 100644 --- a/src/operator/tensor/control_flow_op.cc +++ b/src/operator/tensor/control_flow_op.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * \file control_flow_op.cc * \brief CPU Implementation of flow control */ diff --git a/src/operator/tensor/control_flow_op.cu b/src/operator/tensor/control_flow_op.cu index dbea01c43d3d..da2c47247f28 100644 --- a/src/operator/tensor/control_flow_op.cu +++ b/src/operator/tensor/control_flow_op.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * \file control_flow_op.cu * \brief */ diff --git a/src/operator/tensor/control_flow_op.h b/src/operator/tensor/control_flow_op.h index c7fcda0f0c01..c240247202e8 100644 --- a/src/operator/tensor/control_flow_op.h +++ b/src/operator/tensor/control_flow_op.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * \file control_flow.h * \brief Function definitions of operators for controlling flow */ diff --git a/src/operator/tensor/elemwise_binary_broadcast_op.h b/src/operator/tensor/elemwise_binary_broadcast_op.h index be0d27d1e334..851a1c5cdf2f 100644 --- a/src/operator/tensor/elemwise_binary_broadcast_op.h +++ b/src/operator/tensor/elemwise_binary_broadcast_op.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file elementwise_binary_broadcast_op.h * \brief Function definition of elementwise unary operators */ diff --git a/src/operator/tensor/elemwise_binary_broadcast_op_basic.cc b/src/operator/tensor/elemwise_binary_broadcast_op_basic.cc index 27a4b5f25c82..c80d46a883ea 100644 --- a/src/operator/tensor/elemwise_binary_broadcast_op_basic.cc +++ b/src/operator/tensor/elemwise_binary_broadcast_op_basic.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file elemwise_binary_scalar_op.cc * \brief CPU Implementation of unary function. */ diff --git a/src/operator/tensor/elemwise_binary_broadcast_op_basic.cu b/src/operator/tensor/elemwise_binary_broadcast_op_basic.cu index ef0e679d6166..bf69132cff14 100644 --- a/src/operator/tensor/elemwise_binary_broadcast_op_basic.cu +++ b/src/operator/tensor/elemwise_binary_broadcast_op_basic.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file elemwise_binary_scalar_op.cu * \brief GPU Implementation of unary function. */ diff --git a/src/operator/tensor/elemwise_binary_broadcast_op_extended.cc b/src/operator/tensor/elemwise_binary_broadcast_op_extended.cc index 127d8c0ec1c5..42da19155ef5 100644 --- a/src/operator/tensor/elemwise_binary_broadcast_op_extended.cc +++ b/src/operator/tensor/elemwise_binary_broadcast_op_extended.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file elemwise_binary_scalar_op.cc * \brief CPU Implementation of unary function. */ diff --git a/src/operator/tensor/elemwise_binary_broadcast_op_extended.cu b/src/operator/tensor/elemwise_binary_broadcast_op_extended.cu index 649e19ba092a..2b7cc70b59a7 100644 --- a/src/operator/tensor/elemwise_binary_broadcast_op_extended.cu +++ b/src/operator/tensor/elemwise_binary_broadcast_op_extended.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file elemwise_binary_scalar_op.cu * \brief GPU Implementation of unary function. */ diff --git a/src/operator/tensor/elemwise_binary_broadcast_op_logic.cc b/src/operator/tensor/elemwise_binary_broadcast_op_logic.cc index 900f376fe421..957b00b5e774 100644 --- a/src/operator/tensor/elemwise_binary_broadcast_op_logic.cc +++ b/src/operator/tensor/elemwise_binary_broadcast_op_logic.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file elemwise_binary_scalar_op.cc * \brief CPU Implementation of unary function. */ diff --git a/src/operator/tensor/elemwise_binary_broadcast_op_logic.cu b/src/operator/tensor/elemwise_binary_broadcast_op_logic.cu index 167b5d34f00a..8673b4f1f1ed 100644 --- a/src/operator/tensor/elemwise_binary_broadcast_op_logic.cu +++ b/src/operator/tensor/elemwise_binary_broadcast_op_logic.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file elemwise_binary_scalar_op.cu * \brief GPU Implementation of unary function. */ diff --git a/src/operator/tensor/elemwise_binary_op.h b/src/operator/tensor/elemwise_binary_op.h index 6062febe2d9e..87b0d46a63c9 100644 --- a/src/operator/tensor/elemwise_binary_op.h +++ b/src/operator/tensor/elemwise_binary_op.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file elemwise_binary_op.h * \brief Function definition of elementwise binary operators */ diff --git a/src/operator/tensor/elemwise_binary_op_basic.cc b/src/operator/tensor/elemwise_binary_op_basic.cc index 635f2a8692aa..65d4ca9aadd6 100644 --- a/src/operator/tensor/elemwise_binary_op_basic.cc +++ b/src/operator/tensor/elemwise_binary_op_basic.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file elemwise_binary_scalar_op.cc * \brief CPU Implementation of unary function. */ diff --git a/src/operator/tensor/elemwise_binary_op_basic.cu b/src/operator/tensor/elemwise_binary_op_basic.cu index 6355c4e5cf01..429140a63bc5 100644 --- a/src/operator/tensor/elemwise_binary_op_basic.cu +++ b/src/operator/tensor/elemwise_binary_op_basic.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file elemwise_binary_scalar_op.cu * \brief GPU Implementation of unary function. */ diff --git a/src/operator/tensor/elemwise_binary_op_extended.cc b/src/operator/tensor/elemwise_binary_op_extended.cc index c1669c6daf6e..31d977c9fd48 100644 --- a/src/operator/tensor/elemwise_binary_op_extended.cc +++ b/src/operator/tensor/elemwise_binary_op_extended.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file elemwise_binary_scalar_op.cc * \brief CPU Implementation of unary function. */ diff --git a/src/operator/tensor/elemwise_binary_op_extended.cu b/src/operator/tensor/elemwise_binary_op_extended.cu index 7325ebfff819..9a10b05cf901 100644 --- a/src/operator/tensor/elemwise_binary_op_extended.cu +++ b/src/operator/tensor/elemwise_binary_op_extended.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file elemwise_binary_scalar_op.cu * \brief GPU Implementation of unary function. */ diff --git a/src/operator/tensor/elemwise_binary_op_logic.cc b/src/operator/tensor/elemwise_binary_op_logic.cc index 0903b503e1c8..85f2bf11539d 100644 --- a/src/operator/tensor/elemwise_binary_op_logic.cc +++ b/src/operator/tensor/elemwise_binary_op_logic.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file elemwise_binary_scalar_op.cc * \brief CPU Implementation of unary function. */ diff --git a/src/operator/tensor/elemwise_binary_op_logic.cu b/src/operator/tensor/elemwise_binary_op_logic.cu index 9fab912227b1..1a703ed90ed0 100644 --- a/src/operator/tensor/elemwise_binary_op_logic.cu +++ b/src/operator/tensor/elemwise_binary_op_logic.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file elemwise_binary_scalar_op.cu * \brief GPU Implementation of unary function. */ diff --git a/src/operator/tensor/elemwise_binary_scalar_op.h b/src/operator/tensor/elemwise_binary_scalar_op.h index 5e577c6e3d4f..f27df274e5b2 100644 --- a/src/operator/tensor/elemwise_binary_scalar_op.h +++ b/src/operator/tensor/elemwise_binary_scalar_op.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file elemwise_binary_scalar_op.h * \brief Function definition of elementwise binary scalar operators */ diff --git a/src/operator/tensor/elemwise_binary_scalar_op_basic.cc b/src/operator/tensor/elemwise_binary_scalar_op_basic.cc index bd0b5335e3ae..3249bcbaa7ca 100644 --- a/src/operator/tensor/elemwise_binary_scalar_op_basic.cc +++ b/src/operator/tensor/elemwise_binary_scalar_op_basic.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file elemwise_binary_scalar_op.cc * \brief CPU Implementation of unary function. */ diff --git a/src/operator/tensor/elemwise_binary_scalar_op_basic.cu b/src/operator/tensor/elemwise_binary_scalar_op_basic.cu index ae19aa8a72f6..a843f67c2723 100644 --- a/src/operator/tensor/elemwise_binary_scalar_op_basic.cu +++ b/src/operator/tensor/elemwise_binary_scalar_op_basic.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file elemwise_binary_scalar_op.cu * \brief GPU Implementation of unary function. */ diff --git a/src/operator/tensor/elemwise_binary_scalar_op_extended.cc b/src/operator/tensor/elemwise_binary_scalar_op_extended.cc index 6b712fce2dcb..785fce2dcbb6 100644 --- a/src/operator/tensor/elemwise_binary_scalar_op_extended.cc +++ b/src/operator/tensor/elemwise_binary_scalar_op_extended.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file elemwise_binary_scalar_op.cc * \brief CPU Implementation of unary function. */ diff --git a/src/operator/tensor/elemwise_binary_scalar_op_extended.cu b/src/operator/tensor/elemwise_binary_scalar_op_extended.cu index 4623b0572938..74e6b7de478b 100644 --- a/src/operator/tensor/elemwise_binary_scalar_op_extended.cu +++ b/src/operator/tensor/elemwise_binary_scalar_op_extended.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file elemwise_binary_scalar_op.cu * \brief GPU Implementation of unary function. */ diff --git a/src/operator/tensor/elemwise_binary_scalar_op_logic.cc b/src/operator/tensor/elemwise_binary_scalar_op_logic.cc index a0f5c2355f39..6771fff21387 100644 --- a/src/operator/tensor/elemwise_binary_scalar_op_logic.cc +++ b/src/operator/tensor/elemwise_binary_scalar_op_logic.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file elemwise_binary_scalar_op.cc * \brief CPU Implementation of unary function. */ diff --git a/src/operator/tensor/elemwise_binary_scalar_op_logic.cu b/src/operator/tensor/elemwise_binary_scalar_op_logic.cu index e400b3baa8d8..9fee4e9e4c87 100644 --- a/src/operator/tensor/elemwise_binary_scalar_op_logic.cu +++ b/src/operator/tensor/elemwise_binary_scalar_op_logic.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file elemwise_binary_scalar_op.cu * \brief GPU Implementation of unary function. */ diff --git a/src/operator/tensor/elemwise_sum.cc b/src/operator/tensor/elemwise_sum.cc index 7ae7ae97acea..652be72f3fab 100644 --- a/src/operator/tensor/elemwise_sum.cc +++ b/src/operator/tensor/elemwise_sum.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file elemwise_sum.cc * \brief elementwise sum operator */ diff --git a/src/operator/tensor/elemwise_sum.cu b/src/operator/tensor/elemwise_sum.cu index 63c3a96acb6a..ce734ad90c8a 100644 --- a/src/operator/tensor/elemwise_sum.cu +++ b/src/operator/tensor/elemwise_sum.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file elemwise_sum.cu * \brief elementwise sum operator */ diff --git a/src/operator/tensor/elemwise_sum.h b/src/operator/tensor/elemwise_sum.h index 2dd6a133b407..3d6d72511156 100644 --- a/src/operator/tensor/elemwise_sum.h +++ b/src/operator/tensor/elemwise_sum.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file elemwise_sum.h * \brief elementwise sum * \author Bing Xu diff --git a/src/operator/tensor/elemwise_unary_op.cc b/src/operator/tensor/elemwise_unary_op.cc index ff03846ab5b3..defe72d3738c 100644 --- a/src/operator/tensor/elemwise_unary_op.cc +++ b/src/operator/tensor/elemwise_unary_op.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file elemwise_unary_op.cc * \brief CPU Implementation of unary function. */ diff --git a/src/operator/tensor/elemwise_unary_op.cu b/src/operator/tensor/elemwise_unary_op.cu index 67ceb1ce5093..4211ea305b4e 100644 --- a/src/operator/tensor/elemwise_unary_op.cu +++ b/src/operator/tensor/elemwise_unary_op.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file elemwise_unary_op.cu * \brief GPU Implementation of unary function. */ diff --git a/src/operator/tensor/elemwise_unary_op.h b/src/operator/tensor/elemwise_unary_op.h index 97a7e36535f0..b6994844e0fe 100644 --- a/src/operator/tensor/elemwise_unary_op.h +++ b/src/operator/tensor/elemwise_unary_op.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file elementwise_unary_op-inl.h * \brief Function definition of elementwise unary operators */ diff --git a/src/operator/tensor/indexing_op-inl.cuh b/src/operator/tensor/indexing_op-inl.cuh index 2f366c2c3e48..4458151f1782 100644 --- a/src/operator/tensor/indexing_op-inl.cuh +++ b/src/operator/tensor/indexing_op-inl.cuh @@ -1,293 +1,312 @@ -/*! - * Copyright (c) 2017 by Contributors - * \file indexing_op-inl.cuh - * \brief CUDA implementations for indexing_op.h - * \author Antti-Pekka Hynninen -*/ -#ifndef MXNET_OPERATOR_TENSOR_INDEXING_OP_CUH_ -#define MXNET_OPERATOR_TENSOR_INDEXING_OP_CUH_ -#include -#include - -#if CUDA_VERSION >= 9000 -#define FULLMASK 0xFFFFFFFF -#define __ballot(x) __ballot_sync(FULLMASK, (x)) -#define __all(x) __all_sync(FULLMASK, (x)) -#endif - -namespace mxnet { -namespace op { -const int kWarpSize = 32; - -template -__global__ void AddTakeGradLargeBatchKernel(DType* dst, - // If idx_start == NULL, then in-kernel edge - // detection is used - const IdxType *idx_start, - // idx_start_size_ptr ignored if idx_start == NULL - const int* idx_start_size_ptr, - const IdxType *sorted, const IdxType *index, - const DType *src, - int ymax, int xmax) { - // Size of the shared memory is [blockDim.x*SZ*blockDim.y]*sizeof(DType) - extern __shared__ char sh_grad_weight_char[]; - DType* sh_grad_weight = (DType*)sh_grad_weight_char; - - int iidx_end = (idx_start == NULL) ? ymax : *idx_start_size_ptr; - - for (int iidx = blockIdx.y;iidx < iidx_end;iidx += gridDim.y) { - - // Thread block sums up elements in the range [idx_begin, idx_end-1] - int idx_begin, idx_end; - int sorted_value; - if (idx_start == NULL) { - idx_begin = iidx; - sorted_value = static_cast(sorted[idx_begin]); - if (idx_begin > 0 && sorted_value == static_cast(sorted[idx_begin - 1])) continue; - // Algorithm is explained using an example: - // blockDim.x = 32 - // blockDim.y = 4 - // sorted[idx_begin:] = [4 4 4 9] - // (3,4) denotes threadIdx.x=3, threadIdx.y=4, ":" is used for ranges - // (0:31,0:3) sorted_value = 4 - idx_end = idx_begin + 1; - unsigned int* sh_ballot = (unsigned int*)sh_grad_weight_char; - int no_edge = 0; - do { - int idx = idx_end + threadIdx.x + threadIdx.y*blockDim.x; - // Example: - // (0:1,0) sorted_idx = 4 - // (rest) sorted_idx = -1 - int sorted_idx = (idx < ymax) ? static_cast(sorted[idx]) : -1; - // Example: - // (0:31,0) sh_ballot[0] = 0b100 - // (rest) sh_ballot[1...3] = 0 - // sh_ballot[] tells us which thread within the warp found the edge - sh_ballot[threadIdx.y] = __ballot(sorted_value != sorted_idx); - __syncthreads(); - // No edge if sh_ballot[threadIdx.x] == 0 - // NOTE: All warps have the same value for no_edge - // Example: - // (0,:) no_edge = 0 - // (rest) no_edge = 1 - no_edge = (threadIdx.x < blockDim.y) ? (sh_ballot[threadIdx.x] == 0) : 1; - idx_end += blockDim.x*blockDim.y; - // Example: - // __all(no_edge) = 0 since no_edge = 0 for threadIdx.x = 0, hence we leave the loop - } while (__all(no_edge)); - idx_end -= blockDim.x*blockDim.y; - // Find the first edge - // Example: - // (0,:) val = 1 - // (rest) val = 0 - unsigned int val = (threadIdx.x < blockDim.y && sh_ballot[threadIdx.x] != 0) ? - 1 : 0; - // NOTE: Set nth bit if thread n in the warp has val = 1 - // Example: - // (all) val = 1 - val = __ballot( val ); - // __ffs() returns the position of first set bit, 1...32. __ffs(1) = 1 - // j will be the warp index where edge was found - // Example: - // (all) j = 1 - 1 = 0 - int j = __ffs(val) - 1; - // j = warp index where the edge was found - // __ffs(sh_ballot[j]) - 1 = warp lane where the edge was found - // idx_end points to the one over the last value. - // Example: - // idx_end += 0*blockDim.x + _ffs(0b100) - 1 = 0 + 3 - 1 = 2 - // sorted[idx_end] = 9 - idx_end += j*blockDim.x + __ffs(sh_ballot[j]) - 1; - __syncthreads(); - } else { - idx_begin = idx_start[iidx]; - idx_end = ((iidx + 1) < iidx_end) ? idx_start[iidx + 1] : ymax; - sorted_value = static_cast(sorted[idx_begin]); - } - - const int start_feature = threadIdx.x + blockIdx.x * blockDim.x * SZ; - const int dst_row = sorted_value * xmax; - - int num_idx = idx_end - idx_begin; - int idx0 = idx_begin + threadIdx.y*num_idx/blockDim.y; - int idx1 = idx_begin + (threadIdx.y + 1)*num_idx/blockDim.y; - - // Read and sum data into grad_weight[] - DType grad_weight[SZ]; - #pragma unroll - for (int ii = 0; ii < SZ; ii++) { - grad_weight[ii] = (DType)0; - } - for (int idx=idx0; idx < idx1;idx++) { - const int src_row = static_cast(index[idx]) * xmax; - #pragma unroll - for (int ii = 0; ii < SZ; ii++) - { - int feature_dim = start_feature + ii * blockDim.x; - if (feature_dim < xmax) - { - grad_weight[ii] += src[src_row + feature_dim]; - } - } - } - #pragma unroll - for (int ii = 0; ii < SZ; ii++) { - sh_grad_weight[threadIdx.x + ii*blockDim.x + threadIdx.y*blockDim.x*SZ] = grad_weight[ii]; - } - __syncthreads(); - // We now have grad_weight[] values, reduce within thread block - for (int t=1;t < blockDim.y;t <<= 1) { - DType tmp[SZ]; - #pragma unroll - for (int ii = 0; ii < SZ; ii++) { - tmp[ii] = (threadIdx.y + t < blockDim.y) ? - sh_grad_weight[threadIdx.x + ii*blockDim.x + (threadIdx.y + t)*blockDim.x*SZ] : (DType)0; - } - __syncthreads(); - #pragma unroll - for (int ii = 0; ii < SZ; ii++) { - sh_grad_weight[threadIdx.x + ii*blockDim.x + threadIdx.y*blockDim.x*SZ] += tmp[ii]; - } - __syncthreads(); - } - // Result is in sh_grad_weight[threadIdx.x + ii*blockDim.x] - if (threadIdx.y == 0) { - #pragma unroll - for (int ii = 0; ii < SZ; ii++) { - int feature_dim = start_feature + ii * blockDim.x; - if (feature_dim < xmax) { - dst[dst_row + feature_dim] += sh_grad_weight[threadIdx.x + ii*blockDim.x]; - } - } - } - - } -} - -template -inline typename std::enable_if::value, size_t>::type -AddTakeGradLargeBatchWorkspaceSize(size_t num_keys) { - size_t encode_bytes = 0; - cub::DeviceRunLengthEncode::Encode - (NULL, encode_bytes, NULL, NULL, NULL, NULL, num_keys); - size_t exclusivesum_bytes = 0; - cub::DeviceScan::ExclusiveSum(NULL, exclusivesum_bytes, - NULL, NULL, num_keys); - size_t temporary_bytes = std::max(encode_bytes, exclusivesum_bytes); - size_t unique_bytes = num_keys*sizeof(IndexType); - size_t counts_bytes = num_keys*sizeof(IndexType); - size_t num_runs_bytes = 1*sizeof(int); - return (unique_bytes + counts_bytes + num_runs_bytes + temporary_bytes); -} - -template -inline void AddTakeGradLargeBatch(mshadow::Tensor dst, - const mshadow::Tensor& sorted, - const mshadow::Tensor& index, - const mshadow::Tensor &src, - mshadow::Tensor* workspace) { - CHECK_EQ(dst.CheckContiguous(), true); - CHECK_EQ(sorted.CheckContiguous(), true); - CHECK_EQ(index.CheckContiguous(), true); - CHECK_EQ(src.CheckContiguous(), true); - // const int kWarpBits = kMemUnitBits; - cudaStream_t stream = mshadow::Stream::GetStream(dst.stream_); - IndexType* sum_counts_ptr = NULL; - int* num_runs_ptr = NULL; - if (dst.size(0)*4 < src.size(0) && workspace != NULL) { - // Workspace given and potentially loops at least 4 times, use CUB to create sum_counts - CHECK_EQ(workspace->CheckContiguous(), true); - // workspace = [unique_out, counts_out, temporary_storage] - size_t unique_bytes = sorted.size(0)*sizeof(IndexType); - size_t counts_bytes = sorted.size(0)*sizeof(IndexType); - size_t num_runs_bytes = 1*sizeof(int); - - size_t encode_bytes = 0; - cub::DeviceRunLengthEncode::Encode - (NULL, encode_bytes, NULL, NULL, NULL, NULL, sorted.size(0), stream); - size_t exclusivesum_bytes = 0; - cub::DeviceScan::ExclusiveSum - (NULL, exclusivesum_bytes, NULL, NULL, sorted.size(0), stream); - size_t temporary_bytes = std::max(encode_bytes, exclusivesum_bytes); - - // Check that we have enough storage - CHECK_GE(workspace->size(0), unique_bytes + counts_bytes + - num_runs_bytes + temporary_bytes); - - IndexType* unique_out_ptr = reinterpret_cast(workspace->dptr_); - IndexType* counts_out_ptr = reinterpret_cast(workspace->dptr_ + unique_bytes); - num_runs_ptr = reinterpret_cast(workspace->dptr_ + unique_bytes + - counts_bytes); - void* temporary_storage = reinterpret_cast(workspace->dptr_ + unique_bytes + - counts_bytes + num_runs_bytes); - - cub::DeviceRunLengthEncode::Encode - (temporary_storage, temporary_bytes, sorted.dptr_, unique_out_ptr, counts_out_ptr, - num_runs_ptr, sorted.size(0), stream); - - sum_counts_ptr = unique_out_ptr; - cub::DeviceScan::ExclusiveSum - (temporary_storage, temporary_bytes, counts_out_ptr, sum_counts_ptr, - sorted.size(0), stream); - } - - const int num_unique_est = min(dst.size(0), src.size(0)); - const int max_nthread = 128; - const int num_y = max(src.size(0)/num_unique_est, 1); - const int block_dim_x = kWarpSize; - const int block_dim_y = min(num_y, max_nthread/block_dim_x); - const int SZ = min((src.size(1) + block_dim_x - 1) / block_dim_x, 4); - const int grid_dim_x = (src.size(1) + block_dim_x * SZ - 1) / (block_dim_x * SZ); - const int grid_dim_y = min(num_unique_est, mshadow::cuda::kBaseGridNum); - dim3 dimBlock(block_dim_x, block_dim_y); - dim3 dimGrid(grid_dim_x, grid_dim_y); - // Maximum shared memory usage: 128*4*sizeof(DType), which is 4K for 64bit DType elements - int shmem_size = dimBlock.x*SZ*dimBlock.y*sizeof(DType); - - CHECK_EQ(dst.size(1), src.size(1)) << "AddTakeGradLargeBatch: shape mismatch"; - CHECK_EQ(index.size(0), src.size(0)) << "AddTakeGradLargeBatch: shape mismatch"; - mshadow::cuda::CheckLaunchParam(dimGrid, dimBlock, "AddTakeGradLargeBatch"); - - switch (SZ) { - case 1: - AddTakeGradLargeBatchKernel<1, DType> - <<>> - (dst.dptr_, sum_counts_ptr, num_runs_ptr, - sorted.dptr_, index.dptr_, src.dptr_, - static_cast(src.size(0)), - static_cast(src.size(1))); - break; - case 2: - AddTakeGradLargeBatchKernel<2, DType> - <<>> - (dst.dptr_, sum_counts_ptr, num_runs_ptr, - sorted.dptr_, index.dptr_, src.dptr_, - static_cast(src.size(0)), - static_cast(src.size(1))); - break; - case 3: - AddTakeGradLargeBatchKernel<3, DType> - <<>> - (dst.dptr_, sum_counts_ptr, num_runs_ptr, - sorted.dptr_, index.dptr_, src.dptr_, - static_cast(src.size(0)), - static_cast(src.size(1))); - break; - case 4: - AddTakeGradLargeBatchKernel<4, DType> - <<>> - (dst.dptr_, sum_counts_ptr, num_runs_ptr, - sorted.dptr_, index.dptr_, src.dptr_, - static_cast(src.size(0)), - static_cast(src.size(1))); - break; - default: - LOG(FATAL) << "AddTakeGradLargeBatch, incorrect value SZ " << SZ; - break; - } - MSHADOW_CUDA_POST_KERNEL_CHECK(AddTakeGradLargeBatchKernel); -} - -} // namespace op -} // namespace mxnet -#endif // MXNET_OPERATOR_TENSOR_INDEXING_OP_CUH_ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * Copyright (c) 2017 by Contributors + * \file indexing_op-inl.cuh + * \brief CUDA implementations for indexing_op.h + * \author Antti-Pekka Hynninen +*/ +#ifndef MXNET_OPERATOR_TENSOR_INDEXING_OP_CUH_ +#define MXNET_OPERATOR_TENSOR_INDEXING_OP_CUH_ +#include +#include + +#if CUDA_VERSION >= 9000 +#define FULLMASK 0xFFFFFFFF +#define __ballot(x) __ballot_sync(FULLMASK, (x)) +#define __all(x) __all_sync(FULLMASK, (x)) +#endif + +namespace mxnet { +namespace op { +const int kWarpSize = 32; + +template +__global__ void AddTakeGradLargeBatchKernel(DType* dst, + // If idx_start == NULL, then in-kernel edge + // detection is used + const IdxType *idx_start, + // idx_start_size_ptr ignored if idx_start == NULL + const int* idx_start_size_ptr, + const IdxType *sorted, const IdxType *index, + const DType *src, + int ymax, int xmax) { + // Size of the shared memory is [blockDim.x*SZ*blockDim.y]*sizeof(DType) + extern __shared__ char sh_grad_weight_char[]; + DType* sh_grad_weight = (DType*)sh_grad_weight_char; + + int iidx_end = (idx_start == NULL) ? ymax : *idx_start_size_ptr; + + for (int iidx = blockIdx.y;iidx < iidx_end;iidx += gridDim.y) { + + // Thread block sums up elements in the range [idx_begin, idx_end-1] + int idx_begin, idx_end; + int sorted_value; + if (idx_start == NULL) { + idx_begin = iidx; + sorted_value = static_cast(sorted[idx_begin]); + if (idx_begin > 0 && sorted_value == static_cast(sorted[idx_begin - 1])) continue; + // Algorithm is explained using an example: + // blockDim.x = 32 + // blockDim.y = 4 + // sorted[idx_begin:] = [4 4 4 9] + // (3,4) denotes threadIdx.x=3, threadIdx.y=4, ":" is used for ranges + // (0:31,0:3) sorted_value = 4 + idx_end = idx_begin + 1; + unsigned int* sh_ballot = (unsigned int*)sh_grad_weight_char; + int no_edge = 0; + do { + int idx = idx_end + threadIdx.x + threadIdx.y*blockDim.x; + // Example: + // (0:1,0) sorted_idx = 4 + // (rest) sorted_idx = -1 + int sorted_idx = (idx < ymax) ? static_cast(sorted[idx]) : -1; + // Example: + // (0:31,0) sh_ballot[0] = 0b100 + // (rest) sh_ballot[1...3] = 0 + // sh_ballot[] tells us which thread within the warp found the edge + sh_ballot[threadIdx.y] = __ballot(sorted_value != sorted_idx); + __syncthreads(); + // No edge if sh_ballot[threadIdx.x] == 0 + // NOTE: All warps have the same value for no_edge + // Example: + // (0,:) no_edge = 0 + // (rest) no_edge = 1 + no_edge = (threadIdx.x < blockDim.y) ? (sh_ballot[threadIdx.x] == 0) : 1; + idx_end += blockDim.x*blockDim.y; + // Example: + // __all(no_edge) = 0 since no_edge = 0 for threadIdx.x = 0, hence we leave the loop + } while (__all(no_edge)); + idx_end -= blockDim.x*blockDim.y; + // Find the first edge + // Example: + // (0,:) val = 1 + // (rest) val = 0 + unsigned int val = (threadIdx.x < blockDim.y && sh_ballot[threadIdx.x] != 0) ? + 1 : 0; + // NOTE: Set nth bit if thread n in the warp has val = 1 + // Example: + // (all) val = 1 + val = __ballot( val ); + // __ffs() returns the position of first set bit, 1...32. __ffs(1) = 1 + // j will be the warp index where edge was found + // Example: + // (all) j = 1 - 1 = 0 + int j = __ffs(val) - 1; + // j = warp index where the edge was found + // __ffs(sh_ballot[j]) - 1 = warp lane where the edge was found + // idx_end points to the one over the last value. + // Example: + // idx_end += 0*blockDim.x + _ffs(0b100) - 1 = 0 + 3 - 1 = 2 + // sorted[idx_end] = 9 + idx_end += j*blockDim.x + __ffs(sh_ballot[j]) - 1; + __syncthreads(); + } else { + idx_begin = idx_start[iidx]; + idx_end = ((iidx + 1) < iidx_end) ? idx_start[iidx + 1] : ymax; + sorted_value = static_cast(sorted[idx_begin]); + } + + const int start_feature = threadIdx.x + blockIdx.x * blockDim.x * SZ; + const int dst_row = sorted_value * xmax; + + int num_idx = idx_end - idx_begin; + int idx0 = idx_begin + threadIdx.y*num_idx/blockDim.y; + int idx1 = idx_begin + (threadIdx.y + 1)*num_idx/blockDim.y; + + // Read and sum data into grad_weight[] + DType grad_weight[SZ]; + #pragma unroll + for (int ii = 0; ii < SZ; ii++) { + grad_weight[ii] = (DType)0; + } + for (int idx=idx0; idx < idx1;idx++) { + const int src_row = static_cast(index[idx]) * xmax; + #pragma unroll + for (int ii = 0; ii < SZ; ii++) + { + int feature_dim = start_feature + ii * blockDim.x; + if (feature_dim < xmax) + { + grad_weight[ii] += src[src_row + feature_dim]; + } + } + } + #pragma unroll + for (int ii = 0; ii < SZ; ii++) { + sh_grad_weight[threadIdx.x + ii*blockDim.x + threadIdx.y*blockDim.x*SZ] = grad_weight[ii]; + } + __syncthreads(); + // We now have grad_weight[] values, reduce within thread block + for (int t=1;t < blockDim.y;t <<= 1) { + DType tmp[SZ]; + #pragma unroll + for (int ii = 0; ii < SZ; ii++) { + tmp[ii] = (threadIdx.y + t < blockDim.y) ? + sh_grad_weight[threadIdx.x + ii*blockDim.x + (threadIdx.y + t)*blockDim.x*SZ] : (DType)0; + } + __syncthreads(); + #pragma unroll + for (int ii = 0; ii < SZ; ii++) { + sh_grad_weight[threadIdx.x + ii*blockDim.x + threadIdx.y*blockDim.x*SZ] += tmp[ii]; + } + __syncthreads(); + } + // Result is in sh_grad_weight[threadIdx.x + ii*blockDim.x] + if (threadIdx.y == 0) { + #pragma unroll + for (int ii = 0; ii < SZ; ii++) { + int feature_dim = start_feature + ii * blockDim.x; + if (feature_dim < xmax) { + dst[dst_row + feature_dim] += sh_grad_weight[threadIdx.x + ii*blockDim.x]; + } + } + } + + } +} + +template +inline typename std::enable_if::value, size_t>::type +AddTakeGradLargeBatchWorkspaceSize(size_t num_keys) { + size_t encode_bytes = 0; + cub::DeviceRunLengthEncode::Encode + (NULL, encode_bytes, NULL, NULL, NULL, NULL, num_keys); + size_t exclusivesum_bytes = 0; + cub::DeviceScan::ExclusiveSum(NULL, exclusivesum_bytes, + NULL, NULL, num_keys); + size_t temporary_bytes = std::max(encode_bytes, exclusivesum_bytes); + size_t unique_bytes = num_keys*sizeof(IndexType); + size_t counts_bytes = num_keys*sizeof(IndexType); + size_t num_runs_bytes = 1*sizeof(int); + return (unique_bytes + counts_bytes + num_runs_bytes + temporary_bytes); +} + +template +inline void AddTakeGradLargeBatch(mshadow::Tensor dst, + const mshadow::Tensor& sorted, + const mshadow::Tensor& index, + const mshadow::Tensor &src, + mshadow::Tensor* workspace) { + CHECK_EQ(dst.CheckContiguous(), true); + CHECK_EQ(sorted.CheckContiguous(), true); + CHECK_EQ(index.CheckContiguous(), true); + CHECK_EQ(src.CheckContiguous(), true); + // const int kWarpBits = kMemUnitBits; + cudaStream_t stream = mshadow::Stream::GetStream(dst.stream_); + IndexType* sum_counts_ptr = NULL; + int* num_runs_ptr = NULL; + if (dst.size(0)*4 < src.size(0) && workspace != NULL) { + // Workspace given and potentially loops at least 4 times, use CUB to create sum_counts + CHECK_EQ(workspace->CheckContiguous(), true); + // workspace = [unique_out, counts_out, temporary_storage] + size_t unique_bytes = sorted.size(0)*sizeof(IndexType); + size_t counts_bytes = sorted.size(0)*sizeof(IndexType); + size_t num_runs_bytes = 1*sizeof(int); + + size_t encode_bytes = 0; + cub::DeviceRunLengthEncode::Encode + (NULL, encode_bytes, NULL, NULL, NULL, NULL, sorted.size(0), stream); + size_t exclusivesum_bytes = 0; + cub::DeviceScan::ExclusiveSum + (NULL, exclusivesum_bytes, NULL, NULL, sorted.size(0), stream); + size_t temporary_bytes = std::max(encode_bytes, exclusivesum_bytes); + + // Check that we have enough storage + CHECK_GE(workspace->size(0), unique_bytes + counts_bytes + + num_runs_bytes + temporary_bytes); + + IndexType* unique_out_ptr = reinterpret_cast(workspace->dptr_); + IndexType* counts_out_ptr = reinterpret_cast(workspace->dptr_ + unique_bytes); + num_runs_ptr = reinterpret_cast(workspace->dptr_ + unique_bytes + + counts_bytes); + void* temporary_storage = reinterpret_cast(workspace->dptr_ + unique_bytes + + counts_bytes + num_runs_bytes); + + cub::DeviceRunLengthEncode::Encode + (temporary_storage, temporary_bytes, sorted.dptr_, unique_out_ptr, counts_out_ptr, + num_runs_ptr, sorted.size(0), stream); + + sum_counts_ptr = unique_out_ptr; + cub::DeviceScan::ExclusiveSum + (temporary_storage, temporary_bytes, counts_out_ptr, sum_counts_ptr, + sorted.size(0), stream); + } + + const int num_unique_est = min(dst.size(0), src.size(0)); + const int max_nthread = 128; + const int num_y = max(src.size(0)/num_unique_est, 1); + const int block_dim_x = kWarpSize; + const int block_dim_y = min(num_y, max_nthread/block_dim_x); + const int SZ = min((src.size(1) + block_dim_x - 1) / block_dim_x, 4); + const int grid_dim_x = (src.size(1) + block_dim_x * SZ - 1) / (block_dim_x * SZ); + const int grid_dim_y = min(num_unique_est, mshadow::cuda::kBaseGridNum); + dim3 dimBlock(block_dim_x, block_dim_y); + dim3 dimGrid(grid_dim_x, grid_dim_y); + // Maximum shared memory usage: 128*4*sizeof(DType), which is 4K for 64bit DType elements + int shmem_size = dimBlock.x*SZ*dimBlock.y*sizeof(DType); + + CHECK_EQ(dst.size(1), src.size(1)) << "AddTakeGradLargeBatch: shape mismatch"; + CHECK_EQ(index.size(0), src.size(0)) << "AddTakeGradLargeBatch: shape mismatch"; + mshadow::cuda::CheckLaunchParam(dimGrid, dimBlock, "AddTakeGradLargeBatch"); + + switch (SZ) { + case 1: + AddTakeGradLargeBatchKernel<1, DType> + <<>> + (dst.dptr_, sum_counts_ptr, num_runs_ptr, + sorted.dptr_, index.dptr_, src.dptr_, + static_cast(src.size(0)), + static_cast(src.size(1))); + break; + case 2: + AddTakeGradLargeBatchKernel<2, DType> + <<>> + (dst.dptr_, sum_counts_ptr, num_runs_ptr, + sorted.dptr_, index.dptr_, src.dptr_, + static_cast(src.size(0)), + static_cast(src.size(1))); + break; + case 3: + AddTakeGradLargeBatchKernel<3, DType> + <<>> + (dst.dptr_, sum_counts_ptr, num_runs_ptr, + sorted.dptr_, index.dptr_, src.dptr_, + static_cast(src.size(0)), + static_cast(src.size(1))); + break; + case 4: + AddTakeGradLargeBatchKernel<4, DType> + <<>> + (dst.dptr_, sum_counts_ptr, num_runs_ptr, + sorted.dptr_, index.dptr_, src.dptr_, + static_cast(src.size(0)), + static_cast(src.size(1))); + break; + default: + LOG(FATAL) << "AddTakeGradLargeBatch, incorrect value SZ " << SZ; + break; + } + MSHADOW_CUDA_POST_KERNEL_CHECK(AddTakeGradLargeBatchKernel); +} + +} // namespace op +} // namespace mxnet +#endif // MXNET_OPERATOR_TENSOR_INDEXING_OP_CUH_ diff --git a/src/operator/tensor/indexing_op.cc b/src/operator/tensor/indexing_op.cc index 5f010fdfc62c..e5cb41088e22 100644 --- a/src/operator/tensor/indexing_op.cc +++ b/src/operator/tensor/indexing_op.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * \file indexing_op.cc * \brief * \author Siyi Li, Chi Zhang diff --git a/src/operator/tensor/indexing_op.cu b/src/operator/tensor/indexing_op.cu index 287ec25d70be..d57628a4389c 100644 --- a/src/operator/tensor/indexing_op.cu +++ b/src/operator/tensor/indexing_op.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * \file indexing_op.cu * \brief * \author Siyi Li, Chi Zhang diff --git a/src/operator/tensor/indexing_op.h b/src/operator/tensor/indexing_op.h index 5fd6e81d0b2f..ef42b01fb5b6 100644 --- a/src/operator/tensor/indexing_op.h +++ b/src/operator/tensor/indexing_op.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * \file indexing_op.h * \brief * \author Bing Xu, Siyi Li, Chi Zhang diff --git a/src/operator/tensor/init_op.cc b/src/operator/tensor/init_op.cc index 16f71fc7e4e3..8dac22a64966 100644 --- a/src/operator/tensor/init_op.cc +++ b/src/operator/tensor/init_op.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file init_op.cc * \brief CPU Implementation of init op */ diff --git a/src/operator/tensor/init_op.cu b/src/operator/tensor/init_op.cu index a798f26db60d..6e2b65cc8519 100644 --- a/src/operator/tensor/init_op.cu +++ b/src/operator/tensor/init_op.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file init_op.cu * \brief GPU Implementation of init op */ diff --git a/src/operator/tensor/init_op.h b/src/operator/tensor/init_op.h index 5ce132d4bebf..bdc74d332491 100644 --- a/src/operator/tensor/init_op.h +++ b/src/operator/tensor/init_op.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file init_op.h * \brief Function definition of initialization op */ diff --git a/src/operator/tensor/la_op.cc b/src/operator/tensor/la_op.cc index 859e761c7409..1b726ced906b 100644 --- a/src/operator/tensor/la_op.cc +++ b/src/operator/tensor/la_op.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * \file la_op.cc * \brief CPU-Operators for advanced linear algebra. */ @@ -293,7 +311,7 @@ NNVM_REGISTER_OP(_backward_linalg_trmm) .set_attr("FCompute", LaOpBackward); NNVM_REGISTER_OP(linalg_trsm) -.describe(R"code(Solves matrix equations involving a triangular matrix. +.describe(R"code(Solves matrix equations involving a triangular matrix. Input are two tensors *A*, *B* each of dimension *n >= 2* and each having the same shape on the leading *n-2* dimensions. For every *n-2* dimensional index *i* let *A*\ :sub:`i`\ , *B*\ :sub:`i`\ be the matrices given by the last *2* dimensions. diff --git a/src/operator/tensor/la_op.h b/src/operator/tensor/la_op.h index 488a3edd6bd9..977998855263 100644 --- a/src/operator/tensor/la_op.h +++ b/src/operator/tensor/la_op.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * \file la_op.h * \brief Operators for advanced linear algebra. */ diff --git a/src/operator/tensor/la_op_inline.h b/src/operator/tensor/la_op_inline.h index b83bf81e6e2d..a032988edb75 100644 --- a/src/operator/tensor/la_op_inline.h +++ b/src/operator/tensor/la_op_inline.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * \file la_op_inline.h * \brief Operators for advanced linear algebra. */ diff --git a/src/operator/tensor/matrix_op-inl.h b/src/operator/tensor/matrix_op-inl.h index 26f409a43525..af0de593c1be 100644 --- a/src/operator/tensor/matrix_op-inl.h +++ b/src/operator/tensor/matrix_op-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file matrix_op-inl.h * \brief Function definition of matrix related operators */ diff --git a/src/operator/tensor/matrix_op.cc b/src/operator/tensor/matrix_op.cc index 4832b13f56c5..e7e8f5548a1c 100644 --- a/src/operator/tensor/matrix_op.cc +++ b/src/operator/tensor/matrix_op.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file matrix_op.cc * \brief CPU Implementation of matrix operations */ diff --git a/src/operator/tensor/matrix_op.cu b/src/operator/tensor/matrix_op.cu index 8cf656e999b8..ca40419a9367 100644 --- a/src/operator/tensor/matrix_op.cu +++ b/src/operator/tensor/matrix_op.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file matrix_op.cu * \brief GPU Implementation of matrix operations */ diff --git a/src/operator/tensor/ordering_op-inl.h b/src/operator/tensor/ordering_op-inl.h index a8cee5940838..eb28b010cbd3 100644 --- a/src/operator/tensor/ordering_op-inl.h +++ b/src/operator/tensor/ordering_op-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file ordering_op-inl.h * \brief Function definition of matrix related operators */ diff --git a/src/operator/tensor/ordering_op.cc b/src/operator/tensor/ordering_op.cc index 3308836c1840..22712a82b4c9 100644 --- a/src/operator/tensor/ordering_op.cc +++ b/src/operator/tensor/ordering_op.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file ordering.cc * \brief CPU Implementation of the ordering operations */ diff --git a/src/operator/tensor/ordering_op.cu b/src/operator/tensor/ordering_op.cu index 29df70f9e6b4..8e40b4a350d9 100644 --- a/src/operator/tensor/ordering_op.cu +++ b/src/operator/tensor/ordering_op.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file matrix_op.cu * \brief GPU Implementation of matrix operations */ diff --git a/src/operator/tensor/sort_op-inl.cuh b/src/operator/tensor/sort_op-inl.cuh index 725f9b8c5f9e..5ad31053f92e 100644 --- a/src/operator/tensor/sort_op-inl.cuh +++ b/src/operator/tensor/sort_op-inl.cuh @@ -1,129 +1,148 @@ -/*! - * Copyright (c) 2017 by Contributors - * \file sort_op-inl.cuh - * \brief CUDA implementations for sort_op.h - */ -#ifndef MXNET_OPERATOR_TENSOR_SORT_OP_INL_CUH_ -#define MXNET_OPERATOR_TENSOR_SORT_OP_INL_CUH_ -#include -#include -#if defined(_MSC_VER) && __CUDACC_VER_MAJOR__ == 8 && __CUDACC_VER_BUILD__ != 44 -// Many CUDA 8 compilers other than V8.0.44 crash on Windows -#pragma warning("Potential crash on CUDA compiler detected. Switching sorting from CUB to Thrust") -#define SORT_WITH_THRUST -#else -#include -#undef SORT_WITH_THRUST -#endif -#if CUDA_VERSION >= 7000 -#include -#endif - -namespace mxnet { -namespace op { - -template -inline typename std::enable_if::value, size_t>::type -SortByKeyWorkspaceSize(const size_t num_keys) { -#ifdef SORT_WITH_THRUST - return 0; -#else - size_t sortpairs_bytes = 0; - cub::DeviceRadixSort::SortPairs(NULL, sortpairs_bytes, - NULL, NULL, NULL, NULL, num_keys); - size_t keys_bytes = num_keys*sizeof(KDType); - size_t values_bytes = num_keys*sizeof(VDType); - return (keys_bytes + values_bytes + sortpairs_bytes); -#endif -} - -template -inline void SortByKey(mshadow::Tensor keys, mshadow::Tensor values, - bool is_ascend, mshadow::Tensor* workspace, - const int begin_bit, const int end_bit) { - CHECK_EQ(keys.CheckContiguous(), true); - CHECK_EQ(values.CheckContiguous(), true); -#if CUDA_VERSION >= 7000 - cudaStream_t stream = mshadow::Stream::GetStream(keys.stream_); -#ifndef SORT_WITH_THRUST - if (workspace != NULL) { - // Workspace given, sort using CUB - CHECK_EQ(workspace->CheckContiguous(), true); - // workspace = [keys_out, values_out, temporary_storage] - size_t keys_bytes = keys.size(0)*sizeof(KDType); - size_t values_bytes = keys.size(0)*sizeof(VDType); - // Get the size of internal storage (for checking purposes only) - size_t sortpairs_bytes = 0; - if (is_ascend) { - cub::DeviceRadixSort::SortPairs(NULL, sortpairs_bytes, - NULL, NULL, NULL, NULL, - keys.size(0), begin_bit, end_bit, stream); - } else { - cub::DeviceRadixSort::SortPairsDescending(NULL, sortpairs_bytes, - NULL, NULL, NULL, NULL, - keys.size(0), begin_bit, end_bit, stream); - } - // Check that we have enough storage - CHECK_GE(workspace->size(0), keys_bytes + values_bytes + sortpairs_bytes); - // - KDType* keys_out_ptr = reinterpret_cast(workspace->dptr_); - VDType* values_out_ptr = reinterpret_cast(workspace->dptr_ + keys_bytes); - void* temp_storage = reinterpret_cast(workspace->dptr_ + keys_bytes + values_bytes); - // Sort - if (is_ascend) { - cub::DeviceRadixSort::SortPairs(temp_storage, sortpairs_bytes, - keys.dptr_, keys_out_ptr, values.dptr_, values_out_ptr, - keys.size(0), begin_bit, end_bit, stream); - } else { - cub::DeviceRadixSort::SortPairsDescending(temp_storage, sortpairs_bytes, - keys.dptr_, keys_out_ptr, values.dptr_, values_out_ptr, - keys.size(0), begin_bit, end_bit, stream); - } - // Copy result back to [keys, values] - mshadow::Tensor keys_out(keys_out_ptr, mshadow::Shape1(keys.size(0)), - keys.stream_); - mshadow::Tensor values_out(values_out_ptr, mshadow::Shape1(keys.size(0)), - keys.stream_); - mshadow::Copy(keys, keys_out, keys.stream_); - mshadow::Copy(values, values_out, values.stream_); - } else { -#endif // SORT_WITH_THRUST - // No workspace, sort using thrust - thrust::device_ptr key_iter = thrust::device_pointer_cast(keys.dptr_); - thrust::device_ptr value_iter = thrust::device_pointer_cast(values.dptr_); - if (is_ascend) { - thrust::stable_sort_by_key( - thrust::cuda::par.on(stream), - key_iter, key_iter + keys.size(0), value_iter, thrust::less()); - } else { - thrust::stable_sort_by_key( - thrust::cuda::par.on(stream), - key_iter, key_iter + keys.size(0), value_iter, thrust::greater()); - } -#ifndef SORT_WITH_THRUST - } -#endif // SORT_WITH_THRUST - MSHADOW_CUDA_POST_KERNEL_CHECK(SortByKey); -#else - LOG(FATAL) << "SortByKey is only supported for CUDA version >=7.0!"; -#endif -} - -template -inline void SortByKey(mshadow::Tensor keys, - mshadow::Tensor values, bool is_ascend, - mshadow::Tensor* workspace, const int begin_bit, const int end_bit) { - LOG(FATAL) << "SortByKey for half_t is not implemented!"; -} - -template -inline void SortByKey(mshadow::Tensor keys, - mshadow::Tensor values, bool is_ascend, - mshadow::Tensor* workspace, const int begin_bit, const int end_bit) { - LOG(FATAL) << "SortByKey for half_t is not implemented!"; -} - -} // namespace op -} // namespace mxnet - -#endif // MXNET_OPERATOR_TENSOR_SORT_OP_INL_CUH_ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * Copyright (c) 2017 by Contributors + * \file sort_op-inl.cuh + * \brief CUDA implementations for sort_op.h + */ +#ifndef MXNET_OPERATOR_TENSOR_SORT_OP_INL_CUH_ +#define MXNET_OPERATOR_TENSOR_SORT_OP_INL_CUH_ +#include +#include +#if defined(_MSC_VER) && __CUDACC_VER_MAJOR__ == 8 && __CUDACC_VER_BUILD__ != 44 +// Many CUDA 8 compilers other than V8.0.44 crash on Windows +#pragma warning("Potential crash on CUDA compiler detected. Switching sorting from CUB to Thrust") +#define SORT_WITH_THRUST +#else +#include +#undef SORT_WITH_THRUST +#endif +#if CUDA_VERSION >= 7000 +#include +#endif + +namespace mxnet { +namespace op { + +template +inline typename std::enable_if::value, size_t>::type +SortByKeyWorkspaceSize(const size_t num_keys) { +#ifdef SORT_WITH_THRUST + return 0; +#else + size_t sortpairs_bytes = 0; + cub::DeviceRadixSort::SortPairs(NULL, sortpairs_bytes, + NULL, NULL, NULL, NULL, num_keys); + size_t keys_bytes = num_keys*sizeof(KDType); + size_t values_bytes = num_keys*sizeof(VDType); + return (keys_bytes + values_bytes + sortpairs_bytes); +#endif +} + +template +inline void SortByKey(mshadow::Tensor keys, mshadow::Tensor values, + bool is_ascend, mshadow::Tensor* workspace, + const int begin_bit, const int end_bit) { + CHECK_EQ(keys.CheckContiguous(), true); + CHECK_EQ(values.CheckContiguous(), true); +#if CUDA_VERSION >= 7000 + cudaStream_t stream = mshadow::Stream::GetStream(keys.stream_); +#ifndef SORT_WITH_THRUST + if (workspace != NULL) { + // Workspace given, sort using CUB + CHECK_EQ(workspace->CheckContiguous(), true); + // workspace = [keys_out, values_out, temporary_storage] + size_t keys_bytes = keys.size(0)*sizeof(KDType); + size_t values_bytes = keys.size(0)*sizeof(VDType); + // Get the size of internal storage (for checking purposes only) + size_t sortpairs_bytes = 0; + if (is_ascend) { + cub::DeviceRadixSort::SortPairs(NULL, sortpairs_bytes, + NULL, NULL, NULL, NULL, + keys.size(0), begin_bit, end_bit, stream); + } else { + cub::DeviceRadixSort::SortPairsDescending(NULL, sortpairs_bytes, + NULL, NULL, NULL, NULL, + keys.size(0), begin_bit, end_bit, stream); + } + // Check that we have enough storage + CHECK_GE(workspace->size(0), keys_bytes + values_bytes + sortpairs_bytes); + // + KDType* keys_out_ptr = reinterpret_cast(workspace->dptr_); + VDType* values_out_ptr = reinterpret_cast(workspace->dptr_ + keys_bytes); + void* temp_storage = reinterpret_cast(workspace->dptr_ + keys_bytes + values_bytes); + // Sort + if (is_ascend) { + cub::DeviceRadixSort::SortPairs(temp_storage, sortpairs_bytes, + keys.dptr_, keys_out_ptr, values.dptr_, values_out_ptr, + keys.size(0), begin_bit, end_bit, stream); + } else { + cub::DeviceRadixSort::SortPairsDescending(temp_storage, sortpairs_bytes, + keys.dptr_, keys_out_ptr, values.dptr_, values_out_ptr, + keys.size(0), begin_bit, end_bit, stream); + } + // Copy result back to [keys, values] + mshadow::Tensor keys_out(keys_out_ptr, mshadow::Shape1(keys.size(0)), + keys.stream_); + mshadow::Tensor values_out(values_out_ptr, mshadow::Shape1(keys.size(0)), + keys.stream_); + mshadow::Copy(keys, keys_out, keys.stream_); + mshadow::Copy(values, values_out, values.stream_); + } else { +#endif // SORT_WITH_THRUST + // No workspace, sort using thrust + thrust::device_ptr key_iter = thrust::device_pointer_cast(keys.dptr_); + thrust::device_ptr value_iter = thrust::device_pointer_cast(values.dptr_); + if (is_ascend) { + thrust::stable_sort_by_key( + thrust::cuda::par.on(stream), + key_iter, key_iter + keys.size(0), value_iter, thrust::less()); + } else { + thrust::stable_sort_by_key( + thrust::cuda::par.on(stream), + key_iter, key_iter + keys.size(0), value_iter, thrust::greater()); + } +#ifndef SORT_WITH_THRUST + } +#endif // SORT_WITH_THRUST + MSHADOW_CUDA_POST_KERNEL_CHECK(SortByKey); +#else + LOG(FATAL) << "SortByKey is only supported for CUDA version >=7.0!"; +#endif +} + +template +inline void SortByKey(mshadow::Tensor keys, + mshadow::Tensor values, bool is_ascend, + mshadow::Tensor* workspace, const int begin_bit, const int end_bit) { + LOG(FATAL) << "SortByKey for half_t is not implemented!"; +} + +template +inline void SortByKey(mshadow::Tensor keys, + mshadow::Tensor values, bool is_ascend, + mshadow::Tensor* workspace, const int begin_bit, const int end_bit) { + LOG(FATAL) << "SortByKey for half_t is not implemented!"; +} + +} // namespace op +} // namespace mxnet + +#endif // MXNET_OPERATOR_TENSOR_SORT_OP_INL_CUH_ diff --git a/src/operator/tensor/sort_op.h b/src/operator/tensor/sort_op.h index ac8a69846ea1..a0425a5afe1e 100644 --- a/src/operator/tensor/sort_op.h +++ b/src/operator/tensor/sort_op.h @@ -1,87 +1,105 @@ -/*! - * Copyright (c) 2017 by Contributors - * \file sort_op.h - * \brief SortByKey function - */ -#ifndef MXNET_OPERATOR_TENSOR_SORT_OP_H_ -#define MXNET_OPERATOR_TENSOR_SORT_OP_H_ - -#include -#include -#include -#include - -namespace mxnet { -namespace op { -/*! - * \brief CPU/GPU: Sort key-value pairs stored in separate places. (Stable sort is performed!) - * \param keys the keys to sort - * \param values the values that sorts w.r.t the key - * \param is_ascend whether to sort key in ascending order - */ -template -inline void SortByKey(mshadow::Tensor keys, mshadow::Tensor values, - bool is_ascend = true, mshadow::Tensor* workspace = NULL, - const int begin_bit = 0, const int end_bit = sizeof(KDType)*8) { - CHECK_EQ(keys.CheckContiguous(), true); - CHECK_EQ(values.CheckContiguous(), true); - CHECK_EQ(keys.size(0), values.size(0)) - << "The sizes of key/value are not equal! keys_size: " << keys.size(0) - << "values_size: " << values.size(0); - std::vector idx(keys.size(0)); - std::vector keys_vec(keys.size(0)); - std::vector values_vec(values.size(0)); - for (index_t i = 0; i < keys.size(0); i++) { - idx[i] = i; - keys_vec[i] = keys[i]; - values_vec[i] = values[i]; - } - if (is_ascend) { - std::stable_sort(idx.begin(), idx.end(), - [&keys_vec](size_t i1, size_t i2) - {return keys_vec[i1] < keys_vec[i2]; }); - } else { - std::stable_sort(idx.begin(), idx.end(), - [&keys_vec](size_t i1, size_t i2) - {return keys_vec[i1] > keys_vec[i2]; }); - } - for (index_t i = 0; i < values.size(0); i++) { - keys[i] = keys_vec[idx[i]]; - values[i] = values_vec[idx[i]]; - } -} - -/*! - * \brief CPU/GPU: Return the amount of temporary storage in bytes required for SortByKey - * \param num_keys number of keys to sort - */ -template -inline typename std::enable_if::value, size_t>::type -SortByKeyWorkspaceSize(const size_t num_keys) { - return 0; -} - -/*! - * \brief CPU/GPU: Sort key-value pairs stored in separate places. (Stable sort is performed!) - * \param keys the keys to sort - * \param values the values that sorts w.r.t the key - * \param is_ascend whether to sort key in ascending order - */ -template -inline void SortByKey(mshadow::Tensor keys, mshadow::Tensor values, - bool is_ascend = true, mshadow::Tensor* workspace = NULL, - const int begin_bit = 0, const int end_bit = sizeof(KDType)*8); -/*! - * \brief CPU/GPU: Return the amount of temporary storage in bytes required for SortByKey - * \param num_keys number of keys to sort - */ -template -inline typename std::enable_if::value, size_t>::type -SortByKeyWorkspaceSize(const size_t num_keys); - -} // namespace op -} // namespace mxnet -#ifdef __CUDACC__ -#include "./sort_op-inl.cuh" -#endif -#endif // MXNET_OPERATOR_TENSOR_SORT_OP_H_ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file sort_op.h + * \brief SortByKey function + */ +#ifndef MXNET_OPERATOR_TENSOR_SORT_OP_H_ +#define MXNET_OPERATOR_TENSOR_SORT_OP_H_ + +#include +#include +#include +#include + +namespace mxnet { +namespace op { +/*! + * \brief CPU/GPU: Sort key-value pairs stored in separate places. (Stable sort is performed!) + * \param keys the keys to sort + * \param values the values that sorts w.r.t the key + * \param is_ascend whether to sort key in ascending order + */ +template +inline void SortByKey(mshadow::Tensor keys, mshadow::Tensor values, + bool is_ascend = true, mshadow::Tensor* workspace = NULL, + const int begin_bit = 0, const int end_bit = sizeof(KDType)*8) { + CHECK_EQ(keys.CheckContiguous(), true); + CHECK_EQ(values.CheckContiguous(), true); + CHECK_EQ(keys.size(0), values.size(0)) + << "The sizes of key/value are not equal! keys_size: " << keys.size(0) + << "values_size: " << values.size(0); + std::vector idx(keys.size(0)); + std::vector keys_vec(keys.size(0)); + std::vector values_vec(values.size(0)); + for (index_t i = 0; i < keys.size(0); i++) { + idx[i] = i; + keys_vec[i] = keys[i]; + values_vec[i] = values[i]; + } + if (is_ascend) { + std::stable_sort(idx.begin(), idx.end(), + [&keys_vec](size_t i1, size_t i2) + {return keys_vec[i1] < keys_vec[i2]; }); + } else { + std::stable_sort(idx.begin(), idx.end(), + [&keys_vec](size_t i1, size_t i2) + {return keys_vec[i1] > keys_vec[i2]; }); + } + for (index_t i = 0; i < values.size(0); i++) { + keys[i] = keys_vec[idx[i]]; + values[i] = values_vec[idx[i]]; + } +} + +/*! + * \brief CPU/GPU: Return the amount of temporary storage in bytes required for SortByKey + * \param num_keys number of keys to sort + */ +template +inline typename std::enable_if::value, size_t>::type +SortByKeyWorkspaceSize(const size_t num_keys) { + return 0; +} + +/*! + * \brief CPU/GPU: Sort key-value pairs stored in separate places. (Stable sort is performed!) + * \param keys the keys to sort + * \param values the values that sorts w.r.t the key + * \param is_ascend whether to sort key in ascending order + */ +template +inline void SortByKey(mshadow::Tensor keys, mshadow::Tensor values, + bool is_ascend = true, mshadow::Tensor* workspace = NULL, + const int begin_bit = 0, const int end_bit = sizeof(KDType)*8); +/*! + * \brief CPU/GPU: Return the amount of temporary storage in bytes required for SortByKey + * \param num_keys number of keys to sort + */ +template +inline typename std::enable_if::value, size_t>::type +SortByKeyWorkspaceSize(const size_t num_keys); + +} // namespace op +} // namespace mxnet +#ifdef __CUDACC__ +#include "./sort_op-inl.cuh" +#endif +#endif // MXNET_OPERATOR_TENSOR_SORT_OP_H_ diff --git a/src/operator/upsampling-inl.h b/src/operator/upsampling-inl.h index a10ccb1f7626..fec0f74f14c8 100644 --- a/src/operator/upsampling-inl.h +++ b/src/operator/upsampling-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file upsampling-inl.h * \brief * \author Bing Xu diff --git a/src/operator/upsampling.cc b/src/operator/upsampling.cc index ad89d4ace137..653b5709f120 100644 --- a/src/operator/upsampling.cc +++ b/src/operator/upsampling.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file upsampling_nearest.cc * \brief * \author Bing Xu diff --git a/src/operator/upsampling.cu b/src/operator/upsampling.cu index 70466d438449..8152535233e4 100644 --- a/src/operator/upsampling.cu +++ b/src/operator/upsampling.cu @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file upsampling_nearest.cc * \brief * \author Bing Xu diff --git a/src/optimizer/sgd-inl.h b/src/optimizer/sgd-inl.h index 36b45c375b95..01a330bece8c 100644 --- a/src/optimizer/sgd-inl.h +++ b/src/optimizer/sgd-inl.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file sgd-inl.h * \brief Operator interface of mxnet. * \author Junyuan Xie diff --git a/src/resource.cc b/src/resource.cc index 60e40d1837a1..4c2dbee33f2b 100644 --- a/src/resource.cc +++ b/src/resource.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file resource.cc * \brief Implementation of resource manager. */ diff --git a/src/storage/cpu_device_storage.h b/src/storage/cpu_device_storage.h index 2afb658bb9c6..ead00dafbf44 100644 --- a/src/storage/cpu_device_storage.h +++ b/src/storage/cpu_device_storage.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file cpu_device_storage.h * \brief CPU storage implementation. */ diff --git a/src/storage/gpu_device_storage.h b/src/storage/gpu_device_storage.h index 10684905a861..3c4f732c80dc 100644 --- a/src/storage/gpu_device_storage.h +++ b/src/storage/gpu_device_storage.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file gpu_device_storage.h * \brief GPU storage implementation. */ diff --git a/src/storage/naive_storage_manager.h b/src/storage/naive_storage_manager.h index 05a8b10c2bb1..731f374bbfd2 100644 --- a/src/storage/naive_storage_manager.h +++ b/src/storage/naive_storage_manager.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file naive_storage_manager.h * \brief Naive storage manager. */ diff --git a/src/storage/pinned_memory_storage.h b/src/storage/pinned_memory_storage.h index 5b0df1041909..69e05f7cf90c 100644 --- a/src/storage/pinned_memory_storage.h +++ b/src/storage/pinned_memory_storage.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file cpu_device_storage.h * \brief CPU storage with pinned memory */ diff --git a/src/storage/pooled_storage_manager.h b/src/storage/pooled_storage_manager.h index 5e0050c04b2f..b2c6633a8082 100644 --- a/src/storage/pooled_storage_manager.h +++ b/src/storage/pooled_storage_manager.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file pooled_storage_manager.h * \brief Storage manager with a memory pool. */ diff --git a/src/storage/storage.cc b/src/storage/storage.cc index 353d72d41cd9..fa15a44b4fb6 100644 --- a/src/storage/storage.cc +++ b/src/storage/storage.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors */ #include #include diff --git a/src/storage/storage_manager.h b/src/storage/storage_manager.h index de08688c5aac..924d2ed48b1a 100644 --- a/src/storage/storage_manager.h +++ b/src/storage/storage_manager.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file storage_manager.h * \brief Storage manager. */ diff --git a/tests/ci_build/ci_build.sh b/tests/ci_build/ci_build.sh index fa2d37ee718f..79fcd86a5df0 100755 --- a/tests/ci_build/ci_build.sh +++ b/tests/ci_build/ci_build.sh @@ -1,4 +1,22 @@ #!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # # Execute command within a docker container # diff --git a/tests/ci_build/install/install_julia.sh b/tests/ci_build/install/install_julia.sh index 80232a10fe53..5007c949537a 100644 --- a/tests/ci_build/install/install_julia.sh +++ b/tests/ci_build/install/install_julia.sh @@ -1,5 +1,23 @@ #!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + set -e wget https://julialang.s3.amazonaws.com/bin/linux/x64/0.5/julia-0.5.0-linux-x86_64.tar.gz diff --git a/tests/ci_build/install/install_library.sh b/tests/ci_build/install/install_library.sh index d65ab21ca4c0..399f6a093793 100644 --- a/tests/ci_build/install/install_library.sh +++ b/tests/ci_build/install/install_library.sh @@ -1,5 +1,23 @@ #!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + yum install graphviz pip install graphviz pip install opencv-python diff --git a/tests/ci_build/install/install_maven.sh b/tests/ci_build/install/install_maven.sh index 66459be8b8d2..666ebde06263 100644 --- a/tests/ci_build/install/install_maven.sh +++ b/tests/ci_build/install/install_maven.sh @@ -1,5 +1,23 @@ #!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + set -e wget http://mirrors.ocf.berkeley.edu/apache/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz diff --git a/tests/ci_build/install/install_openblas.sh b/tests/ci_build/install/install_openblas.sh index 3ac642146026..2ec5eeb4498e 100644 --- a/tests/ci_build/install/install_openblas.sh +++ b/tests/ci_build/install/install_openblas.sh @@ -1,5 +1,23 @@ #!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + set -e git clone https://github.com/xianyi/OpenBLAS diff --git a/tests/ci_build/install/install_opencv.sh b/tests/ci_build/install/install_opencv.sh index 70c7ddbdc382..08a4d9b53cd0 100644 --- a/tests/ci_build/install/install_opencv.sh +++ b/tests/ci_build/install/install_opencv.sh @@ -1,5 +1,23 @@ #!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + set -e yum groupinstall -y "Development Tools" diff --git a/tests/ci_build/install/install_python2.sh b/tests/ci_build/install/install_python2.sh index c818c5d0a7db..ec4bbb9a9b87 100644 --- a/tests/ci_build/install/install_python2.sh +++ b/tests/ci_build/install/install_python2.sh @@ -1,5 +1,23 @@ #!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + set -e yum groupinstall -y "Development Tools" diff --git a/tests/ci_build/install/install_python3.sh b/tests/ci_build/install/install_python3.sh index 5aa1d80bf6f9..ee89161da793 100644 --- a/tests/ci_build/install/install_python3.sh +++ b/tests/ci_build/install/install_python3.sh @@ -1,5 +1,23 @@ #!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + set -e wget https://bootstrap.pypa.io/get-pip.py || exit 1 diff --git a/tests/ci_build/install/install_testdeps.sh b/tests/ci_build/install/install_testdeps.sh index 975eec41b63f..c77734805388 100644 --- a/tests/ci_build/install/install_testdeps.sh +++ b/tests/ci_build/install/install_testdeps.sh @@ -1,5 +1,23 @@ #!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + set -e pip install cpplint 'pylint==1.4.4' 'astroid==1.3.6' diff --git a/tests/ci_build/install/ubuntu_install_core.sh b/tests/ci_build/install/ubuntu_install_core.sh index 9ee76976e9e7..49475740d210 100755 --- a/tests/ci_build/install/ubuntu_install_core.sh +++ b/tests/ci_build/install/ubuntu_install_core.sh @@ -1,4 +1,22 @@ #!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # install libraries for building mxnet c++ core on ubuntu apt-get update && apt-get install -y \ diff --git a/tests/ci_build/install/ubuntu_install_perl.sh b/tests/ci_build/install/ubuntu_install_perl.sh index da4df67a464a..a981746bc18d 100755 --- a/tests/ci_build/install/ubuntu_install_perl.sh +++ b/tests/ci_build/install/ubuntu_install_perl.sh @@ -1,4 +1,22 @@ #!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # install libraries for mxnet's perl package on ubuntu apt-get update && apt-get install -y libmouse-perl pdl cpanminus swig libgraphviz-perl cpanm -q Function::Parameters diff --git a/tests/ci_build/install/ubuntu_install_python.sh b/tests/ci_build/install/ubuntu_install_python.sh index 973523d0c8f3..bb67e3401a89 100755 --- a/tests/ci_build/install/ubuntu_install_python.sh +++ b/tests/ci_build/install/ubuntu_install_python.sh @@ -1,4 +1,22 @@ #!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # install libraries for mxnet's python package on ubuntu apt-get update && apt-get install -y python-dev python3-dev diff --git a/tests/ci_build/install/ubuntu_install_r.sh b/tests/ci_build/install/ubuntu_install_r.sh index 10851a6d0276..38d89a3e4783 100755 --- a/tests/ci_build/install/ubuntu_install_r.sh +++ b/tests/ci_build/install/ubuntu_install_r.sh @@ -1,4 +1,22 @@ #!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # install libraries for mxnet's r package on ubuntu echo "deb http://cran.rstudio.com/bin/linux/ubuntu trusty/" >> /etc/apt/sources.list diff --git a/tests/ci_build/install/ubuntu_install_scala.sh b/tests/ci_build/install/ubuntu_install_scala.sh index dcdd4bc72b5d..712eff98b02a 100755 --- a/tests/ci_build/install/ubuntu_install_scala.sh +++ b/tests/ci_build/install/ubuntu_install_scala.sh @@ -1,4 +1,22 @@ #!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # install libraries for mxnet's scala package on ubuntu apt-get update && apt-get install -y \ diff --git a/tests/cpp/engine/threaded_engine_test.cc b/tests/cpp/engine/threaded_engine_test.cc index 73dc53060b63..58b7e57a509c 100644 --- a/tests/cpp/engine/threaded_engine_test.cc +++ b/tests/cpp/engine/threaded_engine_test.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * \file threaded_engine_test.cc * \brief threaded engine tests */ diff --git a/tests/cpp/include/test_op.h b/tests/cpp/include/test_op.h index 57fda19e4c9e..d8f90df8447e 100644 --- a/tests/cpp/include/test_op.h +++ b/tests/cpp/include/test_op.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * \file test_op.h * \brief operator unit test utility functions * \author Chris Olivier diff --git a/tests/cpp/include/test_perf.h b/tests/cpp/include/test_perf.h index 93b7863de694..d74d4d5a8976 100644 --- a/tests/cpp/include/test_perf.h +++ b/tests/cpp/include/test_perf.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * \file test_perf.h * \brief operator unit test utility functions * \author Chris Olivier diff --git a/tests/cpp/include/test_util.h b/tests/cpp/include/test_util.h index 3fa82688c115..3f5f4ecbb5bb 100644 --- a/tests/cpp/include/test_util.h +++ b/tests/cpp/include/test_util.h @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * \file test_util.h * \brief unit test performance analysis functions * \author Chris Olivier diff --git a/tests/cpp/operator/batchnorm_test.cc b/tests/cpp/operator/batchnorm_test.cc index 719980b5d4f5..3fef28f79a0a 100644 --- a/tests/cpp/operator/batchnorm_test.cc +++ b/tests/cpp/operator/batchnorm_test.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * \file batchnorm_test.cc * \brief operator unit test utility functions * \author Chris Olivier diff --git a/tests/cpp/operator/krprod_test.cc b/tests/cpp/operator/krprod_test.cc index 6e10221f7ab3..31b8ab9dd781 100644 --- a/tests/cpp/operator/krprod_test.cc +++ b/tests/cpp/operator/krprod_test.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * \file krprod_test.cc * \brief Test Khatri-Rao product * \author Jencir Lee diff --git a/tests/cpp/storage/storage_test.cc b/tests/cpp/storage/storage_test.cc index b2bbc4918599..8af3984eb40f 100644 --- a/tests/cpp/storage/storage_test.cc +++ b/tests/cpp/storage/storage_test.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * \file storage_test.cc * \brief cpu/gpu storage tests */ diff --git a/tests/cpp/test_main.cc b/tests/cpp/test_main.cc index 28cdf7b6ec2a..b8ffbbd0ad04 100644 --- a/tests/cpp/test_main.cc +++ b/tests/cpp/test_main.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * \file test_main.cc * \brief operator unit test utility functions * \author Chris Olivier diff --git a/tests/jenkins/run_as_user.sh b/tests/jenkins/run_as_user.sh index db90f0bd0088..7ed3cdf5c937 100755 --- a/tests/jenkins/run_as_user.sh +++ b/tests/jenkins/run_as_user.sh @@ -1,5 +1,23 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + # Exit script with error if any errors occur set -e diff --git a/tests/jenkins/run_test.sh b/tests/jenkins/run_test.sh index a8564326443f..bc69ca1d7f39 100755 --- a/tests/jenkins/run_test.sh +++ b/tests/jenkins/run_test.sh @@ -1,5 +1,23 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + # Exit script with error if any errors occur echo "BUILD make" diff --git a/tests/jenkins/run_test_amzn_linux_gpu.sh b/tests/jenkins/run_test_amzn_linux_gpu.sh index 42c037e67a37..ecfb5211b9e6 100755 --- a/tests/jenkins/run_test_amzn_linux_gpu.sh +++ b/tests/jenkins/run_test_amzn_linux_gpu.sh @@ -1,5 +1,23 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + # Exit script with error if any errors occur echo "BUILD make" diff --git a/tests/jenkins/run_test_installation_docs.sh b/tests/jenkins/run_test_installation_docs.sh index 921c89a16605..90eecb95a325 100755 --- a/tests/jenkins/run_test_installation_docs.sh +++ b/tests/jenkins/run_test_installation_docs.sh @@ -1,5 +1,23 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + set -e # Given an array of numbers, removes any numbers of it that fall outside a given range. @@ -30,9 +48,9 @@ function remove_out_of_range() { echo "Error: Min must be less than or equal to Max" exit 1 fi - + return_arr=() - + for number in "${lineno_array[@]}" do if (( ${number} > ${min} && ${number} < ${max} )) diff --git a/tests/jenkins/run_test_pip_installations.sh b/tests/jenkins/run_test_pip_installations.sh index de235a0a6359..44788bfaf772 100755 --- a/tests/jenkins/run_test_pip_installations.sh +++ b/tests/jenkins/run_test_pip_installations.sh @@ -1,5 +1,23 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + set -e if (( $# < 1 )); then diff --git a/tests/jenkins/run_test_ubuntu.sh b/tests/jenkins/run_test_ubuntu.sh index 2e458b52599a..cdddd2865ddc 100755 --- a/tests/jenkins/run_test_ubuntu.sh +++ b/tests/jenkins/run_test_ubuntu.sh @@ -1,5 +1,23 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + set -e echo "BUILD make" diff --git a/tests/jenkins/set_user_permissions.sh b/tests/jenkins/set_user_permissions.sh index d03a97bfdd73..51034c4365b6 100644 --- a/tests/jenkins/set_user_permissions.sh +++ b/tests/jenkins/set_user_permissions.sh @@ -1,5 +1,23 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + # Exit script with error if any errors occur set -e diff --git a/tests/nightly/TestDoc/doc_spell_checker.py b/tests/nightly/TestDoc/doc_spell_checker.py index 20a5c07afdae..a7b8b250c928 100644 --- a/tests/nightly/TestDoc/doc_spell_checker.py +++ b/tests/nightly/TestDoc/doc_spell_checker.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + #pylint: disable=no-member, too-many-instance-attributes """This script uses pyenchant to check spelling for MXNet documentation website. diff --git a/tests/nightly/TestDoc/doc_spell_grammar.sh b/tests/nightly/TestDoc/doc_spell_grammar.sh index 05b0be138fd6..77c7b86eb7aa 100755 --- a/tests/nightly/TestDoc/doc_spell_grammar.sh +++ b/tests/nightly/TestDoc/doc_spell_grammar.sh @@ -1,4 +1,22 @@ #!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + echo "BUILD make" cp ./make/config.mk . echo "USE_CUDA=0" >> ./config.mk diff --git a/tests/nightly/compilation_warnings/compilation_warnings.sh b/tests/nightly/compilation_warnings/compilation_warnings.sh index 871d849b9bae..a6c4863f4b58 100644 --- a/tests/nightly/compilation_warnings/compilation_warnings.sh +++ b/tests/nightly/compilation_warnings/compilation_warnings.sh @@ -1,4 +1,22 @@ #!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + set -e runme() { cmd=$* diff --git a/tests/nightly/compilation_warnings/process_output.py b/tests/nightly/compilation_warnings/process_output.py index 622598d72292..5f85af592f6c 100644 --- a/tests/nightly/compilation_warnings/process_output.py +++ b/tests/nightly/compilation_warnings/process_output.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import re import sys import operator diff --git a/tests/nightly/dist_lenet.py b/tests/nightly/dist_lenet.py index a7ae84cb3200..35f55c8157d2 100644 --- a/tests/nightly/dist_lenet.py +++ b/tests/nightly/dist_lenet.py @@ -1,4 +1,22 @@ #!/usr/bin/env python + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # distributed lenet import os, sys curr_path = os.path.abspath(os.path.dirname(__file__)) diff --git a/tests/nightly/dist_sync_kvstore.py b/tests/nightly/dist_sync_kvstore.py index ebed6c57586d..3fbf9f910879 100644 --- a/tests/nightly/dist_sync_kvstore.py +++ b/tests/nightly/dist_sync_kvstore.py @@ -1,4 +1,22 @@ #!/usr/bin/env python + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: skip-file import sys sys.path.insert(0, "../../python/") diff --git a/tests/nightly/download.sh b/tests/nightly/download.sh index 56f822e6ad42..d07fc6f4ab8a 100644 --- a/tests/nightly/download.sh +++ b/tests/nightly/download.sh @@ -1,5 +1,23 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + dmlc_download() { url=http://data.mxnet.io/mxnet/datasets/ dir=$1 diff --git a/tests/nightly/multi_lenet.py b/tests/nightly/multi_lenet.py index 1fb2dfa6377e..687588bacbe9 100644 --- a/tests/nightly/multi_lenet.py +++ b/tests/nightly/multi_lenet.py @@ -1,4 +1,22 @@ #!/usr/bin/env python + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # lenet with multiple gpus # # using different kvstore will get almost identical results diff --git a/tests/nightly/mxnet_keras_integration_tests/assertion_util.py b/tests/nightly/mxnet_keras_integration_tests/assertion_util.py index 1fad6a1eb326..eb3d3bd85fda 100644 --- a/tests/nightly/mxnet_keras_integration_tests/assertion_util.py +++ b/tests/nightly/mxnet_keras_integration_tests/assertion_util.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from nose.tools import assert_true diff --git a/tests/nightly/mxnet_keras_integration_tests/model_util.py b/tests/nightly/mxnet_keras_integration_tests/model_util.py index 9f73ab60b062..bb9d6374af8f 100644 --- a/tests/nightly/mxnet_keras_integration_tests/model_util.py +++ b/tests/nightly/mxnet_keras_integration_tests/model_util.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import os from keras import backend as K from keras.models import Model diff --git a/tests/nightly/mxnet_keras_integration_tests/profiler.py b/tests/nightly/mxnet_keras_integration_tests/profiler.py index 4b6446a9b8cc..b0d39e19aa00 100644 --- a/tests/nightly/mxnet_keras_integration_tests/profiler.py +++ b/tests/nightly/mxnet_keras_integration_tests/profiler.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import os import signal import time diff --git a/tests/nightly/mxnet_keras_integration_tests/test_mnist_mlp.py b/tests/nightly/mxnet_keras_integration_tests/test_mnist_mlp.py index 7a0c6298d736..89bd2805ce78 100644 --- a/tests/nightly/mxnet_keras_integration_tests/test_mnist_mlp.py +++ b/tests/nightly/mxnet_keras_integration_tests/test_mnist_mlp.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + ''' This code is forked from https://github.com/fchollet/keras/blob/master/examples/mnist_mlp.py and modified to use as MXNet-Keras integration testing for functionality and sanity performance diff --git a/tests/nightly/sh2ju.sh b/tests/nightly/sh2ju.sh index 84c1427227e7..4465cd8f843f 100644 --- a/tests/nightly/sh2ju.sh +++ b/tests/nightly/sh2ju.sh @@ -1,4 +1,22 @@ #!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + ### Copyright 2010 Manuel Carrasco Moñino. (manolo at apache.org) ### ### Licensed under the Apache License, Version 2.0. diff --git a/tests/nightly/test_all.sh b/tests/nightly/test_all.sh index 33c39f5f4bd1..32913c9f5f5b 100755 --- a/tests/nightly/test_all.sh +++ b/tests/nightly/test_all.sh @@ -1,5 +1,23 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + # setup export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH cd `pwd`/`dirname $0` diff --git a/tests/nightly/test_kvstore.py b/tests/nightly/test_kvstore.py index c954c1859d64..b39ec89cf728 100644 --- a/tests/nightly/test_kvstore.py +++ b/tests/nightly/test_kvstore.py @@ -1,4 +1,22 @@ #!/usr/bin/env python + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import sys sys.path.insert(0, "../../python/") import mxnet as mx diff --git a/tests/nightly/test_mxnet_keras_integration_cpu.sh b/tests/nightly/test_mxnet_keras_integration_cpu.sh index 25a1da4ddf46..95cc0d0760e2 100755 --- a/tests/nightly/test_mxnet_keras_integration_cpu.sh +++ b/tests/nightly/test_mxnet_keras_integration_cpu.sh @@ -1,4 +1,22 @@ #!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + set -e ### Build MXNet with CPU support echo "BUILD make" diff --git a/tests/nightly/test_mxnet_keras_integration_gpu.sh b/tests/nightly/test_mxnet_keras_integration_gpu.sh index 86fb37acfc47..5d541fa5b7a4 100755 --- a/tests/nightly/test_mxnet_keras_integration_gpu.sh +++ b/tests/nightly/test_mxnet_keras_integration_gpu.sh @@ -1,4 +1,22 @@ #!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + set -e ### Install git diff --git a/tests/nightly/test_tutorial.py b/tests/nightly/test_tutorial.py index 304642c73a23..56b530a59bb4 100644 --- a/tests/nightly/test_tutorial.py +++ b/tests/nightly/test_tutorial.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + #pylint: disable=no-member, too-many-locals, too-many-branches, no-self-use, broad-except, lost-exception, too-many-nested-blocks, too-few-public-methods, invalid-name """ This script converts all python tutorials into python script diff --git a/tests/python/common/get_data.py b/tests/python/common/get_data.py index e385a7186848..35482f8de584 100644 --- a/tests/python/common/get_data.py +++ b/tests/python/common/get_data.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: skip-file import os, gzip import pickle as pickle diff --git a/tests/python/common/models.py b/tests/python/common/models.py index 2c998afcd1db..b563adc1d760 100644 --- a/tests/python/common/models.py +++ b/tests/python/common/models.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """This file defines various models used in the test""" import mxnet as mx diff --git a/tests/python/doctest/test_docstring.py b/tests/python/doctest/test_docstring.py index e457e7b9ca55..23a29588c5af 100644 --- a/tests/python/doctest/test_docstring.py +++ b/tests/python/doctest/test_docstring.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import doctest import logging import mxnet diff --git a/tests/python/gpu/test_forward.py b/tests/python/gpu/test_forward.py index dc2c129f5326..cddf9afb9cb2 100644 --- a/tests/python/gpu/test_forward.py +++ b/tests/python/gpu/test_forward.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import os import numpy as np import mxnet as mx diff --git a/tests/python/gpu/test_operator_gpu.py b/tests/python/gpu/test_operator_gpu.py index 6fef4e24a24c..c80b9e375306 100644 --- a/tests/python/gpu/test_operator_gpu.py +++ b/tests/python/gpu/test_operator_gpu.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import sys import os import time diff --git a/tests/python/gpu/test_rtc.py b/tests/python/gpu/test_rtc.py index d38f038ffd26..756c3d752faa 100644 --- a/tests/python/gpu/test_rtc.py +++ b/tests/python/gpu/test_rtc.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: skip-file import mxnet as mx import numpy as np @@ -13,4 +30,4 @@ s_rec[threadIdx.x] = x[threadIdx.x]; y[threadIdx.x] = expf(s_rec[threadIdx.x]*5.0);""") rtc.push([x], [y], (1, 1, 1), (10,1,1)) - assert_allclose(y.asnumpy(), np.exp(x.asnumpy()*5.0)) \ No newline at end of file + assert_allclose(y.asnumpy(), np.exp(x.asnumpy()*5.0)) diff --git a/tests/python/predict/mxnet_predict_example.py b/tests/python/predict/mxnet_predict_example.py index 745a1f87b17c..1db3f5c29954 100644 --- a/tests/python/predict/mxnet_predict_example.py +++ b/tests/python/predict/mxnet_predict_example.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import sys, os curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__))) sys.path.append("../../../amalgamation/python/") diff --git a/tests/python/train/common.py b/tests/python/train/common.py index 1622e0294e69..38718fa78175 100644 --- a/tests/python/train/common.py +++ b/tests/python/train/common.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import sys, os curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__))) sys.path.append(os.path.join(curr_path, '../common/')) diff --git a/tests/python/train/test_autograd.py b/tests/python/train/test_autograd.py index 8d67bfb8205a..c9921ecf4f89 100644 --- a/tests/python/train/test_autograd.py +++ b/tests/python/train/test_autograd.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: skip-file from __future__ import print_function diff --git a/tests/python/train/test_bucketing.py b/tests/python/train/test_bucketing.py index 85ea107c5ca2..1303db09cb54 100644 --- a/tests/python/train/test_bucketing.py +++ b/tests/python/train/test_bucketing.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: skip-file import numpy as np import mxnet as mx diff --git a/tests/python/train/test_conv.py b/tests/python/train/test_conv.py index 039790e5612d..46e06848f8ba 100644 --- a/tests/python/train/test_conv.py +++ b/tests/python/train/test_conv.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: skip-file import sys sys.path.insert(0, '../../python') diff --git a/tests/python/train/test_dtype.py b/tests/python/train/test_dtype.py index 3371f4bcaf4e..b0a524815c6c 100644 --- a/tests/python/train/test_dtype.py +++ b/tests/python/train/test_dtype.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: skip-file import sys sys.path.insert(0, '../../python') @@ -173,6 +190,6 @@ def test_cifar10(): (train, val) = get_iterator_uint8(kv) run_cifar10(train, val, use_module=False) run_cifar10(train, val, use_module=True) - + if __name__ == "__main__": test_cifar10() diff --git a/tests/python/train/test_mlp.py b/tests/python/train/test_mlp.py index c983b6eeac4f..a0a45b41e19e 100644 --- a/tests/python/train/test_mlp.py +++ b/tests/python/train/test_mlp.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: skip-file import mxnet as mx import numpy as np diff --git a/tests/python/unittest/common.py b/tests/python/unittest/common.py index 38b4bd6b0227..12ed60d2bc24 100644 --- a/tests/python/unittest/common.py +++ b/tests/python/unittest/common.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import sys, os curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__))) sys.path.append(os.path.join(curr_path, '../common/')) diff --git a/tests/python/unittest/test_attr.py b/tests/python/unittest/test_attr.py index 4cdecaf9146f..0d7e67dced2d 100644 --- a/tests/python/unittest/test_attr.py +++ b/tests/python/unittest/test_attr.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import os import mxnet as mx from common import models diff --git a/tests/python/unittest/test_autograd.py b/tests/python/unittest/test_autograd.py index 7ee35009a164..5be3d356e34b 100644 --- a/tests/python/unittest/test_autograd.py +++ b/tests/python/unittest/test_autograd.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import functools import mxnet.ndarray as nd from mxnet.ndarray import zeros_like diff --git a/tests/python/unittest/test_contrib_autograd.py b/tests/python/unittest/test_contrib_autograd.py index e7b0ce3af752..a144c3433280 100644 --- a/tests/python/unittest/test_contrib_autograd.py +++ b/tests/python/unittest/test_contrib_autograd.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet.ndarray as nd from mxnet.contrib.autograd import * from mxnet.test_utils import * diff --git a/tests/python/unittest/test_executor.py b/tests/python/unittest/test_executor.py index c1cc013b81c0..e3d977df65de 100644 --- a/tests/python/unittest/test_executor.py +++ b/tests/python/unittest/test_executor.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import numpy as np import mxnet as mx diff --git a/tests/python/unittest/test_gluon.py b/tests/python/unittest/test_gluon.py index 8256c719c386..4fff23f2c5a4 100644 --- a/tests/python/unittest/test_gluon.py +++ b/tests/python/unittest/test_gluon.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx from mxnet import gluon from mxnet.gluon import nn diff --git a/tests/python/unittest/test_gluon_data.py b/tests/python/unittest/test_gluon_data.py index 2407f8e25991..da1de6ba4b58 100644 --- a/tests/python/unittest/test_gluon_data.py +++ b/tests/python/unittest/test_gluon_data.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import os import mxnet as mx import numpy as np diff --git a/tests/python/unittest/test_gluon_model_zoo.py b/tests/python/unittest/test_gluon_model_zoo.py index db26fd4a96ab..6fbcf8b3dac8 100644 --- a/tests/python/unittest/test_gluon_model_zoo.py +++ b/tests/python/unittest/test_gluon_model_zoo.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from __future__ import print_function import mxnet as mx from mxnet.gluon import nn diff --git a/tests/python/unittest/test_gluon_rnn.py b/tests/python/unittest/test_gluon_rnn.py index c5c941408fef..ac671e5f8840 100644 --- a/tests/python/unittest/test_gluon_rnn.py +++ b/tests/python/unittest/test_gluon_rnn.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx from mxnet import gluon import numpy as np diff --git a/tests/python/unittest/test_image.py b/tests/python/unittest/test_image.py index 638dbf06a683..04b878dc80b0 100644 --- a/tests/python/unittest/test_image.py +++ b/tests/python/unittest/test_image.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx import numpy as np from mxnet.test_utils import * diff --git a/tests/python/unittest/test_infer_shape.py b/tests/python/unittest/test_infer_shape.py index 35598bc55be8..d7f52e216659 100644 --- a/tests/python/unittest/test_infer_shape.py +++ b/tests/python/unittest/test_infer_shape.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: skip-file import mxnet as mx from common import models diff --git a/tests/python/unittest/test_init.py b/tests/python/unittest/test_init.py index 79862269795d..e642e65ec3d6 100644 --- a/tests/python/unittest/test_init.py +++ b/tests/python/unittest/test_init.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx import numpy as np diff --git a/tests/python/unittest/test_io.py b/tests/python/unittest/test_io.py index 18326754c851..c0f2acd4ed47 100644 --- a/tests/python/unittest/test_io.py +++ b/tests/python/unittest/test_io.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: skip-file import mxnet as mx import numpy as np diff --git a/tests/python/unittest/test_kvstore.py b/tests/python/unittest/test_kvstore.py index 87e5e0027241..f1e10c757fad 100644 --- a/tests/python/unittest/test_kvstore.py +++ b/tests/python/unittest/test_kvstore.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: skip-file import mxnet as mx import numpy as np diff --git a/tests/python/unittest/test_loss.py b/tests/python/unittest/test_loss.py index 7d4c586c2aa8..8eced7bc00d1 100644 --- a/tests/python/unittest/test_loss.py +++ b/tests/python/unittest/test_loss.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx import numpy as np from mxnet import gluon diff --git a/tests/python/unittest/test_metric.py b/tests/python/unittest/test_metric.py index 54b58b2b4d73..7ae93bf36299 100644 --- a/tests/python/unittest/test_metric.py +++ b/tests/python/unittest/test_metric.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx import json diff --git a/tests/python/unittest/test_model_parallel.py b/tests/python/unittest/test_model_parallel.py index 96990e72075e..8ff09d5fcb56 100644 --- a/tests/python/unittest/test_model_parallel.py +++ b/tests/python/unittest/test_model_parallel.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import numpy as np import mxnet as mx diff --git a/tests/python/unittest/test_module.py b/tests/python/unittest/test_module.py index 766995dd2ac9..f522f29dae39 100644 --- a/tests/python/unittest/test_module.py +++ b/tests/python/unittest/test_module.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx import mxnet.ndarray as nd import numpy as np diff --git a/tests/python/unittest/test_multi_device_exec.py b/tests/python/unittest/test_multi_device_exec.py index 8956c4edebac..6f8eb17ff34e 100644 --- a/tests/python/unittest/test_multi_device_exec.py +++ b/tests/python/unittest/test_multi_device_exec.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import os import mxnet as mx diff --git a/tests/python/unittest/test_ndarray.py b/tests/python/unittest/test_ndarray.py index 79a022b17ac4..eae364eeaecf 100644 --- a/tests/python/unittest/test_ndarray.py +++ b/tests/python/unittest/test_ndarray.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import os import mxnet as mx import numpy as np diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py index 62a064a49e0d..718e3df04871 100644 --- a/tests/python/unittest/test_operator.py +++ b/tests/python/unittest/test_operator.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: skip-file from __future__ import print_function import numpy as np diff --git a/tests/python/unittest/test_optimizer.py b/tests/python/unittest/test_optimizer.py index cf7b82eaaa88..3b3b92b372d8 100644 --- a/tests/python/unittest/test_optimizer.py +++ b/tests/python/unittest/test_optimizer.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import numpy as np import mxnet as mx import math diff --git a/tests/python/unittest/test_profiler.py b/tests/python/unittest/test_profiler.py index 9a0deabdd9f8..724ed3a38790 100644 --- a/tests/python/unittest/test_profiler.py +++ b/tests/python/unittest/test_profiler.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from __future__ import print_function import mxnet as mx from mxnet import profiler diff --git a/tests/python/unittest/test_random.py b/tests/python/unittest/test_random.py index 1f88b6b4fbe0..6b8311c145f5 100644 --- a/tests/python/unittest/test_random.py +++ b/tests/python/unittest/test_random.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import os import mxnet as mx import numpy as np diff --git a/tests/python/unittest/test_recordio.py b/tests/python/unittest/test_recordio.py index f4489bdfe641..7de582e236dd 100644 --- a/tests/python/unittest/test_recordio.py +++ b/tests/python/unittest/test_recordio.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: skip-file import sys import mxnet as mx @@ -68,4 +85,4 @@ def test_recordio_pack_label(): if __name__ == '__main__': test_recordio_pack_label() test_recordio() - test_indexed_recordio() \ No newline at end of file + test_indexed_recordio() diff --git a/tests/python/unittest/test_rnn.py b/tests/python/unittest/test_rnn.py index e8176bb468c2..9fe22ae72df6 100644 --- a/tests/python/unittest/test_rnn.py +++ b/tests/python/unittest/test_rnn.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx import numpy as np from numpy.testing import assert_allclose diff --git a/tests/python/unittest/test_symbol.py b/tests/python/unittest/test_symbol.py index ee9e9dcf3a55..c570325a6b66 100644 --- a/tests/python/unittest/test_symbol.py +++ b/tests/python/unittest/test_symbol.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import copy import os import re diff --git a/tests/python/unittest/test_viz.py b/tests/python/unittest/test_viz.py index 79c86681035d..73cfa94ba030 100644 --- a/tests/python/unittest/test_viz.py +++ b/tests/python/unittest/test_viz.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import mxnet as mx def test_print_summary(): diff --git a/tests/travis/is_core_changed.sh b/tests/travis/is_core_changed.sh index 1e32d60b2fcc..7b9eb6123847 100755 --- a/tests/travis/is_core_changed.sh +++ b/tests/travis/is_core_changed.sh @@ -1,5 +1,23 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + # this is a util script to test whether the "core" of # mxnet has changed. Please modify the regex patterns here # to ensure the components are covered if you add new "core" diff --git a/tests/travis/run_test.sh b/tests/travis/run_test.sh index cff4196b6043..fb1869f842b1 100755 --- a/tests/travis/run_test.sh +++ b/tests/travis/run_test.sh @@ -1,5 +1,23 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + if ! tests/travis/is_core_changed.sh then exit 0 diff --git a/tests/travis/setup.sh b/tests/travis/setup.sh index ec071009bda5..94d674f3943e 100755 --- a/tests/travis/setup.sh +++ b/tests/travis/setup.sh @@ -1,5 +1,23 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + if ! tests/travis/is_core_changed.sh then exit 0 diff --git a/tests/travis/travis_after_failure.sh b/tests/travis/travis_after_failure.sh index 5a3940a89b7a..50754c9546cd 100755 --- a/tests/travis/travis_after_failure.sh +++ b/tests/travis/travis_after_failure.sh @@ -1,5 +1,23 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + if [ ${TASK} == "r_test" ]; then echo "Print the install log..." cat mxnet.Rcheck/*.out diff --git a/tools/accnn/acc_conv.py b/tools/accnn/acc_conv.py index 095e386beebc..07717c7b47c9 100644 --- a/tools/accnn/acc_conv.py +++ b/tools/accnn/acc_conv.py @@ -1,77 +1,94 @@ -import numpy as np -from scipy import linalg as LA -import mxnet as mx -import argparse -import utils - -def conv_vh_decomposition(model, args): - W = model.arg_params[args.layer+'_weight'].asnumpy() - N, C, y, x = W.shape - b = model.arg_params[args.layer+'_bias'].asnumpy() - W = W.transpose((1,2,0,3)).reshape((C*y, -1)) - - U, D, Q = np.linalg.svd(W, full_matrices=False) - sqrt_D = LA.sqrtm(np.diag(D)) - K = args.K - V = U[:,:K].dot(sqrt_D[:K, :K]) - H = Q.T[:,:K].dot(sqrt_D[:K, :K]) - V = V.T.reshape(K, C, y, 1) - b_1 = np.zeros((K, )) - H = H.reshape(N, x, 1, K).transpose((0,3,2,1)) - b_2 = b - - W1, b1, W2, b2 = V, b_1, H, b_2 - def sym_handle(data, node): - kernel = eval(node['param']['kernel']) - pad = eval(node['param']['pad']) - name = node['name'] - - name1 = name + '_v' - kernel1 = tuple((kernel[0], 1)) - pad1 = tuple((pad[0], 0)) - num_filter = W1.shape[0] - sym1 = mx.symbol.Convolution(data=data, kernel=kernel1, pad=pad1, num_filter=num_filter, name=name1) - - name2 = name + '_h' - kernel2 = tuple((1, kernel[1])) - pad2 = tuple((0, pad[1])) - num_filter = W2.shape[0] - sym2 = mx.symbol.Convolution(data=sym1, kernel=kernel2, pad=pad2, num_filter=num_filter, name=name2) - return sym2 - - def arg_handle(arg_shape_dic, arg_params): - name1 = args.layer + '_v' - name2 = args.layer + '_h' - weight1 = mx.ndarray.array(W1) - bias1 = mx.ndarray.array(b1) - weight2 = mx.ndarray.array(W2) - bias2 = mx.ndarray.array(b2) - assert weight1.shape == arg_shape_dic[name1+'_weight'], 'weight1' - assert weight2.shape == arg_shape_dic[name2+'_weight'], 'weight2' - assert bias1.shape == arg_shape_dic[name1+'_bias'], 'bias1' - assert bias2.shape == arg_shape_dic[name2+'_bias'], 'bias2' - - arg_params[name1 + '_weight'] = weight1 - arg_params[name1 + '_bias'] = bias1 - arg_params[name2 + '_weight'] = weight2 - arg_params[name2 + '_bias'] = bias2 - - new_model = utils.replace_conv_layer(args.layer, model, sym_handle, arg_handle) - return new_model - -def main(): - model = utils.load_model(args) - new_model = conv_vh_decomposition(model, args) - new_model.save(args.save_model) - -if __name__ == '__main__': - parser=argparse.ArgumentParser() - parser.add_argument('-m', '--model', help='the model to speed up') - parser.add_argument('-g', '--gpus', default='0', help='the gpus to be used in ctx') - parser.add_argument('--load-epoch',type=int,default=1) - parser.add_argument('--layer') - parser.add_argument('--K', type=int) - parser.add_argument('--save-model') - args = parser.parse_args() - main() - +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import numpy as np +from scipy import linalg as LA +import mxnet as mx +import argparse +import utils + +def conv_vh_decomposition(model, args): + W = model.arg_params[args.layer+'_weight'].asnumpy() + N, C, y, x = W.shape + b = model.arg_params[args.layer+'_bias'].asnumpy() + W = W.transpose((1,2,0,3)).reshape((C*y, -1)) + + U, D, Q = np.linalg.svd(W, full_matrices=False) + sqrt_D = LA.sqrtm(np.diag(D)) + K = args.K + V = U[:,:K].dot(sqrt_D[:K, :K]) + H = Q.T[:,:K].dot(sqrt_D[:K, :K]) + V = V.T.reshape(K, C, y, 1) + b_1 = np.zeros((K, )) + H = H.reshape(N, x, 1, K).transpose((0,3,2,1)) + b_2 = b + + W1, b1, W2, b2 = V, b_1, H, b_2 + def sym_handle(data, node): + kernel = eval(node['param']['kernel']) + pad = eval(node['param']['pad']) + name = node['name'] + + name1 = name + '_v' + kernel1 = tuple((kernel[0], 1)) + pad1 = tuple((pad[0], 0)) + num_filter = W1.shape[0] + sym1 = mx.symbol.Convolution(data=data, kernel=kernel1, pad=pad1, num_filter=num_filter, name=name1) + + name2 = name + '_h' + kernel2 = tuple((1, kernel[1])) + pad2 = tuple((0, pad[1])) + num_filter = W2.shape[0] + sym2 = mx.symbol.Convolution(data=sym1, kernel=kernel2, pad=pad2, num_filter=num_filter, name=name2) + return sym2 + + def arg_handle(arg_shape_dic, arg_params): + name1 = args.layer + '_v' + name2 = args.layer + '_h' + weight1 = mx.ndarray.array(W1) + bias1 = mx.ndarray.array(b1) + weight2 = mx.ndarray.array(W2) + bias2 = mx.ndarray.array(b2) + assert weight1.shape == arg_shape_dic[name1+'_weight'], 'weight1' + assert weight2.shape == arg_shape_dic[name2+'_weight'], 'weight2' + assert bias1.shape == arg_shape_dic[name1+'_bias'], 'bias1' + assert bias2.shape == arg_shape_dic[name2+'_bias'], 'bias2' + + arg_params[name1 + '_weight'] = weight1 + arg_params[name1 + '_bias'] = bias1 + arg_params[name2 + '_weight'] = weight2 + arg_params[name2 + '_bias'] = bias2 + + new_model = utils.replace_conv_layer(args.layer, model, sym_handle, arg_handle) + return new_model + +def main(): + model = utils.load_model(args) + new_model = conv_vh_decomposition(model, args) + new_model.save(args.save_model) + +if __name__ == '__main__': + parser=argparse.ArgumentParser() + parser.add_argument('-m', '--model', help='the model to speed up') + parser.add_argument('-g', '--gpus', default='0', help='the gpus to be used in ctx') + parser.add_argument('--load-epoch',type=int,default=1) + parser.add_argument('--layer') + parser.add_argument('--K', type=int) + parser.add_argument('--save-model') + args = parser.parse_args() + main() + diff --git a/tools/accnn/acc_fc.py b/tools/accnn/acc_fc.py index dcc255452b1d..b66b328e5b7b 100644 --- a/tools/accnn/acc_fc.py +++ b/tools/accnn/acc_fc.py @@ -1,57 +1,74 @@ -import numpy as np -from scipy import linalg as LA -import mxnet as mx -import argparse -import utils -import pdb - -def fc_decomposition(model, args): - W = model.arg_params[args.layer+'_weight'].asnumpy() - b = model.arg_params[args.layer+'_bias'].asnumpy() - W = W.reshape((W.shape[0],-1)) - b = b.reshape((b.shape[0],-1)) - u, s, v = LA.svd(W, full_matrices=False) - s = np.diag(s) - t = u.dot(s.dot(v)) - rk = args.K - P = u[:,:rk] - Q = s[:rk,:rk].dot(v[:rk,:]) - - name1 = args.layer + '_red' - name2 = args.layer + '_rec' - def sym_handle(data, node): - W1, W2 = Q, P - sym1 = mx.symbol.FullyConnected(data=data, num_hidden=W1.shape[0], no_bias=True, name=name1) - sym2 = mx.symbol.FullyConnected(data=sym1, num_hidden=W2.shape[0], no_bias=False, name=name2) - return sym2 - - def arg_handle(arg_shape_dic, arg_params): - W1, W2 = Q, P - W1 = W1.reshape(arg_shape_dic[name1+'_weight']) - weight1 = mx.ndarray.array(W1) - W2 = W2.reshape(arg_shape_dic[name2+'_weight']) - b2 = b.reshape(arg_shape_dic[name2+'_bias']) - weight2 = mx.ndarray.array(W2) - bias2 = mx.ndarray.array(b2) - arg_params[name1 + '_weight'] = weight1 - arg_params[name2 + '_weight'] = weight2 - arg_params[name2 + '_bias'] = bias2 - - new_model = utils.replace_conv_layer(args.layer, model, sym_handle, arg_handle) - return new_model - -def main(): - model = utils.load_model(args) - new_model = fc_decomposition(model, args) - new_model.save(args.save_model) - -if __name__ == '__main__': - parser=argparse.ArgumentParser() - parser.add_argument('-m', '--model', help='the model to speed up') - parser.add_argument('-g', '--gpus', default='0', help='the gpus to be used in ctx') - parser.add_argument('--load-epoch',type=int,default=1) - parser.add_argument('--layer') - parser.add_argument('--K', type=int) - parser.add_argument('--save-model') - args = parser.parse_args() - main() +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import numpy as np +from scipy import linalg as LA +import mxnet as mx +import argparse +import utils +import pdb + +def fc_decomposition(model, args): + W = model.arg_params[args.layer+'_weight'].asnumpy() + b = model.arg_params[args.layer+'_bias'].asnumpy() + W = W.reshape((W.shape[0],-1)) + b = b.reshape((b.shape[0],-1)) + u, s, v = LA.svd(W, full_matrices=False) + s = np.diag(s) + t = u.dot(s.dot(v)) + rk = args.K + P = u[:,:rk] + Q = s[:rk,:rk].dot(v[:rk,:]) + + name1 = args.layer + '_red' + name2 = args.layer + '_rec' + def sym_handle(data, node): + W1, W2 = Q, P + sym1 = mx.symbol.FullyConnected(data=data, num_hidden=W1.shape[0], no_bias=True, name=name1) + sym2 = mx.symbol.FullyConnected(data=sym1, num_hidden=W2.shape[0], no_bias=False, name=name2) + return sym2 + + def arg_handle(arg_shape_dic, arg_params): + W1, W2 = Q, P + W1 = W1.reshape(arg_shape_dic[name1+'_weight']) + weight1 = mx.ndarray.array(W1) + W2 = W2.reshape(arg_shape_dic[name2+'_weight']) + b2 = b.reshape(arg_shape_dic[name2+'_bias']) + weight2 = mx.ndarray.array(W2) + bias2 = mx.ndarray.array(b2) + arg_params[name1 + '_weight'] = weight1 + arg_params[name2 + '_weight'] = weight2 + arg_params[name2 + '_bias'] = bias2 + + new_model = utils.replace_conv_layer(args.layer, model, sym_handle, arg_handle) + return new_model + +def main(): + model = utils.load_model(args) + new_model = fc_decomposition(model, args) + new_model.save(args.save_model) + +if __name__ == '__main__': + parser=argparse.ArgumentParser() + parser.add_argument('-m', '--model', help='the model to speed up') + parser.add_argument('-g', '--gpus', default='0', help='the gpus to be used in ctx') + parser.add_argument('--load-epoch',type=int,default=1) + parser.add_argument('--layer') + parser.add_argument('--K', type=int) + parser.add_argument('--save-model') + args = parser.parse_args() + main() diff --git a/tools/accnn/accnn.py b/tools/accnn/accnn.py index 1af78ef880de..ec5b101838f7 100644 --- a/tools/accnn/accnn.py +++ b/tools/accnn/accnn.py @@ -1,38 +1,55 @@ -import mxnet as mx -import argparse -import utils -import acc_conv -import acc_fc -import rank_selection -import collections -import json -import sys - -parser = argparse.ArgumentParser() -parser.add_argument('-m', '--model', help='the model to speed up') -parser.add_argument('-g', '--gpus', default='0', help='the gpus will be used, e.g "0,1,2,3"') -parser.add_argument('--load-epoch',type=int, default=1, help="load the model on an epoch using the model-prefix") -parser.add_argument('--save-model', type=str, default='new-model', help='output model prefix') -parser.add_argument('--config', default=None, help='specify the config file') -parser.add_argument('--ratio', type=float, default=2, help='speed up ratio') -args = parser.parse_args() - -model = utils.load_model(args) -if args.config: - args.config = json.load(open(args.config, 'r')) -else: - config = {} - config['conv_params'] = rank_selection.get_ranksel(model, args.ratio) - config['fc_params'] = {} - json.dump(config, open('config-rksel-%.1f.json'%(args.ratio), 'w'), indent=2) - args.config = config - -new_model = model -Args = collections.namedtuple('ConvArgs', 'layer K') -for layer, K in args.config['conv_params'].items(): - arg = Args(layer=layer, K=K) - new_model = acc_conv.conv_vh_decomposition(new_model, arg) -for layer, K in args.config['fc_params'].items(): - arg = Args(layer=layer, K=K) - new_model = acc_fc.fc_decomposition(new_model, arg) -new_model.save(args.save_model, 1) +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import mxnet as mx +import argparse +import utils +import acc_conv +import acc_fc +import rank_selection +import collections +import json +import sys + +parser = argparse.ArgumentParser() +parser.add_argument('-m', '--model', help='the model to speed up') +parser.add_argument('-g', '--gpus', default='0', help='the gpus will be used, e.g "0,1,2,3"') +parser.add_argument('--load-epoch',type=int, default=1, help="load the model on an epoch using the model-prefix") +parser.add_argument('--save-model', type=str, default='new-model', help='output model prefix') +parser.add_argument('--config', default=None, help='specify the config file') +parser.add_argument('--ratio', type=float, default=2, help='speed up ratio') +args = parser.parse_args() + +model = utils.load_model(args) +if args.config: + args.config = json.load(open(args.config, 'r')) +else: + config = {} + config['conv_params'] = rank_selection.get_ranksel(model, args.ratio) + config['fc_params'] = {} + json.dump(config, open('config-rksel-%.1f.json'%(args.ratio), 'w'), indent=2) + args.config = config + +new_model = model +Args = collections.namedtuple('ConvArgs', 'layer K') +for layer, K in args.config['conv_params'].items(): + arg = Args(layer=layer, K=K) + new_model = acc_conv.conv_vh_decomposition(new_model, arg) +for layer, K in args.config['fc_params'].items(): + arg = Args(layer=layer, K=K) + new_model = acc_fc.fc_decomposition(new_model, arg) +new_model.save(args.save_model, 1) diff --git a/tools/accnn/rank_selection.py b/tools/accnn/rank_selection.py index ee3eca91f934..66937b2859b9 100644 --- a/tools/accnn/rank_selection.py +++ b/tools/accnn/rank_selection.py @@ -1,87 +1,104 @@ -import numpy as np -import mxnet as mx -import json -import utils -import math -import sys - -def calc_complexity(ishape, node): - y, x = map(int, eval(node['param']['kernel'])) - N = int(node['param']['num_filter']) - C, Y, X = ishape - return x*(N+C)*X*Y, x*y*N*C*X*Y - -def calc_eigenvalue(model, node): - W = model.arg_params[node['name'] + '_weight'].asnumpy() - N, C, y, x = W.shape - W = W.transpose((1,2,0,3)).reshape((C*y, -1)) - U, D, Q = np.linalg.svd(W, full_matrices=False) - return D - -def get_ranksel(model, ratio): - conf = json.loads(model.symbol.tojson()) - _, output_shapes, _ = model.symbol.get_internals().infer_shape(data=(1,3,224,224)) - out_names = model.symbol.get_internals().list_outputs() - out_shape_dic = dict(zip(out_names, output_shapes)) - nodes = conf['nodes'] - nodes = utils.topsort(nodes) - C = [] - D = [] - S = [] - conv_names = [] - EC = 0 - for node in nodes: - if node['op'] == 'Convolution': - input_nodes = [nodes[int(j[0])] for j in node['inputs']] - data = [input_node for input_node in input_nodes\ - if not input_node['name'].startswith(node['name'])][0] - - if utils.is_input(data): - ishape = (3, 224, 224) - else: - ishape = out_shape_dic[data['name'] + '_output'][1:] - C.append(calc_complexity(ishape, node)) - D.append(int(node['param']['num_filter'])) - S.append(calc_eigenvalue(model, node)) - conv_names.append(node['name']) - EC += C[-1][1] - for s in S: - ss = sum(s) - for i in xrange(1, len(s)): - s[i] += s[i-1] - n = len(C) - EC /= ratio - dp = [{}, {}] - dpc = [{} for _ in xrange(n)] - now, nxt = 0, 1 - dp[now][0] = 0 - for i in xrange(n): - dp[nxt] = {} - sys.stdout.flush() - for now_c, now_v in dp[now].items(): - for d in xrange(min(len(S[i]), D[i])): - nxt_c = now_c + (d+1)*C[i][0] - if nxt_c > EC: - continue - nxt_v = dp[now][now_c] + math.log(S[i][d]) - if dp[nxt].has_key(nxt_c): - if nxt_v > dp[nxt][nxt_c]: - dp[nxt][nxt_c] = nxt_v - dpc[i][nxt_c] = (d,now_c) - else: - dp[nxt][nxt_c] = nxt_v - dpc[i][nxt_c] = (d,now_c) - now, nxt = nxt, now - maxv = -1e9 - target_c = 0 - for c,v in dp[now].items(): - assert c <= EC, 'False' - if v > maxv: - maxv = v - target_c = c - res = [0]*n - nowc = target_c - for i in xrange(n-1,-1,-1): - res[i] = dpc[i][nowc][0] + 1 - nowc = dpc[i][nowc][1] - return dict(zip(conv_names, res)) +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import numpy as np +import mxnet as mx +import json +import utils +import math +import sys + +def calc_complexity(ishape, node): + y, x = map(int, eval(node['param']['kernel'])) + N = int(node['param']['num_filter']) + C, Y, X = ishape + return x*(N+C)*X*Y, x*y*N*C*X*Y + +def calc_eigenvalue(model, node): + W = model.arg_params[node['name'] + '_weight'].asnumpy() + N, C, y, x = W.shape + W = W.transpose((1,2,0,3)).reshape((C*y, -1)) + U, D, Q = np.linalg.svd(W, full_matrices=False) + return D + +def get_ranksel(model, ratio): + conf = json.loads(model.symbol.tojson()) + _, output_shapes, _ = model.symbol.get_internals().infer_shape(data=(1,3,224,224)) + out_names = model.symbol.get_internals().list_outputs() + out_shape_dic = dict(zip(out_names, output_shapes)) + nodes = conf['nodes'] + nodes = utils.topsort(nodes) + C = [] + D = [] + S = [] + conv_names = [] + EC = 0 + for node in nodes: + if node['op'] == 'Convolution': + input_nodes = [nodes[int(j[0])] for j in node['inputs']] + data = [input_node for input_node in input_nodes\ + if not input_node['name'].startswith(node['name'])][0] + + if utils.is_input(data): + ishape = (3, 224, 224) + else: + ishape = out_shape_dic[data['name'] + '_output'][1:] + C.append(calc_complexity(ishape, node)) + D.append(int(node['param']['num_filter'])) + S.append(calc_eigenvalue(model, node)) + conv_names.append(node['name']) + EC += C[-1][1] + for s in S: + ss = sum(s) + for i in xrange(1, len(s)): + s[i] += s[i-1] + n = len(C) + EC /= ratio + dp = [{}, {}] + dpc = [{} for _ in xrange(n)] + now, nxt = 0, 1 + dp[now][0] = 0 + for i in xrange(n): + dp[nxt] = {} + sys.stdout.flush() + for now_c, now_v in dp[now].items(): + for d in xrange(min(len(S[i]), D[i])): + nxt_c = now_c + (d+1)*C[i][0] + if nxt_c > EC: + continue + nxt_v = dp[now][now_c] + math.log(S[i][d]) + if dp[nxt].has_key(nxt_c): + if nxt_v > dp[nxt][nxt_c]: + dp[nxt][nxt_c] = nxt_v + dpc[i][nxt_c] = (d,now_c) + else: + dp[nxt][nxt_c] = nxt_v + dpc[i][nxt_c] = (d,now_c) + now, nxt = nxt, now + maxv = -1e9 + target_c = 0 + for c,v in dp[now].items(): + assert c <= EC, 'False' + if v > maxv: + maxv = v + target_c = c + res = [0]*n + nowc = target_c + for i in xrange(n-1,-1,-1): + res[i] = dpc[i][nowc][0] + 1 + nowc = dpc[i][nowc][1] + return dict(zip(conv_names, res)) diff --git a/tools/accnn/utils.py b/tools/accnn/utils.py index 4c0290a0643a..25fb18895620 100644 --- a/tools/accnn/utils.py +++ b/tools/accnn/utils.py @@ -1,101 +1,118 @@ -import mxnet as mx -import copy -import json -import ast - -def load_model(args): - devs = mx.cpu() if args.gpus == None else [mx.gpu(int(i)) for i in args.gpus.split(',')] - return mx.model.FeedForward.load(args.model, args.load_epoch, ctx=devs) - -def topsort(nodes): - n = len(nodes) - deg = [0]*n - g = [[] for _ in xrange(n)] - for i,node in enumerate(nodes): - if node.has_key('inputs'): - for j in node['inputs']: - deg[i] += 1 - g[j[0]].append(i) - from collections import deque - q = deque([i for i in xrange(n) if deg[i]==0]) - res = [] - for its in xrange(n): - i = q.popleft() - res.append(nodes[i]) - for j in g[i]: - deg[j] -= 1 - if deg[j] == 0: - q.append(j) - new_ids=dict([(node['name'],i) for i,node in enumerate(res)]) - for node in res: - if node.has_key('inputs'): - for j in node['inputs']: - j[0]=new_ids[nodes[j[0]]['name']] - return res - -def is_input(node): - name = node['name'] - return len(node['inputs']) == 0 and ('weight' not in name) and ('bias' not in name) and ('label' not in name) - -def sym_factory(node, data): - name = node['name'] - params = {} - if 'param' in node: - for k, v in node['param'].items(): - try: - params[k] = ast.literal_eval(v) - except ValueError, e: - params[k] = v - return getattr(mx.symbol, node['op'])(data=data, name=name, **params) - -def replace_conv_layer(layer_name, old_model, sym_handle, arg_handle): - conf = json.loads(old_model.symbol.tojson()) - sym_dict = {} - nodes = conf['nodes'] - nodes = topsort(nodes) - res_sym = None - new_model = old_model - for i,node in enumerate(nodes): - sym = None - if is_input(node): - sym = mx.symbol.Variable(name='data') - elif node['op'] != 'null': - input_nodes = [nodes[int(j[0])] for j in node['inputs']] - datas = [input_node['name'] for input_node in input_nodes\ - if not input_node['name'].startswith(node['name'])] - try: - data=sym_dict[datas[0]] - except Exception, e: - print 'can not find symbol %s'%(datas[0]) - raise e - if node['name'] == layer_name: - sym = sym_handle(data, node) - else: - sym = sym_factory(node, data) - if sym: - sym_dict[node['name']] = sym - res_sym = sym - - arg_params = copy.deepcopy(old_model.arg_params) - if layer_name: - arg_shapes, _, _ = res_sym.infer_shape(data=(1,3,224,224)) - arg_names = res_sym.list_arguments() - arg_shape_dic = dict(zip(arg_names, arg_shapes)) - try: - arg_handle(arg_shape_dic, arg_params) - except Exception, e: - raise Exception('Exception in arg_handle') - - new_model = mx.model.FeedForward( - symbol=res_sym, - ctx=old_model.ctx, - num_epoch=1, - epoch_size=old_model.epoch_size, - optimizer='sgd', - initializer=old_model.initializer, - numpy_batch_size=old_model.numpy_batch_size, - arg_params=arg_params, - aux_params=old_model.aux_params, - allow_extra_params=True, - begin_epoch=old_model.begin_epoch) - return new_model +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import mxnet as mx +import copy +import json +import ast + +def load_model(args): + devs = mx.cpu() if args.gpus == None else [mx.gpu(int(i)) for i in args.gpus.split(',')] + return mx.model.FeedForward.load(args.model, args.load_epoch, ctx=devs) + +def topsort(nodes): + n = len(nodes) + deg = [0]*n + g = [[] for _ in xrange(n)] + for i,node in enumerate(nodes): + if node.has_key('inputs'): + for j in node['inputs']: + deg[i] += 1 + g[j[0]].append(i) + from collections import deque + q = deque([i for i in xrange(n) if deg[i]==0]) + res = [] + for its in xrange(n): + i = q.popleft() + res.append(nodes[i]) + for j in g[i]: + deg[j] -= 1 + if deg[j] == 0: + q.append(j) + new_ids=dict([(node['name'],i) for i,node in enumerate(res)]) + for node in res: + if node.has_key('inputs'): + for j in node['inputs']: + j[0]=new_ids[nodes[j[0]]['name']] + return res + +def is_input(node): + name = node['name'] + return len(node['inputs']) == 0 and ('weight' not in name) and ('bias' not in name) and ('label' not in name) + +def sym_factory(node, data): + name = node['name'] + params = {} + if 'param' in node: + for k, v in node['param'].items(): + try: + params[k] = ast.literal_eval(v) + except ValueError, e: + params[k] = v + return getattr(mx.symbol, node['op'])(data=data, name=name, **params) + +def replace_conv_layer(layer_name, old_model, sym_handle, arg_handle): + conf = json.loads(old_model.symbol.tojson()) + sym_dict = {} + nodes = conf['nodes'] + nodes = topsort(nodes) + res_sym = None + new_model = old_model + for i,node in enumerate(nodes): + sym = None + if is_input(node): + sym = mx.symbol.Variable(name='data') + elif node['op'] != 'null': + input_nodes = [nodes[int(j[0])] for j in node['inputs']] + datas = [input_node['name'] for input_node in input_nodes\ + if not input_node['name'].startswith(node['name'])] + try: + data=sym_dict[datas[0]] + except Exception, e: + print 'can not find symbol %s'%(datas[0]) + raise e + if node['name'] == layer_name: + sym = sym_handle(data, node) + else: + sym = sym_factory(node, data) + if sym: + sym_dict[node['name']] = sym + res_sym = sym + + arg_params = copy.deepcopy(old_model.arg_params) + if layer_name: + arg_shapes, _, _ = res_sym.infer_shape(data=(1,3,224,224)) + arg_names = res_sym.list_arguments() + arg_shape_dic = dict(zip(arg_names, arg_shapes)) + try: + arg_handle(arg_shape_dic, arg_params) + except Exception, e: + raise Exception('Exception in arg_handle') + + new_model = mx.model.FeedForward( + symbol=res_sym, + ctx=old_model.ctx, + num_epoch=1, + epoch_size=old_model.epoch_size, + optimizer='sgd', + initializer=old_model.initializer, + numpy_batch_size=old_model.numpy_batch_size, + arg_params=arg_params, + aux_params=old_model.aux_params, + allow_extra_params=True, + begin_epoch=old_model.begin_epoch) + return new_model diff --git a/tools/bandwidth/measure.py b/tools/bandwidth/measure.py index 0cac3de26731..66ef7371f11e 100644 --- a/tools/bandwidth/measure.py +++ b/tools/bandwidth/measure.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import os, sys curr_path = os.path.abspath(os.path.dirname(__file__)) sys.path.insert(0, os.path.join(curr_path, "../../python")) diff --git a/tools/bandwidth/test_measure.py b/tools/bandwidth/test_measure.py index b490af1cb75c..375290fe6853 100644 --- a/tools/bandwidth/test_measure.py +++ b/tools/bandwidth/test_measure.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ test measure.py """ diff --git a/tools/caffe_converter/caffe_parser.py b/tools/caffe_converter/caffe_parser.py index d4abc8eac614..2ff490c7c6aa 100644 --- a/tools/caffe_converter/caffe_parser.py +++ b/tools/caffe_converter/caffe_parser.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Parse caffe's protobuf """ import re diff --git a/tools/caffe_converter/caffe_proto_utils.py b/tools/caffe_converter/caffe_proto_utils.py index 940e8a47a1d6..8d6183457637 100644 --- a/tools/caffe_converter/caffe_proto_utils.py +++ b/tools/caffe_converter/caffe_proto_utils.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Helper functions for parsing caffe prototxt into a workable DAG """ diff --git a/tools/caffe_converter/compare_layers.py b/tools/caffe_converter/compare_layers.py index bb4451d1b5fc..12568ed2060a 100644 --- a/tools/caffe_converter/compare_layers.py +++ b/tools/caffe_converter/compare_layers.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Test converted models layer by layer """ import os diff --git a/tools/caffe_converter/convert_caffe_modelzoo.py b/tools/caffe_converter/convert_caffe_modelzoo.py index cb6d3ba7fc48..ab9042fcc532 100644 --- a/tools/caffe_converter/convert_caffe_modelzoo.py +++ b/tools/caffe_converter/convert_caffe_modelzoo.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Convert Caffe's modelzoo """ import os diff --git a/tools/caffe_converter/convert_mean.py b/tools/caffe_converter/convert_mean.py index 69cf50c65bd6..3b6dc42a7afc 100644 --- a/tools/caffe_converter/convert_mean.py +++ b/tools/caffe_converter/convert_mean.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Convert caffe mean """ import argparse diff --git a/tools/caffe_converter/convert_model.py b/tools/caffe_converter/convert_model.py index d1e4cd07c155..c04a2aa63cd2 100644 --- a/tools/caffe_converter/convert_model.py +++ b/tools/caffe_converter/convert_model.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Convert caffe model """ from __future__ import print_function diff --git a/tools/caffe_converter/convert_symbol.py b/tools/caffe_converter/convert_symbol.py index 100a64fe63c6..dde3c26b5ca2 100644 --- a/tools/caffe_converter/convert_symbol.py +++ b/tools/caffe_converter/convert_symbol.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Convert caffe prototxt to symbol """ from __future__ import print_function diff --git a/tools/caffe_converter/run.sh b/tools/caffe_converter/run.sh index 65876cc42934..bdf5481624d7 100755 --- a/tools/caffe_converter/run.sh +++ b/tools/caffe_converter/run.sh @@ -1,4 +1,22 @@ #!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + if [[ $# -ne 1 ]]; then echo "usage: $0 model_name" echo " model_name: [vgg16|vgg19], ..." diff --git a/tools/caffe_converter/test_converter.py b/tools/caffe_converter/test_converter.py index 7572d2937186..db17c64e54b7 100644 --- a/tools/caffe_converter/test_converter.py +++ b/tools/caffe_converter/test_converter.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Test converted models """ import os diff --git a/tools/im2rec.cc b/tools/im2rec.cc index a7ccfb667a12..856814024037 100644 --- a/tools/im2rec.cc +++ b/tools/im2rec.cc @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors * \file im2rec.cc * \brief convert images into image recordio format * Image Record Format: zeropad[64bit] imid[64bit] img-binary-content diff --git a/tools/im2rec.py b/tools/im2rec.py index 30ee3ec92a5e..ec6de1969414 100644 --- a/tools/im2rec.py +++ b/tools/im2rec.py @@ -1,307 +1,324 @@ -# -*- coding: utf-8 -*- -from __future__ import print_function -import os -import sys - -curr_path = os.path.abspath(os.path.dirname(__file__)) -sys.path.append(os.path.join(curr_path, "../python")) -import mxnet as mx -import random -import argparse -import cv2 -import time -import traceback -from builtins import range - -try: - import multiprocessing -except ImportError: - multiprocessing = None - -def list_image(root, recursive, exts): - i = 0 - if recursive: - cat = {} - for path, dirs, files in os.walk(root, followlinks=True): - dirs.sort() - files.sort() - for fname in files: - fpath = os.path.join(path, fname) - suffix = os.path.splitext(fname)[1].lower() - if os.path.isfile(fpath) and (suffix in exts): - if path not in cat: - cat[path] = len(cat) - yield (i, os.path.relpath(fpath, root), cat[path]) - i += 1 - for k, v in sorted(cat.items(), key=lambda x: x[1]): - print(os.path.relpath(k, root), v) - else: - for fname in sorted(os.listdir(root)): - fpath = os.path.join(root, fname) - suffix = os.path.splitext(fname)[1].lower() - if os.path.isfile(fpath) and (suffix in exts): - yield (i, os.path.relpath(fpath, root), 0) - i += 1 - -def write_list(path_out, image_list): - with open(path_out, 'w') as fout: - for i, item in enumerate(image_list): - line = '%d\t' % item[0] - for j in item[2:]: - line += '%f\t' % j - line += '%s\n' % item[1] - fout.write(line) - -def make_list(args): - image_list = list_image(args.root, args.recursive, args.exts) - image_list = list(image_list) - if args.shuffle is True: - random.seed(100) - random.shuffle(image_list) - N = len(image_list) - chunk_size = (N + args.chunks - 1) // args.chunks - for i in range(args.chunks): - chunk = image_list[i * chunk_size:(i + 1) * chunk_size] - if args.chunks > 1: - str_chunk = '_%d' % i - else: - str_chunk = '' - sep = int(chunk_size * args.train_ratio) - sep_test = int(chunk_size * args.test_ratio) - if args.train_ratio == 1.0: - write_list(args.prefix + str_chunk + '.lst', chunk) - else: - if args.test_ratio: - write_list(args.prefix + str_chunk + '_test.lst', chunk[:sep_test]) - if args.train_ratio + args.test_ratio < 1.0: - write_list(args.prefix + str_chunk + '_val.lst', chunk[sep_test + sep:]) - write_list(args.prefix + str_chunk + '_train.lst', chunk[sep_test:sep_test + sep]) - -def read_list(path_in): - with open(path_in) as fin: - while True: - line = fin.readline() - if not line: - break - line = [i.strip() for i in line.strip().split('\t')] - line_len = len(line) - if line_len < 3: - print('lst should at least has three parts, but only has %s parts for %s' %(line_len, line)) - continue - try: - item = [int(line[0])] + [line[-1]] + [float(i) for i in line[1:-1]] - except Exception as e: - print('Parsing lst met error for %s, detail: %s' %(line, e)) - continue - yield item - -def image_encode(args, i, item, q_out): - fullpath = os.path.join(args.root, item[1]) - - if len(item) > 3 and args.pack_label: - header = mx.recordio.IRHeader(0, item[2:], item[0], 0) - else: - header = mx.recordio.IRHeader(0, item[2], item[0], 0) - - if args.pass_through: - try: - with open(fullpath, 'rb') as fin: - img = fin.read() - s = mx.recordio.pack(header, img) - q_out.put((i, s, item)) - except Exception as e: - traceback.print_exc() - print('pack_img error:', item[1], e) - q_out.put((i, None, item)) - return - - try: - img = cv2.imread(fullpath, args.color) - except: - traceback.print_exc() - print('imread error trying to load file: %s ' % fullpath) - q_out.put((i, None, item)) - return - if img is None: - print('imread read blank (None) image for file: %s' % fullpath) - q_out.put((i, None, item)) - return - if args.center_crop: - if img.shape[0] > img.shape[1]: - margin = (img.shape[0] - img.shape[1]) // 2; - img = img[margin:margin + img.shape[1], :] - else: - margin = (img.shape[1] - img.shape[0]) // 2; - img = img[:, margin:margin + img.shape[0]] - if args.resize: - if img.shape[0] > img.shape[1]: - newsize = (args.resize, img.shape[0] * args.resize // img.shape[1]) - else: - newsize = (img.shape[1] * args.resize // img.shape[0], args.resize) - img = cv2.resize(img, newsize) - - try: - s = mx.recordio.pack_img(header, img, quality=args.quality, img_fmt=args.encoding) - q_out.put((i, s, item)) - except Exception as e: - traceback.print_exc() - print('pack_img error on file: %s' % fullpath, e) - q_out.put((i, None, item)) - return - -def read_worker(args, q_in, q_out): - while True: - deq = q_in.get() - if deq is None: - break - i, item = deq - image_encode(args, i, item, q_out) - -def write_worker(q_out, fname, working_dir): - pre_time = time.time() - count = 0 - fname = os.path.basename(fname) - fname_rec = os.path.splitext(fname)[0] + '.rec' - fname_idx = os.path.splitext(fname)[0] + '.idx' - record = mx.recordio.MXIndexedRecordIO(os.path.join(working_dir, fname_idx), - os.path.join(working_dir, fname_rec), 'w') - buf = {} - more = True - while more: - deq = q_out.get() - if deq is not None: - i, s, item = deq - buf[i] = (s, item) - else: - more = False - while count in buf: - s, item = buf[count] - del buf[count] - if s is not None: - record.write_idx(item[0], s) - - if count % 1000 == 0: - cur_time = time.time() - print('time:', cur_time - pre_time, ' count:', count) - pre_time = cur_time - count += 1 - -def parse_args(): - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - description='Create an image list or \ - make a record database by reading from an image list') - parser.add_argument('prefix', help='prefix of input/output lst and rec files.') - parser.add_argument('root', help='path to folder containing images.') - - cgroup = parser.add_argument_group('Options for creating image lists') - cgroup.add_argument('--list', type=bool, default=False, - help='If this is set im2rec will create image list(s) by traversing root folder\ - and output to .lst.\ - Otherwise im2rec will read .lst and create a database at .rec') - cgroup.add_argument('--exts', nargs='+', default=['.jpeg', '.jpg'], - help='list of acceptable image extensions.') - cgroup.add_argument('--chunks', type=int, default=1, help='number of chunks.') - cgroup.add_argument('--train-ratio', type=float, default=1.0, - help='Ratio of images to use for training.') - cgroup.add_argument('--test-ratio', type=float, default=0, - help='Ratio of images to use for testing.') - cgroup.add_argument('--recursive', type=bool, default=False, - help='If true recursively walk through subdirs and assign an unique label\ - to images in each folder. Otherwise only include images in the root folder\ - and give them label 0.') - cgroup.add_argument('--shuffle', type=bool, default=True, help='If this is set as True, \ - im2rec will randomize the image order in .lst') - - rgroup = parser.add_argument_group('Options for creating database') - rgroup.add_argument('--pass-through', type=bool, default=False, - help='whether to skip transformation and save image as is') - rgroup.add_argument('--resize', type=int, default=0, - help='resize the shorter edge of image to the newsize, original images will\ - be packed by default.') - rgroup.add_argument('--center-crop', type=bool, default=False, - help='specify whether to crop the center image to make it rectangular.') - rgroup.add_argument('--quality', type=int, default=95, - help='JPEG quality for encoding, 1-100; or PNG compression for encoding, 1-9') - rgroup.add_argument('--num-thread', type=int, default=1, - help='number of thread to use for encoding. order of images will be different\ - from the input list if >1. the input list will be modified to match the\ - resulting order.') - rgroup.add_argument('--color', type=int, default=1, choices=[-1, 0, 1], - help='specify the color mode of the loaded image.\ - 1: Loads a color image. Any transparency of image will be neglected. It is the default flag.\ - 0: Loads image in grayscale mode.\ - -1:Loads image as such including alpha channel.') - rgroup.add_argument('--encoding', type=str, default='.jpg', choices=['.jpg', '.png'], - help='specify the encoding of the images.') - rgroup.add_argument('--pack-label', type=bool, default=False, - help='Whether to also pack multi dimensional label in the record file') - args = parser.parse_args() - args.prefix = os.path.abspath(args.prefix) - args.root = os.path.abspath(args.root) - return args - -if __name__ == '__main__': - args = parse_args() - if args.list: - make_list(args) - else: - if os.path.isdir(args.prefix): - working_dir = args.prefix - else: - working_dir = os.path.dirname(args.prefix) - files = [os.path.join(working_dir, fname) for fname in os.listdir(working_dir) - if os.path.isfile(os.path.join(working_dir, fname))] - count = 0 - for fname in files: - if fname.startswith(args.prefix) and fname.endswith('.lst'): - print('Creating .rec file from', fname, 'in', working_dir) - count += 1 - image_list = read_list(fname) - # -- write_record -- # - if args.num_thread > 1 and multiprocessing is not None: - q_in = [multiprocessing.Queue(1024) for i in range(args.num_thread)] - q_out = multiprocessing.Queue(1024) - read_process = [multiprocessing.Process(target=read_worker, args=(args, q_in[i], q_out)) \ - for i in range(args.num_thread)] - for p in read_process: - p.start() - write_process = multiprocessing.Process(target=write_worker, args=(q_out, fname, working_dir)) - write_process.start() - - for i, item in enumerate(image_list): - q_in[i % len(q_in)].put((i, item)) - for q in q_in: - q.put(None) - for p in read_process: - p.join() - - q_out.put(None) - write_process.join() - else: - print('multiprocessing not available, fall back to single threaded encoding') - try: - import Queue as queue - except ImportError: - import queue - q_out = queue.Queue() - fname = os.path.basename(fname) - fname_rec = os.path.splitext(fname)[0] + '.rec' - fname_idx = os.path.splitext(fname)[0] + '.idx' - record = mx.recordio.MXIndexedRecordIO(os.path.join(working_dir, fname_idx), - os.path.join(working_dir, fname_rec), 'w') - cnt = 0 - pre_time = time.time() - for i, item in enumerate(image_list): - image_encode(args, i, item, q_out) - if q_out.empty(): - continue - _, s, _ = q_out.get() - record.write_idx(item[0], s) - if cnt % 1000 == 0: - cur_time = time.time() - print('time:', cur_time - pre_time, ' count:', cnt) - pre_time = cur_time - cnt += 1 - if not count: - print('Did not find and list file with prefix %s'%args.prefix) +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# -*- coding: utf-8 -*- +from __future__ import print_function +import os +import sys + +curr_path = os.path.abspath(os.path.dirname(__file__)) +sys.path.append(os.path.join(curr_path, "../python")) +import mxnet as mx +import random +import argparse +import cv2 +import time +import traceback +from builtins import range + +try: + import multiprocessing +except ImportError: + multiprocessing = None + +def list_image(root, recursive, exts): + i = 0 + if recursive: + cat = {} + for path, dirs, files in os.walk(root, followlinks=True): + dirs.sort() + files.sort() + for fname in files: + fpath = os.path.join(path, fname) + suffix = os.path.splitext(fname)[1].lower() + if os.path.isfile(fpath) and (suffix in exts): + if path not in cat: + cat[path] = len(cat) + yield (i, os.path.relpath(fpath, root), cat[path]) + i += 1 + for k, v in sorted(cat.items(), key=lambda x: x[1]): + print(os.path.relpath(k, root), v) + else: + for fname in sorted(os.listdir(root)): + fpath = os.path.join(root, fname) + suffix = os.path.splitext(fname)[1].lower() + if os.path.isfile(fpath) and (suffix in exts): + yield (i, os.path.relpath(fpath, root), 0) + i += 1 + +def write_list(path_out, image_list): + with open(path_out, 'w') as fout: + for i, item in enumerate(image_list): + line = '%d\t' % item[0] + for j in item[2:]: + line += '%f\t' % j + line += '%s\n' % item[1] + fout.write(line) + +def make_list(args): + image_list = list_image(args.root, args.recursive, args.exts) + image_list = list(image_list) + if args.shuffle is True: + random.seed(100) + random.shuffle(image_list) + N = len(image_list) + chunk_size = (N + args.chunks - 1) // args.chunks + for i in range(args.chunks): + chunk = image_list[i * chunk_size:(i + 1) * chunk_size] + if args.chunks > 1: + str_chunk = '_%d' % i + else: + str_chunk = '' + sep = int(chunk_size * args.train_ratio) + sep_test = int(chunk_size * args.test_ratio) + if args.train_ratio == 1.0: + write_list(args.prefix + str_chunk + '.lst', chunk) + else: + if args.test_ratio: + write_list(args.prefix + str_chunk + '_test.lst', chunk[:sep_test]) + if args.train_ratio + args.test_ratio < 1.0: + write_list(args.prefix + str_chunk + '_val.lst', chunk[sep_test + sep:]) + write_list(args.prefix + str_chunk + '_train.lst', chunk[sep_test:sep_test + sep]) + +def read_list(path_in): + with open(path_in) as fin: + while True: + line = fin.readline() + if not line: + break + line = [i.strip() for i in line.strip().split('\t')] + line_len = len(line) + if line_len < 3: + print('lst should at least has three parts, but only has %s parts for %s' %(line_len, line)) + continue + try: + item = [int(line[0])] + [line[-1]] + [float(i) for i in line[1:-1]] + except Exception as e: + print('Parsing lst met error for %s, detail: %s' %(line, e)) + continue + yield item + +def image_encode(args, i, item, q_out): + fullpath = os.path.join(args.root, item[1]) + + if len(item) > 3 and args.pack_label: + header = mx.recordio.IRHeader(0, item[2:], item[0], 0) + else: + header = mx.recordio.IRHeader(0, item[2], item[0], 0) + + if args.pass_through: + try: + with open(fullpath, 'rb') as fin: + img = fin.read() + s = mx.recordio.pack(header, img) + q_out.put((i, s, item)) + except Exception as e: + traceback.print_exc() + print('pack_img error:', item[1], e) + q_out.put((i, None, item)) + return + + try: + img = cv2.imread(fullpath, args.color) + except: + traceback.print_exc() + print('imread error trying to load file: %s ' % fullpath) + q_out.put((i, None, item)) + return + if img is None: + print('imread read blank (None) image for file: %s' % fullpath) + q_out.put((i, None, item)) + return + if args.center_crop: + if img.shape[0] > img.shape[1]: + margin = (img.shape[0] - img.shape[1]) // 2; + img = img[margin:margin + img.shape[1], :] + else: + margin = (img.shape[1] - img.shape[0]) // 2; + img = img[:, margin:margin + img.shape[0]] + if args.resize: + if img.shape[0] > img.shape[1]: + newsize = (args.resize, img.shape[0] * args.resize // img.shape[1]) + else: + newsize = (img.shape[1] * args.resize // img.shape[0], args.resize) + img = cv2.resize(img, newsize) + + try: + s = mx.recordio.pack_img(header, img, quality=args.quality, img_fmt=args.encoding) + q_out.put((i, s, item)) + except Exception as e: + traceback.print_exc() + print('pack_img error on file: %s' % fullpath, e) + q_out.put((i, None, item)) + return + +def read_worker(args, q_in, q_out): + while True: + deq = q_in.get() + if deq is None: + break + i, item = deq + image_encode(args, i, item, q_out) + +def write_worker(q_out, fname, working_dir): + pre_time = time.time() + count = 0 + fname = os.path.basename(fname) + fname_rec = os.path.splitext(fname)[0] + '.rec' + fname_idx = os.path.splitext(fname)[0] + '.idx' + record = mx.recordio.MXIndexedRecordIO(os.path.join(working_dir, fname_idx), + os.path.join(working_dir, fname_rec), 'w') + buf = {} + more = True + while more: + deq = q_out.get() + if deq is not None: + i, s, item = deq + buf[i] = (s, item) + else: + more = False + while count in buf: + s, item = buf[count] + del buf[count] + if s is not None: + record.write_idx(item[0], s) + + if count % 1000 == 0: + cur_time = time.time() + print('time:', cur_time - pre_time, ' count:', count) + pre_time = cur_time + count += 1 + +def parse_args(): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + description='Create an image list or \ + make a record database by reading from an image list') + parser.add_argument('prefix', help='prefix of input/output lst and rec files.') + parser.add_argument('root', help='path to folder containing images.') + + cgroup = parser.add_argument_group('Options for creating image lists') + cgroup.add_argument('--list', type=bool, default=False, + help='If this is set im2rec will create image list(s) by traversing root folder\ + and output to .lst.\ + Otherwise im2rec will read .lst and create a database at .rec') + cgroup.add_argument('--exts', nargs='+', default=['.jpeg', '.jpg'], + help='list of acceptable image extensions.') + cgroup.add_argument('--chunks', type=int, default=1, help='number of chunks.') + cgroup.add_argument('--train-ratio', type=float, default=1.0, + help='Ratio of images to use for training.') + cgroup.add_argument('--test-ratio', type=float, default=0, + help='Ratio of images to use for testing.') + cgroup.add_argument('--recursive', type=bool, default=False, + help='If true recursively walk through subdirs and assign an unique label\ + to images in each folder. Otherwise only include images in the root folder\ + and give them label 0.') + cgroup.add_argument('--shuffle', type=bool, default=True, help='If this is set as True, \ + im2rec will randomize the image order in .lst') + + rgroup = parser.add_argument_group('Options for creating database') + rgroup.add_argument('--pass-through', type=bool, default=False, + help='whether to skip transformation and save image as is') + rgroup.add_argument('--resize', type=int, default=0, + help='resize the shorter edge of image to the newsize, original images will\ + be packed by default.') + rgroup.add_argument('--center-crop', type=bool, default=False, + help='specify whether to crop the center image to make it rectangular.') + rgroup.add_argument('--quality', type=int, default=95, + help='JPEG quality for encoding, 1-100; or PNG compression for encoding, 1-9') + rgroup.add_argument('--num-thread', type=int, default=1, + help='number of thread to use for encoding. order of images will be different\ + from the input list if >1. the input list will be modified to match the\ + resulting order.') + rgroup.add_argument('--color', type=int, default=1, choices=[-1, 0, 1], + help='specify the color mode of the loaded image.\ + 1: Loads a color image. Any transparency of image will be neglected. It is the default flag.\ + 0: Loads image in grayscale mode.\ + -1:Loads image as such including alpha channel.') + rgroup.add_argument('--encoding', type=str, default='.jpg', choices=['.jpg', '.png'], + help='specify the encoding of the images.') + rgroup.add_argument('--pack-label', type=bool, default=False, + help='Whether to also pack multi dimensional label in the record file') + args = parser.parse_args() + args.prefix = os.path.abspath(args.prefix) + args.root = os.path.abspath(args.root) + return args + +if __name__ == '__main__': + args = parse_args() + if args.list: + make_list(args) + else: + if os.path.isdir(args.prefix): + working_dir = args.prefix + else: + working_dir = os.path.dirname(args.prefix) + files = [os.path.join(working_dir, fname) for fname in os.listdir(working_dir) + if os.path.isfile(os.path.join(working_dir, fname))] + count = 0 + for fname in files: + if fname.startswith(args.prefix) and fname.endswith('.lst'): + print('Creating .rec file from', fname, 'in', working_dir) + count += 1 + image_list = read_list(fname) + # -- write_record -- # + if args.num_thread > 1 and multiprocessing is not None: + q_in = [multiprocessing.Queue(1024) for i in range(args.num_thread)] + q_out = multiprocessing.Queue(1024) + read_process = [multiprocessing.Process(target=read_worker, args=(args, q_in[i], q_out)) \ + for i in range(args.num_thread)] + for p in read_process: + p.start() + write_process = multiprocessing.Process(target=write_worker, args=(q_out, fname, working_dir)) + write_process.start() + + for i, item in enumerate(image_list): + q_in[i % len(q_in)].put((i, item)) + for q in q_in: + q.put(None) + for p in read_process: + p.join() + + q_out.put(None) + write_process.join() + else: + print('multiprocessing not available, fall back to single threaded encoding') + try: + import Queue as queue + except ImportError: + import queue + q_out = queue.Queue() + fname = os.path.basename(fname) + fname_rec = os.path.splitext(fname)[0] + '.rec' + fname_idx = os.path.splitext(fname)[0] + '.idx' + record = mx.recordio.MXIndexedRecordIO(os.path.join(working_dir, fname_idx), + os.path.join(working_dir, fname_rec), 'w') + cnt = 0 + pre_time = time.time() + for i, item in enumerate(image_list): + image_encode(args, i, item, q_out) + if q_out.empty(): + continue + _, s, _ = q_out.get() + record.write_idx(item[0], s) + if cnt % 1000 == 0: + cur_time = time.time() + print('time:', cur_time - pre_time, ' count:', cnt) + pre_time = cur_time + cnt += 1 + if not count: + print('Did not find and list file with prefix %s'%args.prefix) diff --git a/tools/ipynb2md.py b/tools/ipynb2md.py index 426fa727764a..227174c25eee 100755 --- a/tools/ipynb2md.py +++ b/tools/ipynb2md.py @@ -1,5 +1,23 @@ #!/usr/bin/env python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + """ Convert jupyter notebook into the markdown format. The notebook outputs will be removed. diff --git a/tools/kill-mxnet.py b/tools/kill-mxnet.py index 2bdf949893b0..2a4a4303400b 100644 --- a/tools/kill-mxnet.py +++ b/tools/kill-mxnet.py @@ -1,5 +1,23 @@ #!/usr/bin/env python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + import os, sys import subprocess diff --git a/tools/launch.py b/tools/launch.py index f5366657c2cc..de42ea2a7dd3 100755 --- a/tools/launch.py +++ b/tools/launch.py @@ -1,4 +1,22 @@ #!/usr/bin/env python + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ Launch a distributed job """ @@ -54,7 +72,7 @@ def main(): args.num_servers = args.num_workers args = dmlc_opts(args) - + if args.host_file is None or args.host_file == 'None': if args.cluster == 'yarn': from dmlc_tracker import yarn diff --git a/tools/license_header.py b/tools/license_header.py new file mode 100644 index 000000000000..d0782b2b06fd --- /dev/null +++ b/tools/license_header.py @@ -0,0 +1,157 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Add or check license header + +Usuage: + +- add the default license header to source files that do not contain a valid + license: + + python license_header.py add + +- check if every files has a license header + + python license_header.py check +""" + +import re +import os +import argparse + +# the default apache license +_LICENSE = """Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License.""" + +# if a file contains any str in the list, then consider it has been licensed +_LICENSE_PATTERNS = ['Licensed to the Apache Software Foundation'] + +# the folders or files that will be ignored +_WHITE_LIST = ['R-package/', + 'cub/', + 'dlpack/', + 'dmlc-core/', + 'mshadow/', + 'nnvm', + 'ps-lite', + 'src/operator/mkl/', + 'src/operator/contrib/ctc_include/'] + +# language extensions and the according commment mark +_LANGS = {'.cc':'*', '.h':'*', '.cu':'*', '.cuh':'*', '.py':'#', + '.pm':'#', '.scala':'*', '.cc':'*', '.sh':'#', '.cmake':'#'} + +# Previous license header, which will be removed +_OLD_LICENSE = re.compile('.*Copyright.*by Contributors') + +def _has_license(lines): + return any([any([p in l.decode('utf-8') for p in _LICENSE_PATTERNS]) for l in lines]) + +def _get_license(comment_mark): + if comment_mark == '*': + body = '/*\n' + else: + body = '' + for l in _LICENSE.split('\n'): + if comment_mark == '*': + body += ' ' + body += comment_mark + if len(l): + body += ' ' + l + body += '\n' + + if comment_mark == '*': + body += ' */\n' + body += '\n' + return body + +def _valid_file(fname, verbose=False): + if any([l in fname for l in _WHITE_LIST]): + if verbose: + print('skip ' + fname + ', it matches the white list') + return False + _, ext = os.path.splitext(fname) + if ext not in _LANGS: + if verbose: + print('skip ' + fname + ', unknown file extension') + return False + return True + +def process_file(fname, action, verbose=False): + if not _valid_file(fname, verbose): + return True + with open(fname, 'rb') as f: + lines = f.readlines() + if not lines: + return True + if _has_license(lines): + return True + elif action == 'check': + return False + _, ext = os.path.splitext(fname) + # remove old license + if ext == '.h' or ext == '.cc' or ext == '.cu': + for i, l in enumerate(lines): + if _OLD_LICENSE.match(l.decode('utf-8')): + del lines[i] + break + with open(fname, 'wb') as f: + # shebang line + if lines[0].startswith(b'#!'): + f.write(lines[0].rstrip()+b'\n\n') + del lines[0] + f.write(str.encode(_get_license(_LANGS[ext]))) + for l in lines: + f.write(l.rstrip()+b'\n') + print('added license header to ' + fname) + return False + +def process_folder(root, action): + excepts = [] + for root, _, files in os.walk(root): + for f in files: + fname = os.path.normpath(os.path.join(root, f)) + if not process_file(fname, action): + excepts.append(fname) + if action == 'check' and excepts: + raise Exception('The following files do not contain a valid license, '+ + 'you can use `python tools/license_header.py add` to add'+ + 'them automatically', excepts) + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='Add or check source license header') + parser.add_argument( + 'action', nargs=1, type=str, + choices=['add', 'check'], default='add', + help = 'add or check') + args = parser.parse_args() + process_folder(os.path.join(os.path.dirname(__file__), '..'), args.action[0]) diff --git a/tools/parse_log.py b/tools/parse_log.py index 070f770b8cf6..f0ce53dbe76f 100755 --- a/tools/parse_log.py +++ b/tools/parse_log.py @@ -1,4 +1,22 @@ #!/usr/bin/env python + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ parse mxnet output log into a markdown table """ diff --git a/tools/pip_package/make_pip_package.sh b/tools/pip_package/make_pip_package.sh index a1af18bad528..46b4938b0785 100755 --- a/tools/pip_package/make_pip_package.sh +++ b/tools/pip_package/make_pip_package.sh @@ -1,5 +1,23 @@ #!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + # Assuming the script is run at mxnet/tools/pip_package # This script builds from scratch the dependencies of mxnet into static # librareis and statically links them to produce a (mostly) standalone diff --git a/tools/pip_package/setup.py b/tools/pip_package/setup.py index 45d761e53dea..e4bf48236bde 100644 --- a/tools/pip_package/setup.py +++ b/tools/pip_package/setup.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # pylint: disable=invalid-name, exec-used """Setup mxnet package.""" from __future__ import absolute_import From 44c2bfe6133ae51cfb0531600ccd6408654434e6 Mon Sep 17 00:00:00 2001 From: Hessel Tuinhof Date: Wed, 9 Aug 2017 02:25:48 +0200 Subject: [PATCH 348/834] Clarify definition of cross-entropy metric in the documentation (clean up PR #7291) (#7365) * [R] switch order of LRN and pooling layer Original paper (section 3.5) performs local response normalization of relu. * clearify definition of cross entropy * fix small type * fixed lint, trailing wspace --- python/mxnet/metric.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/python/mxnet/metric.py b/python/mxnet/metric.py index 1a8e67da5396..00cc2da61f3c 100644 --- a/python/mxnet/metric.py +++ b/python/mxnet/metric.py @@ -854,10 +854,14 @@ def update(self, labels, preds): class CrossEntropy(EvalMetric): """Computes Cross Entropy loss. - The cross entropy is given by + The cross entropy over a batch of sample size :math:`N` is given by .. math:: - -y\\log \\hat{y} + (1-y)\\log (1-\\hat{y}) + -\\sum_{n=1}^{N}\\sum_{k=1}^{K}t_{nk}\\log (y_{nk}), + + where :math:`t_{nk}=1` if and only if sample :math:`n` belongs to class :math:`k`. + :math:`y_{nk}` denotes the probability of sample :math:`n` belonging to + class :math:`k`. Parameters ---------- From 8fb640781a8ab76189ee0a22428f33d0d895caf0 Mon Sep 17 00:00:00 2001 From: Kenta Kubo Date: Wed, 9 Aug 2017 14:30:59 +0900 Subject: [PATCH 349/834] Update CONTRIBUTORS.md (#7391) Because my patches have been merged in #6908 and #6927. --- CONTRIBUTORS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 2ee54f07c85e..8cae93854e19 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -145,3 +145,4 @@ List of Contributors * [Guodong Zhang](https://github.com/gd-zhang/) * [Xizhou Zhu](https://github.com/einsiedler0408/) * [Jean Kossaifi](https://github.com/JeanKossaifi/) +* [Kenta Kubo](https://github.com/kkk669/) From 89e3ee3ea7c223db8c65ddd8c94c6e787d7c52df Mon Sep 17 00:00:00 2001 From: "Qiang Kou (KK)" Date: Wed, 9 Aug 2017 06:44:58 +0000 Subject: [PATCH 350/834] [R] im2rec in R. close #7273 (#7389) --- Makefile | 3 +- R-package/R/util.R | 49 ++++- R-package/src/Makevars | 2 +- R-package/src/export.cc | 2 +- R-package/src/im2rec.cc | 269 +++++++++++++++++++++++ R-package/src/im2rec.h | 42 ++++ R-package/src/mxnet.cc | 3 + R-package/vignettes/CatsDogsFinetune.Rmd | 55 +++-- 8 files changed, 402 insertions(+), 23 deletions(-) create mode 100644 R-package/src/im2rec.cc create mode 100644 R-package/src/im2rec.h diff --git a/Makefile b/Makefile index 5c7f54dcd759..ed742144ef7c 100644 --- a/Makefile +++ b/Makefile @@ -379,6 +379,7 @@ rcpplint: rpkg: mkdir -p R-package/inst mkdir -p R-package/inst/libs + cp src/io/image_recordio.h R-package/src cp -rf lib/libmxnet.so R-package/inst/libs mkdir -p R-package/inst/include cp -rf include/* R-package/inst/include @@ -442,7 +443,7 @@ clean: cyclean $(EXTRA_PACKAGES_CLEAN) else clean: cyclean testclean $(EXTRA_PACKAGES_CLEAN) $(RM) -r build lib bin *~ */*~ */*/*~ */*/*/*~ R-package/NAMESPACE R-package/man R-package/R/mxnet_generated.R \ - R-package/inst R-package/src/*.o R-package/src/*.so mxnet_*.tar.gz + R-package/inst R-package/src/image_recordio.h R-package/src/*.o R-package/src/*.so mxnet_*.tar.gz cd $(DMLC_CORE); $(MAKE) clean; cd - cd $(PS_PATH); $(MAKE) clean; cd - cd $(NNVM_PATH); $(MAKE) clean; cd - diff --git a/R-package/R/util.R b/R-package/R/util.R index 2b292d1ad786..acc9510ccfd4 100644 --- a/R-package/R/util.R +++ b/R-package/R/util.R @@ -9,5 +9,52 @@ mx.util.filter.null <- function(lst) { #' #' @export mxnet.export <- function(path) { - mxnet.internal.export(path.expand(path)) + mx.internal.export(path.expand(path)) +} + +#' Convert images into image recordio format +#' @param image_lst +#' The image lst file +#' @param root +#' The root folder for image files +#' @param output_rec +#' The output rec file +#' @param label_width +#' The label width in the list file. Default is 1. +#' @param pack_label +#' Whether to also pack multi dimenional label in the record file. Default is 0. +#' @param new_size +#' The shorter edge of image will be resized to the newsize. +#' Original images will be packed by default. +#' @param nsplit +#' It is used for part generation, logically split the image.lst to NSPLIT parts by position. +#' Default is 1. +#' @param partid +#' It is used for part generation, pack the images from the specific part in image.lst. +#' Default is 0. +#' @param center_crop +#' Whether to crop the center image to make it square. Default is 0. +#' @param quality +#' JPEG quality for encoding (1-100, default: 95) or PNG compression for encoding (1-9, default: 3). +#' @param color_mode +#' Force color (1), gray image (0) or keep source unchanged (-1). Default is 1. +#' @param unchanged +#' Keep the original image encoding, size and color. If set to 1, it will ignore the others parameters. +#' @param inter_method +#' NN(0), BILINEAR(1), CUBIC(2), AREA(3), LANCZOS4(4), AUTO(9), RAND(10). Default is 1. +#' @param encoding +#' The encoding type for images. It can be '.jpg' or '.png'. Default is '.jpg'. +#' @export +im2rec <- function(image_lst, root, output_rec, label_width = 1L, + pack_label = 0L, new_size = -1L, nsplit = 1L, + partid = 0L, center_crop = 0L, quality = 95L, + color_mode = 1L, unchanged = 0L, inter_method = 1L, + encoding = ".jpg") { + image_lst <- path.expand(image_lst) + root <- path.expand(root) + output_rec <- path.expand(output_rec) + mx.internal.im2rec(image_lst, root, output_rec, label_width, + pack_label, new_size, nsplit, partid, + center_crop, quality, color_mode, unchanged, + inter_method, encoding) } diff --git a/R-package/src/Makevars b/R-package/src/Makevars index a9cdabfd9e00..c089c093389b 100644 --- a/R-package/src/Makevars +++ b/R-package/src/Makevars @@ -1,3 +1,3 @@ - +CXX_STD = CXX11 PKG_CPPFLAGS = -I../inst/include PKG_LIBS = $(LAPACK_LIBS) $(BLAS_LIBS) diff --git a/R-package/src/export.cc b/R-package/src/export.cc index 2377a02fbc86..ef77d25fdf89 100644 --- a/R-package/src/export.cc +++ b/R-package/src/export.cc @@ -41,7 +41,7 @@ Exporter* Exporter::Get() { void Exporter::InitRcppModule() { using namespace Rcpp; // NOLINT(*) Exporter::Get()->scope_ = ::getCurrentScope(); - function("mxnet.internal.export", &Exporter::Export, + function("mx.internal.export", &Exporter::Export, Rcpp::List::create(_["path"]), "Internal function of mxnet, used to export generated functions file."); } diff --git a/R-package/src/im2rec.cc b/R-package/src/im2rec.cc new file mode 100644 index 000000000000..0c6bea964a50 --- /dev/null +++ b/R-package/src/im2rec.cc @@ -0,0 +1,269 @@ +/*! + * Copyright (c) 2017 by Contributors + * \file export.h + * \brief Export module that takes charge of code generation and document + * Generation for functions exported from R-side + */ + +#include +#include +#include +#include +#include +#include +#include +#include "dmlc/base.h" +#include "dmlc/io.h" +#include "dmlc/timer.h" +#include "dmlc/logging.h" +#include "dmlc/recordio.h" +#include +#include "image_recordio.h" +#include "base.h" +#include "im2rec.h" + +namespace mxnet { +namespace R { + +int GetInterMethod(int inter_method, int old_width, int old_height, + int new_width, int new_height, std::mt19937& prnd) { // NOLINT(*) + if (inter_method == 9) { + if (new_width > old_width && new_height > old_height) { + return 2; // CV_INTER_CUBIC for enlarge + } else if (new_width rand_uniform_int(0, 4); + return rand_uniform_int(prnd); + } else { + return inter_method; + } +} + +IM2REC* IM2REC::Get() { + static IM2REC inst; + return &inst; +} + +void IM2REC::InitRcppModule() { + using namespace Rcpp; // NOLINT(*) + IM2REC::Get()->scope_ = ::getCurrentScope(); + function("mx.internal.im2rec", &IM2REC::im2rec, + Rcpp::List::create(_["image_lst"], + _["root"], + _["output_rec"], + _["label_width"], + _["pack_label"], + _["new_size"], + _["nsplit"], + _["partid"], + _["center_crop"], + _["quality"], + _["color_mode"], + _["unchanged"], + _["inter_method"], + _["encoding"]), + ""); +} + +void IM2REC::im2rec(const std::string & image_lst, const std::string & root, + const std::string & output_rec, + int label_width, int pack_label, int new_size, int nsplit, + int partid, int center_crop, int quality, + int color_mode, int unchanged, + int inter_method, std::string encoding) { + // Check parameters ranges + if (color_mode != -1 && color_mode != 0 && color_mode != 1) { + Rcpp::stop("Color mode must be -1, 0 or 1."); + } + if (encoding != std::string(".jpg") && encoding != std::string(".png")) { + Rcpp::stop("Encoding mode must be .jpg or .png."); + } + if (label_width <= 1 && pack_label) { + Rcpp::stop("pack_label can only be used when label_width > 1"); + } + if (new_size > 0) { + LOG(INFO) << "New Image Size: Short Edge " << new_size; + } else { + LOG(INFO) << "Keep origin image size"; + } + if (center_crop) { + LOG(INFO) << "Center cropping to square"; + } + if (color_mode == 0) { + LOG(INFO) << "Use gray images"; + } + if (color_mode == -1) { + LOG(INFO) << "Keep original color mode"; + } + LOG(INFO) << "Encoding is " << encoding; + + if (encoding == std::string(".png") && quality > 9) { + quality = 3; + } + if (inter_method != 1) { + switch (inter_method) { + case 0: + LOG(INFO) << "Use inter_method CV_INTER_NN"; + break; + case 2: + LOG(INFO) << "Use inter_method CV_INTER_CUBIC"; + break; + case 3: + LOG(INFO) << "Use inter_method CV_INTER_AREA"; + break; + case 4: + LOG(INFO) << "Use inter_method CV_INTER_LANCZOS4"; + break; + case 9: + LOG(INFO) << "Use inter_method mod auto(cubic for enlarge, area for shrink)"; + break; + case 10: + LOG(INFO) << "Use inter_method mod rand(nn/bilinear/cubic/area/lanczos4)"; + break; + } + } + std::random_device rd; + std::mt19937 prnd(rd()); + using namespace dmlc; + static const size_t kBufferSize = 1 << 20UL; + mxnet::io::ImageRecordIO rec; + size_t imcnt = 0; + double tstart = dmlc::GetTime(); + dmlc::InputSplit *flist = + dmlc::InputSplit::Create(image_lst.c_str(), partid, nsplit, "text"); + std::ostringstream os; + if (nsplit == 1) { + os << output_rec; + } else { + os << output_rec << ".part" << std::setw(3) << std::setfill('0') << partid; + } + LOG(INFO) << "Write to output: " << os.str(); + dmlc::Stream *fo = dmlc::Stream::Create(os.str().c_str(), "w"); + LOG(INFO) << "Output: " << os.str(); + dmlc::RecordIOWriter writer(fo); + std::string fname, path, blob; + std::vector decode_buf; + std::vector encode_buf; + std::vector encode_params; + if (encoding == std::string(".png")) { + encode_params.push_back(CV_IMWRITE_PNG_COMPRESSION); + encode_params.push_back(quality); + LOG(INFO) << "PNG encoding compression: " << quality; + } else { + encode_params.push_back(CV_IMWRITE_JPEG_QUALITY); + encode_params.push_back(quality); + LOG(INFO) << "JPEG encoding quality: " << quality; + } + dmlc::InputSplit::Blob line; + std::vector label_buf(label_width, 0.f); + + while (flist->NextRecord(&line)) { + std::string sline(static_cast(line.dptr), line.size); + std::istringstream is(sline); + if (!(is >> rec.header.image_id[0] >> rec.header.label)) continue; + label_buf[0] = rec.header.label; + for (int k = 1; k < label_width; ++k) { + RCHECK(is >> label_buf[k]) + << "Invalid ImageList, did you provide the correct label_width?"; + } + if (pack_label) rec.header.flag = label_width; + rec.SaveHeader(&blob); + if (pack_label) { + size_t bsize = blob.size(); + blob.resize(bsize + label_buf.size()*sizeof(float)); + memcpy(BeginPtr(blob) + bsize, + BeginPtr(label_buf), label_buf.size()*sizeof(float)); + } + RCHECK(std::getline(is, fname)); + // eliminate invalid chars in the end + while (fname.length() != 0 && + (isspace(*fname.rbegin()) || !isprint(*fname.rbegin()))) { + fname.resize(fname.length() - 1); + } + // eliminate invalid chars in beginning. + const char *p = fname.c_str(); + while (isspace(*p)) ++p; + path = root + p; + // use "r" is equal to rb in dmlc::Stream + dmlc::Stream *fi = dmlc::Stream::Create(path.c_str(), "r"); + decode_buf.clear(); + size_t imsize = 0; + while (true) { + decode_buf.resize(imsize + kBufferSize); + size_t nread = fi->Read(BeginPtr(decode_buf) + imsize, kBufferSize); + imsize += nread; + decode_buf.resize(imsize); + if (nread != kBufferSize) break; + } + delete fi; + + + if (unchanged != 1) { + cv::Mat img = cv::imdecode(decode_buf, color_mode); + RCHECK(img.data != NULL) << "OpenCV decode fail:" << path; + cv::Mat res = img; + if (new_size > 0) { + if (center_crop) { + if (img.rows > img.cols) { + int margin = (img.rows - img.cols)/2; + img = img(cv::Range(margin, margin+img.cols), cv::Range(0, img.cols)); + } else { + int margin = (img.cols - img.rows)/2; + img = img(cv::Range(0, img.rows), cv::Range(margin, margin + img.rows)); + } + } + int interpolation_method = 1; + if (img.rows > img.cols) { + if (img.cols != new_size) { + interpolation_method = GetInterMethod(inter_method, img.cols, img.rows, + new_size, + img.rows * new_size / img.cols, prnd); + cv::resize(img, res, cv::Size(new_size, + img.rows * new_size / img.cols), + 0, 0, interpolation_method); + } else { + res = img.clone(); + } + } else { + if (img.rows != new_size) { + interpolation_method = GetInterMethod(inter_method, img.cols, + img.rows, new_size * img.cols / img.rows, + new_size, prnd); + cv::resize(img, res, cv::Size(new_size * img.cols / img.rows, + new_size), 0, 0, interpolation_method); + } else { + res = img.clone(); + } + } + } + encode_buf.clear(); + RCHECK(cv::imencode(encoding, res, encode_buf, encode_params)); + + // write buffer + size_t bsize = blob.size(); + blob.resize(bsize + encode_buf.size()); + memcpy(BeginPtr(blob) + bsize, + BeginPtr(encode_buf), encode_buf.size()); + } else { + size_t bsize = blob.size(); + blob.resize(bsize + decode_buf.size()); + memcpy(BeginPtr(blob) + bsize, + BeginPtr(decode_buf), decode_buf.size()); + } + writer.WriteRecord(BeginPtr(blob), blob.size()); + // write header + ++imcnt; + if (imcnt % 1000 == 0) { + LOG(INFO) << imcnt << " images processed, " << GetTime() - tstart << " sec elapsed"; + } + } + LOG(INFO) << "Total: " << imcnt << " images processed, " << GetTime() - tstart << " sec elapsed"; + delete fo; + delete flist; +} +} // namespace R +} // namespace mxnet diff --git a/R-package/src/im2rec.h b/R-package/src/im2rec.h new file mode 100644 index 000000000000..a98a73327b97 --- /dev/null +++ b/R-package/src/im2rec.h @@ -0,0 +1,42 @@ +/*! + * Copyright (c) 2017 by Contributors + * \file export.h + * \brief Export module that takes charge of code generation and document + * Generation for functions exported from R-side + */ + +#ifndef MXNET_RCPP_IM2REC_H_ +#define MXNET_RCPP_IM2REC_H_ + +#include +#include + +namespace mxnet { +namespace R { + +class IM2REC { + public: + /*! + * \brief Export the generated file into path. + * \param path The path to be exported. + */ + static void im2rec(const std::string & image_lst, const std::string & root, + const std::string & output_rec, + int label_width = 1, int pack_label = 0, int new_size = -1, int nsplit = 1, + int partid = 0, int center_crop = 0, int quality = 95, + int color_mode = 1, int unchanged = 0, + int inter_method = 1, std::string encoding = ".jpg"); + // intialize the Rcpp module + static void InitRcppModule(); + + public: + // get the singleton of exporter + static IM2REC* Get(); + /*! \brief The scope of current module to export */ + Rcpp::Module* scope_; +}; + +} // namespace R +} // namespace mxnet + +#endif // MXNET_RCPP_IM2REC_H_ diff --git a/R-package/src/mxnet.cc b/R-package/src/mxnet.cc index 9d16190b3bd3..9f8239b94aa8 100644 --- a/R-package/src/mxnet.cc +++ b/R-package/src/mxnet.cc @@ -12,6 +12,7 @@ #include "./io.h" #include "./kvstore.h" #include "./export.h" +#include "./im2rec.h" namespace mxnet { namespace R { @@ -56,4 +57,6 @@ RCPP_MODULE(mxnet) { DataIterCreateFunction::InitRcppModule(); KVStore::InitRcppModule(); Exporter::InitRcppModule(); + IM2REC::InitRcppModule(); } + diff --git a/R-package/vignettes/CatsDogsFinetune.Rmd b/R-package/vignettes/CatsDogsFinetune.Rmd index e30b5137a2f1..95f90beec519 100644 --- a/R-package/vignettes/CatsDogsFinetune.Rmd +++ b/R-package/vignettes/CatsDogsFinetune.Rmd @@ -104,12 +104,30 @@ Map(function(x, y) { }, x = files, y = new_names) ``` -### Creating .rec files using im2rec.py - -```{bash, eval = FALSE} -python im2rec.py --list=1 --recursive=1 --train-ratio=0.8 cats_dogs train_pad_224x224 -python im2rec.py --num-thread=4 --pass-through=1 cats_dogs_train.lst train_pad_224x224 -python im2rec.py --num-thread=4 --pass-through=1 cats_dogs_val.lst train_pad_224x224 +### Creating .rec files + +```{r, eval = FALSE} +cat_files <- list.files("train_pad_224x224/cat/", recursive=TRUE) +cat_files <- paste0("cat/", cat_files) + +dog_files <- list.files("train_pad_224x224/dog/", recursive=TRUE) +dog_files <- paste0("dog/", dog_files) + +train_ind <- sample(length(cat_files), length(cat_files) * 0.8) +train_data <- c(1:(length(train_ind) * 2)) +train_data <- cbind(train_data, c(rep(0, length(train_ind)), rep(1, length(train_ind)))) +train_data <- cbind(train_data, c(cat_files[train_ind], dog_files[train_ind])) +train_data <- train_data[sample(nrow(train_data)),] +write.table(train_data, "cats_dogs_train.lst", quote = FALSE, sep = "\t", row.names = FALSE, col.names = FALSE) +im2rec("cats_dogs_train.lst", "train_pad_224x224/", "cats_dogs_train.rec") + +val_ind <- c(1:length(cat_files))[!c(1:length(cat_files)) %in% train_ind] +val_data <- c(1:(length(val_ind) * 2)) +val_data <- cbind(val_data, c(rep(0, length(val_ind)), rep(1, length(val_ind)))) +val_data <- cbind(val_data, c(cat_files[val_ind], dog_files[val_ind])) +val_data <- val_data[sample(nrow(val_data)),] +write.table(val_data, "cats_dogs_val.lst", quote = FALSE, sep = "\t", row.names = FALSE, col.names = FALSE) +im2rec("cats_dogs_val.lst", "train_pad_224x224/", "cats_dogs_val.rec") ``` ## The data iterator @@ -215,21 +233,20 @@ preprocImage<- function(src, # URL or file location num_channels = 3, # 3 for RGB, 1 for grayscale mult_by = 1, # set to 255 for normalized image crop = FALSE) { # no crop by default - - im <- load.image(src) - - if (crop) { - shape <- dim(im) + im <- load.image(src) + + if (crop) { + shape <- dim(im) short_edge <- min(shape[1:2]) - xx <- floor((shape[1] - short_edge) / 2) - yy <- floor((shape[2] - short_edge) / 2) + xx <- floor((shape[1] - short_edge) / 2) + yy <- floor((shape[2] - short_edge) / 2) im <- crop.borders(im, xx, yy) - } - - resized <- resize(im, size_x = width, size_y = height) - arr <- as.array(resized) * mult_by - dim(arr) <- c(width, height, num_channels, 1) - return(arr) + } + + resized <- resize(im, size_x = width, size_y = height) + arr <- as.array(resized) * mult_by + dim(arr) <- c(width, height, num_channels, 1) + return(arr) } ``` From bd845aa58c08329192fc142f598d045275f43342 Mon Sep 17 00:00:00 2001 From: Zack Chase Lipton Date: Wed, 9 Aug 2017 13:07:57 -0700 Subject: [PATCH 351/834] Drafted documentation for autograd. (#7395) * added autograd documentation * adding output for snippet * adding output for snippet * typo fix * typo fix * revised the autograd doc * typo * added sentence about autograd.pause(): * typo * further deflation * Update autograd.md --- docs/api/python/autograd.md | 58 +++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/docs/api/python/autograd.md b/docs/api/python/autograd.md index d204a2ce4464..5c849648e3cc 100644 --- a/docs/api/python/autograd.md +++ b/docs/api/python/autograd.md @@ -9,6 +9,64 @@ .. warning:: This package is currently experimental and may change in the near future. ``` +## Overview + +The `autograd` package enables automatic +differentiation of NDArray operations. +In machine learning applications, +`autograd` is often used to calculate the gradients +of loss functions with respect to parameters. + + +### Record vs Pause + +`autograd` records computation history on the fly to calculate gradients later. +This is only enabled inside a `with autograd.record():` block. +A `with auto_grad.pause()` block can be used inside a `record()` block +to temporarily disable recording. + +To compute gradient with respect to an `NDArray` `x`, first call `x.attach_grad()` +to allocate space for the gradient. Then, start a `with autograd.record()` block, +and do some computation. Finally, call `backward()` on the result: + +```python +>>> x = mx.nd.array([1,2,3,4]) +>>> x.attach_grad() +>>> with mx.autograd.record(): +... y = x * x + 1 +>>> print(x.grad) +[ 2. 4. 6. 8.] + +``` + + +## Train mode and Predict Mode + +Some operators (Dropout, BatchNorm, etc) behave differently in +when training and when making predictions. +This can be controled with `train_mode` and `predict_mode` scope. + +By default, MXNet is in `predict_mode`. +A `with autograd.record()` block by default turns on `train_mode` +(equivalent to ``with autograd.record(train_mode=True)``). +To compute a gradient in prediction mode (as when generating adversarial examples), +call record with `train_mode=False` and then call `backward(train_mode=False)` + +Although training usually coincides with recording, +this isn't always the case. +To control *training* vs *predict_mode* without changing +*recording* vs *not recording*, +Use a `with autograd.train_mode():` +or `with autograd.predict_mode():` block. + +Detailed tutorials are available in Part 1 of +[the MXNet gluon book](http://gluon.mxnet.io/). + + + + + + ## Autograd From a33ded796325c8da4eebab07a18ddc616ad28098 Mon Sep 17 00:00:00 2001 From: joey2014 Date: Wed, 9 Aug 2017 15:19:21 -0500 Subject: [PATCH 352/834] fix [#7368] Caffe converter test fails, causing CI to halt for all PRs (#7381) * support convert mtcnn and MobileNet model * pass python lint * put "import re" before "import caffe_parser" as lint required * correct missed checkin and pass pylint * When converting vgg16 model, the layer fc6 input name pool5 is changed to flatten_0 by _parse_proto(). It misleads adding the layer to group. * revert disable vgg-16 resnet converter check (#7369) --- tools/caffe_converter/convert_model.py | 13 ++++++++++++- tools/caffe_converter/convert_symbol.py | 21 ++++++++++----------- tools/caffe_converter/test_converter.py | 2 +- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/tools/caffe_converter/convert_model.py b/tools/caffe_converter/convert_model.py index c04a2aa63cd2..1624a017fe0d 100644 --- a/tools/caffe_converter/convert_model.py +++ b/tools/caffe_converter/convert_model.py @@ -26,6 +26,17 @@ import numpy as np from convert_symbol import convert_symbol +def prob_label(arg_names): + candidates = [arg for arg in arg_names if + not arg.endswith('data') and + not arg.endswith('_weight') and + not arg.endswith('_bias') and + not arg.endswith('_gamma') and + not arg.endswith('_beta')] + if len(candidates) == 0: + return 'prob_label' + return candidates[-1] + def convert_model(prototxt_fname, caffemodel_fname, output_prefix=None): """Convert caffe model @@ -198,7 +209,7 @@ def convert_model(prototxt_fname, caffemodel_fname, output_prefix=None): assert len(layer_blobs) == 0 if output_prefix is not None: - model = mx.mod.Module(symbol=sym, label_names=[arg_names[-1], ]) + model = mx.mod.Module(symbol=sym, label_names=[prob_label(arg_names), ]) model.bind(data_shapes=[('data', tuple(input_dim))]) model.init_params(arg_params=arg_params, aux_params=aux_params) model.save_checkpoint(output_prefix, 0) diff --git a/tools/caffe_converter/convert_symbol.py b/tools/caffe_converter/convert_symbol.py index dde3c26b5ca2..13b55fef1296 100644 --- a/tools/caffe_converter/convert_symbol.py +++ b/tools/caffe_converter/convert_symbol.py @@ -145,6 +145,16 @@ def _parse_proto(prototxt_fname): param_string = '' skip_layer = False name = re.sub('[-/]', '_', layer.name) + for k in range(len(layer.bottom)): + if layer.bottom[k] in _output_name: + _output_name[layer.bottom[k]]['count'] = _output_name[layer.bottom[k]]['count']+1 + else: + _output_name[layer.bottom[k]] = {'count':0} + for k in range(len(layer.top)): + if layer.top[k] in _output_name: + _output_name[layer.top[k]]['count'] = _output_name[layer.top[k]]['count']+1 + else: + _output_name[layer.top[k]] = {'count':0, 'name':name} if layer.type == 'Convolution' or layer.type == 4: type_string = 'mx.symbol.Convolution' param_string = _convert_conv_param(layer.convolution_param) @@ -270,17 +280,6 @@ def _parse_proto(prototxt_fname): for j in range(len(layer.top)): mapping[layer.top[j]] = name output_name = name - for k in range(len(layer.bottom)): - if layer.bottom[k] in _output_name: - _output_name[layer.bottom[k]]['count'] = _output_name[layer.bottom[k]]['count']+1 - else: - _output_name[layer.bottom[k]] = {'count':0} - for k in range(len(layer.top)): - if layer.top[k] in _output_name: - _output_name[layer.top[k]]['count'] = _output_name[layer.top[k]]['count']+1 - else: - _output_name[layer.top[k]] = {'count':0, 'name':name} - output_name = [] for i in _output_name: if 'name' in _output_name[i] and _output_name[i]['count'] == 0: diff --git a/tools/caffe_converter/test_converter.py b/tools/caffe_converter/test_converter.py index db17c64e54b7..cdf833198eff 100644 --- a/tools/caffe_converter/test_converter.py +++ b/tools/caffe_converter/test_converter.py @@ -95,7 +95,7 @@ def main(): assert gpus, 'At least one GPU is needed to run test_converter in GPU mode' batch_size = 32 * len(gpus) - models = ['bvlc_googlenet'] + models = ['bvlc_googlenet', 'vgg-16', 'resnet-50'] val = download_data() for m in models: From f674bc40dbdb834919a37bd12af3003d7a427307 Mon Sep 17 00:00:00 2001 From: Sergey Kolychev Date: Wed, 9 Aug 2017 13:20:04 -0700 Subject: [PATCH 353/834] 1) Fixes for ImageIter (#7357) 2) Convolutional RNN 3) Improved Visualization 4) PearsonCorrelation metric 5) Fixed tests. --- perl-package/AI-MXNet/Changes | 3 + perl-package/AI-MXNet/MANIFEST | 1 - perl-package/AI-MXNet/META.json | 4 +- perl-package/AI-MXNet/META.yml | 4 +- perl-package/AI-MXNet/Makefile.PL | 4 +- perl-package/AI-MXNet/README | 2 +- perl-package/AI-MXNet/lib/AI/MXNet.pm | 2 +- perl-package/AI-MXNet/lib/AI/MXNet/Base.pm | 2 +- perl-package/AI-MXNet/lib/AI/MXNet/Image.pm | 8 +- perl-package/AI-MXNet/lib/AI/MXNet/Metric.pm | 50 +++ perl-package/AI-MXNet/lib/AI/MXNet/Module.pm | 8 - perl-package/AI-MXNet/lib/AI/MXNet/RNN.pm | 3 + .../AI-MXNet/lib/AI/MXNet/RNN/Cell.pm | 307 +++++++++++++++++- perl-package/AI-MXNet/lib/AI/MXNet/Symbol.pm | 2 +- perl-package/AI-MXNet/lib/AI/MXNet/Types.pm | 4 +- .../AI-MXNet/lib/AI/MXNet/Visualization.pm | 8 + perl-package/AI-MXNet/t/test_model_parallel.t | 74 +++++ perl-package/AI-MXNet/t/test_rnn.t | 62 +++- perl-package/AI-MXNetCAPI/Changes | 3 + perl-package/AI-MXNetCAPI/META.json | 2 +- perl-package/AI-MXNetCAPI/META.yml | 2 +- perl-package/AI-MXNetCAPI/README | 2 +- perl-package/AI-MXNetCAPI/lib/AI/MXNetCAPI.pm | 2 +- perl-package/AI-MXNetCAPI/mxnet.i | 7 + 24 files changed, 537 insertions(+), 29 deletions(-) create mode 100644 perl-package/AI-MXNet/t/test_model_parallel.t diff --git a/perl-package/AI-MXNet/Changes b/perl-package/AI-MXNet/Changes index 5d5c5a280b70..f8ecc7509737 100644 --- a/perl-package/AI-MXNet/Changes +++ b/perl-package/AI-MXNet/Changes @@ -1,5 +1,8 @@ Revision history for Perl extension AI::MXNet +1.0102 Sun Aug 6 16:55:08 PDT 2017 + - bugfixes in Image.pm, updated tests, added PearsonCorrelation metric, added Convolutional RNN modules. + 1.0101 Sun Jul 2 17:16:01 PDT 2017 - reworked CachedOp, two new optimizers, auto module reshape, using strings to index the kvstore. diff --git a/perl-package/AI-MXNet/MANIFEST b/perl-package/AI-MXNet/MANIFEST index 7a6d78bf0b3f..48cb31dd6b8e 100644 --- a/perl-package/AI-MXNet/MANIFEST +++ b/perl-package/AI-MXNet/MANIFEST @@ -10,7 +10,6 @@ examples/cudnn_lstm_bucketing.pl Makefile.PL Changes META.json -t/test_autograd.t t/test_recordio.t t/test_random.t t/test_init.t diff --git a/perl-package/AI-MXNet/META.json b/perl-package/AI-MXNet/META.json index 54545928e20a..692f1ddaae39 100644 --- a/perl-package/AI-MXNet/META.json +++ b/perl-package/AI-MXNet/META.json @@ -30,7 +30,7 @@ }, "runtime" : { "requires" : { - "AI::MXNetCAPI" : "1.0101", + "AI::MXNetCAPI" : "1.0102", "AI::NNVMCAPI" : "1.01", "Function::Parameters" : "1.0705", "GraphViz" : "2.14", @@ -43,5 +43,5 @@ } }, "release_status" : "stable", - "version" : "1.0101" + "version" : "1.0102" } diff --git a/perl-package/AI-MXNet/META.yml b/perl-package/AI-MXNet/META.yml index 8c09c96eb685..5b920182f159 100644 --- a/perl-package/AI-MXNet/META.yml +++ b/perl-package/AI-MXNet/META.yml @@ -17,10 +17,10 @@ no_index: - t - inc requires: - AI::MXNetCAPI: '1.0101' + AI::MXNetCAPI: '1.0102' AI::NNVMCAPI: '1.01' Function::Parameters: '1.0705' GraphViz: '2.14' Mouse: v2.1.0 PDL: '2.007' -version: '1.0101' +version: '1.0102' diff --git a/perl-package/AI-MXNet/Makefile.PL b/perl-package/AI-MXNet/Makefile.PL index 990176d1b493..2c9bda83330c 100644 --- a/perl-package/AI-MXNet/Makefile.PL +++ b/perl-package/AI-MXNet/Makefile.PL @@ -19,7 +19,7 @@ my %WriteMakefileArgs = ( "LICENSE" => "apache_2_0", "NAME" => "AI::MXNet", "PREREQ_PM" => { - "AI::MXNetCAPI" => "1.0101", + "AI::MXNetCAPI" => "1.0102", "AI::NNVMCAPI" => "1.01", "Function::Parameters" => "1.0705", "Mouse" => "v2.1.0", @@ -35,7 +35,7 @@ my %WriteMakefileArgs = ( my %FallbackPrereqs = ( - "AI::MXNetCAPI" => "1.0101", + "AI::MXNetCAPI" => "1.0102", "AI::NNVMCAPI" => "1.01", "Function::Parameters" => "1.0705", "Mouse" => "v2.1.0", diff --git a/perl-package/AI-MXNet/README b/perl-package/AI-MXNet/README index f275d08f1a11..86b6cf18dbac 100644 --- a/perl-package/AI-MXNet/README +++ b/perl-package/AI-MXNet/README @@ -1,5 +1,5 @@ This archive contains the distribution AI-MXNet, -version 1.0101: +version 1.0102: Perl interface to MXNet machine learning library diff --git a/perl-package/AI-MXNet/lib/AI/MXNet.pm b/perl-package/AI-MXNet/lib/AI/MXNet.pm index 1d2125354019..40e84a6078e6 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet.pm @@ -46,7 +46,7 @@ use AI::MXNet::Image; use AI::MXNet::Contrib; use AI::MXNet::Contrib::AutoGrad; use AI::MXNet::CachedOp; -our $VERSION = '1.0101'; +our $VERSION = '1.0102'; sub import { diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Base.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Base.pm index d5ff0dd08ecc..0c42fa9306cb 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Base.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Base.pm @@ -20,7 +20,7 @@ use strict; use warnings; use PDL; use PDL::Types qw(); -use AI::MXNetCAPI 1.0101; +use AI::MXNetCAPI 1.0102; use AI::NNVMCAPI 1.01; use AI::MXNet::Types; use Time::HiRes; diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Image.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Image.pm index b996b028dff2..18ef42af5525 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Image.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Image.pm @@ -764,7 +764,7 @@ sub BUILD { chomp($line); my @line = split(/\t/, $line); - my $label = AI::MXNet::NDArray->array([@line[1..@line-1]]); + my $label = AI::MXNet::NDArray->array([@line[1..@line-2]]); my $key = $line[0]; $imglist{$key} = [$label, $line[-1]]; push @imgkeys, $key; @@ -838,6 +838,10 @@ sub BUILD { $self->aug_list(AI::MXNet::Image->CreateAugmenter(data_shape => $self->data_shape, %{ $self->kwargs//{} })); } + else + { + $self->aug_list([]); + } $self->cur(0); $self->reset(); } @@ -877,7 +881,7 @@ method next_sample() } else { - my ($label, $fname) = $self->imglist->{$idx}; + my ($label, $fname) = @{ $self->imglist->{$idx} }; if(not defined $self->imgrec) { open(F, $self->path_root . "/$fname") or confess("can't open $fname $!"); diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Metric.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Metric.pm index 6504481ba8ea..c3a3183432d5 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Metric.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Metric.pm @@ -510,6 +510,55 @@ method update(ArrayRef[AI::MXNet::NDArray] $labels, ArrayRef[AI::MXNet::NDArray] }, $labels, $preds); } +package AI::MXNet::PearsonCorrelation; +use Mouse; +use AI::MXNet::Base; +extends 'AI::MXNet::EvalMetric'; +has '+name' => (default => 'pearson-correlation'); + +=head1 NAME + + AI::MXNet::PearsonCorrelation +=cut + +=head1 DESCRIPTION + + Computes Pearson correlation. + + Parameters + ---------- + name : str + Name of this metric instance for display. + + Examples + -------- + >>> $predicts = [mx->nd->array([[0.3, 0.7], [0, 1.], [0.4, 0.6]])] + >>> $labels = [mx->nd->array([[1, 0], [0, 1], [0, 1]])] + >>> $pr = mx->metric->PearsonCorrelation() + >>> $pr->update($labels, $predicts) + >>> print pr->get() + ('pearson-correlation', '0.421637061887229') +=cut + +method update(ArrayRef[AI::MXNet::NDArray] $labels, ArrayRef[AI::MXNet::NDArray] $preds) +{ + AI::MXNet::Metric::check_label_shapes($labels, $preds); + zip(sub { + my ($label, $pred) = @_; + AI::MXNet::Metric::check_label_shapes($label, $pred); + $label = $label->aspdl->flat; + $pred = $pred->aspdl->flat; + my ($label_mean, $label_stdv) = ($label->stats)[0, 6]; + my ($pred_mean, $pred_stdv) = ($pred->stats)[0, 6]; + $self->sum_metric( + $self->sum_metric + + + ((($label-$label_mean)*($pred-$pred_mean))->sum/$label->nelem)/(($label_stdv*$pred_stdv)->at(0)) + ); + $self->num_inst($self->num_inst + 1); + }, $labels, $preds); +} + =head1 DESCRIPTION Custom evaluation metric that takes a sub ref. @@ -574,6 +623,7 @@ my %metrics = qw/ top_k_accuracy AI::MXNet::TopKAccuracy Perplexity AI::MXNet::Perplexity perplexity AI::MXNet::Perplexity + pearsonr AI::MXNet::PearsonCorrelation /; method create(Metric|ArrayRef[Metric] $metric, %kwargs) diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Module.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Module.pm index 967a51142aac..3e4d938bf4e9 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Module.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Module.pm @@ -796,14 +796,6 @@ method forward( ) { assert($self->binded and $self->params_initialized); - # If starting to do the inference, force rebind the module. - if($self->label_shapes and not $data_batch->label) - { - confess( - "If you are trying to do inference, rebind module ". - "with 'force_rebind=True' and 'for_training=False'" - ); - } my @curr_data_shapes = map { $_->shape } @{ $self->data_shapes }; my @new_data_shapes = map { $_->shape } @{ $data_batch->data }; diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/RNN.pm b/perl-package/AI-MXNet/lib/AI/MXNet/RNN.pm index 1ccab31fb7ac..07e72a755723 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/RNN.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/RNN.pm @@ -166,6 +166,9 @@ method SequentialRNNCell(@args) { AI::MXNet::RNN::SequentialCell->new(@args) } method BidirectionalCell(@args) { AI::MXNet::RNN::BidirectionalCell->new(@args) } method DropoutCell(@args) { AI::MXNet::RNN::DropoutCell->new(@args) } method ZoneoutCell(@args) { AI::MXNet::RNN::ZoneoutCell->new(@args) } +method ConvRNNCell(@args) { AI::MXNet::RNN::ConvCell->new(@args) } +method ConvLSTMCell(@args) { AI::MXNet::RNN::ConvLSTMCell->new(@args) } +method ConvGRUCell(@args) { AI::MXNet::RNN::ConvGRUCell->new(@args) } method ResidualCell(@args) { AI::MXNet::RNN::ResidualCell->new(@args) } method encode_sentences(@args) { AI::MXNet::RNN::IO->encode_sentences(@args) } method BucketSentenceIter(@args) diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/RNN/Cell.pm b/perl-package/AI-MXNet/lib/AI/MXNet/RNN/Cell.pm index 0221a90e7157..08c3094aa9c7 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/RNN/Cell.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/RNN/Cell.pm @@ -766,7 +766,7 @@ has '_dropout' => (is => 'ro', isa => 'Num', init_arg => 'dropout', has '_get_next_state' => (is => 'ro', isa => 'Bool', init_arg => 'get_next_state', default => 0); has '_bidirectional' => (is => 'ro', isa => 'Bool', init_arg => 'bidirectional', default => 0); has 'forget_bias' => (is => 'ro', isa => 'Num', default => 1); -has 'initializer' => (is => 'rw', isa => 'Maybe[AI::MXNet::Initializer]'); +has 'initializer' => (is => 'rw', isa => 'Maybe[Initializer]'); has '_mode' => ( is => 'ro', isa => enum([qw/rnn_relu rnn_tanh lstm gru/]), @@ -1429,6 +1429,309 @@ method unroll( return($outputs, $states); } +package AI::MXNet::RNN::ConvCell::Base; +use Mouse; +use AI::MXNet::Base; +extends 'AI::MXNet::RNN::Cell::Base'; + +=head1 NAME + + AI::MXNet::RNN::Conv::Base +=cut + +=head1 DESCRIPTION + + Abstract base class for Convolutional RNN cells + +=cut + +has '_h2h_kernel' => (is => 'ro', isa => 'Shape', init_arg => 'h2h_kernel'); +has '_h2h_dilate' => (is => 'ro', isa => 'Shape', init_arg => 'h2h_dilate'); +has '_h2h_pad' => (is => 'rw', isa => 'Shape', init_arg => undef); +has '_i2h_kernel' => (is => 'ro', isa => 'Shape', init_arg => 'i2h_kernel'); +has '_i2h_stride' => (is => 'ro', isa => 'Shape', init_arg => 'i2h_stride'); +has '_i2h_dilate' => (is => 'ro', isa => 'Shape', init_arg => 'i2h_dilate'); +has '_i2h_pad' => (is => 'ro', isa => 'Shape', init_arg => 'i2h_pad'); +has '_num_hidden' => (is => 'ro', isa => 'DimSize', init_arg => 'num_hidden'); +has '_input_shape' => (is => 'ro', isa => 'Shape', init_arg => 'input_shape'); +has '_conv_layout' => (is => 'ro', isa => 'Str', init_arg => 'conv_layout', default => 'NCHW'); +has '_activation' => (is => 'ro', init_arg => 'activation'); +has '_state_shape' => (is => 'rw', init_arg => undef); +has [qw/i2h_weight_initializer h2h_weight_initializer + i2h_bias_initializer h2h_bias_initializer/] => (is => 'rw', isa => 'Maybe[Initializer]'); + +sub BUILD +{ + my $self = shift; + assert ( + ($self->_h2h_kernel->[0] % 2 == 1 and $self->_h2h_kernel->[1] % 2 == 1), + "Only support odd numbers, got h2h_kernel= (@{[ $self->_h2h_kernel ]})" + ); + $self->_h2h_pad([ + int($self->_h2h_dilate->[0] * ($self->_h2h_kernel->[0] - 1) / 2), + int($self->_h2h_dilate->[1] * ($self->_h2h_kernel->[1] - 1) / 2) + ]); + # Infer state shape + my $data = AI::MXNet::Symbol->Variable('data'); + my $state_shape = AI::MXNet::Symbol->Convolution( + data => $data, + num_filter => $self->_num_hidden, + kernel => $self->_i2h_kernel, + stride => $self->_i2h_stride, + pad => $self->_i2h_pad, + dilate => $self->_i2h_dilate, + layout => $self->_conv_layout + ); + $state_shape = ($state_shape->infer_shape(data=>$self->_input_shape))[1]->[0]; + $state_shape->[0] = 0; + $self->_state_shape($state_shape); +} + +method state_info() +{ + return [ + { shape => $self->_state_shape, __layout__ => $self->_conv_layout }, + { shape => $self->_state_shape, __layout__ => $self->_conv_layout } + ]; +} + +method call($inputs, $states) +{ + confess("AI::MXNet::RNN::ConvCell::Base is abstract class for convolutional RNN"); +} + +package AI::MXNet::RNN::ConvCell; +use Mouse; +extends 'AI::MXNet::RNN::ConvCell::Base'; + +=head1 NAME + + AI::MXNet::RNN::ConvCell +=cut + +=head1 DESCRIPTION + + Convolutional RNN cells + + Parameters + ---------- + input_shape : array ref of int + Shape of input in single timestep. + num_hidden : int + Number of units in output symbol. + h2h_kernel : array ref of int, default (3, 3) + Kernel of Convolution operator in state-to-state transitions. + h2h_dilate : array ref of int, default (1, 1) + Dilation of Convolution operator in state-to-state transitions. + i2h_kernel : array ref of int, default (3, 3) + Kernel of Convolution operator in input-to-state transitions. + i2h_stride : array ref of int, default (1, 1) + Stride of Convolution operator in input-to-state transitions. + i2h_pad : array ref of int, default (1, 1) + Pad of Convolution operator in input-to-state transitions. + i2h_dilate : array ref of int, default (1, 1) + Dilation of Convolution operator in input-to-state transitions. + activation : str or Symbol, + default functools.partial(symbol.LeakyReLU, act_type='leaky', slope=0.2) + Type of activation function. + prefix : str, default 'ConvRNN_' + Prefix for name of layers (and name of weight if params is None). + params : RNNParams, default None + Container for weight sharing between cells. Created if None. + conv_layout : str, , default 'NCHW' + Layout of ConvolutionOp +=cut + +has '+_h2h_kernel' => (default => sub { [3, 3] }); +has '+_h2h_dilate' => (default => sub { [1, 1] }); +has '+_i2h_kernel' => (default => sub { [3, 3] }); +has '+_i2h_stride' => (default => sub { [1, 1] }); +has '+_i2h_dilate' => (default => sub { [1, 1] }); +has '+_i2h_pad' => (default => sub { [1, 1] }); +has '+_prefix' => (default => 'ConvRNN_'); +has '+_activation' => (default => sub { sub { AI::MXNet::Symbol->LeakyReLU(@_, act_type => 'leaky', slope => 0.2) } }); +has '+i2h_bias_initializer' => (default => 'zeros'); +has '+h2h_bias_initializer' => (default => 'zeros'); +has 'forget_bias' => (is => 'ro', isa => 'Num'); +has [qw/_iW _iB + _hW _hB/] => (is => 'rw', init_arg => undef); + + +sub BUILD +{ + my $self = shift; + $self->_iW($self->_params->get('i2h_weight', init => $self->i2h_weight_initializer)); + $self->_hW($self->_params->get('h2h_weight', init => $self->h2h_weight_initializer)); + $self->_iB( + $self->params->get( + 'i2h_bias', + (defined($self->forget_bias and not defined $self->i2h_bias_initializer) + ? (init => AI::MXNet::LSTMBias->new(forget_bias => $self->forget_bias)) + : (init => $self->i2h_bias_initializer) + ) + ) + ); + $self->_hB($self->_params->get('h2h_bias', init => $self->h2h_bias_initializer)); +} + +method _num_gates() +{ + scalar(@{ $self->_gate_names() }); +} + +method _gate_names() +{ + return [''] +} + +method _conv_forward($inputs, $states, $name) +{ + my $i2h = AI::MXNet::Symbol->Convolution( + name => "${name}i2h", + data => $inputs, + num_filter => $self->_num_hidden*$self->_num_gates(), + kernel => $self->_i2h_kernel, + stride => $self->_i2h_stride, + pad => $self->_i2h_pad, + dilate => $self->_i2h_dilate, + weight => $self->_iW, + bias => $self->_iB + ); + my $h2h = AI::MXNet::Symbol->Convolution( + name => "${name}h2h", + data => @{ $states }[0], + num_filter => $self->_num_hidden*$self->_num_gates(), + kernel => $self->_h2h_kernel, + stride => [1, 1], + pad => $self->_h2h_pad, + dilate => $self->_h2h_dilate, + weight => $self->_hW, + bias => $self->_hB + ); + return ($i2h, $h2h); +} + +method call(AI::MXNet::Symbol $inputs, AI::MXNet::Symbol|ArrayRef[AI::MXNet::Symbol] $states) +{ + $self->_counter($self->_counter + 1); + my $name = sprintf('%st%d_', $self->_prefix, $self->_counter); + my ($i2h, $h2h) = $self->_conv_forward($inputs, $states, $name); + my $output = $self->_get_activation($i2h + $h2h, $self->_activation, name => "${name}out"); + return ($output, [$output]); +} + +package AI::MXNet::RNN::ConvLSTMCell; +use Mouse; +extends 'AI::MXNet::RNN::ConvCell'; +has '+forget_bias' => (default => 1); +has '+_prefix' => (default => 'ConvLSTM_'); + +=head1 NAME + + AI::MXNet::RNN::ConvLSTMCell +=cut + +=head1 DESCRIPTION + + Convolutional LSTM network cell. + + Reference: + Xingjian et al. NIPS2015 +=cut + +method _gate_names() +{ + return ['_i', '_f', '_c', '_o']; +} + +method call(AI::MXNet::Symbol $inputs, AI::MXNet::Symbol|ArrayRef[AI::MXNet::Symbol] $states) +{ + $self->_counter($self->_counter + 1); + my $name = sprintf('%st%d_', $self->_prefix, $self->_counter); + my ($i2h, $h2h) = $self->_conv_forward($inputs, $states, $name); + my $gates = $i2h + $h2h; + my @slice_gates = @{ AI::MXNet::Symbol->SliceChannel( + $gates, + num_outputs => 4, + axis => index($self->_conv_layout, 'C'), + name => "${name}slice" + ) }; + my $in_gate = AI::MXNet::Symbol->Activation( + $slice_gates[0], + act_type => "sigmoid", + name => "${name}i" + ); + my $forget_gate = AI::MXNet::Symbol->Activation( + $slice_gates[1], + act_type => "sigmoid", + name => "${name}f" + ); + my $in_transform = $self->_get_activation( + $slice_gates[2], + $self->_activation, + name => "${name}c" + ); + my $out_gate = AI::MXNet::Symbol->Activation( + $slice_gates[3], + act_type => "sigmoid", + name => "${name}o" + ); + my $next_c = AI::MXNet::Symbol->_plus( + $forget_gate * @{$states}[1], + $in_gate * $in_transform, + name => "${name}state" + ); + my $next_h = AI::MXNet::Symbol->_mul( + $out_gate, $self->_get_activation($next_c, $self->_activation), + name => "${name}out" + ); + return ($next_h, [$next_h, $next_c]); +} + +package AI::MXNet::RNN::ConvGRUCell; +use Mouse; +extends 'AI::MXNet::RNN::ConvCell'; +has '+_prefix' => (default => 'ConvGRU_'); + +=head1 NAME + + AI::MXNet::RNN::ConvGRUCell +=cut + +=head1 DESCRIPTION + + Convolutional GRU network cell. +=cut + +method _gate_names() +{ + return ['_r', '_z', '_o']; +} + +method call(AI::MXNet::Symbol $inputs, AI::MXNet::Symbol|ArrayRef[AI::MXNet::Symbol] $states) +{ + $self->_counter($self->_counter + 1); + my $name = sprintf('%st%d_', $self->_prefix, $self->_counter); + my ($i2h, $h2h) = $self->_conv_forward($inputs, $states, $name); + my ($i2h_r, $i2h_z, $h2h_r, $h2h_z); + ($i2h_r, $i2h_z, $i2h) = @{ AI::MXNet::Symbol->SliceChannel($i2h, num_outputs => 3, name => "${name}_i2h_slice") }; + ($h2h_r, $h2h_z, $h2h) = @{ AI::MXNet::Symbol->SliceChannel($h2h, num_outputs => 3, name => "${name}_h2h_slice") }; + my $reset_gate = AI::MXNet::Symbol->Activation( + $i2h_r + $h2h_r, act_type => "sigmoid", + name => "${name}_r_act" + ); + my $update_gate = AI::MXNet::Symbol->Activation( + $i2h_z + $h2h_z, act_type => "sigmoid", + name => "${name}_z_act" + ); + my $next_h_tmp = $self->_get_activation($i2h + $reset_gate * $h2h, $self->_activation, name => "${name}_h_act"); + my $next_h = AI::MXNet::Symbol->_plus( + (1 - $update_gate) * $next_h_tmp, $update_gate * @{$states}[0], + name => "${name}out" + ); + return ($next_h, [$next_h]); +} + package AI::MXNet::RNN::ModifierCell; use Mouse; use AI::MXNet::Base; @@ -1593,7 +1896,7 @@ method call(AI::MXNet::Symbol $inputs, SymbolOrArrayOfSymbols $states) p => $p ); }; - my $prev_output = $self->prev_output || AI::MXNet::Symbol->zeros(shape => [0, 0]); + my $prev_output = $self->prev_output // AI::MXNet::Symbol->zeros(shape => [0, 0]); my $output = $p_outputs != 0 ? AI::MXNet::Symbol->where( &{$mask}($p_outputs, $next_output), diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Symbol.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Symbol.pm index a5298c7bc3af..eed6e93f568b 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Symbol.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Symbol.pm @@ -1232,7 +1232,7 @@ method Variable( Maybe[Num] :$lr_mult=, Maybe[Num] :$wd_mult=, Maybe[Dtype] :$dtype=, - Maybe[AI::MXNet::Initializer] :$init=, + Maybe[Initializer] :$init=, HashRef[Str] :$kwargs={}, Maybe[Str] :$__layout__= ) diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Types.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Types.pm index e48ae3c086ca..b4ec7e9018b3 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Types.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Types.pm @@ -34,6 +34,7 @@ class_type 'AI::MXNet::Callback'; class_type 'AI::MXNet::EvalMetric'; class_type 'AI::MXNet::DataParallelExecutorGroup'; class_type 'AI::MXNet::Optimizer'; +class_type 'AI::MXNet::Initializer'; class_type 'AI::MXNet::InitDesc'; class_type 'AI::MXNet::IRHeader'; subtype "AcceptableInput" => as "Num|PDL|PDL::Matrix|AI::MXNet::NDArray|AI::MXNet::NDArray::Slice|ArrayRef"; @@ -55,6 +56,7 @@ subtype "NameShape" => as "ArrayRef" => where { subtype "Callback" => as "CodeRef|ArrayRef[Coderef]|AI::MXNet::Callback|ArrayRef[AI::MXNet::Callback]"; subtype "EvalMetric" => as "AI::MXNet::EvalMetric|Str|CodeRef"; subtype "Optimizer" => as "AI::MXNet::Optimizer|Str"; -subtype "Activation" => as "AI::MXNet::Symbol|Str"; +subtype "Initializer" => as "AI::MXNet::Initializer|Str"; +subtype "Activation" => as "AI::MXNet::Symbol|Str|CodeRef"; subtype "SymbolOrArrayOfSymbols" => as "AI::MXNet::Symbol|ArrayRef[AI::MXNet::Symbol]"; subtype "NameShapeOrDataDesc" => as "NameShape|AI::MXNet::DataDesc"; diff --git a/perl-package/AI-MXNet/lib/AI/MXNet/Visualization.pm b/perl-package/AI-MXNet/lib/AI/MXNet/Visualization.pm index 4cdc135c4206..e28cd654722d 100644 --- a/perl-package/AI-MXNet/lib/AI/MXNet/Visualization.pm +++ b/perl-package/AI-MXNet/lib/AI/MXNet/Visualization.pm @@ -371,6 +371,7 @@ method plot_network( } $dot->graph->add_node($name, label => $label, %attr); }; + # add edges for my $node (@{ $nodes }) { @@ -395,6 +396,13 @@ method plot_network( { my $key = $input_name; $key .= '_output' if $input_node->{op} ne 'null'; + if($input_node->{op} ne 'null' and exists $input_node->{attr}) + { + if(ref $input_node->{attr} eq 'HASH' and exists $input_node->{attr}{num_outputs}) + { + $key .= ($input_node->{attr}{num_outputs} - 1); + } + } my $end = @{ $shape_dict{$key} }; $attr{label} = join('x', @{ $shape_dict{$key} }[1..$end-1]); } diff --git a/perl-package/AI-MXNet/t/test_model_parallel.t b/perl-package/AI-MXNet/t/test_model_parallel.t new file mode 100644 index 000000000000..6a8aba7aab06 --- /dev/null +++ b/perl-package/AI-MXNet/t/test_model_parallel.t @@ -0,0 +1,74 @@ +use strict; +use warnings; +use Test::More tests => 4; +use AI::MXNet qw(mx); +use AI::MXNet::TestUtils qw(reldiff); +use AI::MXNet::Base; + +sub test_chain +{ + my $ctx1 = mx->cpu(0); + my $ctx2 = mx->cpu(1); + my $n = 2; + my $data1 = mx->sym->Variable('data1'); + my $data2 = mx->sym->Variable('data2'); + my $data3 = mx->sym->Variable('data2'); + my $net; + { + local($mx::AttrScope) = mx->AttrScope(ctx_group=>'dev1'); + $net = $data1 + $data2; + $net = $net * 3; + } + { + local($mx::AttrScope) = mx->AttrScope(ctx_group=>'dev2'); + $net = $net + $data3; + } + + my $arr = []; + my $arr_grad = []; + my $shape = [4, 5]; + { + local($mx::Context) = $ctx1; + for (0..$n-1) + { + push @$arr, mx->nd->empty($shape); + push @$arr_grad, mx->nd->empty($shape); + } + } + { + local($mx::Context) = $ctx2; + push @$arr, mx->nd->empty($shape); + push @$arr_grad, mx->nd->empty($shape); + } + + my $exec1 = $net->bind( + ctx => $ctx1, + args => $arr, + args_grad => $arr_grad, + group2ctx => { dev1 => $ctx1, dev2 => $ctx2 } + ); + $arr->[0] .= 1; + $arr->[1] .= 2; + $arr->[2] .= 3; + my $arr2 = [map { $_->copyto($ctx1) } @$arr]; + my $arr_grad2 = [map { $_->copyto($ctx1) } @$arr_grad]; + my $exec2 = $net->bind( + ctx => $ctx1, + args => $arr2, + args_grad => $arr_grad2 + ); + + $exec1->forward(1); + $exec2->forward(1); + ok(reldiff($exec1->outputs->[0]->aspdl, $exec2->outputs->[0]->aspdl) < 1e-6); + my $out_grad = mx->nd->empty($shape, ctx => $ctx1); + $out_grad .= 1; + $exec1->backward([$out_grad]); + $exec2->backward([$out_grad->copyto($ctx1)]); + zip(sub { + my ($a, $b) = @_; + ok(reldiff($a->aspdl, $b->aspdl) < 1e-6); + }, $arr_grad, $arr_grad2); +} + +test_chain(); diff --git a/perl-package/AI-MXNet/t/test_rnn.t b/perl-package/AI-MXNet/t/test_rnn.t index 77332b156441..76242c0f48c6 100644 --- a/perl-package/AI-MXNet/t/test_rnn.t +++ b/perl-package/AI-MXNet/t/test_rnn.t @@ -3,7 +3,7 @@ use warnings; use AI::MXNet qw(mx); use AI::MXNet::TestUtils qw(same); use PDL; -use Test::More tests => 45; +use Test::More tests => 54; sub test_rnn { @@ -201,6 +201,63 @@ sub test_zoneout is_deeply($outs, [[10, 100], [10, 100], [10, 100]]); } +sub test_convrnn +{ + my $cell = mx->rnn->ConvRNNCell(input_shape => [1, 3, 16, 10], num_hidden=>10, + h2h_kernel=>[3, 3], h2h_dilate=>[1, 1], + i2h_kernel=>[3, 3], i2h_stride=>[1, 1], + i2h_pad=>[1, 1], i2h_dilate=>[1, 1], + prefix=>'rnn_'); + my $inputs = [map { mx->sym->Variable("rnn_t${_}_data") } 0..2]; + my ($outputs) = $cell->unroll(3, inputs => $inputs); + $outputs = mx->sym->Group($outputs); + is_deeply( + [sort keys %{ $cell->params->_params }], + ['rnn_h2h_bias', 'rnn_h2h_weight', 'rnn_i2h_bias', 'rnn_i2h_weight'] + ); + is_deeply($outputs->list_outputs(), ['rnn_t0_out_output', 'rnn_t1_out_output', 'rnn_t2_out_output']); + my (undef, $outs) = $outputs->infer_shape(rnn_t0_data=>[1, 3, 16, 10], rnn_t1_data=>[1, 3, 16, 10], rnn_t2_data=>[1, 3, 16, 10]); + is_deeply($outs, [[1, 10, 16, 10], [1, 10, 16, 10], [1, 10, 16, 10]]); +} + +sub test_convlstm +{ + my $cell = mx->rnn->ConvLSTMCell(input_shape => [1, 3, 16, 10], num_hidden=>10, + h2h_kernel=>[3, 3], h2h_dilate=>[1, 1], + i2h_kernel=>[3, 3], i2h_stride=>[1, 1], + i2h_pad=>[1, 1], i2h_dilate=>[1, 1], + prefix=>'rnn_', forget_bias => 1); + my $inputs = [map { mx->sym->Variable("rnn_t${_}_data") } 0..2]; + my ($outputs) = $cell->unroll(3, inputs => $inputs); + $outputs = mx->sym->Group($outputs); + is_deeply( + [sort keys %{ $cell->params->_params }], + ['rnn_h2h_bias', 'rnn_h2h_weight', 'rnn_i2h_bias', 'rnn_i2h_weight'] + ); + is_deeply($outputs->list_outputs(), ['rnn_t0_out_output', 'rnn_t1_out_output', 'rnn_t2_out_output']); + my (undef, $outs) = $outputs->infer_shape(rnn_t0_data=>[1, 3, 16, 10], rnn_t1_data=>[1, 3, 16, 10], rnn_t2_data=>[1, 3, 16, 10]); + is_deeply($outs, [[1, 10, 16, 10], [1, 10, 16, 10], [1, 10, 16, 10]]); +} + +sub test_convgru +{ + my $cell = mx->rnn->ConvGRUCell(input_shape => [1, 3, 16, 10], num_hidden=>10, + h2h_kernel=>[3, 3], h2h_dilate=>[1, 1], + i2h_kernel=>[3, 3], i2h_stride=>[1, 1], + i2h_pad=>[1, 1], i2h_dilate=>[1, 1], + prefix=>'rnn_', forget_bias => 1); + my $inputs = [map { mx->sym->Variable("rnn_t${_}_data") } 0..2]; + my ($outputs) = $cell->unroll(3, inputs => $inputs); + $outputs = mx->sym->Group($outputs); + is_deeply( + [sort keys %{ $cell->params->_params }], + ['rnn_h2h_bias', 'rnn_h2h_weight', 'rnn_i2h_bias', 'rnn_i2h_weight'] + ); + is_deeply($outputs->list_outputs(), ['rnn_t0_out_output', 'rnn_t1_out_output', 'rnn_t2_out_output']); + my (undef, $outs) = $outputs->infer_shape(rnn_t0_data=>[1, 3, 16, 10], rnn_t1_data=>[1, 3, 16, 10], rnn_t2_data=>[1, 3, 16, 10]); + is_deeply($outs, [[1, 10, 16, 10], [1, 10, 16, 10], [1, 10, 16, 10]]); +} + test_rnn(); test_lstm(); test_lstm_forget_bias(); @@ -211,3 +268,6 @@ test_stack(); test_bidirectional(); test_unfuse(); test_zoneout(); +test_convrnn(); +test_convlstm(); +test_convgru(); diff --git a/perl-package/AI-MXNetCAPI/Changes b/perl-package/AI-MXNetCAPI/Changes index 17595b46e538..1a6356c0333d 100644 --- a/perl-package/AI-MXNetCAPI/Changes +++ b/perl-package/AI-MXNetCAPI/Changes @@ -1,5 +1,8 @@ Revision history for Perl extension AI::MXNetCAPI +1.0102 Sun Aug 6 16:55:08 PDT 2017 + - updated autograd calls. + 1.0101 Sun Jul 2 17:16:01 PDT 2017 - refactored CachedOp, using strings to index the kvstore. diff --git a/perl-package/AI-MXNetCAPI/META.json b/perl-package/AI-MXNetCAPI/META.json index a79b1e059107..a6d65fd2d73a 100644 --- a/perl-package/AI-MXNetCAPI/META.json +++ b/perl-package/AI-MXNetCAPI/META.json @@ -37,5 +37,5 @@ } }, "release_status" : "stable", - "version" : "1.0101" + "version" : "1.0102" } diff --git a/perl-package/AI-MXNetCAPI/META.yml b/perl-package/AI-MXNetCAPI/META.yml index 84b7801683a7..0e3bb53c475c 100644 --- a/perl-package/AI-MXNetCAPI/META.yml +++ b/perl-package/AI-MXNetCAPI/META.yml @@ -19,4 +19,4 @@ no_index: - inc requires: Test::More: '0' -version: '1.0101' +version: '1.0102' diff --git a/perl-package/AI-MXNetCAPI/README b/perl-package/AI-MXNetCAPI/README index 07df0c301902..5c531463e83b 100644 --- a/perl-package/AI-MXNetCAPI/README +++ b/perl-package/AI-MXNetCAPI/README @@ -1,4 +1,4 @@ -AI-MXNetCAPI version 1.0101 +AI-MXNetCAPI version 1.0102 ===================== Swig interface to MXNet c api. diff --git a/perl-package/AI-MXNetCAPI/lib/AI/MXNetCAPI.pm b/perl-package/AI-MXNetCAPI/lib/AI/MXNetCAPI.pm index f09205733b1e..0a93d71916f8 100644 --- a/perl-package/AI-MXNetCAPI/lib/AI/MXNetCAPI.pm +++ b/perl-package/AI-MXNetCAPI/lib/AI/MXNetCAPI.pm @@ -18,7 +18,7 @@ package AI::MXNetCAPI; use base qw(DynaLoader); bootstrap AI::MXNetCAPI; -our $VERSION = '1.0101'; +our $VERSION = '1.0102'; 1; __END__ diff --git a/perl-package/AI-MXNetCAPI/mxnet.i b/perl-package/AI-MXNetCAPI/mxnet.i index bf00e6856d64..fd1a471bcf16 100644 --- a/perl-package/AI-MXNetCAPI/mxnet.i +++ b/perl-package/AI-MXNetCAPI/mxnet.i @@ -458,6 +458,13 @@ int MXNDArrayGetDType(NDArrayHandle handle, int MXNDArrayGetContext(NDArrayHandle handle, int *out, int *out); +/*! + * \brief return gradient buffer attached to this NDArray + * \param handle NDArray handle + * \return 0 when success, -1 when failure happens + */ +int MXNDArrayGetGrad(NDArrayHandle handle, NDArrayHandle *out); + /*! * \brief detach and ndarray from computation graph by clearing entry_ * \param handle NDArray handle From e61fa7c86db06ac960e76bbe577480d446084502 Mon Sep 17 00:00:00 2001 From: Dick Carter Date: Wed, 9 Aug 2017 16:54:24 -0700 Subject: [PATCH 354/834] Tensorcore conv deconv support (#7347) * Adds support for TensorCore in conv and deconv. * Style correction: Adding '_' to cudnn_tensor_core member variable in conv and deconv. * Style correction: Adding '_' to cudnn_tensor_core member variable in rnn. * Adding missing includes needed for compile on Windows. * Empty commit to test CI failure repeatability. * Changed cached algo selections to be per sm_arch, not device_id. --- src/common/cuda_utils.h | 126 +++++++ src/operator/convolution.cu | 14 +- src/operator/cudnn_algoreg-inl.h | 51 ++- src/operator/cudnn_convolution-inl.h | 436 +++++++++++++++++-------- src/operator/cudnn_deconvolution-inl.h | 422 ++++++++++++++++-------- src/operator/cudnn_rnn-inl.h | 46 ++- src/operator/deconvolution.cu | 6 +- 7 files changed, 806 insertions(+), 295 deletions(-) diff --git a/src/common/cuda_utils.h b/src/common/cuda_utils.h index 3c4d1a88de8e..2879ab3cbec2 100644 --- a/src/common/cuda_utils.h +++ b/src/common/cuda_utils.h @@ -25,6 +25,8 @@ #define MXNET_COMMON_CUDA_UTILS_H_ #include +#include +#include #include /*! \brief Macros/inlines to assist CLion to parse Cuda files (*.cu, *.cuh) */ @@ -175,6 +177,79 @@ inline const char* CurandGetErrorString(curandStatus_t status) { << "cuRAND: " << common::cuda::CurandGetErrorString(e); \ } +/*! + * \brief Determine major version number of the gpu's cuda compute architecture. + * \param device_id The device index of the cuda-capable gpu of interest. + * \return the major version number of the gpu's cuda compute architecture. + */ +inline int ComputeCapabilityMajor(int device_id) { + int major = 0; + CUDA_CALL(cudaDeviceGetAttribute(&major, + cudaDevAttrComputeCapabilityMajor, device_id)); + return major; +} + +/*! + * \brief Determine minor version number of the gpu's cuda compute architecture. + * \param device_id The device index of the cuda-capable gpu of interest. + * \return the minor version number of the gpu's cuda compute architecture. + */ +inline int ComputeCapabilityMinor(int device_id) { + int minor = 0; + CUDA_CALL(cudaDeviceGetAttribute(&minor, + cudaDevAttrComputeCapabilityMinor, device_id)); + return minor; +} + +/*! + * \brief Return the integer SM architecture (e.g. Volta = 70). + * \param device_id The device index of the cuda-capable gpu of interest. + * \return the gpu's cuda compute architecture as an int. + */ +inline int SMArch(int device_id) { + auto major = ComputeCapabilityMajor(device_id); + auto minor = ComputeCapabilityMinor(device_id); + return 10 * major + minor; +} + +/*! + * \brief Determine whether a cuda-capable gpu's architecture supports float16 math. + * \param device_id The device index of the cuda-capable gpu of interest. + * \return whether the gpu's architecture supports float16 math. + */ +inline bool SupportsFloat16Compute(int device_id) { + // Kepler and most Maxwell GPUs do not support fp16 compute + int computeCapabilityMajor = ComputeCapabilityMajor(device_id); + int computeCapabilityMinor = ComputeCapabilityMinor(device_id); + return (computeCapabilityMajor > 5) || + (computeCapabilityMajor == 5 && computeCapabilityMinor >= 3); +} + +/*! + * \brief Determine whether a cuda-capable gpu's architecture supports Tensor Core math. + * \param device_id The device index of the cuda-capable gpu of interest. + * \return whether the gpu's architecture supports Tensor Core math. + */ +inline bool SupportsTensorCore(int device_id) { + // Volta (sm_70) supports TensorCore algos + int computeCapabilityMajor = ComputeCapabilityMajor(device_id); + return (computeCapabilityMajor >= 7); +} + +// The policy if the user hasn't set the environment variable MXNET_CUDA_ALLOW_TENSOR_CORE +#define MXNET_CUDA_ALLOW_TENSOR_CORE_DEFAULT true + +/*! + * \brief Returns global policy for TensorCore algo use. + * \return whether to allow TensorCore algo (if not specified by the Operator locally). + */ +inline bool GetEnvAllowTensorCore() { + // Use of optional here permits: "0", "1", "true" and "false" to all be legal. + bool default_value = MXNET_CUDA_ALLOW_TENSOR_CORE_DEFAULT; + return dmlc::GetEnv("MXNET_CUDA_ALLOW_TENSOR_CORE", + dmlc::optional(default_value)).value(); +} + #endif // MXNET_USE_CUDA #if MXNET_USE_CUDNN @@ -187,6 +262,57 @@ inline const char* CurandGetErrorString(curandStatus_t status) { CHECK_EQ(e, CUDNN_STATUS_SUCCESS) << "cuDNN: " << cudnnGetErrorString(e); \ } +/*! + * \brief Return max number of perf structs cudnnFindConvolutionForwardAlgorithm() + * may want to populate. + * \param cudnn_handle cudnn handle needed to perform the inquiry. + * \return max number of perf structs cudnnFindConvolutionForwardAlgorithm() may + * want to populate. + */ +inline int MaxForwardAlgos(cudnnHandle_t cudnn_handle) { +#if CUDNN_MAJOR >= 7 + int max_algos = 0; + CUDNN_CALL(cudnnGetConvolutionForwardAlgorithmMaxCount(cudnn_handle, &max_algos)); + return max_algos; +#else + return 10; +#endif +} + +/*! + * \brief Return max number of perf structs cudnnFindConvolutionBackwardFilterAlgorithm() + * may want to populate. + * \param cudnn_handle cudnn handle needed to perform the inquiry. + * \return max number of perf structs cudnnFindConvolutionBackwardFilterAlgorithm() may + * want to populate. + */ +inline int MaxBackwardFilterAlgos(cudnnHandle_t cudnn_handle) { +#if CUDNN_MAJOR >= 7 + int max_algos = 0; + CUDNN_CALL(cudnnGetConvolutionBackwardFilterAlgorithmMaxCount(cudnn_handle, &max_algos)); + return max_algos; +#else + return 10; +#endif +} + +/*! + * \brief Return max number of perf structs cudnnFindConvolutionBackwardDataAlgorithm() + * may want to populate. + * \param cudnn_handle cudnn handle needed to perform the inquiry. + * \return max number of perf structs cudnnFindConvolutionBackwardDataAlgorithm() may + * want to populate. + */ +inline int MaxBackwardDataAlgos(cudnnHandle_t cudnn_handle) { +#if CUDNN_MAJOR >= 7 + int max_algos = 0; + CUDNN_CALL(cudnnGetConvolutionBackwardDataAlgorithmMaxCount(cudnn_handle, &max_algos)); + return max_algos; +#else + return 10; +#endif +} + #endif // MXNET_USE_CUDNN // Overload atomicAdd to work for floats on all architectures diff --git a/src/operator/convolution.cu b/src/operator/convolution.cu index bf5f3053b2de..ab354849600a 100644 --- a/src/operator/convolution.cu +++ b/src/operator/convolution.cu @@ -71,14 +71,14 @@ Operator* CreateOp(ConvolutionParam param, int dtype, int backward_compute_type = desired_backward_compute_type; bool convolutionIsSupported = CuDNNConvolutionOp::Supports(param, forward_compute_type, - backward_compute_type); + backward_compute_type, ctx); // If cuDNN can't handle this case with fp16 backprop kernels, try fp32 backprop. if (!convolutionIsSupported && backward_compute_type == mshadow::kFloat16) { backward_compute_type = mshadow::kFloat32; convolutionIsSupported = CuDNNConvolutionOp::Supports(param, forward_compute_type, - backward_compute_type); + backward_compute_type, ctx); } // If cuDNN can't handle this case with fp16 forward kernels, try fp32 @@ -86,16 +86,16 @@ Operator* CreateOp(ConvolutionParam param, int dtype, forward_compute_type = mshadow::kFloat32; convolutionIsSupported = CuDNNConvolutionOp::Supports(param, forward_compute_type, - backward_compute_type); + backward_compute_type, ctx); } if (!convolutionIsSupported) { LOG(WARNING) << "This convolution is not supported by cudnn, MXNET convolution is applied."; op = new ConvolutionOp(param); } else { - if ((forward_compute_type != desired_forward_compute_type) || - (backward_compute_type != desired_backward_compute_type)) - LOG(WARNING) << "True fp16 convolution by cudnn not supported in this configuration. " << - "Falling back to pseudo fp16."; + if (forward_compute_type != desired_forward_compute_type) + LOG(WARNING) << "Requested forward compute precision not supported, using fp32."; + if (backward_compute_type != desired_backward_compute_type) + LOG(WARNING) << "Requested backward compute precision not supported, using fp32."; op = new CuDNNConvolutionOp(param, forward_compute_type, backward_compute_type, diff --git a/src/operator/cudnn_algoreg-inl.h b/src/operator/cudnn_algoreg-inl.h index 1078d658597e..dc5db6bbc8b7 100644 --- a/src/operator/cudnn_algoreg-inl.h +++ b/src/operator/cudnn_algoreg-inl.h @@ -32,11 +32,35 @@ #include "../common/cuda_utils.h" #include "./convolution-inl.h" #include "./deconvolution-inl.h" - namespace mxnet { namespace op { #if MXNET_USE_CUDNN == 1 +/*! + * \brief A cuDNN algorithm: an algo number and whether it should be run in TENSOR CORE mode. + */ +template +class CuDNNAlgo { + public: + CuDNNAlgo() : + algo_number_(static_cast(0)), + is_tensor_core_algo_(false) { } + void Set(CuDNNAlgoType algo, bool is_tensor_core) { + algo_number_ = algo; + is_tensor_core_algo_ = is_tensor_core; + } + CuDNNAlgoType AlgoNumber() const { return algo_number_; } + bool IsTensorCoreAlgo() const { return is_tensor_core_algo_; } + #if CUDNN_MAJOR >= 7 + cudnnMathType_t MathType() { + return IsTensorCoreAlgo() ? CUDNN_TENSOR_OP_MATH : CUDNN_DEFAULT_MATH; + } + #endif + private: + CuDNNAlgoType algo_number_; + bool is_tensor_core_algo_; +}; + class CuDNNAlgoReg { public: template @@ -44,7 +68,8 @@ class CuDNNAlgoReg { const std::vector &out_shape, cudnnDataType_t cudnn_data_type, cudnnDataType_t cudnn_forward_compute_type, - cudnnDataType_t cudnn_backward_compute_type) { + cudnnDataType_t cudnn_backward_compute_type, + int sm_arch) { std::ostringstream oss; oss << "inputs="; for (auto &i : in_shape) @@ -58,12 +83,15 @@ class CuDNNAlgoReg { oss << "cudnn_data_type=" << cudnn_data_type << ";"; oss << "cudnn_forward_compute_type=" << cudnn_forward_compute_type << ";"; oss << "cudnn_backward_compute_type=" << cudnn_backward_compute_type << ";"; + // All GPUs of the same compute capability (SM arch) share an algo selection. + oss << "sm_arch=" << sm_arch << ";"; return oss.str(); } - bool Find(std::string key, cudnnConvolutionFwdAlgo_t *fwd, - cudnnConvolutionBwdDataAlgo_t *bwd, - cudnnConvolutionBwdFilterAlgo_t *flt) { + bool Find(std::string key, + CuDNNAlgo *fwd, + CuDNNAlgo *bwd, + CuDNNAlgo *flt) { std::lock_guard guard(lock_); auto i = reg_.find(key); if (i != reg_.end()) { @@ -75,9 +103,10 @@ class CuDNNAlgoReg { return false; } - void Register(std::string key, cudnnConvolutionFwdAlgo_t fwd, - cudnnConvolutionBwdDataAlgo_t bwd, - cudnnConvolutionBwdFilterAlgo_t flt) { + void Register(std::string key, + const CuDNNAlgo &fwd, + const CuDNNAlgo &bwd, + const CuDNNAlgo &flt) { std::lock_guard guard(lock_); if (reg_.size() % 50 == 0) { LOG(INFO) << "Running performance tests to find the best convolution " @@ -100,9 +129,9 @@ class CuDNNAlgoReg { private: struct CudnnAlgorithms { - cudnnConvolutionFwdAlgo_t fwd; - cudnnConvolutionBwdDataAlgo_t bwd; - cudnnConvolutionBwdFilterAlgo_t flt; + CuDNNAlgo fwd; + CuDNNAlgo bwd; + CuDNNAlgo flt; }; std::mutex lock_; diff --git a/src/operator/cudnn_convolution-inl.h b/src/operator/cudnn_convolution-inl.h index e966b56d2a20..428278498337 100644 --- a/src/operator/cudnn_convolution-inl.h +++ b/src/operator/cudnn_convolution-inl.h @@ -59,6 +59,8 @@ class CuDNNConvolutionOp : public Operator { init_cudnn_ = false; init_temp_size_ = false; dtype_ = DataType::kCudnnFlag; + // TensorCore algos only allowed on fp16-I/O convolutions if permitted by the global policy. + cudnn_tensor_core_ = DataType::kFlag == kFloat16 && GetEnvAllowTensorCore(); #if CUDNN_MAJOR >= 5 MSHADOW_LAYOUT_SWITCH(param_.layout.value(), Layout, { @@ -69,7 +71,7 @@ class CuDNNConvolutionOp : public Operator { << "Need CuDNN > 5.0 for layout support"; #endif // Double check to make sure this class supports the operation - if (!Supports(param, forward_compute_type, backward_compute_type)) + if (!Supports(param, forward_compute_type, backward_compute_type, ctx)) LOG(FATAL) << "Need CuDNN >= 6.0 for dilated convolution."; InitDescriptors(ctx, in_shape, out_shape, @@ -95,7 +97,8 @@ class CuDNNConvolutionOp : public Operator { CUDNN_CALL(cudnnDestroyTensorDescriptor(bias_desc_)); CUDNN_CALL(cudnnDestroyFilterDescriptor(filter_desc_)); CUDNN_CALL(cudnnDestroyConvolutionDescriptor(forward_conv_desc_)); - CUDNN_CALL(cudnnDestroyConvolutionDescriptor(backward_conv_desc_)); + CUDNN_CALL(cudnnDestroyConvolutionDescriptor(back_conv_desc_)); + CUDNN_CALL(cudnnDestroyConvolutionDescriptor(back_conv_desc_w_)); } } @@ -148,7 +151,7 @@ class CuDNNConvolutionOp : public Operator { filter_desc_, wmat_ptr + weight_offset_ * g, forward_conv_desc_, - algo_, + forward_algo_.AlgoNumber(), workspace.dptr_, workspace_size, req[conv::kOut] == kAddTo? &beta_add : &beta, @@ -244,8 +247,8 @@ class CuDNNConvolutionOp : public Operator { data_ptr + data_offset_ * g, out_desc_, grad_ptr + out_offset_ * g, - backward_conv_desc_, - back_algo_w_, + back_conv_desc_w_, + back_algo_w_.AlgoNumber(), workspace.dptr_, workspace_size, req[conv::kWeight] == kAddTo? &beta_add : &beta, @@ -258,8 +261,8 @@ class CuDNNConvolutionOp : public Operator { data_ptr + data_offset_ * g, out_desc_, grad_ptr + out_offset_ * g, - backward_conv_desc_, - back_algo_w_, + back_conv_desc_w_, + back_algo_w_.AlgoNumber(), workspace.dptr_, workspace_size, req[conv::kWeight] == kAddTo? &beta_add : &beta, @@ -275,8 +278,8 @@ class CuDNNConvolutionOp : public Operator { wmat_ptr + weight_offset_ * g, out_desc_, grad_ptr + out_offset_ * g, - backward_conv_desc_, - back_algo_, + back_conv_desc_, + back_algo_.AlgoNumber(), workspace.dptr_, workspace_size, req[conv::kData] == kAddTo? &beta_add : &beta, @@ -289,8 +292,8 @@ class CuDNNConvolutionOp : public Operator { wmat_ptr + weight_offset_ * g, out_desc_, grad_ptr + out_offset_ * g, - backward_conv_desc_, - back_algo_, + back_conv_desc_, + back_algo_.AlgoNumber(), workspace.dptr_, workspace_size, req[conv::kData] == kAddTo? &beta_add : &beta, @@ -308,7 +311,8 @@ class CuDNNConvolutionOp : public Operator { */ static bool Supports(ConvolutionParam param, int forward_compute_type, - int backward_compute_type) { + int backward_compute_type, + const Context &ctx) { using namespace mshadow; // NDHWC not supported, NHWC not supported in true fp16 @@ -318,6 +322,12 @@ class CuDNNConvolutionOp : public Operator { if (layout_val == kNDHWC || layout_val == kNHWC && true_fp16) return false; + // Permits graceful fallback to pseudo-fp16 on heterogenous systems + if (!SupportsFloat16Compute(ctx.dev_id) && + (forward_compute_type == kFloat16 || backward_compute_type == kFloat16)) { + return false; + } + // The factor by which the effective filter size grows based on dilation. auto filterDilationFactor = param.dilate.Size(); @@ -355,7 +365,8 @@ class CuDNNConvolutionOp : public Operator { CUDNN_CALL(cudnnCreateTensorDescriptor(&bias_desc_)); CUDNN_CALL(cudnnCreateFilterDescriptor(&filter_desc_)); CUDNN_CALL(cudnnCreateConvolutionDescriptor(&forward_conv_desc_)); - CUDNN_CALL(cudnnCreateConvolutionDescriptor(&backward_conv_desc_)); + CUDNN_CALL(cudnnCreateConvolutionDescriptor(&back_conv_desc_)); + CUDNN_CALL(cudnnCreateConvolutionDescriptor(&back_conv_desc_w_)); TShape dshape = in_shape[conv::kData]; TShape wshape = in_shape[conv::kWeight]; @@ -379,7 +390,16 @@ class CuDNNConvolutionOp : public Operator { param_.dilate[1], CUDNN_CROSS_CORRELATION, cudnn_forward_compute_type)); - CUDNN_CALL(cudnnSetConvolution2dDescriptor(backward_conv_desc_, + CUDNN_CALL(cudnnSetConvolution2dDescriptor(back_conv_desc_, + param_.pad[0], + param_.pad[1], + param_.stride[0], + param_.stride[1], + param_.dilate[0], + param_.dilate[1], + CUDNN_CROSS_CORRELATION, + cudnn_backward_compute_type)); + CUDNN_CALL(cudnnSetConvolution2dDescriptor(back_conv_desc_w_, param_.pad[0], param_.pad[1], param_.stride[0], @@ -397,7 +417,15 @@ class CuDNNConvolutionOp : public Operator { param_.dilate[0], param_.dilate[1], CUDNN_CROSS_CORRELATION)); - CUDNN_CALL(cudnnSetConvolution2dDescriptor(backward_conv_desc_, + CUDNN_CALL(cudnnSetConvolution2dDescriptor(back_conv_desc_, + param_.pad[0], + param_.pad[1], + param_.stride[0], + param_.stride[1], + param_.dilate[0], + param_.dilate[1], + CUDNN_CROSS_CORRELATION)); + CUDNN_CALL(cudnnSetConvolution2dDescriptor(back_conv_desc_w_, param_.pad[0], param_.pad[1], param_.stride[0], @@ -460,7 +488,15 @@ class CuDNNConvolutionOp : public Operator { CUDNN_CROSS_CORRELATION, cudnn_forward_compute_type)); - CUDNN_CALL(cudnnSetConvolutionNdDescriptor(backward_conv_desc_, + CUDNN_CALL(cudnnSetConvolutionNdDescriptor(back_conv_desc_, + 3, + param_pad_.data(), + param_stride_.data(), + param_dilate_.data(), + CUDNN_CROSS_CORRELATION, + cudnn_backward_compute_type)); + + CUDNN_CALL(cudnnSetConvolutionNdDescriptor(back_conv_desc_w_, 3, param_pad_.data(), param_stride_.data(), @@ -484,6 +520,14 @@ class CuDNNConvolutionOp : public Operator { param_.layout.value(), kNCDHW); oshape = ConvertLayout(oshape.get<5>(), param_.layout.value(), kNCDHW); } + // Set "allow tensor core" flag in convolution descriptors, if available. + #if CUDNN_MAJOR >= 7 + cudnnMathType_t math_type = cudnn_tensor_core_ ? CUDNN_TENSOR_OP_MATH + : CUDNN_DEFAULT_MATH; + CUDNN_CALL(cudnnSetConvolutionMathType(forward_conv_desc_, math_type)); + CUDNN_CALL(cudnnSetConvolutionMathType(back_conv_desc_, math_type)); + CUDNN_CALL(cudnnSetConvolutionMathType(back_conv_desc_w_, math_type)); + #endif dshape[1] /= param_.num_group; oshape[1] /= param_.num_group; weight_offset_ = wshape.Size(); @@ -538,122 +582,234 @@ class CuDNNConvolutionOp : public Operator { cudnnDataType_t cudnn_backward_compute_type) { std::string key = CuDNNAlgoReg::Get()->GetKey(param_, in_shape, out_shape, dtype_, cudnn_forward_compute_type, - cudnn_backward_compute_type); - if (CuDNNAlgoReg::Get()->Find(key, &algo_, &back_algo_, &back_algo_w_)) - return; - - Engine::VarHandle var = Engine::Get()->NewVariable(); - Engine::Get()->PushSync([=](RunContext rctx) { - mshadow::Stream *s = rctx.get_stream(); - CHECK_EQ(s->dnn_handle_ownership_, mshadow::Stream::OwnHandle); - size_t workspace_byte = static_cast(param_.workspace * sizeof(DType)); - if (!param_.cudnn_tune.value()) { - // In cuDNNv6, for kNHWC, only CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_GEMM is - // supported. Hard-coded this since the algo find() or get() throws an FPE. - if (CUDNN_MAJOR == 6 && param_.layout.value() == mshadow::kNHWC) { - algo_ = CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_GEMM; - } else { - CUDNN_CALL(cudnnGetConvolutionForwardAlgorithm(s->dnn_handle_, - in_desc_, - filter_desc_, - forward_conv_desc_, - out_desc_, - CUDNN_CONVOLUTION_FWD_SPECIFY_WORKSPACE_LIMIT, - workspace_byte, - &(this->algo_))); - } - CUDNN_CALL(cudnnGetConvolutionBackwardFilterAlgorithm(s->dnn_handle_, - in_desc_, - out_desc_, - backward_conv_desc_, - filter_desc_, - CUDNN_CONVOLUTION_BWD_FILTER_SPECIFY_WORKSPACE_LIMIT, - workspace_byte, - &(this->back_algo_w_))); - CUDNN_CALL(cudnnGetConvolutionBackwardDataAlgorithm(s->dnn_handle_, - filter_desc_, - out_desc_, - backward_conv_desc_, - in_desc_, - CUDNN_CONVOLUTION_BWD_DATA_SPECIFY_WORKSPACE_LIMIT, - workspace_byte, - &(this->back_algo_))); - } else { - const int kMaxAlgos = 10; - int nalgo = kMaxAlgos; - int i; - - // In cuDNNv6, for kNHWC, only CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_GEMM is - // supported. Hard-coded this since the algo find() or get() throws an FPE. - if (CUDNN_MAJOR == 6 && param_.layout.value() == mshadow::kNHWC) { - algo_ = CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_GEMM; - } else { - cudnnConvolutionFwdAlgoPerf_t fwd_algo[kMaxAlgos]; - CUDNN_CALL(cudnnFindConvolutionForwardAlgorithm(s->dnn_handle_, - in_desc_, - filter_desc_, - forward_conv_desc_, - out_desc_, - kMaxAlgos, - &nalgo, - fwd_algo)); - i = 0; - while (i < nalgo - && (fwd_algo[i].status != CUDNN_STATUS_SUCCESS - || (param_.cudnn_tune.value() == conv::kLimited - && fwd_algo[i].memory > workspace_byte))) ++i; - if (i == nalgo) { - LOG(FATAL) << "Failed to find a forward convolution algorithm."; + cudnn_backward_compute_type, + SMArch(ctx.dev_id)); + if (!CuDNNAlgoReg::Get()->Find(key, &forward_algo_, &back_algo_, &back_algo_w_)) { + // Not in algo registry, must determine via *Get*() or *Find*() + Engine::VarHandle var = Engine::Get()->NewVariable(); + Engine::Get()->PushSync([=](RunContext rctx) { + mshadow::Stream *s = rctx.get_stream(); + CHECK_EQ(s->dnn_handle_ownership_, mshadow::Stream::OwnHandle); + size_t workspace_byte = static_cast(param_.workspace * sizeof(DType)); + #if CUDNN_MAJOR >= 7 + // Starting with cuDNNv7, the algo number returned by *Get*() is not the entire + // story: the notion of whether the algo ran in Tensor Core mode is not known. + // Since we want to report the Tensor Core mode in the verbose output, we switch + // to using the new *Get*_v7() call. Since the function signature of *Get*_v7() matches + // that of *Find*(), we can unify the find-vs-get logic by using function pointers. + + // Forward Algorithm Find/Get() v7 + std::vector fwd_results(MaxForwardAlgos(s->dnn_handle_)); + int actual_fwd_algos = 0; + auto fwd_algo_discoverer = + param_.cudnn_tune.value() == conv::kOff ? cudnnGetConvolutionForwardAlgorithm_v7 + : cudnnFindConvolutionForwardAlgorithm; + CUDNN_CALL((*fwd_algo_discoverer)(s->dnn_handle_, + in_desc_, + filter_desc_, + forward_conv_desc_, + out_desc_, + fwd_results.size(), + &actual_fwd_algos, + fwd_results.data())); + fwd_results.resize(actual_fwd_algos); + AlgoFinalSelect(fwd_results, "forward", + workspace_byte, &forward_algo_); + + // Backprop-to-Filter Algorithm Find/Get() v7 + auto max_bwd_filt_algos = MaxBackwardFilterAlgos(s->dnn_handle_); + std::vector bwd_filt_results(max_bwd_filt_algos); + int actual_bwd_filter_algos = 0; + auto bwd_filter_algo_discoverer = + param_.cudnn_tune.value() == conv::kOff ? cudnnGetConvolutionBackwardFilterAlgorithm_v7 + : cudnnFindConvolutionBackwardFilterAlgorithm; + CUDNN_CALL((*bwd_filter_algo_discoverer)(s->dnn_handle_, + in_desc_, + out_desc_, + back_conv_desc_w_, + filter_desc_, + bwd_filt_results.size(), + &actual_bwd_filter_algos, + bwd_filt_results.data())); + bwd_filt_results.resize(actual_bwd_filter_algos); + AlgoFinalSelect(bwd_filt_results, "backprop-to-filter", + workspace_byte, &back_algo_w_); + + // Backprop-to-Data Algorithm Find/Get() v7 + auto max_bwd_data_algos = MaxBackwardDataAlgos(s->dnn_handle_); + std::vector bwd_data_results(max_bwd_data_algos); + int actual_bwd_data_algos = 0; + auto bwd_data_algo_discoverer = + param_.cudnn_tune.value() == conv::kOff ? cudnnGetConvolutionBackwardDataAlgorithm_v7 + : cudnnFindConvolutionBackwardDataAlgorithm; + CUDNN_CALL((*bwd_data_algo_discoverer)(s->dnn_handle_, + filter_desc_, + out_desc_, + back_conv_desc_, + in_desc_, + bwd_data_results.size(), + &actual_bwd_data_algos, + bwd_data_results.data())); + bwd_data_results.resize(actual_bwd_data_algos); + AlgoFinalSelect(bwd_data_results, "backprop-to-data", + workspace_byte, &back_algo_); + #else + // CUDNN_MAJOR < 7 + const int kMaxAlgos = 10; + int nalgo = kMaxAlgos; + int i = 0; + // Forward Algorithm Find/Get, v6 and earlier + if (CUDNN_MAJOR == 6 && param_.layout.value() == mshadow::kNHWC) { + // In cuDNNv6, for kNHWC, only CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_GEMM is + // supported. Hard-coded this since the algo find() or get() throws an FPE. + forward_algo_.Set(CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_GEMM, false); + } else if (!param_.cudnn_tune.value()) { + cudnnConvolutionFwdAlgo_t fastest_fwd_algo; + CUDNN_CALL(cudnnGetConvolutionForwardAlgorithm(s->dnn_handle_, + in_desc_, + filter_desc_, + forward_conv_desc_, + out_desc_, + CUDNN_CONVOLUTION_FWD_SPECIFY_WORKSPACE_LIMIT, + workspace_byte, + &fastest_fwd_algo)); + forward_algo_.Set(fastest_fwd_algo, false); } else { - this->algo_ = fwd_algo[i].algo; + cudnnConvolutionFwdAlgoPerf_t fwd_algo[kMaxAlgos]; + CUDNN_CALL(cudnnFindConvolutionForwardAlgorithm(s->dnn_handle_, + in_desc_, + filter_desc_, + forward_conv_desc_, + out_desc_, + kMaxAlgos, + &nalgo, + fwd_algo)); + i = 0; + while (i < nalgo + && (fwd_algo[i].status != CUDNN_STATUS_SUCCESS + || (param_.cudnn_tune.value() == conv::kLimited + && fwd_algo[i].memory > workspace_byte))) + ++i; + if (i == nalgo) { + LOG(FATAL) << "Failed to find a forward convolution algorithm."; + } else { + forward_algo_.Set(fwd_algo[i].algo, false); + } } - } - - cudnnConvolutionBwdFilterAlgoPerf_t bwd_filter_algo[kMaxAlgos]; - CUDNN_CALL(cudnnFindConvolutionBackwardFilterAlgorithm(s->dnn_handle_, - in_desc_, - out_desc_, - backward_conv_desc_, - filter_desc_, - kMaxAlgos, - &nalgo, - bwd_filter_algo)); - i = 0; - while (i < nalgo - && (bwd_filter_algo[i].status != CUDNN_STATUS_SUCCESS - || (param_.cudnn_tune.value() == conv::kLimited - && bwd_filter_algo[i].memory > workspace_byte))) ++i; - if (i == nalgo) { - LOG(FATAL) << "Failed to find a backward filter convolution algorithm."; - } else { - this->back_algo_w_ = bwd_filter_algo[i].algo; - } - - cudnnConvolutionBwdDataAlgoPerf_t bwd_data_algo[kMaxAlgos]; - CUDNN_CALL(cudnnFindConvolutionBackwardDataAlgorithm(s->dnn_handle_, - filter_desc_, - out_desc_, - backward_conv_desc_, - in_desc_, - kMaxAlgos, - &nalgo, - bwd_data_algo)); - i = 0; - while (i < nalgo - && (bwd_data_algo[i].status != CUDNN_STATUS_SUCCESS - || (param_.cudnn_tune.value() == conv::kLimited - && bwd_data_algo[i].memory > workspace_byte))) ++i; - if (i == nalgo) { - LOG(FATAL) << "Failed to find a backward data convolution algorithm."; - } else { - this->back_algo_ = bwd_data_algo[i].algo; - } - CuDNNAlgoReg::Get()->Register(key, this->algo_, this->back_algo_, + // Backprop-to-Filter Algorithm Find/Get, v6 and earlier + if (!param_.cudnn_tune.value()) { + cudnnConvolutionBwdFilterAlgo_t fastest_bwd_filt_algo; + CUDNN_CALL(cudnnGetConvolutionBackwardFilterAlgorithm(s->dnn_handle_, + in_desc_, + out_desc_, + back_conv_desc_w_, + filter_desc_, + CUDNN_CONVOLUTION_BWD_FILTER_SPECIFY_WORKSPACE_LIMIT, + workspace_byte, + &fastest_bwd_filt_algo)); + back_algo_w_.Set(fastest_bwd_filt_algo, false); + } else { + cudnnConvolutionBwdFilterAlgoPerf_t bwd_filter_algo[kMaxAlgos]; + CUDNN_CALL(cudnnFindConvolutionBackwardFilterAlgorithm(s->dnn_handle_, + in_desc_, + out_desc_, + back_conv_desc_w_, + filter_desc_, + kMaxAlgos, + &nalgo, + bwd_filter_algo)); + i = 0; + while (i < nalgo + && (bwd_filter_algo[i].status != CUDNN_STATUS_SUCCESS + || (param_.cudnn_tune.value() == conv::kLimited + && bwd_filter_algo[i].memory > workspace_byte))) + ++i; + if (i == nalgo) { + LOG(FATAL) << "Failed to find a backward filter convolution algorithm."; + } else { + back_algo_w_.Set(bwd_filter_algo[i].algo, false); + } + } + // Backprop-to-Data Algorithm Get(), v6 and earlier + if (!param_.cudnn_tune.value()) { + cudnnConvolutionBwdDataAlgo_t fastest_bwd_data_algo; + CUDNN_CALL(cudnnGetConvolutionBackwardDataAlgorithm(s->dnn_handle_, + filter_desc_, + out_desc_, + back_conv_desc_, + in_desc_, + CUDNN_CONVOLUTION_BWD_DATA_SPECIFY_WORKSPACE_LIMIT, + workspace_byte, + &fastest_bwd_data_algo)); + back_algo_.Set(fastest_bwd_data_algo, false); + } else { + cudnnConvolutionBwdDataAlgoPerf_t bwd_data_algo[kMaxAlgos]; + CUDNN_CALL(cudnnFindConvolutionBackwardDataAlgorithm(s->dnn_handle_, + filter_desc_, + out_desc_, + back_conv_desc_, + in_desc_, + kMaxAlgos, + &nalgo, + bwd_data_algo)); + i = 0; + while (i < nalgo + && (bwd_data_algo[i].status != CUDNN_STATUS_SUCCESS + || (param_.cudnn_tune.value() == conv::kLimited + && bwd_data_algo[i].memory > workspace_byte))) + ++i; + if (i == nalgo) { + LOG(FATAL) << "Failed to find a backward data convolution algorithm."; + } else { + back_algo_.Set(bwd_data_algo[i].algo, false); + } + } + #endif // CUDNN_MAJOR < 7 + // An algo specification by the user may be cached here, but another + // convolution will match only if identically specified. + // We're caching results of *Get* as well as *Find*, but these records + // will be held distinctly because param_.cudnn_tune is part of the key. + CuDNNAlgoReg::Get()->Register(key, this->forward_algo_, this->back_algo_, this->back_algo_w_); + }, ctx, {}, {var}); + Engine::Get()->WaitForVar(var); + Engine::Get()->DeleteVariable([](RunContext s) {}, ctx, var); + } + // If we're allowing Tensor Core variants of the algos to be considered in + // *Find*() or *Get*(), but a non-Tensor-Core algo variant is the fastest, + // we must change the descriptor to preclude Tensor Core. Simplest is to + // once again set the mathType in all cases. + #if CUDNN_MAJOR >= 7 + CUDNN_CALL(cudnnSetConvolutionMathType(forward_conv_desc_, forward_algo_.MathType())); + CUDNN_CALL(cudnnSetConvolutionMathType(back_conv_desc_, back_algo_.MathType())); + CUDNN_CALL(cudnnSetConvolutionMathType(back_conv_desc_w_, back_algo_w_.MathType())); + #endif + } + + // Look over the results from *Find*() or *Get*() and pick the fastest algo given possible + // workspace constraints. + template + void AlgoFinalSelect(const std::vector &perf_results, std::string kernel_name, + size_t workspace_byte, CuDNNAlgo *algo) { + // Determine the fastest acceptable algo that matches the algo_preference (-1 = any), + // regardless of mathType. + for (decltype(perf_results.size()) i = 0; i != perf_results.size(); ++i) { + const auto &result = perf_results[i]; + bool algo_is_tensor_core = false; + #if CUDNN_MAJOR >= 7 + algo_is_tensor_core = result.mathType == CUDNN_TENSOR_OP_MATH; + #endif + if (result.status == CUDNN_STATUS_SUCCESS && + (param_.cudnn_tune.value() != conv::kLimited || result.memory <= workspace_byte)) { + algo->Set(result.algo, algo_is_tensor_core); + return; } - }, ctx, {}, {var}); - Engine::Get()->WaitForVar(var); - Engine::Get()->DeleteVariable([](RunContext s) {}, ctx, var); + } + auto mode = param_.cudnn_tune.value() == conv::kOff ? " get " : " find "; + LOG(FATAL) << "Failed to" << mode << "any " << kernel_name << " convolution algorithm."; } void GetTempSize(const OpContext& ctx) { @@ -663,16 +819,16 @@ class CuDNNConvolutionOp : public Operator { CUDNN_CALL(cudnnGetConvolutionBackwardDataWorkspaceSize(s->dnn_handle_, filter_desc_, out_desc_, - backward_conv_desc_, + back_conv_desc_, in_desc_, - back_algo_, + back_algo_.AlgoNumber(), &back_size)); CUDNN_CALL(cudnnGetConvolutionBackwardFilterWorkspaceSize(s->dnn_handle_, in_desc_, out_desc_, - backward_conv_desc_, + back_conv_desc_w_, filter_desc_, - back_algo_w_, + back_algo_w_.AlgoNumber(), &back_size_w)); backward_workspace_byte_ = std::max(back_size, back_size_w); CUDNN_CALL(cudnnGetConvolutionForwardWorkspaceSize(s->dnn_handle_, @@ -680,7 +836,7 @@ class CuDNNConvolutionOp : public Operator { filter_desc_, forward_conv_desc_, out_desc_, - algo_, + forward_algo_.AlgoNumber(), &forward_workspace_byte_)); init_temp_size_ = true; @@ -733,15 +889,19 @@ class CuDNNConvolutionOp : public Operator { cudnnFilterDescriptor_t filter_desc_; // Convolution descriptor for forward inference operation cudnnConvolutionDescriptor_t forward_conv_desc_; - // Convolution descriptor for back-prop operations to data and filter - cudnnConvolutionDescriptor_t backward_conv_desc_; + // Convolution descriptor for back-prop operations to the data + cudnnConvolutionDescriptor_t back_conv_desc_; + // Convolution descriptor for back-prop operations to the weights + cudnnConvolutionDescriptor_t back_conv_desc_w_; // Algorithm for the forward inference operation - cudnnConvolutionFwdAlgo_t algo_; + CuDNNAlgo forward_algo_; // Algorithm for the back-prop operation to the data - cudnnConvolutionBwdDataAlgo_t back_algo_; + CuDNNAlgo back_algo_; // Algorithm for the back-prop operation to the weights - cudnnConvolutionBwdFilterAlgo_t back_algo_w_; + CuDNNAlgo back_algo_w_; cudnnTensorFormat_t format_; + // Allow TensorCore algo policy + bool cudnn_tensor_core_; ConvolutionParam param_; }; #endif // __CUDACC__ && CUDNN diff --git a/src/operator/cudnn_deconvolution-inl.h b/src/operator/cudnn_deconvolution-inl.h index 8c8f0551dde3..de3e70c7d6a7 100644 --- a/src/operator/cudnn_deconvolution-inl.h +++ b/src/operator/cudnn_deconvolution-inl.h @@ -56,6 +56,8 @@ class CuDNNDeconvolutionOp : public Operator { init_cudnn_ = false; init_temp_size_ = false; dtype_ = mshadow::DataType::kCudnnFlag; + // TensorCore algos only allowed on fp16-I/O deconvolutions if permitted by the global policy. + cudnn_tensor_core_ = DataType::kFlag == kFloat16 && GetEnvAllowTensorCore(); #if CUDNN_MAJOR >= 5 MSHADOW_LAYOUT_SWITCH(param_.layout.value(), Layout, { @@ -66,7 +68,7 @@ class CuDNNDeconvolutionOp : public Operator { << "Need CuDNN > 5.0 for layout support"; #endif // Double check to make sure this class supports the operation - if (!Supports(param, forward_compute_type, backward_compute_type)) + if (!Supports(param, forward_compute_type, backward_compute_type, ctx)) LOG(FATAL) << "Need CuDNN >= 6.0 for dilated convolution."; InitDescriptors(ctx, in_shape, out_shape, @@ -92,7 +94,8 @@ class CuDNNDeconvolutionOp : public Operator { CUDNN_CALL(cudnnDestroyTensorDescriptor(bias_desc_)); CUDNN_CALL(cudnnDestroyFilterDescriptor(filter_desc_)); CUDNN_CALL(cudnnDestroyConvolutionDescriptor(forward_conv_desc_)); - CUDNN_CALL(cudnnDestroyConvolutionDescriptor(backward_conv_desc_)); + CUDNN_CALL(cudnnDestroyConvolutionDescriptor(back_conv_desc_)); + CUDNN_CALL(cudnnDestroyConvolutionDescriptor(back_conv_desc_w_)); } } @@ -146,7 +149,7 @@ class CuDNNDeconvolutionOp : public Operator { in_desc_, data_ptr + data_offset_ * g, forward_conv_desc_, // this backward algorithm used for inference - back_algo_, + back_algo_.AlgoNumber(), workspace.dptr_, workspace_size, &beta, @@ -160,7 +163,7 @@ class CuDNNDeconvolutionOp : public Operator { in_desc_, data_ptr + data_offset_ * g, forward_conv_desc_, // this backward algorithm used for inference - back_algo_, + back_algo_.AlgoNumber(), workspace.dptr_, workspace_size, &beta, @@ -270,8 +273,8 @@ class CuDNNDeconvolutionOp : public Operator { grad_ptr + out_offset_ * g, in_desc_, data_ptr + data_offset_ * g, - backward_conv_desc_, - back_algo_w_, + back_conv_desc_, + back_algo_w_.AlgoNumber(), workspace.dptr_, workspace_size, &weight_beta, @@ -285,8 +288,8 @@ class CuDNNDeconvolutionOp : public Operator { grad_ptr + out_offset_ * g, in_desc_, data_ptr + data_offset_ * g, - backward_conv_desc_, - back_algo_w_, + back_conv_desc_, + back_algo_w_.AlgoNumber(), workspace.dptr_, workspace_size, &weight_beta, @@ -301,8 +304,8 @@ class CuDNNDeconvolutionOp : public Operator { grad_ptr + out_offset_ * g, filter_desc_, wmat_ptr + weight_offset_ * g, - backward_conv_desc_, - algo_, + back_conv_desc_, + forward_algo_.AlgoNumber(), workspace.dptr_, workspace_size, &data_beta, @@ -319,7 +322,8 @@ class CuDNNDeconvolutionOp : public Operator { */ static bool Supports(DeconvolutionParam param, int forward_compute_type, - int backward_compute_type) { + int backward_compute_type, + const Context &ctx) { using namespace mshadow; // NDHWC not supported, NHWC not supported in true fp16 @@ -329,6 +333,12 @@ class CuDNNDeconvolutionOp : public Operator { if (layout_val == kNDHWC || layout_val == kNHWC && true_fp16) return false; + // Permits graceful fallback to pseudo-fp16 on heterogenous systems + if (!SupportsFloat16Compute(ctx.dev_id) && + (forward_compute_type == kFloat16 || backward_compute_type == kFloat16)) { + return false; + } + // The factor by which the effective filter size grows based on dilation. auto filterDilationFactor = param.dilate.Size(); @@ -374,7 +384,8 @@ class CuDNNDeconvolutionOp : public Operator { CUDNN_CALL(cudnnCreateTensorDescriptor(&bias_desc_)); CUDNN_CALL(cudnnCreateFilterDescriptor(&filter_desc_)); CUDNN_CALL(cudnnCreateConvolutionDescriptor(&forward_conv_desc_)); - CUDNN_CALL(cudnnCreateConvolutionDescriptor(&backward_conv_desc_)); + CUDNN_CALL(cudnnCreateConvolutionDescriptor(&back_conv_desc_)); + CUDNN_CALL(cudnnCreateConvolutionDescriptor(&back_conv_desc_w_)); TShape dshape = in_shape[deconv::kData]; TShape wshape = in_shape[deconv::kWeight]; @@ -398,7 +409,16 @@ class CuDNNDeconvolutionOp : public Operator { param_.dilate[1], CUDNN_CROSS_CORRELATION, cudnn_forward_compute_type)); - CUDNN_CALL(cudnnSetConvolution2dDescriptor(backward_conv_desc_, + CUDNN_CALL(cudnnSetConvolution2dDescriptor(back_conv_desc_, + o_pad[0], + o_pad[1], + param_.stride[0], + param_.stride[1], + param_.dilate[0], + param_.dilate[1], + CUDNN_CROSS_CORRELATION, + cudnn_backward_compute_type)); + CUDNN_CALL(cudnnSetConvolution2dDescriptor(back_conv_desc_w_, o_pad[0], o_pad[1], param_.stride[0], @@ -416,7 +436,15 @@ class CuDNNDeconvolutionOp : public Operator { param_.dilate[0], param_.dilate[1], CUDNN_CROSS_CORRELATION)); - CUDNN_CALL(cudnnSetConvolution2dDescriptor(backward_conv_desc_, + CUDNN_CALL(cudnnSetConvolution2dDescriptor(back_conv_desc_, + o_pad[0], + o_pad[1], + param_.stride[0], + param_.stride[1], + param_.dilate[0], + param_.dilate[1], + CUDNN_CROSS_CORRELATION)); + CUDNN_CALL(cudnnSetConvolution2dDescriptor(back_conv_desc_w_, o_pad[0], o_pad[1], param_.stride[0], @@ -483,7 +511,15 @@ class CuDNNDeconvolutionOp : public Operator { CUDNN_CROSS_CORRELATION, cudnn_forward_compute_type)); - CUDNN_CALL(cudnnSetConvolutionNdDescriptor(backward_conv_desc_, + CUDNN_CALL(cudnnSetConvolutionNdDescriptor(back_conv_desc_, + 3, + reinterpret_cast(&o_pad[0]), + param_stride_.data(), + param_dilate_.data(), + CUDNN_CROSS_CORRELATION, + cudnn_backward_compute_type)); + + CUDNN_CALL(cudnnSetConvolutionNdDescriptor(back_conv_desc_w_, 3, reinterpret_cast(&o_pad[0]), param_stride_.data(), @@ -507,6 +543,14 @@ class CuDNNDeconvolutionOp : public Operator { param_.layout.value(), kNCDHW); oshape = ConvertLayout(oshape.get<5>(), param_.layout.value(), kNCDHW); } + // Set "allow tensor core" flag in convolution descriptors, if available. +#if CUDNN_MAJOR >= 7 + cudnnMathType_t math_type = cudnn_tensor_core_ ? CUDNN_TENSOR_OP_MATH + : CUDNN_DEFAULT_MATH; + CUDNN_CALL(cudnnSetConvolutionMathType(forward_conv_desc_, math_type)); + CUDNN_CALL(cudnnSetConvolutionMathType(back_conv_desc_, math_type)); + CUDNN_CALL(cudnnSetConvolutionMathType(back_conv_desc_w_, math_type)); +#endif dshape[1] /= param_.num_group; oshape[1] /= param_.num_group; weight_offset_ = wshape.Size(); @@ -556,125 +600,242 @@ class CuDNNDeconvolutionOp : public Operator { cudnnDataType_t cudnn_backward_compute_type) { std::string key = CuDNNAlgoReg::Get()->GetKey(param_, in_shape, out_shape, dtype_, cudnn_forward_compute_type, - cudnn_backward_compute_type); - if (CuDNNAlgoReg::Get()->Find(key, &algo_, &back_algo_, &back_algo_w_)) - return; - - Engine::VarHandle var = Engine::Get()->NewVariable(); - Engine::Get()->PushSync([=](RunContext rctx) { - mshadow::Stream *s = rctx.get_stream(); - CHECK_EQ(s->dnn_handle_ownership_, mshadow::Stream::OwnHandle); - size_t workspace_byte = static_cast(param_.workspace * sizeof(DType)); - if (!param_.cudnn_tune.value()) { - // In cuDNNv6, for kNHWC, only CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_GEMM is - // supported. Hard-coded this since the algo find() or get() throws an FPE. - if (CUDNN_MAJOR == 6 && param_.layout.value() == mshadow::kNHWC) { - algo_ = CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_GEMM; - } else { - CUDNN_CALL(cudnnGetConvolutionForwardAlgorithm(s->dnn_handle_, - out_desc_, - filter_desc_, - backward_conv_desc_, // forward algorithm used to backprop-to-data - in_desc_, - CUDNN_CONVOLUTION_FWD_SPECIFY_WORKSPACE_LIMIT, - workspace_byte, - &(this->algo_))); - } - CUDNN_CALL(cudnnGetConvolutionBackwardFilterAlgorithm(s->dnn_handle_, - out_desc_, - in_desc_, - backward_conv_desc_, - filter_desc_, - CUDNN_CONVOLUTION_BWD_FILTER_SPECIFY_WORKSPACE_LIMIT, - workspace_byte, - &(this->back_algo_w_))); - CUDNN_CALL(cudnnGetConvolutionBackwardDataAlgorithm(s->dnn_handle_, - filter_desc_, - in_desc_, - forward_conv_desc_, // this backward algorithm used for inference - out_desc_, - CUDNN_CONVOLUTION_BWD_DATA_SPECIFY_WORKSPACE_LIMIT, - workspace_byte, - &(this->back_algo_))); - } else { + cudnn_backward_compute_type, + SMArch(ctx.dev_id)); + if (!CuDNNAlgoReg::Get()->Find(key, &forward_algo_, &back_algo_, &back_algo_w_)) { + // Not in algo registry, must determine via *Get*() or *Find*() + Engine::VarHandle var = Engine::Get()->NewVariable(); + Engine::Get()->PushSync([=](RunContext rctx) { + mshadow::Stream *s = rctx.get_stream(); + CHECK_EQ(s->dnn_handle_ownership_, mshadow::Stream::OwnHandle); + size_t workspace_byte = static_cast(param_.workspace * sizeof(DType)); + #if CUDNN_MAJOR >= 7 + // Starting with cuDNNv7, the algo number returned by *Get*() is not the entire + // story: the notion of whether the algo ran in Tensor Core mode is not known. + // Since we want to report the Tensor Core mode in the verbose output, we switch + // to using the new *Get*_v7() call. Since the function signature of *Get*_v7() matches + // that of *Find*(), we can unify the find-vs-get logic by using function pointers. + + // Forward Algorithm Find/Get() v7 + std::vector fwd_results(MaxForwardAlgos(s->dnn_handle_)); + int actual_fwd_algos = 0; + auto fwd_algo_discoverer = + param_.cudnn_tune.value() == conv::kOff ? cudnnGetConvolutionForwardAlgorithm_v7 + : cudnnFindConvolutionForwardAlgorithm; + CUDNN_CALL((*fwd_algo_discoverer)(s->dnn_handle_, + out_desc_, + filter_desc_, + back_conv_desc_, // fwd algo used to backprop-to-data + in_desc_, + fwd_results.size(), + &actual_fwd_algos, + fwd_results.data())); + fwd_results.resize(actual_fwd_algos); + AlgoFinalSelect(fwd_results, "forward", + workspace_byte, &forward_algo_); + + // Backprop-to-Filter Algorithm Find/Get() v7 + auto max_bwd_filt_algos = MaxBackwardFilterAlgos(s->dnn_handle_); + std::vector bwd_filt_results(max_bwd_filt_algos); + int actual_bwd_filter_algos = 0; + auto bwd_filter_algo_discoverer = + param_.cudnn_tune.value() == conv::kOff ? cudnnGetConvolutionBackwardFilterAlgorithm_v7 + : cudnnFindConvolutionBackwardFilterAlgorithm; + CUDNN_CALL((*bwd_filter_algo_discoverer)(s->dnn_handle_, + out_desc_, + in_desc_, + back_conv_desc_, + filter_desc_, + bwd_filt_results.size(), + &actual_bwd_filter_algos, + bwd_filt_results.data())); + bwd_filt_results.resize(actual_bwd_filter_algos); + AlgoFinalSelect(bwd_filt_results, "backprop-to-filter", + workspace_byte, &back_algo_w_); + + // Backprop-to-Data Algorithm Find/Get() v7 + auto max_bwd_data_algos = MaxBackwardDataAlgos(s->dnn_handle_); + std::vector bwd_data_results(max_bwd_data_algos); + int actual_bwd_data_algos = 0; + auto bwd_data_algo_discoverer = + param_.cudnn_tune.value() == conv::kOff ? cudnnGetConvolutionBackwardDataAlgorithm_v7 + : cudnnFindConvolutionBackwardDataAlgorithm; + CUDNN_CALL((*bwd_data_algo_discoverer)(s->dnn_handle_, + filter_desc_, + in_desc_, + forward_conv_desc_, // bwd algo used in inference + out_desc_, + bwd_data_results.size(), + &actual_bwd_data_algos, + bwd_data_results.data())); + bwd_data_results.resize(actual_bwd_data_algos); + AlgoFinalSelect(bwd_data_results, "backprop-to-data", + workspace_byte, &back_algo_); + #else + // CUDNN_MAJOR < 7 const int kMaxAlgos = 10; int nalgo = kMaxAlgos; - int i; - - // In cuDNNv6, for kNHWC, only CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_GEMM is - // supported. Hard-coded this since the algo find() or get() throws an FPE. + int i = 0; + // Forward Algorithm Find/Get, v6 and earlier if (CUDNN_MAJOR == 6 && param_.layout.value() == mshadow::kNHWC) { - algo_ = CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_GEMM; + // In cuDNNv6, for kNHWC, only CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_GEMM is + // supported. Hard-coded this since the algo find() or get() throws an FPE. + forward_algo_.Set(CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_GEMM, false); + } else if (!param_.cudnn_tune.value()) { + cudnnConvolutionFwdAlgo_t fastest_fwd_algo; + CUDNN_CALL(cudnnGetConvolutionForwardAlgorithm(s->dnn_handle_, + out_desc_, + filter_desc_, + back_conv_desc_, // fwd algo used in dgrad + in_desc_, + CUDNN_CONVOLUTION_FWD_SPECIFY_WORKSPACE_LIMIT, + workspace_byte, + &fastest_fwd_algo)); + forward_algo_.Set(fastest_fwd_algo, false); } else { cudnnConvolutionFwdAlgoPerf_t fwd_algo[kMaxAlgos]; CUDNN_CALL(cudnnFindConvolutionForwardAlgorithm(s->dnn_handle_, - out_desc_, - filter_desc_, - backward_conv_desc_, // forward algorithm used to backprop-to-data - in_desc_, - kMaxAlgos, - &nalgo, - fwd_algo)); + out_desc_, + filter_desc_, + back_conv_desc_, // fwd algo used in dgrad + in_desc_, + kMaxAlgos, + &nalgo, + fwd_algo)); i = 0; while (i < nalgo - && (fwd_algo[i].status != CUDNN_STATUS_SUCCESS - || (param_.cudnn_tune.value() == deconv::kLimited - && fwd_algo[i].memory > workspace_byte))) ++i; + && (fwd_algo[i].status != CUDNN_STATUS_SUCCESS + || (param_.cudnn_tune.value() == deconv::kLimited + && fwd_algo[i].memory > workspace_byte))) + ++i; if (i == nalgo) { LOG(FATAL) << "Failed to find a 'forward' convolution algorithm " << - "(for use in deconvolution operator backprop-to-data)."; + "(for use in deconvolution operator backprop-to-data)."; } else { - this->algo_ = fwd_algo[i].algo; + forward_algo_.Set(fwd_algo[i].algo, false); } } - - cudnnConvolutionBwdFilterAlgoPerf_t bwd_filter_algo[kMaxAlgos]; - CUDNN_CALL(cudnnFindConvolutionBackwardFilterAlgorithm(s->dnn_handle_, - out_desc_, - in_desc_, - backward_conv_desc_, - filter_desc_, - kMaxAlgos, - &nalgo, - bwd_filter_algo)); - i = 0; - while (i < nalgo - && (bwd_filter_algo[i].status != CUDNN_STATUS_SUCCESS - || (param_.cudnn_tune.value() == deconv::kLimited - && bwd_filter_algo[i].memory > workspace_byte))) ++i; - if (i == nalgo) { - LOG(FATAL) << "Failed to find a backward filter convolution algorithm " << - "(for use in deconvolution operator backprop-to-filter)."; + // Backprop-to-Filter Algorithm Find/Get, v6 and earlier + if (!param_.cudnn_tune.value()) { + cudnnConvolutionBwdFilterAlgo_t fastest_bwd_filt_algo; + CUDNN_CALL(cudnnGetConvolutionBackwardFilterAlgorithm(s->dnn_handle_, + out_desc_, + in_desc_, + back_conv_desc_, + filter_desc_, + CUDNN_CONVOLUTION_BWD_FILTER_SPECIFY_WORKSPACE_LIMIT, + workspace_byte, + &fastest_bwd_filt_algo)); + back_algo_w_.Set(fastest_bwd_filt_algo, false); } else { - this->back_algo_w_ = bwd_filter_algo[i].algo; + cudnnConvolutionBwdFilterAlgoPerf_t bwd_filter_algo[kMaxAlgos]; + CUDNN_CALL(cudnnFindConvolutionBackwardFilterAlgorithm(s->dnn_handle_, + out_desc_, + in_desc_, + back_conv_desc_, + filter_desc_, + kMaxAlgos, + &nalgo, + bwd_filter_algo)); + i = 0; + while (i < nalgo + && (bwd_filter_algo[i].status != CUDNN_STATUS_SUCCESS + || (param_.cudnn_tune.value() == deconv::kLimited + && bwd_filter_algo[i].memory > workspace_byte))) + ++i; + if (i == nalgo) { + LOG(FATAL) << "Failed to find a backward filter convolution algorithm " << + "(for use in deconvolution operator backprop-to-filter)."; + } else { + back_algo_w_.Set(bwd_filter_algo[i].algo, false); + } } - - cudnnConvolutionBwdDataAlgoPerf_t bwd_data_algo[kMaxAlgos]; - CUDNN_CALL(cudnnFindConvolutionBackwardDataAlgorithm(s->dnn_handle_, - filter_desc_, - in_desc_, - forward_conv_desc_, // this backward algorithm used for inference - out_desc_, - kMaxAlgos, - &nalgo, - bwd_data_algo)); - i = 0; - while (i < nalgo - && (bwd_data_algo[i].status != CUDNN_STATUS_SUCCESS - || (param_.cudnn_tune.value() == deconv::kLimited - && bwd_data_algo[i].memory > workspace_byte))) ++i; - if (i == nalgo) { - LOG(FATAL) << "Failed to find a backward data convolution algorithm." << - "(for use in deconvolution operator forward inference)."; + // Backprop-to-Data Algorithm Get(), v6 and earlier + if (!param_.cudnn_tune.value()) { + cudnnConvolutionBwdDataAlgo_t fastest_bwd_data_algo; + CUDNN_CALL(cudnnGetConvolutionBackwardDataAlgorithm(s->dnn_handle_, + filter_desc_, + in_desc_, + forward_conv_desc_, // bwd algo used for inference + out_desc_, + CUDNN_CONVOLUTION_BWD_DATA_SPECIFY_WORKSPACE_LIMIT, + workspace_byte, + &fastest_bwd_data_algo)); + back_algo_.Set(fastest_bwd_data_algo, false); } else { - this->back_algo_ = bwd_data_algo[i].algo; + cudnnConvolutionBwdDataAlgoPerf_t bwd_data_algo[kMaxAlgos]; + CUDNN_CALL(cudnnFindConvolutionBackwardDataAlgorithm(s->dnn_handle_, + filter_desc_, + in_desc_, + forward_conv_desc_, // bwd algo used in inference + out_desc_, + kMaxAlgos, + &nalgo, + bwd_data_algo)); + i = 0; + while (i < nalgo + && (bwd_data_algo[i].status != CUDNN_STATUS_SUCCESS + || (param_.cudnn_tune.value() == deconv::kLimited + && bwd_data_algo[i].memory > workspace_byte))) + ++i; + if (i == nalgo) { + LOG(FATAL) << "Failed to find a backward data convolution algorithm." << + "(for use in deconvolution operator forward inference)."; + } else { + back_algo_.Set(bwd_data_algo[i].algo, false); + } } - CuDNNAlgoReg::Get()->Register(key, this->algo_, this->back_algo_, + #endif // CUDNN_MAJOR < 7 + // An algo specification by the user may be cached here, but another + // convolution will match only if identically specified. + // We're caching results of *Get* as well as *Find*, but these records + // will be held distinctly because param_.cudnn_tune is part of the key. + CuDNNAlgoReg::Get()->Register(key, this->forward_algo_, this->back_algo_, this->back_algo_w_); + }, ctx, {}, {var}); + Engine::Get()->WaitForVar(var); + Engine::Get()->DeleteVariable([](RunContext s) {}, ctx, var); + } + // If we're allowing Tensor Core variants of the algos to be considered in + // *Find*() or *Get*(), but a non-Tensor-Core algo variant is the fastest, + // we must change the descriptor to preclude Tensor Core. Simplest is to + // once again set the mathType in all cases. + #if CUDNN_MAJOR >= 7 + // The next two code lines will look like they have typos, but they don't! + // The forward_conv_desc_ is used during inference, which invokes the back_algo_. + // Thus, the mathType of the back_algo_ should be stored in the forward_conv_desc_. + // Conversely, the back_conv_desc_ is used during training backprop, which invokes + // the forward_algo_. Thus, the mathType of the forward_algo_ should be stored + // in the back_conv_desc_. + CUDNN_CALL(cudnnSetConvolutionMathType(forward_conv_desc_, back_algo_.MathType())); + CUDNN_CALL(cudnnSetConvolutionMathType(back_conv_desc_, forward_algo_.MathType())); + CUDNN_CALL(cudnnSetConvolutionMathType(back_conv_desc_w_, back_algo_w_.MathType())); + #endif + } + + // Look over the results from *Find*() or *Get*() and pick the fastest algo given possible + // workspace constraints and a possible user algo preference. + template + void AlgoFinalSelect(const std::vector &perf_results, std::string kernel_name, + size_t workspace_byte, CuDNNAlgo *algo) { + // Determine the fastest acceptable algo regardless of mathType. + for (decltype(perf_results.size()) i = 0; i != perf_results.size(); ++i) { + const auto &result = perf_results[i]; + bool algo_is_tensor_core = false; + #if CUDNN_MAJOR >= 7 + algo_is_tensor_core = result.mathType == CUDNN_TENSOR_OP_MATH; + #endif + if (result.status == CUDNN_STATUS_SUCCESS && + (param_.cudnn_tune.value() != conv::kLimited || result.memory <= workspace_byte)) { + algo->Set(result.algo, algo_is_tensor_core); + return; } - }, ctx, {}, {var}); - Engine::Get()->WaitForVar(var); - Engine::Get()->DeleteVariable([](RunContext s) {}, ctx, var); + } + auto mode = param_.cudnn_tune.value() == conv::kOff ? " get " : " find "; + LOG(FATAL) << "Failed to" << mode << "any " << kernel_name << " deconvolution algorithm."; } void GetTempSize(const OpContext& ctx) { @@ -688,21 +849,21 @@ class CuDNNDeconvolutionOp : public Operator { in_desc_, forward_conv_desc_, out_desc_, - back_algo_, + back_algo_.AlgoNumber(), &back_data_algo_workspace_size)); CUDNN_CALL(cudnnGetConvolutionBackwardFilterWorkspaceSize(s->dnn_handle_, out_desc_, in_desc_, - backward_conv_desc_, + back_conv_desc_, filter_desc_, - back_algo_w_, + back_algo_w_.AlgoNumber(), &back_filter_algo_workspace_size)); CUDNN_CALL(cudnnGetConvolutionForwardWorkspaceSize(s->dnn_handle_, out_desc_, filter_desc_, - backward_conv_desc_, + back_conv_desc_, in_desc_, - algo_, + forward_algo_.AlgoNumber(), &forward_algo_workspace_size)); forward_workspace_byte_ = back_data_algo_workspace_size; @@ -761,19 +922,24 @@ class CuDNNDeconvolutionOp : public Operator { // Note that in deconvolution, the forward operation is handled // by the cuDNN backprop-to-data kernel. cudnnConvolutionDescriptor_t forward_conv_desc_; - // Convolution descriptor for "back-prop" operations to data and filter. + // Convolution descriptor for "back-prop" operations to data . + // Note that in deconvolution, the backprop-to-data operation is handled + // by the cuDNN forward kernel. + cudnnConvolutionDescriptor_t back_conv_desc_; + // Convolution descriptor for "back-prop" operations to filter. // Note that in deconvolution, the backprop-to-data operation is handled - // by the cuDNN forward kernel, while the backprop-to-filter operation - // stays consistent with the convolution operator and is handled by - // the backprop-to-filter kernel. - cudnnConvolutionDescriptor_t backward_conv_desc_; + // by the backprop-to-filter kernel (so consistent with the treatment + // in convolution). + cudnnConvolutionDescriptor_t back_conv_desc_w_; // Algorithm for the cuDNN forward kernel (used in gradient backprop to input) - cudnnConvolutionFwdAlgo_t algo_; + CuDNNAlgo forward_algo_; // Algorithm for the cuDNN backprop-to-data kernel (used in inference) - cudnnConvolutionBwdDataAlgo_t back_algo_; + CuDNNAlgo back_algo_; // Algorithm for the cuDNN backprop-to-filter kernel - cudnnConvolutionBwdFilterAlgo_t back_algo_w_; + CuDNNAlgo back_algo_w_; cudnnTensorFormat_t format_; + // Allow TensorCore algo policy + bool cudnn_tensor_core_; DeconvolutionParam param_; }; #endif // CUDNN diff --git a/src/operator/cudnn_rnn-inl.h b/src/operator/cudnn_rnn-inl.h index 1122aff033f2..a260cb4ca0e3 100644 --- a/src/operator/cudnn_rnn-inl.h +++ b/src/operator/cudnn_rnn-inl.h @@ -43,6 +43,12 @@ class CuDNNRNNOp : public Operator { this->param_ = param; init_cudnn_ = false; dtype_ = mshadow::DataType::kCudnnFlag; + // TensorCore algos only allowed on fp16-I/O convolutions if permitted by the global policy. + // No tests in place for fp16 RNNs, so leave TensorCore disabled for now. + cudnn_tensor_core_ = false; + // When fp16 RNN tests are introduced, we can enable TensorCore as follows: +// cudnn_tensor_core = +// mshadow::DataType::kFlag == mshadow::kFloat16 && GetEnvAllowTensorCore(); // Defaults input_mode_ = CUDNN_LINEAR_INPUT; // Don't support this yet // RNN Mode @@ -450,14 +456,36 @@ class CuDNNRNNOp : public Operator { seed_)); // RNN descriptors CUDNN_CALL(cudnnCreateRNNDescriptor(&rnn_desc_)); - CUDNN_CALL(cudnnSetRNNDescriptor(rnn_desc_, - param_.state_size, - param_.num_layers, - dropout_desc_, - input_mode_, - direction_, - mode_, - dtype_)); + + #if CUDNN_MAJOR >= 6 + cudnnRNNAlgo_t rnn_algo = CUDNN_RNN_ALGO_STANDARD; + CUDNN_CALL(cudnnSetRNNDescriptor_v6(s->dnn_handle_, + rnn_desc_, + param_.state_size, + param_.num_layers, + dropout_desc_, + input_mode_, + direction_, + mode_, + rnn_algo, + dtype_)); + #else + CUDNN_CALL(cudnnSetRNNDescriptor(rnn_desc_, + param_.state_size, + param_.num_layers, + dropout_desc_, + input_mode_, + direction_, + mode_, + dtype_)); + #endif + #if CUDNN_MAJOR >= 7 + cudnnMathType_t math_type = CUDNN_DEFAULT_MATH; + if (cudnn_tensor_core_ && rnn_algo == CUDNN_RNN_ALGO_STANDARD) { + math_type = CUDNN_TENSOR_OP_MATH; + } + CUDNN_CALL(cudnnSetRNNMatrixMathType(rnn_desc_, math_type)); + #endif // Get temp space sizes CUDNN_CALL(cudnnGetRNNWorkspaceSize(s->dnn_handle_, rnn_desc_, @@ -554,6 +582,8 @@ class CuDNNRNNOp : public Operator { cudnnTensorDescriptor_t dhy_desc_, dcy_desc_; cudnnFilterDescriptor_t w_desc_, dw_desc_; + // Allow TensorCore algo policy + bool cudnn_tensor_core_; #if CUDNN_MAJOR >= 5 cudnnTensorFormat_t format_; diff --git a/src/operator/deconvolution.cu b/src/operator/deconvolution.cu index b9dd1c156187..e9b5cb8e3c7f 100644 --- a/src/operator/deconvolution.cu +++ b/src/operator/deconvolution.cu @@ -70,14 +70,14 @@ Operator* CreateOp(DeconvolutionParam param, int dtype, int backward_compute_type = desired_backward_compute_type; bool deconvolutionIsSupported = CuDNNDeconvolutionOp::Supports(param, forward_compute_type, - backward_compute_type); + backward_compute_type, ctx); // If cuDNN can't handle this case with fp16 backprop kernels, try fp32 backprop. if (!deconvolutionIsSupported && backward_compute_type == mshadow::kFloat16) { backward_compute_type = mshadow::kFloat32; deconvolutionIsSupported = CuDNNDeconvolutionOp::Supports(param, forward_compute_type, - backward_compute_type); + backward_compute_type, ctx); } // If cuDNN can't handle this case with fp16 forward kernels, try fp32 @@ -85,7 +85,7 @@ Operator* CreateOp(DeconvolutionParam param, int dtype, forward_compute_type = mshadow::kFloat32; deconvolutionIsSupported = CuDNNDeconvolutionOp::Supports(param, forward_compute_type, - backward_compute_type); + backward_compute_type, ctx); } if (!deconvolutionIsSupported) { LOG(WARNING) << From 85ad64921f1b89dd6130087612e102b7728f459d Mon Sep 17 00:00:00 2001 From: Dom Divakaruni Date: Wed, 9 Aug 2017 17:16:23 -0700 Subject: [PATCH 355/834] remove dmlc/mxnet logo from readme (#7256) remove dmlc/mxnet logo from readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6b62986d6477..5027f6d3fdb6 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ - *for Deep Learning* +Apache MXNet (incubating) for Deep Learning ===== [![Build Status](https://travis-ci.org/dmlc/mxnet.svg?branch=master)](https://travis-ci.org/dmlc/mxnet) @@ -7,7 +7,7 @@ ![banner](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/image/banner.png) -MXNet is a deep learning framework designed for both *efficiency* and *flexibility*. +Apache MXNet (incubating) is a deep learning framework designed for both *efficiency* and *flexibility*. It allows you to ***mix*** [symbolic and imperative programming](http://mxnet.io/architecture/index.html#deep-learning-system-design-concepts) to ***maximize*** efficiency and productivity. At its core, MXNet contains a dynamic dependency scheduler that automatically parallelizes both symbolic and imperative operations on the fly. From 46039c34e13191c98020c6f046a197b262de9f72 Mon Sep 17 00:00:00 2001 From: Zack Chase Lipton Date: Wed, 9 Aug 2017 20:15:19 -0700 Subject: [PATCH 356/834] Update autograd.md (#7405) Fixing two typos that snuck into this doc. Thanks @fhieber for the careful eyes! --- docs/api/python/autograd.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/api/python/autograd.md b/docs/api/python/autograd.md index 5c849648e3cc..444e01fc9688 100644 --- a/docs/api/python/autograd.md +++ b/docs/api/python/autograd.md @@ -34,6 +34,7 @@ and do some computation. Finally, call `backward()` on the result: >>> x.attach_grad() >>> with mx.autograd.record(): ... y = x * x + 1 +>>> y.backward() >>> print(x.grad) [ 2. 4. 6. 8.] @@ -44,7 +45,7 @@ and do some computation. Finally, call `backward()` on the result: Some operators (Dropout, BatchNorm, etc) behave differently in when training and when making predictions. -This can be controled with `train_mode` and `predict_mode` scope. +This can be controlled with `train_mode` and `predict_mode` scope. By default, MXNet is in `predict_mode`. A `with autograd.record()` block by default turns on `train_mode` From 613132e2b6c6573b184b467aed5cf50dbb91aa47 Mon Sep 17 00:00:00 2001 From: "Qiang Kou (KK)" Date: Thu, 10 Aug 2017 15:19:28 +0000 Subject: [PATCH 357/834] [R][MISC] update Makefile/Jenkinsfile; use mx.ctx.default() in R test (#7401) --- Jenkinsfile | 4 +- Makefile | 1 + R-package/README.md | 6 -- R-package/tests/testthat.R | 4 - R-package/tests/testthat/get_data.R | 13 +++ R-package/tests/testthat/test_img_seg.R | 7 +- R-package/tests/testthat/test_lstm.R | 7 +- R-package/tests/testthat/test_model.R | 117 +++++++++++++++++++++--- R-package/tests/testthat/test_ndarray.R | 9 +- example/captcha/README.md | 2 +- 10 files changed, 142 insertions(+), 28 deletions(-) delete mode 100644 R-package/tests/testthat.R diff --git a/Jenkinsfile b/Jenkinsfile index 370c2b397626..632789ac194a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -325,11 +325,11 @@ try { init_git() unpack_lib('gpu') timeout(time: max_time, unit: 'MINUTES') { - sh "${docker_run} cpu rm -rf .Renviron" + sh "${docker_run} gpu rm -rf .Renviron" sh "${docker_run} gpu mkdir -p /workspace/ut-r-gpu/site-library" sh "${docker_run} gpu make rpkg USE_BLAS=openblas R_LIBS=/workspace/ut-r-gpu/site-library" sh "${docker_run} gpu R CMD INSTALL --library=/workspace/ut-r-gpu/site-library mxnet_current_r.tar.gz" - sh "${docker_run} gpu make rpkgtest R_LIBS=/workspace/ut-r-gpu/site-library" + sh "${docker_run} gpu make rpkgtest R_LIBS=/workspace/ut-r-gpu/site-library R_GPU_ENABLE=1" } } } diff --git a/Makefile b/Makefile index ed742144ef7c..560b77a3e81e 100644 --- a/Makefile +++ b/Makefile @@ -398,6 +398,7 @@ rpkg: Rscript -e "require(roxygen2); roxygen2::roxygenise('R-package')" R CMD build --no-build-vignettes R-package rm -rf mxnet_current_r.tar.gz + rm -rf R-package/src/image_recordio.h mv mxnet_*.tar.gz mxnet_current_r.tar.gz rpkgtest: diff --git a/R-package/README.md b/R-package/README.md index e150f17bb732..6576700e11c6 100644 --- a/R-package/README.md +++ b/R-package/README.md @@ -1,7 +1,5 @@ Deep Learning for R ========================== -[![Build Status](https://travis-ci.org/dmlc/mxnet.svg?branch=master)](https://travis-ci.org/dmlc/mxnet) -[![Documentation Status](https://readthedocs.org/projects/mxnet/badge/?version=latest)](http://mxnet.readthedocs.io/en/latest/api/r/index.html) You have found MXNet R Package! The MXNet R packages brings flexible and efficient GPU computing and state-of-art deep learning to R. @@ -12,10 +10,6 @@ computing and state-of-art deep learning to R. Sounds exciting? This page contains links to all the related documentation of the R package. -Resources ---------- -* [MXNet R Package Document](http://mxnet.io/get_started/install.html) - - Check this out for detailed documents, examples and installation guides. Installation ------------ diff --git a/R-package/tests/testthat.R b/R-package/tests/testthat.R deleted file mode 100644 index f002e4c52c53..000000000000 --- a/R-package/tests/testthat.R +++ /dev/null @@ -1,4 +0,0 @@ -library(testthat) -library(mxnet) - -test_check("mxnet") diff --git a/R-package/tests/testthat/get_data.R b/R-package/tests/testthat/get_data.R index 555e5e9b77b2..6d8de8516ae1 100644 --- a/R-package/tests/testthat/get_data.R +++ b/R-package/tests/testthat/get_data.R @@ -92,3 +92,16 @@ GetISBI_data <- function() { file.remove('data/ISBI.zip') } } + +GetCaptcha_data <- function() { + if (!dir.exists("data")) { + dir.create("data/") + } + if (!file.exists('data/captcha_example/captcha_train.rec') | + !file.exists('data/captcha_example/captcha_test.rec')) { + download.file('https://s3-us-west-2.amazonaws.com/apache-mxnet/R/data/captcha_example.zip', + destfile = 'data/captcha_example.zip') + unzip('data/captcha_example.zip', exdir = 'data/') + file.remove('data/captcha_example.zip') + } +} diff --git a/R-package/tests/testthat/test_img_seg.R b/R-package/tests/testthat/test_img_seg.R index ba5c9cd8369b..fbca92e2a8a2 100644 --- a/R-package/tests/testthat/test_img_seg.R +++ b/R-package/tests/testthat/test_img_seg.R @@ -2,6 +2,11 @@ require(mxnet) source("get_data.R") +if (Sys.getenv("R_GPU_ENABLE") != "" & as.integer(Sys.getenv("R_GPU_ENABLE")) == 1) { + mx.ctx.default(new = mx.gpu()) + message("Using GPU for testing.") +} + print_inferred_shape <- function(net) { slist <- mx.symbol.infer.shape(symbol = net, data = c(168, 168, 1, 2)) print(slist$out.shapes) @@ -116,7 +121,7 @@ test_that("UNET", { train.y.array = train.y dim(train.y.array) = c(IMG_SIZE, IMG_SIZE, 1, 30) - devices <- mx.cpu() + devices <- mx.ctx.default() mx.set.seed(0) net <- get_unet() diff --git a/R-package/tests/testthat/test_lstm.R b/R-package/tests/testthat/test_lstm.R index 24b1a59636dc..4a5cdbeb436f 100644 --- a/R-package/tests/testthat/test_lstm.R +++ b/R-package/tests/testthat/test_lstm.R @@ -1,5 +1,10 @@ require(mxnet) +if (Sys.getenv("R_GPU_ENABLE") != "" & as.integer(Sys.getenv("R_GPU_ENABLE")) == 1) { + mx.ctx.default(new = mx.gpu()) + message("Using GPU for testing.") +} + context("lstm models") get.nll <- function(s) { @@ -26,7 +31,7 @@ test_that("training error decreasing", { X.train <- list(data=array(1:16, dim=c(2,8)), label=array(2:17, dim=c(2,8))) s <- capture.output(model <- mx.lstm( X.train, - ctx=mx.cpu(), + ctx=mx.ctx.default(), num.round=num.round, update.period=update.period, num.lstm.layer=num.lstm.layer, diff --git a/R-package/tests/testthat/test_model.R b/R-package/tests/testthat/test_model.R index 73a212714af8..8cdd396c2525 100644 --- a/R-package/tests/testthat/test_model.R +++ b/R-package/tests/testthat/test_model.R @@ -4,6 +4,11 @@ source("get_data.R") context("models") +if (Sys.getenv("R_GPU_ENABLE") != "" & as.integer(Sys.getenv("R_GPU_ENABLE")) == 1) { + mx.ctx.default(new = mx.gpu()) + message("Using GPU for testing.") +} + test_that("MNIST", { # # Network configuration GetMNIST_ubyte() @@ -36,13 +41,10 @@ test_that("MNIST", { silent=0) mx.set.seed(0) - devices = lapply(1:2, function(i) { - mx.cpu(i) - }) - + # create the model model <- mx.model.FeedForward.create(softmax, X=dtrain, eval.data=dtest, - ctx=devices, num.round=1, + ctx = mx.ctx.default(), num.round=1, learning.rate=0.1, momentum=0.9, initializer=mx.init.uniform(0.07), epoch.end.callback=mx.callback.save.checkpoint("chkpt"), @@ -83,12 +85,30 @@ test_that("Regression", { }) mx.set.seed(0) model <- mx.model.FeedForward.create(lro, X = train.x, y = train.y, - ctx = mx.cpu(), num.round = 5, + ctx = mx.ctx.default(), num.round = 5, array.batch.size = 20, learning.rate = 2e-6, momentum = 0.9, eval.metric = demo.metric.mae) + train.x <- data.matrix(BostonHousing[train.ind, -(13:14)]) + train.y <- BostonHousing[train.ind, c(13:14)] + test.x <- data.matrix(BostonHousing[-train.ind, -(13:14)]) + test.y <- BostonHousing[-train.ind, c(13:14)] + + data <- mx.symbol.Variable("data") + fc2 <- mx.symbol.FullyConnected(data, num_hidden=2) + lro2 <- mx.symbol.LinearRegressionOutput(fc2) + + mx.set.seed(0) + train_iter = mx.io.arrayiter(data = t(train.x), label = t(train.y)) + + model <- mx.model.FeedForward.create(lro2, X = train_iter, + ctx = mx.ctx.default(), + num.round = 50, + array.batch.size = 20, + learning.rate = 2e-6, + momentum = 0.9) }) @@ -141,7 +161,7 @@ test_that("Fine-tune", { arg_params_new[["fc1_bias"]] <- fc1_bias_new #model <- mx.model.FeedForward.create(symbol = new_soft, X = train_iter, eval.data = val_iter, - # ctx = mx.cpu(), eval.metric = mx.metric.accuracy, + # ctx = mx.ctx.default(), eval.metric = mx.metric.accuracy, # num.round = 2, learning.rate = 0.05, momentum = 0.9, # wd = 0.00001, kvstore = "local", # batch.end.callback = mx.callback.log.train.metric(50), @@ -171,9 +191,7 @@ test_that("Matrix Factorization", { pred1 <- mx.symbol.sum_axis(pred, axis = 1, name = "pred1") pred2 <- mx.symbol.Flatten(pred1, name = "pred2") pred3 <- mx.symbol.LinearRegressionOutput(data = pred2, label = score, name = "pred3") - devices = lapply(1:2, function(i) { - mx.cpu(i) - }) + mx.set.seed(123) CustomIter <- setRefClass( "CustomIter", fields = c("iter1", "iter2"), @@ -216,7 +234,7 @@ test_that("Matrix Factorization", { train_iter <- CustomIter$new(user_iter, item_iter) - model <- mx.model.FeedForward.create(pred3, X = train_iter, ctx = devices, + model <- mx.model.FeedForward.create(pred3, X = train_iter, ctx = mx.ctx.default(), num.round = 5, initializer = mx.init.uniform(0.07), learning.rate = 0.07, eval.metric = mx.metric.rmse, @@ -225,3 +243,80 @@ test_that("Matrix Factorization", { input.names = c("user", "item"), output.names = "score") }) + +test_that("Captcha", { + GetCaptcha_data() + data.shape <- c(80, 30, 3) + batch_size <- 40 + train <- mx.io.ImageRecordIter( + path.imgrec = "./data/captcha_example/captcha_train.rec", + path.imglist = "./data/captcha_example/captcha_train.lst", + batch.size = batch_size, + label.width = 4, + data.shape = data.shape, + mean.img = "mean.bin") + + val <- mx.io.ImageRecordIter( + path.imgrec = "./data/captcha_example/captcha_test.rec", + path.imglist = "./data/captcha_example/captcha_test.lst", + batch.size = batch_size, + label.width = 4, + data.shape = data.shape, + mean.img = "mean.bin") + + data <- mx.symbol.Variable("data") + label <- mx.symbol.Variable("label") + conv1 <- mx.symbol.Convolution(data = data, kernel = c(5, 5), num_filter = 32) + pool1 <- mx.symbol.Pooling(data = conv1, pool_type = "max", kernel = c(2, 2), stride = c(1, 1)) + relu1 <- mx.symbol.Activation(data = pool1, act_type = "relu") + + conv2 <- mx.symbol.Convolution(data = relu1, kernel = c(5, 5), num_filter = 32) + pool2 <- mx.symbol.Pooling(data = conv2, pool_type = "avg", kernel = c(2, 2), stride = c(1, 1)) + relu2 <- mx.symbol.Activation(data = pool2, act_type = "relu") + + flatten <- mx.symbol.Flatten(data = relu2) + fc1 <- mx.symbol.FullyConnected(data = flatten, num_hidden = 120) + fc21 <- mx.symbol.FullyConnected(data = fc1, num_hidden = 10) + fc22 <- mx.symbol.FullyConnected(data = fc1, num_hidden = 10) + fc23 <- mx.symbol.FullyConnected(data = fc1, num_hidden = 10) + fc24 <- mx.symbol.FullyConnected(data = fc1, num_hidden = 10) + fc2 <- mx.symbol.Concat(c(fc21, fc22, fc23, fc24), dim = 0, num.args = 4) + label <- mx.symbol.transpose(data = label) + label <- mx.symbol.Reshape(data = label, target_shape = c(0)) + captcha_net <- mx.symbol.SoftmaxOutput(data = fc2, label = label, name = "softmax") + + mx.metric.acc2 <- mx.metric.custom("accuracy", function(label, pred) { + ypred <- max.col(t(pred)) - 1 + ypred <- matrix(ypred, nrow = nrow(label), ncol = ncol(label), byrow = TRUE) + return(sum(colSums(label == ypred) == 4)/ncol(label)) + }) + + mx.set.seed(42) + + train$reset() + train$iter.next() + + input.names <- "data" + input.shape <- sapply(input.names, function(n){dim(train$value()[[n]])}, simplify = FALSE) + arg_names <- arguments(captcha_net) + output.names <- "label" + output.shape <- sapply(output.names, function(n){dim(train$value()[[n]])}, simplify = FALSE) + params <- mx.model.init.params(captcha_net, input.shape, output.shape, + mx.init.Xavier(factor_type = "in", magnitude = 2.34), + mx.cpu()) + + #model <- mx.model.FeedForward.create( + # X = train, + # eval.data = val, + # ctx = mx.ctx.default(), + # symbol = captcha_net, + # eval.metric = mx.metric.acc2, + # num.round = 1, + # learning.rate = 1e-04, + # momentum = 0.9, + # wd = 1e-05, + # batch.end.callback = mx.callback.log.train.metric(50), + # initializer = mx.init.Xavier(factor_type = "in", magnitude = 2.34), + # optimizer = "sgd", + # clip_gradient = 10) +}) diff --git a/R-package/tests/testthat/test_ndarray.R b/R-package/tests/testthat/test_ndarray.R index 142c87e9666b..326ea6ca7f30 100644 --- a/R-package/tests/testthat/test_ndarray.R +++ b/R-package/tests/testthat/test_ndarray.R @@ -2,9 +2,14 @@ require(mxnet) context("ndarray") +if (Sys.getenv("R_GPU_ENABLE") != "" & as.integer(Sys.getenv("R_GPU_ENABLE")) == 1) { + mx.ctx.default(new = mx.gpu()) + message("Using GPU for testing.") +} + test_that("element-wise calculation for vector", { x = 1:10 - mat = mx.nd.array(as.array(x), mx.cpu(0)) + mat = mx.nd.array(as.array(x), mx.ctx.default()) expect_equal(x, as.array(mat)) expect_equal(x + 1, as.array(mat + 1)) expect_equal(x - 10, as.array(mat - 10)) @@ -26,7 +31,7 @@ test_that("element-wise calculation for vector", { test_that("element-wise calculation for matrix", { x = matrix(1:4, 2, 2) - mat = mx.nd.array(as.array(x), mx.cpu(0)) + mat = mx.nd.array(as.array(x), mx.ctx.default()) expect_equal(x, as.array(mat)) expect_equal(x + 1, as.array(mat + 1)) expect_equal(x - 10, as.array(mat - 10)) diff --git a/example/captcha/README.md b/example/captcha/README.md index 588d626d03d0..02e87267ccba 100644 --- a/example/captcha/README.md +++ b/example/captcha/README.md @@ -2,4 +2,4 @@ This is the R version of [captcha recognition](http://blog.xlvector.net/2016-05/ ![](captcha_example.png) -You can download the images and `.rec` files from [here](https://drive.google.com/open?id=0B_52ppM3wSXBdHctQmhUdmlTbDQ). Since each image has 4 labels, please remember to use `label_width=4` when generating the `.rec` files. +You can download the images and `.rec` files from [here](https://s3-us-west-2.amazonaws.com/apache-mxnet/R/data/captcha_example.zip). Since each image has 4 labels, please remember to use `label_width=4` when generating the `.rec` files. From 59d717074aae05cf47f69ab8589747bca572cd49 Mon Sep 17 00:00:00 2001 From: "Joshua Z. Zhang" Date: Thu, 10 Aug 2017 09:54:42 -0700 Subject: [PATCH 358/834] Hotfix mx.image documents (#7404) * fix typo and incorrect doc * Fix doc formats * fix doc con't --- python/mxnet/image/detection.py | 36 ++++++++++++++++----------------- python/mxnet/image/image.py | 31 +++++++++++++++------------- 2 files changed, 35 insertions(+), 32 deletions(-) diff --git a/python/mxnet/image/detection.py b/python/mxnet/image/detection.py index 142ba25cc564..43131f03d488 100644 --- a/python/mxnet/image/detection.py +++ b/python/mxnet/image/detection.py @@ -483,15 +483,15 @@ def CreateDetAugmenter(data_shape, resize=0, rand_crop=0, rand_pad=0, rand_gray= rand_mirror=False, mean=None, std=None, brightness=0, contrast=0, saturation=0, pca_noise=0, hue=0, inter_method=2, min_object_covered=0.1, aspect_ratio_range=(0.75, 1.33), area_range=(0.05, 3.0), - min_eject_coverage=0.3, max_attempts=50, pad_val=(128, 128, 128)): + min_eject_coverage=0.3, max_attempts=50, pad_val=(127, 127, 127)): """Create augmenters for detection. Parameters ---------- data_shape : tuple of int - shape for output data + Shape for output data resize : int - resize shorter edge if larger than 0 at the begining + Resize shorter edge if larger than 0 at the begining rand_crop : float [0, 1], probability to apply random cropping rand_pad : float @@ -499,23 +499,23 @@ def CreateDetAugmenter(data_shape, resize=0, rand_crop=0, rand_pad=0, rand_gray= rand_gray : float [0, 1], probability to convert to grayscale for all channels rand_mirror : bool - whether apply horizontal flip to image with probability 0.5 + Whether to apply horizontal flip to image with probability 0.5 mean : np.ndarray or None - mean pixel values for [r, g, b] + Mean pixel values for [r, g, b] std : np.ndarray or None - standard deviations for [r, g, b] + Standard deviations for [r, g, b] brightness : float - brightness jittering range (percent) + Brightness jittering range (percent) contrast : float - contrast jittering range + Contrast jittering range (percent) saturation : float - saturation jittering range + Saturation jittering range (percent) hue : float - hue jittering range + Hue jittering range (percent) pca_noise : float - pca noise level + Pca noise level (percent) inter_method : int, default=2(Area-based) - interpolation method for all resizing operations + Interpolation method for all resizing operations Possible values: 0: Nearest Neighbors Interpolation. @@ -550,7 +550,7 @@ def CreateDetAugmenter(data_shape, resize=0, rand_crop=0, rand_pad=0, rand_gray= Number of attempts at generating a cropped/padded region of the image of the specified constraints. After max_attempts failures, return the original image. pad_val: float - pixel value to be filled when padding is enabled. pad_val will automatically + Pixel value to be filled when padding is enabled. pad_val will automatically be subtracted by mean and divided by std if applicable. Examples @@ -627,7 +627,7 @@ class ImageDetIter(ImageIter): Parameters ---------- aug_list : list or None - augmenter list for generating distorted images + Augmenter list for generating distorted images batch_size : int Number of examples per batch. data_shape : tuple @@ -657,7 +657,7 @@ class ImageDetIter(ImageIter): data_name : str Data name for provided symbols. label_name : str - name for detection labels + Name for detection labels kwargs : ... More arguments for creating augmenter. See mx.image.CreateDetAugmenter. """ @@ -723,7 +723,7 @@ def _parse_label(self, label): obj_width = int(raw[1]) if (raw.size - header_width) % obj_width != 0: msg = "Label shape %s inconsistent with annotation width %d." \ - %(str(raw.shape, obj_width)) + %(str(raw.shape), obj_width) raise RuntimeError(msg) out = np.reshape(raw[header_width:], (-1, obj_width)) # remove bad ground-truths @@ -738,9 +738,9 @@ def reshape(self, data_shape=None, label_shape=None): Parameters ---------- data_shape : tuple or None - reshape the data_shape to the new shape if not None + Reshape the data_shape to the new shape if not None label_shape : tuple or None - reshape label shape to new shape if not None + Reshape label shape to new shape if not None """ if data_shape is not None: self.check_data_shape(data_shape) diff --git a/python/mxnet/image/image.py b/python/mxnet/image/image.py index 02cd3cd0d551..ce44029d2ca1 100644 --- a/python/mxnet/image/image.py +++ b/python/mxnet/image/image.py @@ -826,31 +826,34 @@ def CreateAugmenter(data_shape, resize=0, rand_crop=False, rand_resize=False, ra Parameters ---------- data_shape : tuple of int - shape for output data + Shape for output data resize : int - resize shorter edge if larger than 0 at the begining - rand_resize : float - [0, 1], probability to apply random resizing + Resize shorter edge if larger than 0 at the begining + rand_crop : bool + Whether to enable random cropping other than center crop + rand_resize : bool + Whether to enable random sized cropping, require rand_crop to be enabled rand_gray : float - [0, 1], probability to convert to grayscale for all channels + [0, 1], probability to convert to grayscale for all channels, the number + of channels will not be reduced to 1 rand_mirror : bool - whether apply horizontal flip to image with probability 0.5 + Whether to apply horizontal flip to image with probability 0.5 mean : np.ndarray or None - mean pixel values for [r, g, b] + Mean pixel values for [r, g, b] std : np.ndarray or None - standard deviations for [r, g, b] + Standard deviations for [r, g, b] brightness : float - brightness jittering range (percent) + Brightness jittering range (percent) contrast : float - contrast jittering range + Contrast jittering range (percent) saturation : float - saturation jittering range + Saturation jittering range (percent) hue : float - hue jittering range + Hue jittering range (percent) pca_noise : float - pca noise level + Pca noise level (percent) inter_method : int, default=2(Area-based) - interpolation method for all resizing operations + Interpolation method for all resizing operations Possible values: 0: Nearest Neighbors Interpolation. From 0d1407fb69c1f3a71ef6c8d717d97d5aa0a44061 Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Thu, 10 Aug 2017 10:20:39 -0700 Subject: [PATCH 359/834] add verification to gluon dataset (#7322) * add verification to gluon dataset * fix * rename variables * add tests * fix doc --- docs/api/python/gluon.md | 4 +- python/mxnet/gluon/data/vision.py | 40 ++++++++++++++----- python/mxnet/gluon/model_zoo/model_store.py | 25 ++++-------- .../mxnet/gluon/model_zoo/vision/__init__.py | 13 ++++-- python/mxnet/gluon/utils.py | 36 +++++++++++++++-- tests/python/unittest/test_gluon_data.py | 4 ++ 6 files changed, 85 insertions(+), 37 deletions(-) diff --git a/docs/api/python/gluon.md b/docs/api/python/gluon.md index 6e213bbe05e0..ac637749f856 100644 --- a/docs/api/python/gluon.md +++ b/docs/api/python/gluon.md @@ -239,6 +239,7 @@ Model zoo provides pre-defined and pre-trained models to help bootstrap machine ```eval_rst .. currentmodule:: mxnet.gluon.model_zoo.vision +.. automodule:: mxnet.gluon.model_zoo.vision ``` ```eval_rst @@ -508,8 +509,7 @@ Model zoo provides pre-defined and pre-trained models to help bootstrap machine .. automodule:: mxnet.gluon.data.vision :members: -.. automodule:: mxnet.gluon.model_zoo.vision - :members: +.. automethod:: mxnet.gluon.model_zoo.vision.get_model .. automethod:: mxnet.gluon.model_zoo.vision.resnet18_v1 .. automethod:: mxnet.gluon.model_zoo.vision.resnet34_v1 .. automethod:: mxnet.gluon.model_zoo.vision.resnet50_v1 diff --git a/python/mxnet/gluon/data/vision.py b/python/mxnet/gluon/data/vision.py index 4ddbbbdf48a2..a16e736b027d 100644 --- a/python/mxnet/gluon/data/vision.py +++ b/python/mxnet/gluon/data/vision.py @@ -26,7 +26,7 @@ import numpy as np from . import dataset -from ..utils import download +from ..utils import download, check_sha1 from ... import nd @@ -67,7 +67,8 @@ class MNIST(_DownloadedDataset): transform=lambda data, label: (data.astype(np.float32)/255, label) """ - def __init__(self, root, train=True, transform=lambda data, label: (data, label)): + def __init__(self, root='~/.mxnet/datasets/', train=True, + transform=lambda data, label: (data, label)): super(MNIST, self).__init__(root, train, transform) def _get_data(self): @@ -75,11 +76,15 @@ def _get_data(self): os.makedirs(self._root) url = 'http://data.mxnet.io/data/mnist/' if self._train: - data_file = download(url+'train-images-idx3-ubyte.gz', self._root) - label_file = download(url+'train-labels-idx1-ubyte.gz', self._root) + data_file = download(url+'train-images-idx3-ubyte.gz', self._root, + sha1_hash='6c95f4b05d2bf285e1bfb0e7960c31bd3b3f8a7d') + label_file = download(url+'train-labels-idx1-ubyte.gz', self._root, + sha1_hash='2a80914081dc54586dbdf242f9805a6b8d2a15fc') else: - data_file = download(url+'t10k-images-idx3-ubyte.gz', self._root) - label_file = download(url+'t10k-labels-idx1-ubyte.gz', self._root) + data_file = download(url+'t10k-images-idx3-ubyte.gz', self._root, + sha1_hash='c3a25af1f52dad7f726cce8cacb138654b760d48') + label_file = download(url+'t10k-labels-idx1-ubyte.gz', self._root, + sha1_hash='763e7fa3757d93b0cdec073cef058b2004252c17') with gzip.open(label_file, 'rb') as fin: struct.unpack(">II", fin.read(8)) @@ -110,7 +115,14 @@ class CIFAR10(_DownloadedDataset): transform=lambda data, label: (data.astype(np.float32)/255, label) """ - def __init__(self, root, train=True, transform=lambda data, label: (data, label)): + def __init__(self, root='~/.mxnet/datasets/', train=True, + transform=lambda data, label: (data, label)): + self._file_hashes = {'data_batch_1.bin': 'aadd24acce27caa71bf4b10992e9e7b2d74c2540', + 'data_batch_2.bin': 'c0ba65cce70568cd57b4e03e9ac8d2a5367c1795', + 'data_batch_3.bin': '1dd00a74ab1d17a6e7d73e185b69dbf31242f295', + 'data_batch_4.bin': 'aab85764eb3584312d3c7f65fd2fd016e36a258e', + 'data_batch_5.bin': '26e2849e66a845b7f1e4614ae70f4889ae604628', + 'test_batch.bin': '67eb016db431130d61cd03c7ad570b013799c88c'} super(CIFAR10, self).__init__(root, train, transform) def _read_batch(self, filename): @@ -123,11 +135,17 @@ def _read_batch(self, filename): def _get_data(self): if not os.path.isdir(self._root): os.makedirs(self._root) - url = 'https://www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz' - filename = download(url, self._root) - with tarfile.open(filename) as tar: - tar.extractall(self._root) + file_paths = [(name, os.path.join(self._root, 'cifar-10-batches-bin/', name)) + for name in self._file_hashes] + if any(not os.path.exists(path) or not check_sha1(path, self._file_hashes[name]) + for name, path in file_paths): + url = 'https://www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz' + filename = download(url, self._root, + sha1_hash='e8aa088b9774a44ad217101d2e2569f823d2d491') + + with tarfile.open(filename) as tar: + tar.extractall(self._root) if self._train: filename = os.path.join(self._root, 'cifar-10-batches-bin/data_batch_%d.bin') diff --git a/python/mxnet/gluon/model_zoo/model_store.py b/python/mxnet/gluon/model_zoo/model_store.py index e3c48ba2235a..67ba572deb8c 100644 --- a/python/mxnet/gluon/model_zoo/model_store.py +++ b/python/mxnet/gluon/model_zoo/model_store.py @@ -19,11 +19,10 @@ """Model zoo for pre-trained models.""" from __future__ import print_function __all__ = ['get_model_file', 'purge'] -import hashlib import os import zipfile -from ...test_utils import download +from ..utils import download, check_sha1 _model_sha1 = {name: checksum for checksum, name in [ ('44335d1f0046b328243b32a26a4fbd62d9057b45', 'alexnet'), @@ -56,21 +55,11 @@ def short_hash(name): raise ValueError('Pretrained model for {name} is not available.'.format(name=name)) return _model_sha1[name][:8] -def verified(file_path, name): - sha1 = hashlib.sha1() - with open(file_path, 'rb') as f: - while True: - data = f.read(1048576) - if not data: - break - sha1.update(data) - - return sha1.hexdigest() == _model_sha1[name] - def get_model_file(name, local_dir=os.path.expanduser('~/.mxnet/models/')): r"""Return location for the pretrained on local file system. This function will download from online model zoo when model cannot be found or has mismatch. + The local_dir directory will be created if it doesn't exist. Parameters ---------- @@ -87,8 +76,9 @@ def get_model_file(name, local_dir=os.path.expanduser('~/.mxnet/models/')): file_name = '{name}-{short_hash}'.format(name=name, short_hash=short_hash(name)) file_path = os.path.join(local_dir, file_name+'.params') + sha1_hash = _model_sha1[name] if os.path.exists(file_path): - if verified(file_path, name): + if check_sha1(file_path, sha1_hash): return file_path else: print('Mismatch in the content of model file detected. Downloading again.') @@ -98,17 +88,16 @@ def get_model_file(name, local_dir=os.path.expanduser('~/.mxnet/models/')): if not os.path.exists(local_dir): os.makedirs(local_dir) + zip_file_path = os.path.join(local_dir, file_name+'.zip') download(_url_format.format(bucket=bucket, file_name=file_name), - fname=file_name+'.zip', - dirname=local_dir, + path=zip_file_path, overwrite=True) - zip_file_path = os.path.join(local_dir, file_name+'.zip') with zipfile.ZipFile(zip_file_path) as zf: zf.extractall(local_dir) os.remove(zip_file_path) - if verified(file_path, name): + if check_sha1(file_path, sha1_hash): return file_path else: raise ValueError('Downloaded file has different hash. Please try again.') diff --git a/python/mxnet/gluon/model_zoo/vision/__init__.py b/python/mxnet/gluon/model_zoo/vision/__init__.py index e4016db2ea20..354236b2d896 100644 --- a/python/mxnet/gluon/model_zoo/vision/__init__.py +++ b/python/mxnet/gluon/model_zoo/vision/__init__.py @@ -18,6 +18,7 @@ # coding: utf-8 # pylint: disable=wildcard-import, arguments-differ r"""Module for pre-defined neural network models. + This module contains definitions for the following model architectures: - `AlexNet`_ - `DenseNet`_ @@ -26,21 +27,26 @@ - `ResNet V2`_ - `SqueezeNet`_ - `VGG`_ + You can construct a model with random weights by calling its constructor: -.. code:: python +.. code:: + import mxnet.gluon.models as models resnet18 = models.resnet18_v1() alexnet = models.alexnet() squeezenet = models.squeezenet1_0() densenet = models.densenet_161() + We provide pre-trained models for all the models except ResNet V2. These can constructed by passing ``pretrained=True``: -.. code:: python +.. code:: + import mxnet.gluon.models as models resnet18 = models.resnet18_v1(pretrained=True) alexnet = models.alexnet(pretrained=True) -Pretrained model is converted from torchvision. + +Pretrained models are converted from torchvision. All pre-trained models expect input images normalized in the same way, i.e. mini-batches of 3-channel RGB images of shape (N x 3 x H x W), where N is the batch size, and H and W are expected to be at least 224. @@ -48,6 +54,7 @@ using ``mean = [0.485, 0.456, 0.406]`` and ``std = [0.229, 0.224, 0.225]``. The transformation should preferrably happen at preprocessing. You can use ``mx.image.color_normalize`` for such transformation:: + image = image/255 normalized = mx.image.color_normalize(image, mean=mx.nd.array([0.485, 0.456, 0.406]), diff --git a/python/mxnet/gluon/utils.py b/python/mxnet/gluon/utils.py index 7d9c378fe76d..cece22b75b14 100644 --- a/python/mxnet/gluon/utils.py +++ b/python/mxnet/gluon/utils.py @@ -19,6 +19,7 @@ # pylint: disable= """Parallelization utility optimizer.""" import os +import hashlib try: import requests except ImportError: @@ -136,7 +137,33 @@ def _indent(s_, numSpaces): return s -def download(url, path=None, overwrite=False): +def check_sha1(filename, sha1_hash): + """Check whether the sha1 hash of the file content matches the expected hash. + + Parameters + ---------- + filename : str + Path to the file. + sha1_hash : str + Expected sha1 hash in hexadecimal digits. + + Returns + ------- + bool + Whether the file content matches the expected hash. + """ + sha1 = hashlib.sha1() + with open(filename, 'rb') as f: + while True: + data = f.read(1048576) + if not data: + break + sha1.update(data) + + return sha1.hexdigest() == sha1_hash + + +def download(url, path=None, overwrite=False, sha1_hash=None): """Download an given URL Parameters @@ -148,11 +175,14 @@ def download(url, path=None, overwrite=False): current directory with same name as in url. overwrite : bool, optional Whether to overwrite destination file if already exists. + sha1_hash : str, optional + Expected sha1 hash in hexadecimal digits. Will ignore existing file when hash is specified + but doesn't match. Returns ------- str - The filename of the downloaded file. + The file path of the downloaded file. """ if path is None: fname = url.split('/')[-1] @@ -161,7 +191,7 @@ def download(url, path=None, overwrite=False): else: fname = path - if overwrite or not os.path.exists(fname): + if overwrite or not os.path.exists(fname) or (sha1_hash and not check_sha1(fname, sha1_hash)): dirname = os.path.dirname(os.path.abspath(os.path.expanduser(fname))) if not os.path.exists(dirname): os.makedirs(dirname) diff --git a/tests/python/unittest/test_gluon_data.py b/tests/python/unittest/test_gluon_data.py index da1de6ba4b58..e9a430124499 100644 --- a/tests/python/unittest/test_gluon_data.py +++ b/tests/python/unittest/test_gluon_data.py @@ -67,6 +67,10 @@ def test_sampler(): rand_batch_keep = gluon.data.BatchSampler(rand_sampler, 3, 'keep') assert sorted(sum(list(rand_batch_keep), [])) == list(range(10)) +def test_datasets(): + assert len(gluon.data.vision.MNIST(root='data')) == 60000 + assert len(gluon.data.vision.CIFAR10(root='data', train=False)) == 10000 + if __name__ == '__main__': import nose nose.runmodule() From c55fc571d22ec458365d87740b589827ddfd86cf Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Thu, 10 Aug 2017 10:24:03 -0700 Subject: [PATCH 360/834] add Sequential compatibility to rnn layers (#7352) --- python/mxnet/gluon/rnn/rnn_layer.py | 46 ++++++++++++++++++------- tests/python/unittest/test_gluon_rnn.py | 29 ++++++++++++++++ 2 files changed, 63 insertions(+), 12 deletions(-) diff --git a/python/mxnet/gluon/rnn/rnn_layer.py b/python/mxnet/gluon/rnn/rnn_layer.py index a9bcee52a6d3..86b7c618e503 100644 --- a/python/mxnet/gluon/rnn/rnn_layer.py +++ b/python/mxnet/gluon/rnn/rnn_layer.py @@ -168,10 +168,13 @@ def begin_state(self, batch_size=0, func=ndarray.zeros, **kwargs): states.append(func(name='%sh0_%d'%(self.prefix, i), **info)) return states - def forward(self, inputs, states): + def forward(self, inputs, states=None): + batch_size = inputs.shape[self._layout.find('N')] + skip_states = states is None + if skip_states: + states = self.begin_state(batch_size) if isinstance(states, ndarray.NDArray): states = [states] - batch_size = inputs.shape[self._layout.find('N')] for state, info in zip(states, self.state_info(batch_size)): if state.shape != info['shape']: raise ValueError( @@ -182,8 +185,12 @@ def forward(self, inputs, states): self.i2h_weight[i].shape = (self._gates*self._hidden_size, inputs.shape[2]) self.i2h_weight[i]._finish_deferred_init() if inputs.context.device_type == 'gpu': - return self._forward_gpu(inputs, states) - return self._forward_cpu(inputs, states) + out = self._forward_gpu(inputs, states) + else: + out = self._forward_cpu(inputs, states) + + # out is (output, state) + return out[0] if skip_states else out def _forward_cpu(self, inputs, states): ns = len(states) @@ -282,10 +289,12 @@ class RNN(_RNNLayer): If `bidirectional` is True, output shape will instead be `(sequence_length, batch_size, 2*num_hidden)` - Recurrent state shape: - The recurrent state's shape is `(num_layers, batch_size, num_hidden)`. - If `bidirectional` is True, state shape will instead be + Recurrent state: + The recurrent state is an NDArray with shape `(num_layers, batch_size, num_hidden)`. + If `bidirectional` is True, the recurrent state shape will instead be `(2*num_layers, batch_size, num_hidden)` + If input recurrent state is None, zeros are used as default begin states, + and the output recurrent state is omitted. Examples @@ -293,6 +302,9 @@ class RNN(_RNNLayer): >>> layer = mx.gluon.rnn.RNN(100, 3) >>> layer.initialize() >>> input = mx.nd.random_uniform(shape=(5, 3, 10)) + >>> # by default zeros are used as begin state + >>> output = layer(input) + >>> # manually specify begin state. >>> h0 = mx.nd.random_uniform(shape=(3, 3, 100)) >>> output, hn = layer(input, h0) """ @@ -379,11 +391,13 @@ class LSTM(_RNNLayer): If `bidirectional` is True, output shape will instead be `(sequence_length, batch_size, 2*num_hidden)` - Recurrent state shape: + Recurrent state: The recurrent state is a list of two NDArrays. Both has shape `(num_layers, batch_size, num_hidden)`. - If `bidirectional` is True, state shape will instead be + If `bidirectional` is True, each recurrent state will instead have shape `(2*num_layers, batch_size, num_hidden)`. + If input recurrent state is None, zeros are used as default begin states, + and the output recurrent state is omitted. Examples @@ -391,6 +405,9 @@ class LSTM(_RNNLayer): >>> layer = mx.gluon.rnn.LSTM(100, 3) >>> layer.initialize() >>> input = mx.nd.random_uniform(shape=(5, 3, 10)) + >>> # by default zeros are used as begin state + >>> output = layer(input) + >>> # manually specify begin state. >>> h0 = mx.nd.random_uniform(shape=(3, 3, 100)) >>> c0 = mx.nd.random_uniform(shape=(3, 3, 100)) >>> output, hn = layer(input, [h0, c0]) @@ -474,10 +491,12 @@ class GRU(_RNNLayer): If `bidirectional` is True, output shape will instead be `(sequence_length, batch_size, 2*num_hidden)` - Recurrent state shape: - The recurrent state's shape is `(num_layers, batch_size, num_hidden)`. - If `bidirectional` is True, state shape will instead be + Recurrent state: + The recurrent state is an NDArray with shape `(num_layers, batch_size, num_hidden)`. + If `bidirectional` is True, the recurrent state shape will instead be `(2*num_layers, batch_size, num_hidden)` + If input recurrent state is None, zeros are used as default begin states, + and the output recurrent state is omitted. Examples @@ -485,6 +504,9 @@ class GRU(_RNNLayer): >>> layer = mx.gluon.rnn.GRU(100, 3) >>> layer.initialize() >>> input = mx.nd.random_uniform(shape=(5, 3, 10)) + >>> # by default zeros are used as begin state + >>> output = layer(input) + >>> # manually specify begin state. >>> h0 = mx.nd.random_uniform(shape=(3, 3, 100)) >>> output, hn = layer(input, h0) """ diff --git a/tests/python/unittest/test_gluon_rnn.py b/tests/python/unittest/test_gluon_rnn.py index ac671e5f8840..40620136645a 100644 --- a/tests/python/unittest/test_gluon_rnn.py +++ b/tests/python/unittest/test_gluon_rnn.py @@ -209,6 +209,35 @@ def test_rnn_cells(): net.add(gluon.rnn.GRUCell(100, input_size=100)) check_rnn_forward(net, mx.nd.ones((8, 3, 200))) +def check_rnn_layer_forward(layer, inputs, states=None): + layer.collect_params().initialize() + with mx.autograd.record(): + out = layer(inputs, states) + if states is not None: + assert isinstance(out, tuple) and len(out) == 2 + out = out[0] + else: + assert isinstance(out, mx.nd.NDArray) + out.backward() + mx.nd.waitall() + +def test_rnn_layers(): + check_rnn_layer_forward(gluon.rnn.RNN(10, 2), mx.nd.ones((8, 3, 20))) + check_rnn_layer_forward(gluon.rnn.RNN(10, 2), mx.nd.ones((8, 3, 20)), mx.nd.ones((2, 3, 10))) + check_rnn_layer_forward(gluon.rnn.LSTM(10, 2), mx.nd.ones((8, 3, 20))) + check_rnn_layer_forward(gluon.rnn.LSTM(10, 2), mx.nd.ones((8, 3, 20)), [mx.nd.ones((2, 3, 10)), mx.nd.ones((2, 3, 10))]) + check_rnn_layer_forward(gluon.rnn.GRU(10, 2), mx.nd.ones((8, 3, 20))) + check_rnn_layer_forward(gluon.rnn.GRU(10, 2), mx.nd.ones((8, 3, 20)), mx.nd.ones((2, 3, 10))) + + net = gluon.nn.Sequential() + net.add(gluon.rnn.LSTM(10, 2, bidirectional=True)) + net.add(gluon.nn.BatchNorm(axis=2)) + net.add(gluon.nn.Flatten()) + net.add(gluon.nn.Dense(3, activation='relu')) + net.collect_params().initialize() + with mx.autograd.record(): + net(mx.nd.ones((2, 3, 10))).backward() + if __name__ == '__main__': import nose From aed297aed61dc96afc93565fda2dda85f1a2749b Mon Sep 17 00:00:00 2001 From: Yao Wang Date: Thu, 10 Aug 2017 13:58:20 -0700 Subject: [PATCH 361/834] Add disclaimer and download link (#7402) * Add disclaimer and download link * Fix --- docs/_static/mxnet-theme/index.html | 7 +++ docs/_static/mxnet.css | 14 ++++++ docs/build_version_doc/AddPackageLink.py | 58 ++++++++++++++++++++++++ docs/build_version_doc/build_doc.sh | 2 + docs/get_started/install.md | 2 + 5 files changed, 83 insertions(+) create mode 100644 docs/build_version_doc/AddPackageLink.py diff --git a/docs/_static/mxnet-theme/index.html b/docs/_static/mxnet-theme/index.html index c4f3c48fcaad..b39e710d6155 100644 --- a/docs/_static/mxnet-theme/index.html +++ b/docs/_static/mxnet-theme/index.html @@ -127,3 +127,10 @@

    Model Zoo

    + +
    + +

    + Apache MXNet is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that the project has yet to be fully endorsed by the ASF. +

    +
    \ No newline at end of file diff --git a/docs/_static/mxnet.css b/docs/_static/mxnet.css index 6f6d8cda1351..db2c5a275c53 100644 --- a/docs/_static/mxnet.css +++ b/docs/_static/mxnet.css @@ -717,6 +717,14 @@ li.dropdown-submenu ul.dropdown-menu a { filter: grayscale(0%); } +.section-disclaimer { + padding: 3em 3em 3em; +} + +.section-disclaimer p { + padding-top: 2em; +} + .footer{ padding-top: 40px; } @@ -1244,3 +1252,9 @@ div.download_btn a:hover { padding-bottom: 3px; font-style: italic; } + +/*------------Download source-----------------*/ +#download-source-package { + display: none; + padding-top: 40px; +} diff --git a/docs/build_version_doc/AddPackageLink.py b/docs/build_version_doc/AddPackageLink.py new file mode 100644 index 000000000000..8fe04b50b5ce --- /dev/null +++ b/docs/build_version_doc/AddPackageLink.py @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os +import argparse +from bs4 import BeautifulSoup as bs + +parser = argparse.ArgumentParser(description="Add download package link.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument('--file_path', type=str, default='docs/_build/html/get_started/install.html', + help='file to be modified') +parser.add_argument('--current_version', type=str, default='master', + help='Current version') + +if __name__ == '__main__': + args = parser.parse_args() + tag = args.current_version + + src_url = "http://www.apache.org/dyn/closer.cgi/incubator/" \ + "mxnet/%s-incubating/apache-mxnet-src-%s-incubating.tar.gz" % (tag, tag) + pgp_url = "http://www.apache.org/dyn/closer.cgi/incubator/" \ + "mxnet/%s-incubating/apache-mxnet-src-%s-incubating.tar.gz.asc" % (tag, tag) + sha_url = "http://www.apache.org/dyn/closer.cgi/incubator/" \ + "mxnet/%s-incubating/apache-mxnet-src-%s-incubating.tar.gz.sha" % (tag, tag) + md5_url = "http://www.apache.org/dyn/closer.cgi/incubator/" \ + "mxnet/%s-incubating/apache-mxnet-src-%s-incubating.tar.gz.md5" % (tag, tag) + + download_str = "
    " + download_str += "
    " \ + "" \ + " Source for %s
    " % (src_url, tag) + download_str += "
    PGP
    " % (pgp_url) + download_str += "
    SHA-256
    " % (sha_url) + download_str += "
    MD5
    " % (md5_url) + download_str += "
    " + + with open(args.file_path, 'r') as html_file: + content = bs(html_file, 'html.parser') + download_div = content.find(id="download-source-package") + download_div['style'] = "display:block" + download_div.append(download_str) + outstr = str(content).replace('<', '<').replace('>', '>') + with open(args.file_path, 'w') as outf: + outf.write(outstr) \ No newline at end of file diff --git a/docs/build_version_doc/build_doc.sh b/docs/build_version_doc/build_doc.sh index c5b59ba1df92..5a4f15d33c9c 100755 --- a/docs/build_version_doc/build_doc.sh +++ b/docs/build_version_doc/build_doc.sh @@ -57,6 +57,8 @@ then cat $tag_list_file tests/ci_build/ci_build.sh doc python docs/build_version_doc/AddVersion.py --file_path "docs/_build/html/" \ --current_version "$latest_tag" --root_url "http://mxnet.incubator.apache.org/" + tests/ci_build/ci_build.sh doc python docs/build_version_doc/AddPackageLink.py \ + --file_path "docs/_build/html/get_started/install.html" --current_version "$latest_tag" cp -a "docs/_build/html/." "$local_build" cp $tag_list_file "$local_build/tag.txt" rm -rf "$web_folder/.git" diff --git a/docs/get_started/install.md b/docs/get_started/install.md index 063d419f5c3b..0e88a0d2a2ee 100644 --- a/docs/get_started/install.md +++ b/docs/get_started/install.md @@ -1462,3 +1462,5 @@ Will be available soon.
    + +# Download Source Package \ No newline at end of file From 1a3faa63f2a24820427e6454f5d6eaa72ea636c1 Mon Sep 17 00:00:00 2001 From: Krishna Sridhar <1875987+srikris@users.noreply.github.com> Date: Thu, 10 Aug 2017 15:22:07 -0700 Subject: [PATCH 362/834] Initial commit of an MXNet converter. (#7413) --- tools/coreml/__init__.py | 18 ++ tools/coreml/_layers.py | 397 +++++++++++++++++++++++ tools/coreml/_mxnet_converter.py | 210 ++++++++++++ tools/coreml/test_mxnet_converer.py | 477 ++++++++++++++++++++++++++++ 4 files changed, 1102 insertions(+) create mode 100644 tools/coreml/__init__.py create mode 100644 tools/coreml/_layers.py create mode 100644 tools/coreml/_mxnet_converter.py create mode 100644 tools/coreml/test_mxnet_converer.py diff --git a/tools/coreml/__init__.py b/tools/coreml/__init__.py new file mode 100644 index 000000000000..e56490a472cf --- /dev/null +++ b/tools/coreml/__init__.py @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from _mxnet_converter import * diff --git a/tools/coreml/_layers.py b/tools/coreml/_layers.py new file mode 100644 index 000000000000..51489849cd28 --- /dev/null +++ b/tools/coreml/_layers.py @@ -0,0 +1,397 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import numpy as _np + +def _get_input_output_name(net, node, index = 0): + name = node['name'] + inputs = node['inputs'] + + if index == 'all': + input_name = [_get_node_name(net, inputs[id][0]) for id in range(len(inputs))] + elif type(index) == int: + input_name = _get_node_name(net, inputs[0][0]) + else: + input_name = [_get_node_name(net, inputs[id][0]) for id in index] + return input_name, name + +def _get_node_name(net, node_id): + return net['nodes'][node_id]['name'] + +def _get_node_shape(net, node_id): + return net['nodes'][node_id]['shape'] + +def convert_transpose(net, node, model, builder): + """Convert a transpose layer from mxnet to coreml. + + Parameters + ---------- + network: net + A mxnet network object. + + layer: node + Node to convert. + + model: model + An model for MXNet + + builder: NeuralNetworkBuilder + A neural network builder object. + """ + input_name, output_name = _get_input_output_name(net, node) + name = node['name'] + param = node['attr'] + from ast import literal_eval + axes = literal_eval(param['axes']) + builder.add_permute(name, input_name, output_name, axes) + +def convert_flatten(net, node, model, builder): + """Convert a flatten layer from mxnet to coreml. + + Parameters + ---------- + network: net + A mxnet network object. + + layer: node + Node to convert. + + model: model + An model for MXNet + + builder: NeuralNetworkBuilder + A neural network builder object. + """ + input_name, output_name = _get_input_output_name(net, node) + name = node['name'] + builder.add_flatten(0, name, input_name, output_name) + +def convert_softmax(net, node, model, builder): + """Convert a softmax layer from mxnet to coreml. + + Parameters + ---------- + network: net + A mxnet network object. + + layer: node + Node to convert. + + model: model + An model for MXNet + + builder: NeuralNetworkBuilder + A neural network builder object. + """ + input_name, output_name = _get_input_output_name(net, node) + name = node['name'] + builder.add_softmax(name = name, + input_name = input_name, + output_name = output_name) + +def convert_activation(net, node, model, builder): + """Convert an activation layer from mxnet to coreml. + + Parameters + ---------- + network: net + A mxnet network object. + + layer: node + Node to convert. + + model: model + An model for MXNet + + builder: NeuralNetworkBuilder + A neural network builder object. + """ + input_name, output_name = _get_input_output_name(net, node) + name = node['name'] + mx_non_linearity = node['attr']['act_type'] + if mx_non_linearity == 'relu': + non_linearity = 'RELU' + elif mx_non_linearity == 'tanh': + non_linearity = 'TANH' + elif mx_non_linearity == 'sigmoid': + non_linearity = 'SIGMOID' + else: + raise TypeError('Unknown activation type %s' % mx_non_linearity) + builder.add_activation(name = name, + non_linearity = non_linearity, + input_name = input_name, + output_name = output_name) + +def convert_elementwise_add(net, node, model, builder): + """Convert an elementwise add layer from mxnet to coreml. + + Parameters + ---------- + network: net + A mxnet network object. + + layer: node + Node to convert. + + model: model + An model for MXNet + + builder: NeuralNetworkBuilder + A neural network builder object. + """ + + input_names, output_name = _get_input_output_name(net, node,[0,1]) + name = node['name'] + + builder.add_elementwise(name, input_names, output_name, 'ADD') + +def convert_dense(net, node, model, builder): + """Convert a dense layer from mxnet to coreml. + + Parameters + ---------- + network: net + A mxnet network object. + + layer: node + Node to convert. + + model: model + An model for MXNet + + builder: NeuralNetworkBuilder + A neural network builder object. + """ + input_name, output_name = _get_input_output_name(net, node) + param = node['attr'] + has_bias = True + name = node['name'] + + inputs = node['inputs'] + outputs = node['outputs'] + args = model.arg_params + W = args[_get_node_name(net, inputs[1][0])].asnumpy() + if has_bias: + Wb = args[_get_node_name(net, inputs[2][0])].asnumpy() + else: + Wb = None + nC, nB = W.shape + + builder.add_inner_product(name = name, + W = W, + Wb = Wb, + nB = nB, + nC = nC, + has_bias = has_bias, + input_name = input_name, + output_name = output_name) + +def convert_convolution(net, node, model, builder): + """Convert a convolution layer from mxnet to coreml. + + Parameters + ---------- + network: net + A mxnet network object. + + layer: node + Node to convert. + + model: model + An model for MXNet + + builder: NeuralNetworkBuilder + A neural network builder object. + """ + input_name, output_name = _get_input_output_name(net, node) + name = node['name'] + param = node['attr'] + inputs = node['inputs'] + outputs = node['outputs'] + args = model.arg_params + + from ast import literal_eval + + if 'no_bias' in param.keys(): + has_bias = not literal_eval(param['no_bias']) + else: + has_bias = True + + border_mode = "same" if literal_eval(param['pad']) != (0, 0) else 'valid' + border_mode = "valid" + n_filters = int(param['num_filter']) + output_shape = None # (needed for de-conv) + + W = args[_get_node_name(net, inputs[1][0])].asnumpy() + if has_bias: + Wb = args[_get_node_name(net, inputs[2][0])].asnumpy() + else: + Wb = None + + n_filters, channels = W.shape[0:2] + stride_height, stride_width = literal_eval(param['stride']) + kernel_height, kernel_width = literal_eval(param['kernel']) + + W = W.transpose((2, 3, 1, 0)) + builder.add_convolution(name = name, + kernelChannels = channels, + outputChannels = n_filters, + height = kernel_height, + width = kernel_width, + stride_height = stride_height, + stride_width = stride_width, + borderMode = border_mode, + groups = 1, + W = W, + b = Wb, + has_bias = has_bias, + is_deconv = False, + output_shape = output_shape, + input_name = input_name, + output_name = output_name) + + # Add padding if there is any + convLayer = builder.nn_spec.layers[-1].convolution + pad = literal_eval(param['pad']) + for i in range(len(pad)): + convLayer.valid.paddingAmounts.borderAmounts[i].startEdgeSize = pad[i] + convLayer.valid.paddingAmounts.borderAmounts[i].endEdgeSize = pad[i] + +def convert_pooling(net, node, model, builder): + """Convert a pooling layer from mxnet to coreml. + + Parameters + ---------- + network: net + A mxnet network object. + + layer: node + Node to convert. + + model: model + An model for MXNet + + builder: NeuralNetworkBuilder + A neural network builder object. + """ + input_name, output_name = _get_input_output_name(net, node) + name = node['name'] + inputs = node['inputs'] + param = node['attr'] + outputs = node['outputs'] + args = model.arg_params + + layer_type_mx = param['pool_type'] + if layer_type_mx == 'max': + layer_type= 'MAX' + elif layer_type_mx == 'avg': + layer_type = 'AVERAGE' + else: + raise TypeError("Pooling type %s not supported" % layer_type_mx) + + from ast import literal_eval + stride_height, stride_width = literal_eval(param['stride']) + kernel_width, kernel_height = literal_eval(param['kernel']) + + padding_type = 'VALID' + if 'global_pool' in param.keys(): + is_global = literal_eval(param['global_pool']) + else: + is_global = False + builder.add_pooling(name = name, + height = kernel_height, + width = kernel_width, + stride_height = stride_height, + stride_width = stride_width, + layer_type = layer_type, + padding_type = padding_type, + exclude_pad_area = False, + is_global = is_global, + input_name = input_name, + output_name = output_name) + + # Add padding if there is any + poolingLayer = builder.nn_spec.layers[-1].pooling + pad = literal_eval(param['pad']) + for i in range(len(pad)): + poolingLayer.valid.paddingAmounts.borderAmounts[i].startEdgeSize = pad[i] + poolingLayer.valid.paddingAmounts.borderAmounts[i].endEdgeSize = pad[i] + +def convert_batchnorm(net, node, model, builder): + """Convert a transpose layer from mxnet to coreml. + + Parameters + ---------- + network: net + A mxnet network object. + + layer: node + Node to convert. + + model: model + An model for MXNet + + builder: NeuralNetworkBuilder + A neural network builder object. + """ + input_name, output_name = _get_input_output_name(net, node) + name = node['name'] + param = node['attr'] + inputs = node['inputs'] + outputs = node['outputs'] + args = model.arg_params + aux = model.aux_params + + gamma = args[_get_node_name(net, inputs[1][0])].asnumpy() + beta = args[_get_node_name(net, inputs[2][0])].asnumpy() + mean = aux[_get_node_name(net, inputs[3][0])].asnumpy() + variance = aux[_get_node_name(net, inputs[4][0])].asnumpy() + + nb_channels = gamma.shape[0] + + builder.add_batchnorm( + name = name, + channels = nb_channels, + gamma = gamma, + beta = beta, + mean = mean, + variance = variance, + input_name = input_name, + output_name = output_name) + +def convert_concat(net, node, model, builder): + """Convert concat layer from mxnet to coreml. + + Parameters + ---------- + network: net + A mxnet network object. + + layer: node + Node to convert. + + model: model + An model for MXNet + + builder: NeuralNetworkBuilder + A neural network builder object. + """ + # Get input and output names + input_names, output_name = _get_input_output_name(net, node, 'all') + name = node['name'] + mode = 'CONCAT' + builder.add_elementwise(name = name, input_names = input_names, + output_name = output_name, mode = mode) diff --git a/tools/coreml/_mxnet_converter.py b/tools/coreml/_mxnet_converter.py new file mode 100644 index 000000000000..88a980c61c1b --- /dev/null +++ b/tools/coreml/_mxnet_converter.py @@ -0,0 +1,210 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import _layers +import coremltools as _coremltools +import coremltools.models.datatypes as _datatypes +from coremltools.models import neural_network as _neural_network + +import json as _json +import mxnet as _mxnet +import numpy as _np + +_MXNET_LAYER_REGISTRY = { + 'FullyConnected' : _layers.convert_dense, + 'Activation' : _layers.convert_activation, + 'SoftmaxOutput' : _layers.convert_softmax, + 'Convolution' : _layers.convert_convolution, + 'Pooling' : _layers.convert_pooling, + 'Flatten' : _layers.convert_flatten, + 'transpose' : _layers.convert_transpose, + 'Concat' : _layers.convert_concat, + 'BatchNorm' : _layers.convert_batchnorm, + 'elemwise_add' : _layers.convert_elementwise_add, +} + +_MXNET_SKIP_LAYERS = [ + '_MulScalar', +] + +def _mxnet_remove_batch(input_data): + for blob in input_data: + input_data[blob] = _np.reshape(input_data[blob], input_data[blob].shape[1:]) + return input_data + +def check_error(model, path, shapes, output = 'softmax_output', verbose = True): + """ + Check the difference between predictions from MXNet and CoreML. + """ + coreml_model = _coremltools.models.MLModel(path) + input_data = {} + input_data_copy = {} + for ip in shapes: + input_data[ip] = _np.random.rand(*shapes[ip]).astype('f') + input_data_copy[ip] = _np.copy(input_data[ip]) + + dataIter = _mxnet.io.NDArrayIter(input_data_copy) + mx_out = model.predict(dataIter).flatten() + + e_out_dict = coreml_model.predict(_mxnet_remove_batch(input_data)) + e_out = e_out_dict[output].flatten() + error = _np.linalg.norm(e_out - mx_out) + + if verbose: + print "First few predictions from CoreML : %s" % e_out[0:10] + print "First few predictions from MXNet : %s" % e_out[0:10] + print "L2 Error on random data %s" % error + return error + +def _set_input_output_layers(builder, input_names, output_names): + input_layers_indices = [] + output_layers_indices = [] + spec = builder.spec + layers = builder.spec.neuralNetwork.layers + for idx, l in enumerate(layers): + if set(input_names).intersection(l.input): + input_layers_indices.append(idx) + if set(output_names).intersection(l.output): + output_layers_indices.append(idx) + + builder.input_layers_indices = input_layers_indices + builder.output_layers_indices = output_layers_indices + builder.input_layers_is1d = [False for i in input_names] + builder.output_layers_is1d = [False for i in output_names] + +def _get_layer_converter_fn(layer): + """Get the right converter function for MXNet + """ + if layer in _MXNET_LAYER_REGISTRY: + return _MXNET_LAYER_REGISTRY[layer] + else: + raise TypeError("MXNet layer of type %s is not supported." % layer) + +def convert(model, order = None, **kwargs): + """Convert a keras model to the protobuf spec. + + Parameters + ---------- + model: MXNet model + A trained MXNet neural network model. + + order: Order of inputs + + **kwargs : + Provide keyword arguments of known shapes. + + Returns + ------- + model_spec: An object of type ModelSpec_pb. + Protobuf representation of the model + """ + if not kwargs: + raise TypeError("Must provide input shape to be able to perform conversion") + + def remove_batch(dim): + return dim[1:] + + if order is None: + input_names = kwargs.keys() + input_dims = map(remove_batch, kwargs.values()) + else: + names = kwargs.keys() + shapes = map(remove_batch, kwargs.values()) + input_names = [names[i] for i in order] + input_dims = [shapes[i] for i in order] + + net = model.symbol + + # Infer shapes and store in a dictionary + shapes = net.infer_shape(**kwargs) + arg_names = net.list_arguments() + output_names = net.list_outputs() + aux_names = net.list_auxiliary_states() + shape_dict = {} + for idx, op in enumerate(arg_names): + shape_dict[op] = shapes[0][idx] + for idx, op in enumerate(output_names): + shape_dict[op] = shapes[1][idx] + for idx, op in enumerate(aux_names): + shape_dict[op] = shapes[2][idx] + + + # Get the inputs and outputs + output_dims = shapes[1] + input_types = [_datatypes.Array(*dim) for dim in input_dims] + output_types = [_datatypes.Array(*dim) for dim in output_dims] + + # Make the builder + input_features = zip(input_names, input_types) + output_features = zip(output_names, output_types) + builder = _neural_network.NeuralNetworkBuilder(input_features, output_features) + + # Get out the layers + net = _json.loads(net.tojson()) + nodes = net['nodes'] + for i, node in enumerate(nodes): + node['id'] = i + + if node['name'] in shape_dict: + node['shape'] = shape_dict[node['name']] + + node['outputs'] = [] + if 'inputs' in node: + for ip in node['inputs']: + nodes[ip[0]]['outputs'].append([i, 0]) + else: + node['inputs'] = [] + + # Mark the head nodes + for head in net['heads']: + head_id = head[0] + head_node = nodes[head_id] + head_node['outputs'] = [head] + head_node['name'] += "_output" + head_node['shape'] = shape_dict[head_node['name']] + + # For skipped layers, make sure nodes are modified + for iter, node in enumerate(nodes): + op = node['op'] + inputs = node['inputs'] + outputs = node['outputs'] + if op in _MXNET_SKIP_LAYERS: + nodes[inputs[0][0]]['outputs'][0] = outputs[0] + nodes[outputs[0][0]]['inputs'][0] = inputs[0] + + # Find the input and output names for this node + for iter, node in enumerate(nodes): + op = node['op'] + if op == 'null' or op in _MXNET_SKIP_LAYERS: + continue + name = node['name'] + print("%d : %s, %s" % (iter, name, op)) + converter_func = _get_layer_converter_fn(op) + converter_func(net, node, model, builder) + + spec = builder.spec + layers = spec.neuralNetwork.layers + + # Set the right inputs and outputs + _set_input_output_layers(builder, input_names, output_names) + builder.set_input(input_names, input_dims) + builder.set_output(output_names, output_dims) + + # Return the spec + spec = builder.spec + layers = spec.neuralNetwork.layers + return spec diff --git a/tools/coreml/test_mxnet_converer.py b/tools/coreml/test_mxnet_converer.py new file mode 100644 index 000000000000..179d04a10930 --- /dev/null +++ b/tools/coreml/test_mxnet_converer.py @@ -0,0 +1,477 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import unittest +import mxnet as mx +import numpy as np +import tempfile +import os +import mxnet_converter +import coremltools + +def _mxnet_remove_batch(input_data): + for blob in input_data: + input_data[blob] = np.reshape(input_data[blob], input_data[blob].shape[1:]) + return input_data + +def _get_coreml_model(net, engine, model_path, input_shape, + input_names = ['data'], output_names = ['output']): + model = mx.model.FeedForward(net, engine, arg_params = engine.arg_dict) + spec = mxnet_converter.convert(model, **input_shape) + return coremltools.models.MLModel(spec) + +def set_weights(net, engine, mode = 'random'): + for arg in net.list_arguments(): + if mode == 'random': + engine.arg_dict[arg][:] = np.random.uniform(-0.1, 0.1, engine.arg_dict[arg].shape) + elif mode == 'zeros': + engine.arg_dict[arg][:] = np.zeros(engine.arg_dict[arg].shape) + elif mode == 'ones': + engine.arg_dict[arg][:] = np.ones(engine.arg_dict[arg].shape) + return net + +class MXNetSingleLayerTest(unittest.TestCase): + """ + Unit test class for testing mxnet converter. + """ + def _test_mxnet_model(self, net, engine, delta = 1e-3, **input_shape): + + # Generate some dummy data + input_data = {} + for ip in input_shape: + input_data[ip] = engine.arg_dict[ip].asnumpy() + output_blob = net.list_outputs()[0] + + # Make predictions from mxnet (only works on single output for now) + mxnet_preds = engine.forward()[0].asnumpy().flatten() + + # Get predictions from coreml + model_path = os.path.join(tempfile.mkdtemp(), 'mxnet.mlmodel') + model = _get_coreml_model(net, engine, model_path, input_shape, input_data.keys()) + coreml_preds = model.predict(_mxnet_remove_batch(input_data)).values()[0].flatten() + + # Check prediction accuracy + self.assertEquals(len(mxnet_preds), len(coreml_preds)) + for i in range(len(mxnet_preds)): + self.assertAlmostEquals(mxnet_preds[i], coreml_preds[i], delta = delta) + + def test_tiny_inner_product_zero_input(self): + np.random.seed(1988) + input_shape = (1, 10) + + # Define a model + net = mx.sym.Variable('data') + net = mx.sym.FullyConnected(data = net, name = 'fc1', num_hidden = 5) + engine = net.simple_bind(ctx=mx.cpu(), data=input_shape) + + # Set some random weights + set_weights(net, engine, mode = 'zeros') + + # Test the mxnet model + self._test_mxnet_model(net, engine, data = input_shape) + + def test_really_tiny_inner_product_ones_input(self): + np.random.seed(1988) + input_shape = (1, 1) + + # Define a model + net = mx.sym.Variable('data') + net = mx.sym.FullyConnected(data = net, name = 'fc1', num_hidden = 1) + engine = net.simple_bind(ctx=mx.cpu(), data=input_shape) + + # Set some random weights + set_weights(net, engine, mode = 'ones') + + # Test the mxnet model + self._test_mxnet_model(net, engine, data = input_shape) + + def test_really_tiny_2_inner_product_ones_input(self): + np.random.seed(1988) + input_shape = (1, 1) + + # Define a model + net = mx.sym.Variable('data') + net = mx.sym.FullyConnected(data = net, name = 'fc1', num_hidden = 5) + engine = net.simple_bind(ctx=mx.cpu(), data=input_shape) + + # Set some random weights + set_weights(net, engine, mode = 'ones') + + # Test the mxnet model + self._test_mxnet_model(net, engine, data = input_shape) + + def test_tiny_inner_product_ones_input(self): + np.random.seed(1988) + input_shape = (1, 10) + + # Define a model + net = mx.sym.Variable('data') + net = mx.sym.FullyConnected(data = net, name = 'fc1', num_hidden = 5) + engine = net.simple_bind(ctx=mx.cpu(), data=input_shape) + + # Set some random weights + set_weights(net, engine, mode = 'ones') + + # Test the mxnet model + self._test_mxnet_model(net, engine, data = input_shape) + + def test_tiny_inner_product_random_input(self): + np.random.seed(1988) + input_shape = (1, 10) + + # Define a model + net = mx.sym.Variable('data') + net = mx.sym.FullyConnected(data = net, name = 'fc1', num_hidden = 5) + engine = net.simple_bind(ctx=mx.cpu(), data=input_shape) + + # Set some random weights + set_weights(net, engine, mode = 'random') + + # Test the mxnet model + self._test_mxnet_model(net, engine, data = input_shape) + + def test_tiny_softmax_random_input(self): + np.random.seed(1988) + input_shape = (1, 10) + + # Define a model + net = mx.sym.Variable('data') + net = mx.sym.FullyConnected(data = net, name = 'fc1', num_hidden = 5) + net = mx.sym.SoftmaxOutput(net, name = 'softmax') + engine = net.simple_bind(ctx = mx.cpu(), data = input_shape) + + # Set some random weights + set_weights(net, engine, mode = 'random') + + # Test the mxnet model + self._test_mxnet_model(net, engine, data = input_shape) + + def test_tiny_relu_activation_random_input(self): + np.random.seed(1988) + input_shape = (1, 10) + + # Define a model + net = mx.sym.Variable('data') + net = mx.sym.FullyConnected(data = net, name = 'fc1', num_hidden = 5) + net = mx.sym.Activation(net, name = 'relu1', act_type = "relu") + engine = net.simple_bind(ctx = mx.cpu(), data = input_shape) + + # Set some random weights + set_weights(net, engine, mode = 'random') + + # Test the mxnet model + self._test_mxnet_model(net, engine, data = input_shape) + + def test_tiny_sigmoid_activation_random_input(self): + np.random.seed(1988) + input_shape = (1, 10) + + # Define a model + net = mx.sym.Variable('data') + net = mx.sym.FullyConnected(data = net, name = 'fc1', num_hidden = 5) + net = mx.sym.Activation(net, name = 'sigmoid1', act_type = "sigmoid") + engine = net.simple_bind(ctx = mx.cpu(), data = input_shape) + + # Set some random weights + set_weights(net, engine, mode = 'random') + + # Test the mxnet model + self._test_mxnet_model(net, engine, data = input_shape) + + def test_tiny_tanh_activation_random_input(self): + np.random.seed(1988) + input_shape = (1, 10) + + # Define a model + net = mx.sym.Variable('data') + net = mx.sym.FullyConnected(data = net, name = 'fc1', num_hidden = 5) + net = mx.sym.Activation(net, name = 'tanh1', act_type = "tanh") + engine = net.simple_bind(ctx = mx.cpu(), data = input_shape) + + # Set some random weights + set_weights(net, engine, mode = 'random') + + # Test the mxnet model + self._test_mxnet_model(net, engine, data = input_shape) + + def test_really_tiny_conv_random_input(self): + np.random.seed(1988) + input_shape = (1, 1, 10, 10) + num_filter = 1 + kernel = (1 ,1) + stride = (1, 1) + pad = (0, 0) + + # Define a model + net = mx.sym.Variable('data') + net = mx.symbol.Convolution(data = net, num_filter = num_filter, kernel=kernel, + stride = stride, pad = pad, name = 'conv_1') + engine = net.simple_bind(ctx = mx.cpu(), data = input_shape) + + # Set some random weights + set_weights(net, engine, mode = 'random') + + # Test the mxnet model + self._test_mxnet_model(net, engine, data = input_shape) + + def test_tiny_conv_ones_input(self): + np.random.seed(1988) + input_shape = (1, 1, 10, 10) + num_filter = 1 + kernel = (5 ,5) + stride = (1, 1) + pad = (0, 0) + + # Define a model + net = mx.sym.Variable('data') + net = mx.symbol.Convolution(data = net, num_filter = num_filter, kernel=kernel, + stride = stride, pad = pad, name = 'conv_1') + engine = net.simple_bind(ctx = mx.cpu(), data = input_shape) + + # Set some random weights + set_weights(net, engine, mode = 'ones') + + # Test the mxnet model + self._test_mxnet_model(net, engine, data = input_shape) + + def test_tiny_conv_random_input(self): + np.random.seed(1988) + input_shape = (1, 1, 10, 10) + num_filter = 1 + kernel = (5 ,5) + stride = (1, 1) + pad = (0, 0) + + # define a model + net = mx.sym.Variable('data') + net = mx.symbol.Convolution(data = net, num_filter = num_filter, kernel=kernel, + stride = stride, pad = pad, name = 'conv_1') + engine = net.simple_bind(ctx = mx.cpu(), data = input_shape) + + # set some random weights + set_weights(net, engine, mode = 'random') + + # test the mxnet model + self._test_mxnet_model(net, engine, data = input_shape) + + def test_tiny_asym_conv_random_input(self): + np.random.seed(1988) + input_shape = (1, 1, 10, 10) + num_filter = 1 + kernel = (5 ,3) + stride = (1, 1) + pad = (0, 0) + + # define a model + net = mx.sym.Variable('data') + net = mx.symbol.Convolution(data = net, num_filter = num_filter, kernel=kernel, + stride = stride, pad = pad, name = 'conv_1') + engine = net.simple_bind(ctx = mx.cpu(), data = input_shape) + + # set some random weights + set_weights(net, engine, mode = 'random') + + # test the mxnet model + self._test_mxnet_model(net, engine, data = input_shape) + + def test_tiny_asym_conv_random_asym_input(self): + np.random.seed(1988) + input_shape = (1, 1, 28, 18) + num_filter = 16 + kernel = (5 ,3) + stride = (1, 1) + pad = (0, 0) + dilate = (1, 1) + + # define a model + net = mx.sym.Variable('data') + net = mx.symbol.Convolution(data = net, num_filter = num_filter, kernel=kernel, + stride = stride, pad = pad, name = 'conv_1', dilate = dilate) + net = mx.sym.Activation(net, name = 'tanh', act_type = "tanh") + engine = net.simple_bind(ctx = mx.cpu(), data = input_shape) + + # set some random weights + set_weights(net, engine, mode = 'random') + + # test the mxnet model + self._test_mxnet_model(net, engine, data = input_shape) + + def test_tiny_conv_pooling_random_input(self): + np.random.seed(1988) + input_shape = (1, 1, 10, 10) + num_filter = 1 + kernel = (5 ,5) + stride = (1, 1) + pad = (0, 0) + + # define a model + net = mx.sym.Variable('data') + net = mx.symbol.Convolution(data = net, num_filter = num_filter, kernel=kernel, + stride = stride, pad = pad, name = 'conv_1') + net = mx.symbol.Pooling(data = net, kernel=kernel, + stride = stride, pad = pad, name = 'pool_1', pool_type = 'max') + engine = net.simple_bind(ctx = mx.cpu(), data = input_shape) + + # set some random weights + set_weights(net, engine, mode = 'random') + + # test the mxnet model + self._test_mxnet_model(net, engine, data = input_shape) + + def test_really_tiny_conv_random_3d_input(self): + np.random.seed(1988) + input_shape = (1, 3, 10, 10) + num_filter = 1 + kernel = (1 ,1) + stride = (1, 1) + pad = (0, 0) + + # define a model + net = mx.sym.Variable('data') + net = mx.symbol.Convolution(data = net, num_filter = num_filter, kernel=kernel, + stride = stride, pad = pad, name = 'conv_1') + engine = net.simple_bind(ctx = mx.cpu(), data = input_shape) + + # set some random weights + set_weights(net, engine, mode = 'random') + + # test the mxnet model + self._test_mxnet_model(net, engine, data = input_shape) + + def test_really_tiny_conv_random_input_multi_filter(self): + np.random.seed(1988) + input_shape = (1, 1, 10, 10) + num_filter = 64 + kernel = (1 ,1) + stride = (1, 1) + pad = (0, 0) + + # define a model + net = mx.sym.Variable('data') + net = mx.symbol.Convolution(data = net, num_filter = num_filter, kernel=kernel, + stride = stride, pad = pad, name = 'conv_1') + engine = net.simple_bind(ctx = mx.cpu(), data = input_shape) + + # set some random weights + set_weights(net, engine, mode = 'random') + + # test the mxnet model + self._test_mxnet_model(net, engine, data = input_shape) + + def test_tiny_conv_random_3d_input(self): + np.random.seed(1988) + input_shape = (1, 3, 10, 10) + num_filter = 1 + kernel = (5 ,5) + stride = (1, 1) + pad = (0, 0) + + # define a model + net = mx.sym.Variable('data') + net = mx.symbol.Convolution(data = net, num_filter = num_filter, kernel=kernel, + stride = stride, pad = pad, name = 'conv_1') + engine = net.simple_bind(ctx = mx.cpu(), data = input_shape) + + # set some random weights + set_weights(net, engine, mode = 'random') + + # test the mxnet model + self._test_mxnet_model(net, engine, data = input_shape) + + def test_tiny_conv_random_input_multi_filter(self): + np.random.seed(1988) + input_shape = (1, 1, 10, 10) + num_filter = 64 + kernel = (5 ,5) + stride = (1, 1) + pad = (0, 0) + + # define a model + net = mx.sym.Variable('data') + net = mx.symbol.Convolution(data = net, num_filter = num_filter, kernel=kernel, + stride = stride, pad = pad, name = 'conv_1') + engine = net.simple_bind(ctx = mx.cpu(), data = input_shape) + + # set some random weights + set_weights(net, engine, mode = 'random') + + # test the mxnet model + self._test_mxnet_model(net, engine, data = input_shape) + + def test_conv_random(self): + np.random.seed(1988) + input_shape = (1, 3, 10, 10) + num_filter = 64 + kernel = (5 ,5) + stride = (1, 1) + pad = (0, 0) + + # define a model + net = mx.sym.Variable('data') + net = mx.symbol.Convolution(data = net, num_filter = num_filter, kernel=kernel, + stride = stride, pad = pad, name = 'conv_1') + engine = net.simple_bind(ctx = mx.cpu(), data = input_shape) + + # set some random weights + set_weights(net, engine, mode = 'random') + + # test the mxnet model + self._test_mxnet_model(net, engine, data = input_shape) + + def test_flatten(self): + np.random.seed(1988) + input_shape = (1, 3, 10, 10) + num_filter = 64 + kernel = (5 ,5) + stride = (1, 1) + pad = (0, 0) + + # define a model + net = mx.sym.Variable('data') + net = mx.symbol.Convolution(data = net, num_filter = num_filter, kernel=kernel, + stride = stride, pad = pad, name = 'conv_1') + net = mx.sym.Flatten(data = net, name = 'flatten1') + net = mx.sym.FullyConnected(data = net, name = 'fc1', num_hidden = 5) + net = mx.sym.SoftmaxOutput(net, name = 'softmax') + engine = net.simple_bind(ctx = mx.cpu(), data = input_shape) + + # set some random weights + set_weights(net, engine, mode = 'random') + + # test the mxnet model + self._test_mxnet_model(net, engine, data = input_shape) + + def test_transpose(self): + np.random.seed(1988) + input_shape = (1, 3, 10, 10) + num_filter = 64 + kernel = (5 ,5) + stride = (1, 1) + pad = (0, 0) + + # define a model + net = mx.sym.Variable('data') + net = mx.sym.transpose(data = net, name = 'transpose', axes = (0, 1, 2, 3)) + net = mx.symbol.Convolution(data = net, num_filter = num_filter, kernel=kernel, + stride = stride, pad = pad, name = 'conv_1') + engine = net.simple_bind(ctx = mx.cpu(), data = input_shape) + + # set some random weights + set_weights(net, engine, mode = 'random') + + # test the mxnet model + self._test_mxnet_model(net, engine, data = input_shape) From 8ae2970083aa5bcdf37c3d85fcfead39c1802f40 Mon Sep 17 00:00:00 2001 From: Eric Junyuan Xie Date: Fri, 11 Aug 2017 13:26:48 -0700 Subject: [PATCH 363/834] Add autograd function (#7403) * fix optimizer * add imread * add autograd Function * add function * fix * fix * fix * fix * fix * fix --- docs/api/python/autograd.md | 1 + include/mxnet/c_api.h | 35 +++- include/mxnet/ndarray.h | 8 + include/mxnet/op_attr_types.h | 1 + python/mxnet/autograd.py | 167 ++++++++++++++++++- python/mxnet/base.py | 10 ++ python/mxnet/gluon/data/dataset.py | 26 +-- python/mxnet/gluon/data/vision.py | 109 ++++++++++++- python/mxnet/image/detection.py | 12 +- python/mxnet/image/image.py | 89 +++++++--- python/mxnet/operator.py | 11 +- src/c_api/c_api_function.cc | 199 +++++++++++++++++++++++ src/c_api/c_api_ndarray.cc | 8 + src/executor/attach_op_execs_pass.cc | 2 +- src/io/image_io.cc | 160 +++++++++++++----- src/ndarray/autograd.cc | 18 +- src/operator/custom/custom.cc | 12 +- tests/python/unittest/test_autograd.py | 47 ++++++ tests/python/unittest/test_gluon_data.py | 31 ++-- 19 files changed, 809 insertions(+), 137 deletions(-) create mode 100644 src/c_api/c_api_function.cc diff --git a/docs/api/python/autograd.md b/docs/api/python/autograd.md index 444e01fc9688..de8188446b7c 100644 --- a/docs/api/python/autograd.md +++ b/docs/api/python/autograd.md @@ -86,6 +86,7 @@ Detailed tutorials are available in Part 1 of set_recording is_recording mark_variables + Function ``` ## API Reference diff --git a/include/mxnet/c_api.h b/include/mxnet/c_api.h index 7a45099b8da0..2289354e8a5e 100644 --- a/include/mxnet/c_api.h +++ b/include/mxnet/c_api.h @@ -145,6 +145,7 @@ enum CustomOpPropCallbacks { kCustomOpPropInferType }; + typedef int (*CustomOpFBFunc)(int /*size*/, void** /*ptrs*/, int* /*tags*/, const int* /*reqs*/, const int /*is_train*/, void* /*state*/); @@ -164,6 +165,17 @@ typedef int (*CustomOpPropCreator)(const char* /*op_type*/, const int /*num_kwar const char** /*keys*/, const char** /*values*/, struct MXCallbackList* /*ret*/); + +enum CustomFunctionCallbacks { + kCustomFunctionBackward, + kCustomFunctionDelete +}; + +typedef int (*CustomFunctionBwdFunc)(int /*num_ograds*/, int /*num_igrads*/, void** /*ptrs*/, + const int* /*reqs*/, const int /*is_train*/, + void* /*state*/); +typedef int (*CustomFunctionDelFunc)(void* /*state*/); + /*! * \brief return str message of the last error * all function in this file will return 0 when success @@ -639,6 +651,12 @@ MXNET_DLL int MXAutogradBackwardEx(mx_uint num_output, NDArrayHandle* ograd_handles, int retain_graph, int is_train); +/* + * \brief get the graph constructed by autograd. + * \param handle ndarray handle + * \param out output symbol handle + */ +MXNET_DLL int MXAutogradGetSymbol(NDArrayHandle handle, SymbolHandle *out); /*! * \brief create cached operator */ @@ -1686,8 +1704,23 @@ MXNET_DLL int MXRtcPush(RtcHandle handle, mx_uint num_input, mx_uint num_output, * \brief Delete a MXRtc object */ MXNET_DLL int MXRtcFree(RtcHandle handle); - +/* + * \brief register custom operators from frontend. + * \param op_type name of custom op + * \param creator + */ MXNET_DLL int MXCustomOpRegister(const char* op_type, CustomOpPropCreator creator); +/* + * \brief record custom function for backward later. + * \param num_inputs number of input NDArrays. + * \param inputs handle to input NDArrays. + * \param num_outputs number of output NDArrays. + * \param outputs handle to output NDArrays. + * \param callbacks callbacks for backward function. + */ +MXNET_DLL int MXCustomFunctionRecord(int num_inputs, NDArrayHandle *inputs, + int num_outputs, NDArrayHandle *outputs, + MXCallbackList *callbacks); #ifdef __cplusplus } diff --git a/include/mxnet/ndarray.h b/include/mxnet/ndarray.h index d40b549d8740..d7dff4098b27 100644 --- a/include/mxnet/ndarray.h +++ b/include/mxnet/ndarray.h @@ -329,6 +329,14 @@ class NDArray { ret.entry_ = autograd::AGNodeEntry{nullptr, 0, 0}; return ret; } + + nnvm::Symbol get_autograd_symbol() { + CHECK(!entry_.is_none()) + << "NDArray is not part of a computation graph. Did you forget to turn on recording?"; + nnvm::Symbol ret; + ret.outputs.emplace_back(entry_.nn_entry()); + return ret; + } /*! * \brief Allocate the space if it is delayed allocated. * This is an internal function used by system that normal user should not use diff --git a/include/mxnet/op_attr_types.h b/include/mxnet/op_attr_types.h index 1ba07374d894..1bcae0d29348 100644 --- a/include/mxnet/op_attr_types.h +++ b/include/mxnet/op_attr_types.h @@ -35,6 +35,7 @@ #include "./base.h" #include "./ndarray.h" #include "./engine.h" +#include "./resource.h" namespace mxnet { diff --git a/python/mxnet/autograd.py b/python/mxnet/autograd.py index 7340851cdef6..292bcc2308fc 100644 --- a/python/mxnet/autograd.py +++ b/python/mxnet/autograd.py @@ -20,11 +20,14 @@ from __future__ import absolute_import from __future__ import division +from threading import Lock +import traceback import ctypes +from ctypes import c_int, c_void_p, CFUNCTYPE, POINTER, cast from .base import _LIB, check_call, string_types -from .base import mx_uint, NDArrayHandle, c_array +from .base import mx_uint, NDArrayHandle, c_array, MXCallbackList, SymbolHandle from .ndarray import NDArray -from .symbol import _GRAD_REQ_MAP +from .symbol import _GRAD_REQ_MAP, Symbol def set_recording(is_recording): #pylint: disable=redefined-outer-name @@ -265,3 +268,163 @@ def backward(heads, head_grads=None, retain_graph=False, train_mode=True): #pyli c_array(NDArrayHandle, ograd_handles), ctypes.c_int(retain_graph), ctypes.c_int(train_mode))) + + +def get_symbol(x): + """Retrieve recorded computation history as `Symbol`. + + Parameters + ---------- + x : NDArray + Array representing the head of computation graph. + + Returns + ------- + Symbol + The retrieved Symbol. + """ + hdl = SymbolHandle() + check_call(_LIB.MXAutogradGetSymbol(x.handle, ctypes.byref(hdl))) + return Symbol(hdl) + + +class Function(object): + """User-defined differentiable function. + + Function allows defining both forward and backward computation for + custom operators. During gradient computation, the used-defined + backward function will be used instead of the default chain-rule. + You can also cast to numpy array and back for some operations in + forward and backward. + + For example, a stable sigmoid function can be defined as:: + + class sigmoid(Function): + def forward(self, x): + y = 1 / (1 + mx.nd.exp(-x)) + self.save_for_backward(y) + return y + + def backward(self, dy): + # backward takes as many inputs as forward's return value, + # and returns as many NDArrays as forward's arguments. + y, = self.saved_tensors + return y * (1-y) + """ + _bwd_functype = CFUNCTYPE(c_int, c_int, c_int, POINTER(c_void_p), + POINTER(c_int), c_int, c_void_p) + _del_functype = CFUNCTYPE(c_int, c_void_p) + class _Registry(object): + """CustomOp registry.""" + def __init__(self): + self.ref_holder = {} + self.counter = 0 + self.lock = Lock() + + def inc(self): + """Get index for new entry.""" + self.lock.acquire() + cur = self.counter + self.counter += 1 + self.lock.release() + return cur + + _registry = _Registry() + + def __init__(self): + self._used = False + self.saved_tensors = () + + def save_for_backward(self, *args): + self.saved_tensors = args + + def __call__(self, *inputs): + assert not self._used, \ + "Each Function instance can only be called once. "\ + "Please create another instance." + self._used = True + + prev_recording = set_recording(False) + outputs = self.forward(*inputs) + set_recording(prev_recording) + + if not prev_recording: + return outputs + + ret_outputs = outputs + if isinstance(outputs, NDArray): + outputs = (outputs,) + + key = Function._registry.inc() + + def backward_entry(num_ograds, num_igrads, ptrs, reqs, is_train, _): + """entry point for backward.""" + # pylint: disable=W0613 + try: + output_grads = [NDArray(ctypes.cast(i, NDArrayHandle), writable=False) \ + for i in ptrs[:num_ograds]] + input_grads = [NDArray(ctypes.cast(i, NDArrayHandle), writable=True) \ + for i in ptrs[num_ograds:num_ograds+num_igrads]] + reqs = [reqs[i] for i in range(num_igrads)] + rets = self.backward(*output_grads) + if isinstance(rets, NDArray): + rets = (rets,) + assert len(rets) == len(input_grads), \ + "%s.backward must return exactly the same number " \ + "of NDArrays as the number of NDArrays arguments to forward." \ + "Expecting %d got %d"%(self.__class__.name, len(input_grads), len(rets)) + for igrad, ret, req in zip(input_grads, rets, reqs): + assert isinstance(ret, NDArray), \ + "autograd.Function.backward must return NDArrays, not %s"%type(ret) + if req == 0: # null + return + elif req == 1 or req == 2: # write or inplace + igrad[:] = ret + elif req == 'add': + igrad[:] += ret + except Exception: # pylint: disable=broad-except + print('Error in Function.backward: %s' % traceback.format_exc()) + return False + return True + + def delete_entry(_): + """C Callback for CustomFunction::delete""" + try: + del Function._registry.ref_holder[key] + except Exception: # pylint: disable=broad-except + print('Error in autograd.Function.delete: %s' % traceback.format_exc()) + return False + return True + + input_handles = [x.handle for x in inputs] + output_handles = [x.handle for x in outputs] + callbacks = [Function._bwd_functype(backward_entry), + Function._del_functype(delete_entry)] + callbacks = [cast(i, CFUNCTYPE(c_int)) for i in callbacks] + context = MXCallbackList(c_int(len(callbacks)), + cast(c_array(CFUNCTYPE(c_int), callbacks), + POINTER(CFUNCTYPE(c_int))), + cast(c_array(c_void_p, [None]*len(callbacks)), + POINTER(c_void_p))) + check_call(_LIB.MXCustomFunctionRecord( + c_int(len(inputs)), + c_array(NDArrayHandle, input_handles), + c_int(len(outputs)), + c_array(NDArrayHandle, output_handles), + ctypes.byref(context))) + + Function._registry.ref_holder[key] = context + + return ret_outputs + + def forward(self, *inputs): + """Forward computation.""" + raise NotImplementedError + + def backward(self, *output_grads): + """Backward computation. + + Takes as many inputs as forward's outputs, + and returns as many NDArrays as forward's inputs. + """ + raise NotImplementedError diff --git a/python/mxnet/base.py b/python/mxnet/base.py index 7d5a5bf8f889..aad0580e7d07 100644 --- a/python/mxnet/base.py +++ b/python/mxnet/base.py @@ -72,6 +72,16 @@ def __str__(self): msg += ' is not implemented for Symbol and only available in NDArray.' return msg + +class MXCallbackList(ctypes.Structure): + """Structure that holds Callback information. Passed to CustomOpProp.""" + _fields_ = [ + ('num_callbacks', ctypes.c_int), + ('callbacks', ctypes.POINTER(ctypes.CFUNCTYPE(ctypes.c_int))), + ('contexts', ctypes.POINTER(ctypes.c_void_p)) + ] + + def _load_lib(): """Load library by searching possible path.""" lib_path = libinfo.find_lib_path() diff --git a/python/mxnet/gluon/data/dataset.py b/python/mxnet/gluon/data/dataset.py index f3dd691962bc..37d103266d8f 100644 --- a/python/mxnet/gluon/data/dataset.py +++ b/python/mxnet/gluon/data/dataset.py @@ -20,7 +20,7 @@ """Dataset container.""" import os -from ... import recordio, image +from ... import recordio class Dataset(object): """Abstract dataset class. All datasets should have this interface. @@ -80,27 +80,3 @@ def __getitem__(self, idx): def __len__(self): return len(self._record.keys) - - -class ImageRecordDataset(RecordFileDataset): - """A dataset wrapping over a RecordIO file containing images. - - Each sample is an image and its corresponding label. - - Parameters - ---------- - filename : str - Path to rec file. - flag : {0, 1}, default 1 - If 0, always convert images to greyscale. - - If 1, always convert images to colored (RGB). - """ - def __init__(self, filename, flag=1): - super(ImageRecordDataset, self).__init__(filename) - self._flag = flag - - def __getitem__(self, idx): - record = super(ImageRecordDataset, self).__getitem__(idx) - header, img = recordio.unpack(record) - return image.imdecode(img, self._flag), header.label diff --git a/python/mxnet/gluon/data/vision.py b/python/mxnet/gluon/data/vision.py index a16e736b027d..b63624508124 100644 --- a/python/mxnet/gluon/data/vision.py +++ b/python/mxnet/gluon/data/vision.py @@ -23,11 +23,12 @@ import gzip import tarfile import struct +import warnings import numpy as np from . import dataset from ..utils import download, check_sha1 -from ... import nd +from ... import nd, image, recordio class _DownloadedDataset(dataset.Dataset): @@ -42,7 +43,9 @@ def __init__(self, root, train, transform): self._get_data() def __getitem__(self, idx): - return self._transform(self._data[idx], self._label[idx]) + if self._transform is not None: + return self._transform(self._data[idx], self._label[idx]) + return self._data[idx], self._label[idx] def __len__(self): return len(self._label) @@ -68,7 +71,7 @@ class MNIST(_DownloadedDataset): transform=lambda data, label: (data.astype(np.float32)/255, label) """ def __init__(self, root='~/.mxnet/datasets/', train=True, - transform=lambda data, label: (data, label)): + transform=None): super(MNIST, self).__init__(root, train, transform) def _get_data(self): @@ -116,7 +119,7 @@ class CIFAR10(_DownloadedDataset): transform=lambda data, label: (data.astype(np.float32)/255, label) """ def __init__(self, root='~/.mxnet/datasets/', train=True, - transform=lambda data, label: (data, label)): + transform=None): self._file_hashes = {'data_batch_1.bin': 'aadd24acce27caa71bf4b10992e9e7b2d74c2540', 'data_batch_2.bin': 'c0ba65cce70568cd57b4e03e9ac8d2a5367c1795', 'data_batch_3.bin': '1dd00a74ab1d17a6e7d73e185b69dbf31242f295', @@ -158,3 +161,101 @@ def _get_data(self): self._data = [nd.array(x, dtype=x.dtype) for x in data] self._label = label + + +class ImageRecordDataset(dataset.RecordFileDataset): + """A dataset wrapping over a RecordIO file containing images. + + Each sample is an image and its corresponding label. + + Parameters + ---------- + filename : str + Path to rec file. + flag : {0, 1}, default 1 + If 0, always convert images to greyscale. + + If 1, always convert images to colored (RGB). + transform : function + A user defined callback that transforms each instance. For example:: + + transform=lambda data, label: (data.astype(np.float32)/255, label) + """ + def __init__(self, filename, flag=1, transform=None): + super(ImageRecordDataset, self).__init__(filename) + self._flag = flag + self._transform = transform + + def __getitem__(self, idx): + record = super(ImageRecordDataset, self).__getitem__(idx) + header, img = recordio.unpack(record) + if self._transform is not None: + return self._transform(image.imdecode(img, self._flag), header.label) + return image.imdecode(img, self._flag), header.label + + +class ImageFolderDataset(dataset.Dataset): + """A dataset for loading image files stored in a folder structure like:: + + root/car/0001.jpg + root/car/xxxa.jpg + root/car/yyyb.jpg + root/bus/123.jpg + root/bus/023.jpg + root/bus/wwww.jpg + + Parameters + ---------- + root : str + Path to root directory. + flag : {0, 1}, default 1 + If 0, always convert loaded images to greyscale (1 channel). + If 1, always convert loaded images to colored (3 channels). + transform : callable + A function that takes data and label and transforms them:: + + transform = lambda data, label: (data.astype(np.float32)/255, label) + + Attributes + ---------- + synsets : list + List of class names. `synsets[i]` is the name for the integer label `i` + items : list of tuples + List of all images in (filename, label) pairs. + """ + def __init__(self, root, flag=1, transform=None): + self._root = os.path.expanduser(root) + self._flag = flag + self._transform = transform + self._exts = ['.jpg', '.jpeg', '.png'] + self._list_iamges(self._root) + + def _list_iamges(self, root): + self.synsets = [] + self.items = [] + + for folder in sorted(os.listdir(root)): + path = os.path.join(root, folder) + if not os.path.isdir(path): + warnings.warn('Ignoring %s, which is not a directory.'%path, stacklevel=3) + continue + label = len(self.synsets) + self.synsets.append(folder) + for filename in sorted(os.listdir(path)): + filename = os.path.join(path, filename) + ext = os.path.splitext(filename)[1] + if ext.lower() not in self._exts: + warnings.warn('Ignoring %s of type %s. Only support %s'%( + filename, ext, ', '.join(self._exts))) + continue + self.items.append((filename, label)) + + def __getitem__(self, idx): + img = image.imread(self.items[idx][0], self._flag) + label = self.items[idx][1] + if self._transform is not None: + return self._transform(img, label) + return img, label + + def __len__(self): + return len(self.items) diff --git a/python/mxnet/image/detection.py b/python/mxnet/image/detection.py index 43131f03d488..8ac1aebe72dd 100644 --- a/python/mxnet/image/detection.py +++ b/python/mxnet/image/detection.py @@ -81,7 +81,7 @@ def dumps(self): def __call__(self, src, label): """Augmenter implementation body""" - src = self.augmenter(src)[0] + src = self.augmenter(src) return (src, label) @@ -275,7 +275,7 @@ def _random_crop_proposal(self, label, height, width): from math import sqrt if not self.enabled or height <= 0 or width <= 0: - return None + return () min_area = self.area_range[0] * height * width max_area = self.area_range[1] * height * width for _ in range(self.max_attempts): @@ -317,7 +317,7 @@ def _random_crop_proposal(self, label, height, width): new_label = self._update_labels(label, (x, y, w, h), height, width) if new_label is not None: return (x, y, w, h, new_label) - return None + return () class DetRandomPadAug(DetAugmenter): @@ -386,7 +386,7 @@ def _random_pad_proposal(self, label, height, width): """Generate random padding region""" from math import sqrt if not self.enabled or height <= 0 or width <= 0: - return None + return () min_area = self.area_range[0] * height * width max_area = self.area_range[1] * height * width for _ in range(self.max_attempts): @@ -411,7 +411,7 @@ def _random_pad_proposal(self, label, height, width): x = random.randint(0, max(0, w - width)) new_label = self._update_labels(label, (x, y, w, h), height, width) return (x, y, w, h, new_label) - return None + return () def CreateMultiRandCropAugmenter(min_object_covered=0.1, aspect_ratio_range=(0.75, 1.33), @@ -771,7 +771,7 @@ def next(self): continue for datum in [data]: assert i < batch_size, 'Batch size must be multiples of augmenter output length' - batch_data[i][:] = self.postprocess_data(datum) + batch_data[i] = self.postprocess_data(datum) num_object = label.shape[0] batch_label[i][0:num_object] = nd.array(label) if num_object < batch_label[i].shape[0]: diff --git a/python/mxnet/image/image.py b/python/mxnet/image/image.py index ce44029d2ca1..2e40019971ac 100644 --- a/python/mxnet/image/image.py +++ b/python/mxnet/image/image.py @@ -41,11 +41,52 @@ from .. import recordio +def imread(filename, *args, **kwargs): + """Read and decode an image to an NDArray. + + Note: `imread` uses OpenCV (not the CV2 Python library). + MXNet must have been built with USE_OPENCV=1 for `imdecode` to work. + + Parameters + ---------- + filename : str + Name of the image file to be loaded. + flag : {0, 1}, default 1 + 1 for three channel color output. 0 for grayscale output. + to_rgb : bool, default True + True for RGB formatted output (MXNet default). + False for BGR formatted output (OpenCV default). + out : NDArray, optional + Output buffer. Use `None` for automatic allocation. + + Returns + ------- + NDArray + An `NDArray` containing the image. + + Example + ------- + >>> mx.img.imread("flower.jpg") + + + Set `flag` parameter to 0 to get grayscale output + + >>> mx.img.imdecode("flower.jpg", flag=0) + + + Set `to_rgb` parameter to 0 to get output in OpenCV format (BGR) + + >>> mx.img.imdecode(str_image, to_rgb=0) + + """ + return _internal._cvimread(filename, *args, **kwargs) + + def imdecode(buf, *args, **kwargs): """Decode an image to an NDArray. Note: `imdecode` uses OpenCV (not the CV2 Python library). - MXNet must have been built with OpenCV for `imdecode` to work. + MXNet must have been built with USE_OPENCV=1 for `imdecode` to work. Parameters ---------- @@ -130,7 +171,7 @@ def scale_down(src_size, size): return int(w), int(h) -def _get_interp_method(interp, sizes=None): +def _get_interp_method(interp, sizes=()): """Get the interpolation method for resize functions. The major purpose of this function is to wrap a random interp method selection and a auto-estimation method. @@ -481,7 +522,7 @@ def __init__(self, size, interp=2): def __call__(self, src): """Augmenter body""" - return [resize_short(src, self.size, self.interp)] + return resize_short(src, self.size, self.interp) class ForceResizeAug(Augmenter): @@ -502,7 +543,7 @@ def __init__(self, size, interp=2): def __call__(self, src): """Augmenter body""" sizes = (src.shape[0], src.shape[1], self.size[1], self.size[0]) - return [imresize(src, *self.size, interp=_get_interp_method(self.interp, sizes))] + return imresize(src, *self.size, interp=_get_interp_method(self.interp, sizes)) class RandomCropAug(Augmenter): @@ -522,7 +563,7 @@ def __init__(self, size, interp=2): def __call__(self, src): """Augmenter body""" - return [random_crop(src, self.size, self.interp)[0]] + return random_crop(src, self.size, self.interp)[0] class RandomSizedCropAug(Augmenter): @@ -549,7 +590,7 @@ def __init__(self, size, min_area, ratio, interp=2): def __call__(self, src): """Augmenter body""" - return [random_size_crop(src, self.size, self.min_area, self.ratio, self.interp)[0]] + return random_size_crop(src, self.size, self.min_area, self.ratio, self.interp)[0] class CenterCropAug(Augmenter): @@ -569,7 +610,7 @@ def __init__(self, size, interp=2): def __call__(self, src): """Augmenter body""" - return [center_crop(src, self.size, self.interp)[0]] + return center_crop(src, self.size, self.interp)[0] class RandomOrderAug(Augmenter): @@ -590,10 +631,9 @@ def dumps(self): def __call__(self, src): """Augmenter body""" - src = [src] random.shuffle(self.ts) for t in self.ts: - src = [j for i in src for j in t(i)] + src = t(src) return src @@ -613,7 +653,7 @@ def __call__(self, src): """Augmenter body""" alpha = 1.0 + random.uniform(-self.brightness, self.brightness) src *= alpha - return [src] + return src class ContrastJitterAug(Augmenter): @@ -636,7 +676,7 @@ def __call__(self, src): gray = (3.0 * (1.0 - alpha) / gray.size) * nd.sum(gray) src *= alpha src += gray - return [src] + return src class SaturationJitterAug(Augmenter): @@ -660,7 +700,7 @@ def __call__(self, src): gray *= (1.0 - alpha) src *= alpha src += gray - return [src] + return src class HueJitterAug(Augmenter): @@ -694,7 +734,7 @@ def __call__(self, src): [0.0, vsw, vsu]]) t = np.dot(np.dot(self.tyiq, bt), self.ityiq).T src = nd.dot(src, nd.array(t)) - return [src] + return src class ColorJitterAug(RandomOrderAug): @@ -743,7 +783,7 @@ def __call__(self, src): alpha = np.random.normal(0, self.alphastd, size=(3,)) rgb = np.dot(self.eigvec * alpha, self.eigval) src += nd.array(rgb) - return [src] + return src class ColorNormalizeAug(Augmenter): @@ -763,7 +803,7 @@ def __init__(self, mean, std): def __call__(self, src): """Augmenter body""" - return [color_normalize(src, self.mean, self.std)] + return color_normalize(src, self.mean, self.std) class RandomGrayAug(Augmenter): @@ -785,7 +825,7 @@ def __call__(self, src): """Augmenter body""" if random.random() < self.p: src = nd.dot(src, self.mat) - return [src] + return src class HorizontalFlipAug(Augmenter): @@ -804,7 +844,7 @@ def __call__(self, src): """Augmenter body""" if random.random() < self.p: src = nd.flip(src, axis=1) - return [src] + return src class CastAug(Augmenter): @@ -815,7 +855,7 @@ def __init__(self): def __call__(self, src): """Augmenter body""" src = src.astype(np.float32) - return [src] + return src def CreateAugmenter(data_shape, resize=0, rand_crop=False, rand_resize=False, rand_mirror=False, @@ -1108,18 +1148,17 @@ def next(self): try: while i < batch_size: label, s = self.next_sample() - data = [self.imdecode(s)] + data = self.imdecode(s) try: self.check_valid_image(data) except RuntimeError as e: logging.debug('Invalid image, skipping: %s', str(e)) continue data = self.augmentation_transform(data) - for datum in data: - assert i < batch_size, 'Batch size must be multiples of augmenter output length' - batch_data[i][:] = self.postprocess_data(datum) - batch_label[i][:] = label - i += 1 + assert i < batch_size, 'Batch size must be multiples of augmenter output length' + batch_data[i] = self.postprocess_data(data) + batch_label[i] = label + i += 1 except StopIteration: if not i: raise StopIteration @@ -1157,7 +1196,7 @@ def read_image(self, fname): def augmentation_transform(self, data): """Transforms input data with specified augmentation.""" for aug in self.auglist: - data = [ret for src in data for ret in aug(src)] + data = aug(data) return data def postprocess_data(self, datum): diff --git a/python/mxnet/operator.py b/python/mxnet/operator.py index 692c7fe827ee..1337bbccc3c8 100644 --- a/python/mxnet/operator.py +++ b/python/mxnet/operator.py @@ -26,7 +26,7 @@ from ctypes import CFUNCTYPE, POINTER, Structure, pointer from ctypes import c_void_p, c_int, c_char, c_char_p, cast, c_bool -from .base import _LIB, check_call +from .base import _LIB, check_call, MXCallbackList from .base import c_array, c_str, mx_uint, mx_float, ctypes2numpy_shared, NDArrayHandle, py_str from . import symbol, context from .ndarray import NDArray, _DTYPE_NP_TO_MX, _DTYPE_MX_TO_NP @@ -594,15 +594,6 @@ def register(reg_name): """Register a subclass of CustomOpProp to the registry with name reg_name.""" def do_register(prop_cls): """Register a subclass of CustomOpProp to the registry.""" - - class MXCallbackList(Structure): - """Structure that holds Callback information. Passed to CustomOpProp.""" - _fields_ = [ - ('num_callbacks', c_int), - ('callbacks', POINTER(CFUNCTYPE(c_int))), - ('contexts', POINTER(c_void_p)) - ] - fb_functype = CFUNCTYPE(c_int, c_int, POINTER(c_void_p), POINTER(c_int), POINTER(c_int), c_int, c_void_p) del_functype = CFUNCTYPE(c_int, c_void_p) diff --git a/src/c_api/c_api_function.cc b/src/c_api/c_api_function.cc new file mode 100644 index 000000000000..3d8b5328c1a0 --- /dev/null +++ b/src/c_api/c_api_function.cc @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file custom.cc + * \brief + * \author Junyuan Xie +*/ +#include +#include +#include + +#include "./c_api_common.h" +#include "../ndarray/autograd.h" + +namespace mxnet { +namespace custom_function { + +struct CustomFunctionParam { + size_t num_args, num_outs; + std::shared_ptr info; + std::vector out_shapes; + std::vector out_dtypes; +}; + +std::vector Gradient( + const nnvm::NodePtr& n, + const std::vector& out_grads) { + const CustomFunctionParam& params = nnvm::get(n->attrs.parsed); + + nnvm::NodePtr g = nnvm::Node::Create(); + g->attrs.op = nnvm::Op::Get("_backward_CustomFunction"); + g->attrs.name = n->attrs.name + "_backward"; + g->attrs.parsed = params; + g->control_deps.emplace_back(n); + + g->inputs = out_grads; + + std::vector ret; + for (index_t i = 0; i < g->num_outputs(); ++i) { + ret.emplace_back(nnvm::NodeEntry{g, i, 0}); + } + + return ret; +} + +OpStatePtr CreateState(const nnvm::NodeAttrs& attrs, + Context ctx, + const std::vector& ishape, + const std::vector& itype) { + LOG(FATAL) << "Not reached"; + return OpStatePtr::Create(nullptr); +} + +void Forward(const OpStatePtr& state, + const OpContext& ctx, + const std::vector& inputs, + const std::vector& req, + const std::vector& outputs) { + LOG(FATAL) << "Not reached"; +} + +void Backward(const OpStatePtr& state, + const OpContext& ctx, + const std::vector& inputs, + const std::vector& req, + const std::vector& outputs) { + const CustomFunctionParam& params = state.get_state(); + + std::vector ptrs; + + for (const auto& i : inputs) { + NDArray* nd = new NDArray(i.Detach()); + ptrs.push_back(reinterpret_cast(nd)); + } + for (const auto& i : outputs) { + NDArray* nd = new NDArray(i.Detach()); + ptrs.push_back(reinterpret_cast(nd)); + } + + bool prev_recording = autograd::AutogradRuntime::Get()->SetIsRecording(false); + bool prev_training = autograd::AutogradRuntime::Get()->SetIsTraining(ctx.is_train); + + CHECK(reinterpret_cast( + params.info->callbacks[kCustomFunctionBackward])( + inputs.size(), outputs.size(), ptrs.data(), + reinterpret_cast(req.data()), ctx.is_train, + params.info->contexts[kCustomFunctionBackward])); + + autograd::AutogradRuntime::Get()->SetIsTraining(prev_training); + autograd::AutogradRuntime::Get()->SetIsRecording(prev_recording); +} + + +NNVM_REGISTER_OP(_CustomFunction) +.set_num_inputs([](const NodeAttrs& attrs) { + const CustomFunctionParam& params = nnvm::get(attrs.parsed); + return params.num_args; + }) +.set_num_outputs([](const NodeAttrs& attrs) { + const CustomFunctionParam& params = nnvm::get(attrs.parsed); + return params.num_outs; + }) +.set_attr("FInferShape", + [](const NodeAttrs& attrs, std::vector *in_shape, + std::vector *out_shape) { + const CustomFunctionParam& params = nnvm::get(attrs.parsed); + *out_shape = params.out_shapes; + return true; + }) +.set_attr("FInferType", + [](const NodeAttrs& attrs, std::vector *in_type, + std::vector *out_type) { + const CustomFunctionParam& params = nnvm::get(attrs.parsed); + *out_type = params.out_dtypes; + return true; + }) +.set_attr("FCreateOpState", CreateState) +.set_attr("FGradient", Gradient) +.set_attr("FStatefulComputeEx", Forward) +.set_attr("FStatefulComputeEx", Forward); + + +NNVM_REGISTER_OP(_backward_CustomFunction) +.set_num_inputs([](const NodeAttrs& attrs) { + const CustomFunctionParam& params = nnvm::get(attrs.parsed); + return params.num_outs; + }) +.set_num_outputs([](const NodeAttrs& attrs) { + const CustomFunctionParam& params = nnvm::get(attrs.parsed); + return params.num_args; + }) +.set_attr("TIsBackward", true) +.set_attr("TIsLayerOpBackward", true) +.set_attr("FExecType", [](const NodeAttrs& attrs) { + return ExecType::kLocal; + }) +.set_attr("FStatefulComputeEx", Backward) +.set_attr("FStatefulComputeEx", Backward); + +} // namespace custom_function +} // namespace mxnet + +int MXCustomFunctionRecord(int num_inputs, NDArrayHandle *inputs, + int num_outputs, NDArrayHandle *outputs, + MXCallbackList *callbacks) { + using namespace mxnet; + using namespace mxnet::custom_function; + using mxnet::autograd::AutogradRuntime; + API_BEGIN(); + CHECK(AutogradRuntime::Get()->IsRecording()); + std::vector ndinputs, ndoutputs; + for (int i = 0; i < num_inputs; ++i) { + ndinputs.emplace_back(*reinterpret_cast(inputs[i])); + } + for (int i = 0; i < num_outputs; ++i) { + ndoutputs.emplace_back(*reinterpret_cast(outputs[i])); + } + CustomFunctionParam params; + params.num_args = num_inputs; + params.num_outs = num_outputs; + params.info.reset(callbacks, [](MXCallbackList* ptr){ + reinterpret_cast(ptr->callbacks[kCustomFunctionDelete])( + ptr->contexts[kCustomFunctionDelete]); + }); + for (const auto& i : ndoutputs) { + params.out_shapes.emplace_back(i.shape()); + params.out_dtypes.emplace_back(i.dtype()); + } + nnvm::NodeAttrs attrs; + attrs.op = nnvm::Op::Get("_CustomFunction"); + attrs.parsed = params; + // TODO(piiswrong): remove state by using FComputeEx + auto state = OpStatePtr::Create(params); + AutogradRuntime::Get()->RecordImperativeOperator( + state, attrs.op, attrs, &ndinputs, &ndoutputs); + + for (size_t i = 0; i < ndoutputs.size(); ++i) { + *reinterpret_cast(outputs[i]) = ndoutputs[i]; + } + + API_END(); +} diff --git a/src/c_api/c_api_ndarray.cc b/src/c_api/c_api_ndarray.cc index 89605183e748..3202f55abea7 100644 --- a/src/c_api/c_api_ndarray.cc +++ b/src/c_api/c_api_ndarray.cc @@ -621,3 +621,11 @@ int MXAutogradBackwardEx(mx_uint num_output, AutogradRuntime::Get()->ComputeGradient(outputs, ograds, retain_graph, is_train); API_END(); } + +int MXAutogradGetSymbol(NDArrayHandle handle, SymbolHandle *out) { + API_BEGIN(); + NDArray *head = reinterpret_cast(handle); + auto sym = new nnvm::Symbol(head->get_autograd_symbol()); + *out = reinterpret_cast(sym); + API_END(); +} diff --git a/src/executor/attach_op_execs_pass.cc b/src/executor/attach_op_execs_pass.cc index 13b0018b6dae..046460b85900 100644 --- a/src/executor/attach_op_execs_pass.cc +++ b/src/executor/attach_op_execs_pass.cc @@ -243,7 +243,7 @@ Graph AttachOpExecs(Graph g) { ret[i] = std::make_shared( inode.source->attrs, fcompute, exec_type); } else { - LOG(INFO) << "FCompute not registered " << op->name; + LOG(FATAL) << "FCompute not registered " << op->name; } } } diff --git a/src/io/image_io.cc b/src/io/image_io.cc index f9d7f33a5a44..e6b5a624448e 100644 --- a/src/io/image_io.cc +++ b/src/io/image_io.cc @@ -33,6 +33,8 @@ #include #include +#include + #include "../operator/elemwise_op_common.h" #if MXNET_USE_OPENCV @@ -108,8 +110,66 @@ struct ImdecodeParam : public dmlc::Parameter { "(instead of opencv's default BGR)."); } }; + DMLC_REGISTER_PARAMETER(ImdecodeParam); +struct ImreadParam : public dmlc::Parameter { + std::string filename; + int flag; + bool to_rgb; + DMLC_DECLARE_PARAMETER(ImreadParam) { + DMLC_DECLARE_FIELD(filename) + .describe("Name of the image file to be loaded."); + DMLC_DECLARE_FIELD(flag) + .set_lower_bound(0) + .set_default(1) + .describe("Convert decoded image to grayscale (0) or color (1)."); + DMLC_DECLARE_FIELD(to_rgb) + .set_default(true) + .describe("Whether to convert decoded image to mxnet's default RGB format " + "(instead of opencv's default BGR)."); + } +}; + +DMLC_REGISTER_PARAMETER(ImreadParam); + + +#if MXNET_USE_OPENCV +void ImdecodeImpl(int flag, bool to_rgb, void* data, size_t size, + NDArray* out) { + cv::Mat buf(1, size, CV_8U, data); + cv::Mat dst; + if (out->is_none()) { + cv::Mat res = cv::imdecode(buf, flag); + if (res.empty()) { + LOG(INFO) << "Invalid image file. Only supports png and jpg."; + *out = NDArray(); + return; + } + *out = NDArray(mshadow::Shape3(res.rows, res.cols, flag == 0 ? 1 : 3), + Context::CPU(), false, mshadow::kUint8); + dst = cv::Mat(out->shape()[0], out->shape()[1], flag == 0 ? CV_8U : CV_8UC3, + out->data().dptr_); + res.copyTo(dst); + } else { + dst = cv::Mat(out->shape()[0], out->shape()[1], flag == 0 ? CV_8U : CV_8UC3, + out->data().dptr_); +#if (CV_MAJOR_VERSION > 2 || (CV_MAJOR_VERSION == 2 && CV_MINOR_VERSION >=4)) + cv::imdecode(buf, flag, &dst); +#else + cv::Mat tmp = cv::imdecode(buf, flag); + CHECK(!tmp.empty()); + tmp.copyTo(dst); +#endif + } + CHECK(!dst.empty()); + CHECK_EQ(static_cast(dst.ptr()), out->data().dptr_); + if (to_rgb && flag != 0) { + cv::cvtColor(dst, dst, CV_BGR2RGB); + } +} +#endif // MXNET_USE_OPENCV + void Imdecode(const nnvm::NodeAttrs& attrs, const std::vector& inputs, std::vector* outputs) { @@ -118,63 +178,71 @@ void Imdecode(const nnvm::NodeAttrs& attrs, CHECK_EQ(inputs[0].ctx().dev_mask(), cpu::kDevMask) << "Only supports cpu input"; CHECK_EQ(inputs[0].dtype(), mshadow::kUint8) << "Input needs to be uint8 buffer"; - const uint8_t* str_img = reinterpret_cast(inputs[0].data().dptr_); - uint32_t len = inputs[0].shape().Size(); + inputs[0].WaitToRead(); - NDArray ndin = inputs[0]; - ndin.WaitToRead(); + uint8_t* str_img = inputs[0].data().dptr(); + size_t len = inputs[0].shape().Size(); TShape oshape(3); oshape[2] = param.flag == 0 ? 1 : 3; if (get_jpeg_size(str_img, len, &oshape[1], &oshape[0])) { } else if (get_png_size(str_img, len, &oshape[1], &oshape[0])) { } else { - cv::Mat buf(1, ndin.shape().Size(), CV_8U, ndin.data().dptr_); - cv::Mat res = cv::imdecode(buf, param.flag); - if (res.empty()) { - LOG(INFO) << "Invalid image file. Only supports png and jpg."; - (*outputs)[0] = NDArray(); - return; - } - oshape[0] = res.rows; - oshape[1] = res.cols; - NDArray ndout(oshape, Context::CPU(), false, mshadow::kUint8); - cv::Mat dst(ndout.shape()[0], ndout.shape()[1], - param.flag == 0 ? CV_8U : CV_8UC3, - ndout.data().dptr_); - res.copyTo(dst); - if (param.to_rgb && param.flag != 0) { - cv::cvtColor(dst, dst, CV_BGR2RGB); - } - (*outputs)[0] = ndout; + (*outputs)[0] = NDArray(); + ImdecodeImpl(param.flag, param.to_rgb, str_img, len, &((*outputs)[0])); return; } - NDArray ndout(oshape, Context::CPU(), true, mshadow::kUint8); - Engine::Get()->PushSync([ndin, ndout, param](RunContext ctx){ - cv::Mat buf(1, ndin.shape().Size(), CV_8U, ndin.data().dptr_); - cv::Mat dst(ndout.shape()[0], ndout.shape()[1], - param.flag == 0 ? CV_8U : CV_8UC3, - ndout.data().dptr_); -#if (CV_MAJOR_VERSION > 2 || (CV_MAJOR_VERSION == 2 && CV_MINOR_VERSION >=4)) - cv::imdecode(buf, param.flag, &dst); -#else - cv::Mat tmp = cv::imdecode(buf, param.flag); - CHECK(!tmp.empty()); - tmp.copyTo(dst); -#endif - CHECK(!dst.empty()); - CHECK_EQ(static_cast(dst.ptr()), ndout.data().dptr_); - if (param.to_rgb && param.flag != 0) { - cv::cvtColor(dst, dst, CV_BGR2RGB); - } + const NDArray& ndin = inputs[0]; + NDArray& ndout = (*outputs)[0]; + ndout = NDArray(oshape, Context::CPU(), true, mshadow::kUint8); + Engine::Get()->PushSync([ndin, ndout, str_img, len, param](RunContext ctx){ + ImdecodeImpl(param.flag, param.to_rgb, str_img, len, + const_cast(&ndout)); }, ndout.ctx(), {ndin.var()}, {ndout.var()}, FnProperty::kNormal, 0, PROFILER_MESSAGE("Imdecode")); - (*outputs)[0] = ndout; #else LOG(FATAL) << "Build with USE_OPENCV=1 for image io."; #endif // MXNET_USE_OPENCV } +void Imread(const nnvm::NodeAttrs& attrs, + const std::vector& inputs, + std::vector* outputs) { +#if MXNET_USE_OPENCV + const auto& param = nnvm::get(attrs.parsed); + + std::ifstream file(param.filename, std::ios::binary | std::ios::ate); + size_t fsize = file.tellg(); + file.seekg(0, std::ios::beg); + auto buff = new uint8_t[fsize]; + file.read(reinterpret_cast(buff), fsize); + CHECK(file.good()) << "Failed reading image file " << param.filename; + + TShape oshape(3); + oshape[2] = param.flag == 0 ? 1 : 3; + if (get_jpeg_size(buff, fsize, &oshape[1], &oshape[0])) { + } else if (get_png_size(buff, fsize, &oshape[1], &oshape[0])) { + } else { + (*outputs)[0] = NDArray(); + ImdecodeImpl(param.flag, param.to_rgb, buff, fsize, &((*outputs)[0])); + delete buff; + return; + } + + NDArray& ndout = (*outputs)[0]; + ndout = NDArray(oshape, Context::CPU(), true, mshadow::kUint8); + Engine::Get()->PushSync([ndout, buff, fsize, param](RunContext ctx){ + ImdecodeImpl(param.flag, param.to_rgb, buff, fsize, + const_cast(&ndout)); + delete buff; + }, ndout.ctx(), {}, {ndout.var()}, + FnProperty::kNormal, 0, PROFILER_MESSAGE("Imread")); +#else + LOG(FATAL) << "Build with USE_OPENCV=1 for image io."; +#endif // MXNET_USE_OPENCV +} + + struct ResizeParam : public dmlc::Parameter { int w; int h; @@ -301,6 +369,16 @@ NNVM_REGISTER_OP(_cvimdecode) .add_argument("buf", "NDArray", "Buffer containing binary encoded image") .add_arguments(ImdecodeParam::__FIELDS__()); +NNVM_REGISTER_OP(_cvimread) +.describe("Read and decode image with OpenCV. \n" + "Note: return image in RGB by default, " + "instead of OpenCV's default BGR.") +.set_num_inputs(0) +.set_num_outputs(1) +.set_attr_parser(op::ParamParser) +.set_attr("FNDArrayFunction", Imread) +.add_arguments(ImreadParam::__FIELDS__()); + NNVM_REGISTER_OP(_cvimresize) .describe("Resize image with OpenCV. \n") .set_num_inputs(1) diff --git a/src/ndarray/autograd.cc b/src/ndarray/autograd.cc index 33d0d5d307ed..78b98dabc661 100644 --- a/src/ndarray/autograd.cc +++ b/src/ndarray/autograd.cc @@ -82,19 +82,21 @@ void AutogradRuntime::MarkVariables( for (uint32_t i = 0; i < variables.size(); ++i) { std::string str_c(std::to_string(variable_count_++)); - AGNodeEntry e{AGNode::Create(Node::Create()), 0, 0}; + AGNodeEntry e{ + AGNode::Create( + nnvm::Symbol::CreateVariable("var" + str_c).outputs[0].node), 0, 0}; variables[i]->entry_.clear(); e.ag_node->outputs.emplace_back(*variables[i]); - AGNodeEntry ge{AGNode::Create(Node::Create()), 0, 0}; + AGNodeEntry ge{ + AGNode::Create( + nnvm::Symbol::CreateVariable("grad" + str_c).outputs[0].node), 0, 0}; gradients[i]->entry_.clear(); ge.ag_node->outputs.emplace_back(*gradients[i]); - ge.ag_node->nn_node->attrs.name = "grad" + str_c; gradients[i]->entry_ = std::move(ge); e.ag_node->out_grads.emplace_back(*gradients[i]); e.ag_node->grad_req = static_cast(grad_reqs[i]); - e.ag_node->nn_node->attrs.name = "var" + str_c; variables[i]->entry_ = std::move(e); // assign last to prevent cyclic reference } } @@ -141,10 +143,12 @@ AGNodePtr AutogradRuntime::RecordOp(const nnvm::Op* op, for (size_t i = 0; i < inputs.size(); ++i) { if (inputs[i].entry_.is_none()) { - AGNodeEntry e{AGNode::Create(Node::Create()), 0, 0}; + AGNodeEntry e{ + AGNode::Create( + nnvm::Symbol::CreateVariable( + "null" + std::to_string(variable_count_++)).outputs[0].node), 0, 0}; e.ag_node->outputs.emplace_back(inputs[i]); e.ag_node->out_grads.emplace_back(); - e.ag_node->nn_node->attrs.name = "var_" + std::to_string(variable_count_++); inputs[i].entry_ = std::move(e); // assign last to prevent cyclic reference } nn_node->inputs.push_back(inputs[i].entry_.nn_entry()); @@ -177,7 +181,7 @@ void AutogradRuntime::ComputeGradient(const std::vector& outputs, for (const auto& i : outputs) { CHECK(!i.entry_.is_none()) << "Cannot differentiate node because it is not in a computational graph. " - << "You need to set is_training to true or use autograd.record() to save " + << "You need to set is_recording to true or use autograd.record() to save " << "computational graphs for backward. If you want to differentiate the same " << "graph twice, you need to pass retain_graph=True to backward."; heads.emplace_back(i.entry_); diff --git a/src/operator/custom/custom.cc b/src/operator/custom/custom.cc index 7b257ba843c3..59414d30ddc3 100644 --- a/src/operator/custom/custom.cc +++ b/src/operator/custom/custom.cc @@ -286,13 +286,15 @@ void Forward(const OpStatePtr& state, tags.push_back(4); } - bool old = autograd::AutogradRuntime::Get()->SetIsRecording(false); + bool prev_recording = autograd::AutogradRuntime::Get()->SetIsRecording(false); + bool prev_training = autograd::AutogradRuntime::Get()->SetIsTraining(ctx.is_train); CHECK(reinterpret_cast(params.info->callbacks[kCustomOpForward])( ptrs.size(), ptrs.data(), tags.data(), reinterpret_cast(req.data()), static_cast(ctx.is_train), params.info->contexts[kCustomOpForward])); - autograd::AutogradRuntime::Get()->SetIsRecording(old); + autograd::AutogradRuntime::Get()->SetIsTraining(prev_training); + autograd::AutogradRuntime::Get()->SetIsRecording(prev_recording); } @@ -330,13 +332,15 @@ void Backward(const OpStatePtr& state, tags.push_back(4); } - bool old = autograd::AutogradRuntime::Get()->SetIsRecording(false); + bool prev_recording = autograd::AutogradRuntime::Get()->SetIsRecording(false); + bool prev_training = autograd::AutogradRuntime::Get()->SetIsTraining(ctx.is_train); CHECK(reinterpret_cast(params.info->callbacks[kCustomOpBackward])( ptrs.size(), ptrs.data(), tags.data(), reinterpret_cast(req.data()), static_cast(ctx.is_train), params.info->contexts[kCustomOpBackward])); - autograd::AutogradRuntime::Get()->SetIsRecording(old); + autograd::AutogradRuntime::Get()->SetIsTraining(prev_training); + autograd::AutogradRuntime::Get()->SetIsRecording(prev_recording); } diff --git a/tests/python/unittest/test_autograd.py b/tests/python/unittest/test_autograd.py index 5be3d356e34b..30dd662ff1cc 100644 --- a/tests/python/unittest/test_autograd.py +++ b/tests/python/unittest/test_autograd.py @@ -312,6 +312,53 @@ def test_is_train(): assert y.asnumpy().max() == 2 and y.asnumpy().min() == 0 +def test_function(): + class func(Function): + def forward(self, x, y): + m = x / y + n = x * y + self.save_for_backward(x, y) + return m, n + + def backward(self, dm, dn): + x, y = self.saved_tensors + dx = dm/y + dn*y + dy = dn*x - dm * x / y / y + return dx, dy + + f = func() + x = mx.nd.random_uniform(shape=(10,)) + x.attach_grad() + y = mx.nd.random_uniform(shape=(10,)) + y.attach_grad() + with record(): + m, n = f(x, y) + backward([m, n]) + + dx1 = x.grad.asnumpy() + dy1 = y.grad.asnumpy() + + with record(): + backward([x/y, x*y]) + + assert_almost_equal(x.grad.asnumpy(), dx1) + assert_almost_equal(y.grad.asnumpy(), dy1) + + +def test_get_symbol(): + x = mx.nd.ones((1,)) + x.attach_grad() + with record(): + y = x*x + 2*x - 1 + assert len(get_symbol(y).list_arguments()) == 1 + + z = mx.nd.ones((1,)) + z.attach_grad() + with record(): + y = x*x + 2*z - 1 + assert len(get_symbol(y).list_arguments()) == 2 + + if __name__ == "__main__": import nose nose.runmodule() diff --git a/tests/python/unittest/test_gluon_data.py b/tests/python/unittest/test_gluon_data.py index e9a430124499..32298fcd57d5 100644 --- a/tests/python/unittest/test_gluon_data.py +++ b/tests/python/unittest/test_gluon_data.py @@ -16,6 +16,7 @@ # under the License. import os +import tarfile import mxnet as mx import numpy as np from mxnet import gluon @@ -32,23 +33,24 @@ def test_array_dataset(): def prepare_record(): - if not os.path.isdir("data"): - os.makedirs('data') if not os.path.isdir("data/test_images"): - os.system("wget http://data.mxnet.io/data/test_images.tar.gz -O data/test_images.tar.gz") - os.system("tar -xf data/test_images.tar.gz -C data") - imgs = os.listdir('data/test_images') - record = mx.recordio.MXIndexedRecordIO('data/test.idx', 'data/test.rec', 'w') - for i, img in enumerate(imgs): - str_img = open('data/test_images/'+img, 'rb').read() - s = mx.recordio.pack((0, i, i, 0), str_img) - record.write_idx(i, s) + os.makedirs('data/test_images') + if not os.path.isdir("data/test_images/test_images"): + gluon.utils.download("http://data.mxnet.io/data/test_images.tar.gz", "data/test_images.tar.gz") + tarfile.open('data/test_images.tar.gz').extractall('data/test_images/') + if not os.path.exists('data/test.rec'): + imgs = os.listdir('data/test_images/test_images') + record = mx.recordio.MXIndexedRecordIO('data/test.idx', 'data/test.rec', 'w') + for i, img in enumerate(imgs): + str_img = open('data/test_images/test_images/'+img, 'rb').read() + s = mx.recordio.pack((0, i, i, 0), str_img) + record.write_idx(i, s) return 'data/test.rec' def test_recordimage_dataset(): recfile = prepare_record() - dataset = gluon.data.ImageRecordDataset(recfile) + dataset = gluon.data.vision.ImageRecordDataset(recfile) loader = gluon.data.DataLoader(dataset, 1) for i, (x, y) in enumerate(loader): @@ -71,6 +73,13 @@ def test_datasets(): assert len(gluon.data.vision.MNIST(root='data')) == 60000 assert len(gluon.data.vision.CIFAR10(root='data', train=False)) == 10000 +def test_image_folder_dataset(): + prepare_record() + dataset = gluon.data.vision.ImageFolderDataset('data/test_images') + assert dataset.synsets == ['test_images'] + assert len(dataset.items) == 16 + + if __name__ == '__main__': import nose nose.runmodule() From a5373e6af0c31988ada048a9ebbc84a937a7baf2 Mon Sep 17 00:00:00 2001 From: lxn2 Date: Fri, 11 Aug 2017 14:12:47 -0700 Subject: [PATCH 364/834] Add more license files (#7429) * Add more licenses * Revert "Add more licenses" This reverts commit 8395a84d21a0cebaf909ec277b7b5d6feffa1412. * Add license files --- .../jni/org/dmlc/mxnet/MxnetException.java | 19 ++++++++++++++++ .../jni/org/dmlc/mxnet/Predictor.java | 21 +++++++++++++++++- cpp-package/example/alexnet.cpp | 20 ++++++++++++++++- cpp-package/example/charRNN.cpp | 20 ++++++++++++++++- .../feature_extract/feature_extract.cpp | 20 ++++++++++++++++- .../prepare_data_with_opencv.cpp | 20 ++++++++++++++++- cpp-package/example/googlenet.cpp | 20 ++++++++++++++++- cpp-package/example/inception_bn.cpp | 20 ++++++++++++++++- cpp-package/example/lenet.cpp | 20 ++++++++++++++++- cpp-package/example/lenet_with_mxdataiter.cpp | 20 ++++++++++++++++- cpp-package/example/mlp.cpp | 20 ++++++++++++++++- cpp-package/example/mlp_cpu.cpp | 20 ++++++++++++++++- cpp-package/example/mlp_gpu.cpp | 20 ++++++++++++++++- cpp-package/example/resnet.cpp | 20 ++++++++++++++++- cpp-package/example/test_score.cpp | 20 ++++++++++++++++- cpp-package/include/mxnet-cpp/executor.hpp | 20 ++++++++++++++++- cpp-package/include/mxnet-cpp/io.hpp | 20 ++++++++++++++++- cpp-package/include/mxnet-cpp/kvstore.hpp | 20 ++++++++++++++++- cpp-package/include/mxnet-cpp/monitor.hpp | 20 ++++++++++++++++- cpp-package/include/mxnet-cpp/ndarray.hpp | 20 ++++++++++++++++- cpp-package/include/mxnet-cpp/operator.hpp | 20 ++++++++++++++++- cpp-package/include/mxnet-cpp/optimizer.hpp | 20 ++++++++++++++++- cpp-package/include/mxnet-cpp/symbol.hpp | 20 ++++++++++++++++- .../lib/CollapseUnaryTransformer.java | 19 ++++++++++++++++ .../tree_lstm/lib/ConstituencyParse.java | 21 +++++++++++++++++- .../gluon/tree_lstm/lib/DependencyParse.java | 19 ++++++++++++++++ example/rcnn/rcnn/cython/gpu_nms.hpp | 19 ++++++++++++++++ example/rcnn/rcnn/pycocotools/maskApi.c | 19 ++++++++++++++++ .../ssd/tools/caffe_converter/make_win32.bat | 17 ++++++++++++++ perl-package/AI-MXNet/examples/calculator.pl | 18 +++++++++++++++ perl-package/AI-MXNet/examples/char_lstm.pl | 18 +++++++++++++++ .../AI-MXNet/examples/cudnn_lstm_bucketing.pl | 20 ++++++++++++++++- .../AI-MXNet/examples/lstm_bucketing.pl | 18 +++++++++++++++ perl-package/AI-MXNet/examples/mnist.pl | 22 +++++++++++++++++-- .../AI-MXNet/examples/plot_network.pl | 18 +++++++++++++++ setup-utils/install-mxnet-windows-python.bat | 17 ++++++++++++++ tools/caffe_converter/make_win32.bat | 17 ++++++++++++++ tools/license_header.py | 9 +++++--- 38 files changed, 702 insertions(+), 29 deletions(-) diff --git a/amalgamation/jni/org/dmlc/mxnet/MxnetException.java b/amalgamation/jni/org/dmlc/mxnet/MxnetException.java index c342cfaeee1d..08d80d683a4a 100644 --- a/amalgamation/jni/org/dmlc/mxnet/MxnetException.java +++ b/amalgamation/jni/org/dmlc/mxnet/MxnetException.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.dmlc.mxnet; public class MxnetException extends Exception { diff --git a/amalgamation/jni/org/dmlc/mxnet/Predictor.java b/amalgamation/jni/org/dmlc/mxnet/Predictor.java index a91312a4121d..53152dcf7436 100644 --- a/amalgamation/jni/org/dmlc/mxnet/Predictor.java +++ b/amalgamation/jni/org/dmlc/mxnet/Predictor.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.dmlc.mxnet; import android.graphics.Bitmap; @@ -37,7 +56,7 @@ int ctype() { private long handle = 0; public Predictor(byte[] symbol, byte[] params, Device dev, InputNode[] input) { - String[] keys = new String[input.length]; + String[] keys = new String[input.length]; int[][] shapes = new int[input.length][]; for (int i=0; i #include diff --git a/cpp-package/example/charRNN.cpp b/cpp-package/example/charRNN.cpp index d95c97d8e734..f5fff853cbad 100644 --- a/cpp-package/example/charRNN.cpp +++ b/cpp-package/example/charRNN.cpp @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * Hua Zhang mz24cn@hotmail.com * The code implements C++ version charRNN for mxnet\example\rnn\char-rnn.ipynb with MXNet.cpp API. * The generated params file is compatiable with python version. diff --git a/cpp-package/example/feature_extract/feature_extract.cpp b/cpp-package/example/feature_extract/feature_extract.cpp index 21853a3912e7..1886c576400d 100644 --- a/cpp-package/example/feature_extract/feature_extract.cpp +++ b/cpp-package/example/feature_extract/feature_extract.cpp @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors */ #include #include diff --git a/cpp-package/example/feature_extract/prepare_data_with_opencv.cpp b/cpp-package/example/feature_extract/prepare_data_with_opencv.cpp index 20cbe140fc09..a7b4cba0a64a 100644 --- a/cpp-package/example/feature_extract/prepare_data_with_opencv.cpp +++ b/cpp-package/example/feature_extract/prepare_data_with_opencv.cpp @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors */ #include #include diff --git a/cpp-package/example/googlenet.cpp b/cpp-package/example/googlenet.cpp index 2e59fbfe45cd..ac0585e81a70 100644 --- a/cpp-package/example/googlenet.cpp +++ b/cpp-package/example/googlenet.cpp @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors */ #include #include diff --git a/cpp-package/example/inception_bn.cpp b/cpp-package/example/inception_bn.cpp index 4442e006b5a5..de21aadea9b5 100644 --- a/cpp-package/example/inception_bn.cpp +++ b/cpp-package/example/inception_bn.cpp @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors */ #include #include diff --git a/cpp-package/example/lenet.cpp b/cpp-package/example/lenet.cpp index 56f8d2c8743a..05cc4517fe1e 100644 --- a/cpp-package/example/lenet.cpp +++ b/cpp-package/example/lenet.cpp @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors */ #include #include diff --git a/cpp-package/example/lenet_with_mxdataiter.cpp b/cpp-package/example/lenet_with_mxdataiter.cpp index f6301b52a61f..077f55622561 100644 --- a/cpp-package/example/lenet_with_mxdataiter.cpp +++ b/cpp-package/example/lenet_with_mxdataiter.cpp @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors */ #include #include diff --git a/cpp-package/example/mlp.cpp b/cpp-package/example/mlp.cpp index 6152eddc726a..c9c4ff245180 100644 --- a/cpp-package/example/mlp.cpp +++ b/cpp-package/example/mlp.cpp @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2015 by Contributors */ #include diff --git a/cpp-package/example/mlp_cpu.cpp b/cpp-package/example/mlp_cpu.cpp index 358e8348ac5e..748c32e8c274 100644 --- a/cpp-package/example/mlp_cpu.cpp +++ b/cpp-package/example/mlp_cpu.cpp @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2017 by Contributors * Xin Li yakumolx@gmail.com */ #include diff --git a/cpp-package/example/mlp_gpu.cpp b/cpp-package/example/mlp_gpu.cpp index a6281c385dfb..531afbb29db6 100644 --- a/cpp-package/example/mlp_gpu.cpp +++ b/cpp-package/example/mlp_gpu.cpp @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * Xin Li yakumolx@gmail.com */ #include diff --git a/cpp-package/example/resnet.cpp b/cpp-package/example/resnet.cpp index b9766c7a64d0..ca5643de9d81 100644 --- a/cpp-package/example/resnet.cpp +++ b/cpp-package/example/resnet.cpp @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors */ #include #include diff --git a/cpp-package/example/test_score.cpp b/cpp-package/example/test_score.cpp index 35342699558f..254a6d242fd6 100644 --- a/cpp-package/example/test_score.cpp +++ b/cpp-package/example/test_score.cpp @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * Xin Li yakumolx@gmail.com */ #include diff --git a/cpp-package/include/mxnet-cpp/executor.hpp b/cpp-package/include/mxnet-cpp/executor.hpp index 6887956290c2..0aa698174005 100644 --- a/cpp-package/include/mxnet-cpp/executor.hpp +++ b/cpp-package/include/mxnet-cpp/executor.hpp @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file executor.hpp * \brief implementation of the executor * \author Zhang Chen, Chuntao Hong diff --git a/cpp-package/include/mxnet-cpp/io.hpp b/cpp-package/include/mxnet-cpp/io.hpp index 1be7993fbe4f..677c0f6ee1f0 100644 --- a/cpp-package/include/mxnet-cpp/io.hpp +++ b/cpp-package/include/mxnet-cpp/io.hpp @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! -* Copyright (c) 2016 by Contributors * \file operator.hpp * \brief implementation of data iter * \author Zhang Chen diff --git a/cpp-package/include/mxnet-cpp/kvstore.hpp b/cpp-package/include/mxnet-cpp/kvstore.hpp index 4f66c1d637a5..f2b5e74990ce 100644 --- a/cpp-package/include/mxnet-cpp/kvstore.hpp +++ b/cpp-package/include/mxnet-cpp/kvstore.hpp @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file kvstore.hpp * \brief implementation of kvstore * \author Xin Li diff --git a/cpp-package/include/mxnet-cpp/monitor.hpp b/cpp-package/include/mxnet-cpp/monitor.hpp index eef218bff41d..f3584e2e8092 100644 --- a/cpp-package/include/mxnet-cpp/monitor.hpp +++ b/cpp-package/include/mxnet-cpp/monitor.hpp @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! -* Copyright (c) 2017 by Contributors * \file monitor.hpp * \brief monitor implementation * \author Xin Li diff --git a/cpp-package/include/mxnet-cpp/ndarray.hpp b/cpp-package/include/mxnet-cpp/ndarray.hpp index ba0954b3f815..5ed04a547b85 100644 --- a/cpp-package/include/mxnet-cpp/ndarray.hpp +++ b/cpp-package/include/mxnet-cpp/ndarray.hpp @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file ndarray.hpp * \brief implementation of the ndarray * \author Zhang Chen, Chuntao Hong diff --git a/cpp-package/include/mxnet-cpp/operator.hpp b/cpp-package/include/mxnet-cpp/operator.hpp index 17f4885133fc..a0100cd601be 100644 --- a/cpp-package/include/mxnet-cpp/operator.hpp +++ b/cpp-package/include/mxnet-cpp/operator.hpp @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! -* Copyright (c) 2016 by Contributors * \file operator.hpp * \brief implementation of operator * \author Chuntao Hong, Zhang Chen diff --git a/cpp-package/include/mxnet-cpp/optimizer.hpp b/cpp-package/include/mxnet-cpp/optimizer.hpp index 0d6a7be9dd6b..f9c885fc1fdd 100644 --- a/cpp-package/include/mxnet-cpp/optimizer.hpp +++ b/cpp-package/include/mxnet-cpp/optimizer.hpp @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! -* Copyright (c) 2016 by Contributors * \file optimizer.hpp * \brief implementation of optimizer * \author Chuntao Hong, Zhang Chen diff --git a/cpp-package/include/mxnet-cpp/symbol.hpp b/cpp-package/include/mxnet-cpp/symbol.hpp index 7f88e485830f..ee1a11e26a40 100644 --- a/cpp-package/include/mxnet-cpp/symbol.hpp +++ b/cpp-package/include/mxnet-cpp/symbol.hpp @@ -1,5 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /*! - * Copyright (c) 2016 by Contributors * \file symbol.hpp * \brief implementation of the symbol * \author Zhang Chen, Chuntao Hong diff --git a/example/gluon/tree_lstm/lib/CollapseUnaryTransformer.java b/example/gluon/tree_lstm/lib/CollapseUnaryTransformer.java index 590dcb3dfa05..a0ff1936cb88 100644 --- a/example/gluon/tree_lstm/lib/CollapseUnaryTransformer.java +++ b/example/gluon/tree_lstm/lib/CollapseUnaryTransformer.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + import java.util.List; import edu.stanford.nlp.ling.Label; diff --git a/example/gluon/tree_lstm/lib/ConstituencyParse.java b/example/gluon/tree_lstm/lib/ConstituencyParse.java index 7100eccde7f0..346138c6a06d 100644 --- a/example/gluon/tree_lstm/lib/ConstituencyParse.java +++ b/example/gluon/tree_lstm/lib/ConstituencyParse.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + import edu.stanford.nlp.process.WordTokenFactory; import edu.stanford.nlp.ling.HasWord; import edu.stanford.nlp.ling.Word; @@ -212,7 +231,7 @@ public static void main(String[] args) throws Exception { // produce parent pointer representation int[] parents = deps ? processor.depTreeParents(parse, tokens) : processor.constTreeParents(parse); - + // print if (tokPath != null) { processor.printTokens(tokens); diff --git a/example/gluon/tree_lstm/lib/DependencyParse.java b/example/gluon/tree_lstm/lib/DependencyParse.java index e94de7764e3c..445cab805cc9 100644 --- a/example/gluon/tree_lstm/lib/DependencyParse.java +++ b/example/gluon/tree_lstm/lib/DependencyParse.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + import edu.stanford.nlp.process.WordTokenFactory; import edu.stanford.nlp.ling.HasWord; import edu.stanford.nlp.ling.Word; diff --git a/example/rcnn/rcnn/cython/gpu_nms.hpp b/example/rcnn/rcnn/cython/gpu_nms.hpp index 68b6d42cd88b..93d1f90183bb 100644 --- a/example/rcnn/rcnn/cython/gpu_nms.hpp +++ b/example/rcnn/rcnn/cython/gpu_nms.hpp @@ -1,2 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + void _nms(int* keep_out, int* num_out, const float* boxes_host, int boxes_num, int boxes_dim, float nms_overlap_thresh, int device_id); diff --git a/example/rcnn/rcnn/pycocotools/maskApi.c b/example/rcnn/rcnn/pycocotools/maskApi.c index 85e397918278..9dd660de1252 100644 --- a/example/rcnn/rcnn/pycocotools/maskApi.c +++ b/example/rcnn/rcnn/pycocotools/maskApi.c @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + /************************************************************************** * Microsoft COCO Toolbox. version 2.0 * Data, paper, and tutorials available at: http://mscoco.org/ diff --git a/example/ssd/tools/caffe_converter/make_win32.bat b/example/ssd/tools/caffe_converter/make_win32.bat index 7d354dcaeb6c..1ee8e89f018f 100644 --- a/example/ssd/tools/caffe_converter/make_win32.bat +++ b/example/ssd/tools/caffe_converter/make_win32.bat @@ -1,3 +1,20 @@ +rem Licensed to the Apache Software Foundation (ASF) under one +rem or more contributor license agreements. See the NOTICE file +rem distributed with this work for additional information +rem regarding copyright ownership. The ASF licenses this file +rem to you under the Apache License, Version 2.0 (the +rem "License"); you may not use this file except in compliance +rem with the License. You may obtain a copy of the License at +rem +rem http://www.apache.org/licenses/LICENSE-2.0 +rem +rem Unless required by applicable law or agreed to in writing, +rem software distributed under the License is distributed on an +rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +rem KIND, either express or implied. See the License for the +rem specific language governing permissions and limitations +rem under the License. + @protoc --python_out=./ ./caffe_parse/caffe.proto @echo done. @pause diff --git a/perl-package/AI-MXNet/examples/calculator.pl b/perl-package/AI-MXNet/examples/calculator.pl index f41895508450..aadc7cd2641e 100755 --- a/perl-package/AI-MXNet/examples/calculator.pl +++ b/perl-package/AI-MXNet/examples/calculator.pl @@ -1,4 +1,22 @@ #!/usr/bin/perl + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + use strict; use warnings; use AI::MXNet ('mx'); diff --git a/perl-package/AI-MXNet/examples/char_lstm.pl b/perl-package/AI-MXNet/examples/char_lstm.pl index 1b69ee1e93c6..54a9e3672f63 100755 --- a/perl-package/AI-MXNet/examples/char_lstm.pl +++ b/perl-package/AI-MXNet/examples/char_lstm.pl @@ -1,4 +1,22 @@ #!/usr/bin/perl + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + use strict; use warnings; use PDL; diff --git a/perl-package/AI-MXNet/examples/cudnn_lstm_bucketing.pl b/perl-package/AI-MXNet/examples/cudnn_lstm_bucketing.pl index 4cfe51bfd94a..8976e6465003 100755 --- a/perl-package/AI-MXNet/examples/cudnn_lstm_bucketing.pl +++ b/perl-package/AI-MXNet/examples/cudnn_lstm_bucketing.pl @@ -1,4 +1,22 @@ #!/usr/bin/perl + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + use strict; use warnings; use AI::MXNet qw(mx); @@ -280,4 +298,4 @@ =head1 SYNOPSIS else { $train->(); -} \ No newline at end of file +} diff --git a/perl-package/AI-MXNet/examples/lstm_bucketing.pl b/perl-package/AI-MXNet/examples/lstm_bucketing.pl index ffc176dccb79..e6699d79f0b1 100755 --- a/perl-package/AI-MXNet/examples/lstm_bucketing.pl +++ b/perl-package/AI-MXNet/examples/lstm_bucketing.pl @@ -1,4 +1,22 @@ #!/usr/bin/perl + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + use strict; use warnings; use PDL; diff --git a/perl-package/AI-MXNet/examples/mnist.pl b/perl-package/AI-MXNet/examples/mnist.pl index 891b5348039c..ca452cd95444 100755 --- a/perl-package/AI-MXNet/examples/mnist.pl +++ b/perl-package/AI-MXNet/examples/mnist.pl @@ -1,4 +1,22 @@ #!/usr/bin/perl + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + use strict; use warnings; # derived from http://mxnet.io/tutorials/python/mnist.html @@ -115,7 +133,7 @@ sub nn_fc { # Epoch[9] Validation-accuracy=0.964600 my($data) = @_; - # Flatten the data from 4-D shape (batch_size, num_channel, width, height) + # Flatten the data from 4-D shape (batch_size, num_channel, width, height) # into 2-D (batch_size, num_channel*width*height) $data = mx->sym->Flatten(data => $data); @@ -175,7 +193,7 @@ sub nn_conv { ); $model->fit( $train_iter, # training data - num_epoch => 10, # number of data passes for training + num_epoch => 10, # number of data passes for training eval_data => $val_iter, # validation data batch_end_callback => mx->callback->Speedometer($batch_size, 200), # output progress for each 200 data batches optimizer => 'adam', diff --git a/perl-package/AI-MXNet/examples/plot_network.pl b/perl-package/AI-MXNet/examples/plot_network.pl index a0bcf847af1b..fc38ef2baaab 100755 --- a/perl-package/AI-MXNet/examples/plot_network.pl +++ b/perl-package/AI-MXNet/examples/plot_network.pl @@ -1,4 +1,22 @@ #!/usr/bin/perl + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + use strict; use warnings; use AI::MXNet qw(mx); diff --git a/setup-utils/install-mxnet-windows-python.bat b/setup-utils/install-mxnet-windows-python.bat index 206c66c4c008..021baaeff331 100644 --- a/setup-utils/install-mxnet-windows-python.bat +++ b/setup-utils/install-mxnet-windows-python.bat @@ -1,3 +1,20 @@ +rem Licensed to the Apache Software Foundation (ASF) under one +rem or more contributor license agreements. See the NOTICE file +rem distributed with this work for additional information +rem regarding copyright ownership. The ASF licenses this file +rem to you under the Apache License, Version 2.0 (the +rem "License"); you may not use this file except in compliance +rem with the License. You may obtain a copy of the License at +rem +rem http://www.apache.org/licenses/LICENSE-2.0 +rem +rem Unless required by applicable law or agreed to in writing, +rem software distributed under the License is distributed on an +rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +rem KIND, either express or implied. See the License for the +rem specific language governing permissions and limitations +rem under the License. + @echo off setlocal :::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: diff --git a/tools/caffe_converter/make_win32.bat b/tools/caffe_converter/make_win32.bat index 2f3367d000d4..e5bc9143e05c 100644 --- a/tools/caffe_converter/make_win32.bat +++ b/tools/caffe_converter/make_win32.bat @@ -1,3 +1,20 @@ +rem Licensed to the Apache Software Foundation (ASF) under one +rem or more contributor license agreements. See the NOTICE file +rem distributed with this work for additional information +rem regarding copyright ownership. The ASF licenses this file +rem to you under the Apache License, Version 2.0 (the +rem "License"); you may not use this file except in compliance +rem with the License. You may obtain a copy of the License at +rem +rem http://www.apache.org/licenses/LICENSE-2.0 +rem +rem Unless required by applicable law or agreed to in writing, +rem software distributed under the License is distributed on an +rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +rem KIND, either express or implied. See the License for the +rem specific language governing permissions and limitations +rem under the License. + @protoc --python_out=./ ./caffe.proto @echo done. @pause diff --git a/tools/license_header.py b/tools/license_header.py index d0782b2b06fd..db67000837b0 100644 --- a/tools/license_header.py +++ b/tools/license_header.py @@ -67,7 +67,9 @@ # language extensions and the according commment mark _LANGS = {'.cc':'*', '.h':'*', '.cu':'*', '.cuh':'*', '.py':'#', - '.pm':'#', '.scala':'*', '.cc':'*', '.sh':'#', '.cmake':'#'} + '.pm':'#', '.scala':'*', '.cc':'*', '.sh':'#', '.cmake':'#', + '.java':'*', '.sh':'#', '.cpp':'*', '.hpp':'*', '.c':'*', + '.bat':'rem', '.pl':'#'} # Previous license header, which will be removed _OLD_LICENSE = re.compile('.*Copyright.*by Contributors') @@ -105,7 +107,7 @@ def _valid_file(fname, verbose=False): return False return True -def process_file(fname, action, verbose=False): +def process_file(fname, action, verbose=True): if not _valid_file(fname, verbose): return True with open(fname, 'rb') as f: @@ -118,7 +120,8 @@ def process_file(fname, action, verbose=False): return False _, ext = os.path.splitext(fname) # remove old license - if ext == '.h' or ext == '.cc' or ext == '.cu': + if ext == '.h' or ext == '.cc' or ext == '.cu' or ext == '.cpp' \ + or ext == '.hpp': for i, l in enumerate(lines): if _OLD_LICENSE.match(l.decode('utf-8')): del lines[i] From cee48d8e5f9afa0c341eb6d53157853285b6e6dc Mon Sep 17 00:00:00 2001 From: Wei Wu Date: Sat, 12 Aug 2017 08:08:36 +0800 Subject: [PATCH 365/834] fix consistency of cpu/gpu in stn (#7374) * fix consistency of cpu/gpu in stn * add consistent test of stn add consistent test of stn * add consistent test of stn --- src/operator/spatial_transformer.cc | 72 +++++++++++++++++--------- src/operator/spatial_transformer.cu | 74 ++++++++++++++++++--------- tests/python/gpu/test_operator_gpu.py | 14 ++++- 3 files changed, 111 insertions(+), 49 deletions(-) diff --git a/src/operator/spatial_transformer.cc b/src/operator/spatial_transformer.cc index 0d8ee2917637..51b0ebfde1f0 100644 --- a/src/operator/spatial_transformer.cc +++ b/src/operator/spatial_transformer.cc @@ -27,6 +27,10 @@ namespace mshadow { template +bool between(DType value, int lowerBound, int upperBound) { + return (value >= lowerBound && value <= upperBound); +} +template inline void BilinearSamplingForward(const Tensor &output, const Tensor &input, const Tensor grid_src) { @@ -43,19 +47,28 @@ inline void BilinearSamplingForward(const Tensor &output, index_t grid_index = n * o_h * o_w * 2 + h * o_w + w; DType y_real = (*(grid + grid_index + o_h * o_w) + 1) * (i_h - 1) / 2; DType x_real = (*(grid + grid_index) + 1) * (i_w - 1) / 2; - index_t top_left_y = std::min(i_h, std::max(0, static_cast(floor(y_real)))); - index_t top_left_x = std::min(i_w, std::max(0, static_cast(floor(x_real)))); + int top_left_y = static_cast(floor(y_real)); + int top_left_x = static_cast(floor(x_real)); DType top_left_y_w = 1.0 - (y_real - top_left_y); DType top_left_x_w = 1.0 - (x_real - top_left_x); - index_t data_index = n * i_c * i_h * i_w + c * i_h * i_w + top_left_y * i_w + top_left_x; - DType top_left_v = *(data + data_index); - DType top_right_v = *(data + data_index + 1); - DType bottom_left_v = *(data + data_index + i_w); - DType bottom_right_v = *(data + data_index + i_w + 1); + int data_index = n * i_c * i_h * i_w + c * i_h * i_w + + top_left_y * i_w + top_left_x; + DType top_left_v = 0; + DType top_right_v = 0; + DType bottom_left_v = 0; + DType bottom_right_v = 0; + if (between(top_left_x, 0, i_w-1) && between(top_left_y, 0, i_h-1)) + top_left_v = *(data + data_index); + if (between(top_left_x + 1, 0, i_w-1) && between(top_left_y, 0, i_h-1)) + top_right_v = *(data + data_index + 1); + if (between(top_left_x, 0, i_w-1) && between(top_left_y + 1, 0, i_h-1)) + bottom_left_v = *(data + data_index + i_w); + if (between(top_left_x+1, 0, i_w-1) && between(top_left_y + 1, 0, i_h-1)) + bottom_right_v = *(data + data_index + i_w + 1); *(out+out_index) = top_left_v * top_left_y_w * top_left_x_w + - top_right_v * top_left_y_w * (1.0 - top_left_x_w) + - bottom_left_v * (1.0 - top_left_y_w) * top_left_x_w + - bottom_right_v * (1.0 - top_left_y_w) * (1.0 - top_left_x_w); + top_right_v * top_left_y_w * (1.0 - top_left_x_w) + + bottom_left_v * (1.0 - top_left_y_w) * top_left_x_w + + bottom_right_v * (1.0 - top_left_y_w) * (1.0 - top_left_x_w); } } } @@ -82,8 +95,8 @@ inline void BilinearSamplingBackward(const Tensor &input_grad, index_t grid_src_index = n * o_h * o_w * 2 + h * o_w + w; DType y_real = (*(grid_src + grid_src_index + o_h * o_w) + 1) * (i_h - 1) / 2; DType x_real = (*(grid_src + grid_src_index) + 1) * (i_w - 1) / 2; - index_t top_left_y = std::min(i_h, std::max(0, static_cast(floor(y_real)))); - index_t top_left_x = std::min(i_w, std::max(0, static_cast(floor(x_real)))); + index_t top_left_y = static_cast(floor(y_real)); + index_t top_left_x = static_cast(floor(x_real)); DType top_left_y_w = 1.0 - (y_real - top_left_y); DType top_left_x_w = 1.0 - (x_real - top_left_x); for (index_t c = 0; c < static_cast(o_c); ++c) { @@ -91,18 +104,29 @@ inline void BilinearSamplingBackward(const Tensor &input_grad, index_t data_index = n * i_c * i_h * i_w + c * i_h * i_w + top_left_y * i_w + top_left_x; // calc 4 vertex value in input data - DType top_left_v = *(data + data_index); - DType top_right_v = *(data + data_index + 1); - DType bottom_left_v = *(data + data_index + i_w); - DType bottom_right_v = *(data + data_index + i_w + 1); - // calc input grad - *(g_input + data_index) += *(grad + grad_index) * top_left_y_w * top_left_x_w; - *(g_input + data_index + 1) += *(grad + grad_index) * top_left_y_w - * (1.0 - top_left_x_w); - *(g_input + data_index+ i_w) += *(grad + grad_index) * (1.0 - top_left_y_w) - * top_left_x_w; - *(g_input + data_index+ i_w + 1) += *(grad + grad_index) * (1.0 - top_left_y_w) - * (1.0 - top_left_x_w); + DType top_left_v = 0; + DType top_right_v = 0; + DType bottom_left_v = 0; + DType bottom_right_v = 0; + if (between(top_left_x, 0, i_w-1) && between(top_left_y, 0, i_h-1)) { + *(g_input + data_index) += *(grad + grad_index) * top_left_y_w * top_left_x_w; + top_left_v = *(data + data_index); + } + if (between(top_left_x+1, 0, i_w-1) && between(top_left_y, 0, i_h-1)) { + *(g_input + data_index + 1) += *(grad + grad_index) * top_left_y_w + * (1.0 - top_left_x_w); + top_right_v = *(data + data_index + 1); + } + if (between(top_left_x, 0, i_w-1) && between(top_left_y+1, 0, i_h-1)) { + *(g_input + data_index+ i_w) += *(grad + grad_index) * (1.0 - top_left_y_w) + * top_left_x_w; + bottom_left_v = *(data + data_index + i_w); + } + if (between(top_left_x+1, 0, i_w-1) && between(top_left_y+1, 0, i_h-1)) { + *(g_input + data_index+ i_w + 1) += *(grad + grad_index) * (1.0 - top_left_y_w) + * (1.0 - top_left_x_w); + bottom_right_v = *(data + data_index + i_w + 1); + } // calc weight grad of top_left_w, then multiple -1 is the grad of grid_src top_left_y_gw -= *(grad + grad_index) * (top_right_v - bottom_right_v + (top_left_v - top_right_v - bottom_left_v + bottom_right_v) diff --git a/src/operator/spatial_transformer.cu b/src/operator/spatial_transformer.cu index b3d635c5e8ab..d5e4480dc187 100644 --- a/src/operator/spatial_transformer.cu +++ b/src/operator/spatial_transformer.cu @@ -31,6 +31,10 @@ namespace mshadow { template +__device__ bool between(DType value, int lowerBound, int upperBound) { + return (value >= lowerBound && value <= upperBound); +} +template __global__ void BilinearSamplingForwardKernel(const int i_c, const int i_h, const int i_w, const DType* data, const DType* grid, const int o_n, @@ -48,19 +52,27 @@ __global__ void BilinearSamplingForwardKernel(const int i_c, const int i_h, index_t grid_index = n * o_h * o_w * 2 + h * o_w + w; DType y_real = (*(grid + grid_index + o_h * o_w) + 1) * (i_h - 1) / 2; DType x_real = (*(grid + grid_index) + 1) * (i_w - 1) / 2; - index_t top_left_y = min(i_h, max(0, static_cast(floor(y_real)))); - index_t top_left_x = min(i_w, max(0, static_cast(floor(x_real)))); + int top_left_y = static_cast(floor(y_real)); + int top_left_x = static_cast(floor(x_real)); DType top_left_y_w = 1.0 - (y_real - top_left_y); DType top_left_x_w = 1.0 - (x_real - top_left_x); - index_t data_index = n * i_c * i_h * i_w + c * i_h * i_w + top_left_y * i_w + top_left_x; - DType top_left_v = *(data + data_index); - DType top_right_v = *(data + data_index + 1); - DType bottom_left_v = *(data + data_index + i_w); - DType bottom_right_v = *(data + data_index + i_w + 1); + int data_index = n * i_c * i_h * i_w + c * i_h * i_w + top_left_y * i_w + top_left_x; + DType top_left_v = 0; + DType top_right_v = 0; + DType bottom_left_v = 0; + DType bottom_right_v = 0; + if (between(top_left_x, 0, i_w-1) && between(top_left_y, 0, i_h-1)) + top_left_v = *(data + data_index); + if (between(top_left_x + 1, 0, i_w-1) && between(top_left_y, 0, i_h-1)) + top_right_v = *(data + data_index + 1); + if (between(top_left_x, 0, i_w-1) && between(top_left_y + 1, 0, i_h-1)) + bottom_left_v = *(data + data_index + i_w); + if (between(top_left_x+1, 0, i_w-1) && between(top_left_y + 1, 0, i_h-1)) + bottom_right_v = *(data + data_index + i_w + 1); *(out+out_index) = top_left_v * top_left_y_w * top_left_x_w + - top_right_v * top_left_y_w * (1.0 - top_left_x_w) + - bottom_left_v * (1.0 - top_left_y_w) * top_left_x_w + - bottom_right_v * (1.0 - top_left_y_w) * (1.0 - top_left_x_w); + top_right_v * top_left_y_w * (1.0 - top_left_x_w) + + bottom_left_v * (1.0 - top_left_y_w) * top_left_x_w + + bottom_right_v * (1.0 - top_left_y_w) * (1.0 - top_left_x_w); } } @@ -83,29 +95,43 @@ __global__ void BilinearSamplingBackwardKernel(const int i_c, const int i_h, index_t grid_src_index = n * o_h * o_w * 2 + h * o_w + w; DType y_real = (*(grid_src + grid_src_index + o_h * o_w) + 1) * (i_h - 1) / 2; DType x_real = (*(grid_src + grid_src_index) + 1) * (i_w - 1) / 2; - index_t top_left_y = min(i_h, max(0, static_cast(floor(y_real)))); - index_t top_left_x = min(i_w, max(0, static_cast(floor(x_real)))); + int top_left_y = static_cast(floor(y_real)); + int top_left_x = static_cast(floor(x_real)); DType top_left_y_w = 1.0 - (y_real - top_left_y); DType top_left_x_w = 1.0 - (x_real - top_left_x); for (index_t c = 0; c < o_c; ++c) { index_t grad_index = n * o_c * o_h * o_w + c * o_h * o_w + h * o_w + w; index_t data_index = n * i_c * i_h * i_w + c * i_h * i_w + top_left_y * i_w + top_left_x; // calc 4 vertex value in input data - DType top_left_v = *(data + data_index); - DType top_right_v = *(data + data_index + 1); - DType bottom_left_v = *(data + data_index + i_w); - DType bottom_right_v = *(data + data_index + i_w + 1); + DType top_left_v = 0; + DType top_right_v = 0; + DType bottom_left_v = 0; + DType bottom_right_v = 0; // calc input grad - *(g_input + data_index) += *(grad + grad_index) * top_left_y_w * top_left_x_w; - *(g_input + data_index + 1) += *(grad + grad_index) * top_left_y_w * (1.0 - top_left_x_w); - *(g_input + data_index+ i_w) += *(grad + grad_index) * (1.0 - top_left_y_w) * top_left_x_w; - *(g_input + data_index+ i_w + 1) += *(grad + grad_index) * (1.0 - top_left_y_w) * - (1.0 - top_left_x_w); + if (between(top_left_x, 0, i_w-1) && between(top_left_y, 0, i_h-1)) { + *(g_input + data_index) += *(grad + grad_index) * top_left_y_w * top_left_x_w; + top_left_v = *(data + data_index); + } + if (between(top_left_x+1, 0, i_w-1) && between(top_left_y, 0, i_h-1)) { + *(g_input + data_index + 1) += *(grad + grad_index) * top_left_y_w * (1.0 - top_left_x_w); + top_right_v = *(data + data_index + 1); + } + if (between(top_left_x, 0, i_w-1) && between(top_left_y+1, 0, i_h-1)) { + *(g_input + data_index+ i_w) += *(grad + grad_index) * (1.0 - top_left_y_w) * top_left_x_w; + bottom_left_v = *(data + data_index + i_w); + } + if (between(top_left_x+1, 0, i_w-1) && between(top_left_y+1, 0, i_h-1)) { + *(g_input + data_index+ i_w + 1) += *(grad + grad_index) * (1.0 - top_left_y_w) * + (1.0 - top_left_x_w); + bottom_right_v = *(data + data_index + i_w + 1); + } // calc weight grad of top_left_w, then multiple -1 is the grad of grid_src top_left_y_gw -= *(grad + grad_index) * (top_right_v - bottom_right_v + - (top_left_v - top_right_v - bottom_left_v + bottom_right_v) * top_left_x_w); - top_left_x_gw -= *(grad + grad_index) * (bottom_left_v - bottom_right_v + (top_left_v - - top_right_v - bottom_left_v + bottom_right_v) * top_left_y_w); + (top_left_v - top_right_v - bottom_left_v + bottom_right_v) + * top_left_x_w); + top_left_x_gw -= *(grad + grad_index) * (bottom_left_v - bottom_right_v + + (top_left_v - top_right_v - bottom_left_v + bottom_right_v) + * top_left_y_w); } // calc grid_src grad *(grid_src + grid_src_index + o_h * o_w) = top_left_y_gw * (i_h - 1) / 2; diff --git a/tests/python/gpu/test_operator_gpu.py b/tests/python/gpu/test_operator_gpu.py index c80b9e375306..cd8e85ac9157 100644 --- a/tests/python/gpu/test_operator_gpu.py +++ b/tests/python/gpu/test_operator_gpu.py @@ -623,7 +623,6 @@ def test_bilinear_sampler_with_type(): check_consistency(sym, ctx_list) check_consistency(sym, ctx_list, grad_req="add") - def test_grid_generator_with_type(): data = mx.sym.Variable('data') sym = mx.sym.GridGenerator(data=data, transform_type='affine', target_shape=(20, 20)) @@ -637,6 +636,19 @@ def test_grid_generator_with_type(): check_consistency(sym, ctx_list) check_consistency(sym, ctx_list, grad_req="add") +def test_spatial_transformer_with_type(): + np.random.seed(1234) + data = mx.sym.Variable('data') + loc = mx.sym.Flatten(data) + loc = mx.sym.FullyConnected(data=loc, num_hidden=10) + loc = mx.sym.Activation(data=loc, act_type='relu') + loc = mx.sym.FullyConnected(data=loc, num_hidden=6) + sym = mx.sym.SpatialTransformer(data=data, loc=loc, target_shape=(10, 10), + transform_type="affine", sampler_type="bilinear") + ctx_list = [{'ctx': mx.gpu(0), 'data': (1, 5, 10, 10), 'type_dict': {'data': np.float32}}, + {'ctx': mx.cpu(0), 'data': (1, 5, 10, 10), 'type_dict': {'data': np.float32}}] + check_consistency(sym, ctx_list) + check_consistency(sym, ctx_list, grad_req="add") # Checking max pooling consistency over the data sets of different float types is problematic # as one max value in a float32 data set may not be the max value in a float16 data set. From 89e29e2677f1f76992641a14869519dc6ed5dcf4 Mon Sep 17 00:00:00 2001 From: lxn2 Date: Fri, 11 Aug 2017 18:02:37 -0700 Subject: [PATCH 366/834] Prepare for v0.11.0 release (#7432) * Bump up version to 0.11.0 * Update NEWS & README for v0.11.0 * Make headers * Change to rc0 and link to release notes * Make headers * Change to rc0 and link to release notes --- NEWS.md | 32 +++++++++++++++++++ R-package/DESCRIPTION | 2 +- README.md | 1 + include/mxnet/base.h | 4 +-- python/mxnet/libinfo.py | 2 +- .../assembly/linux-x86_64-cpu/pom.xml | 4 +-- .../assembly/linux-x86_64-gpu/pom.xml | 4 +-- scala-package/assembly/osx-x86_64-cpu/pom.xml | 4 +-- scala-package/assembly/pom.xml | 4 +-- scala-package/core/pom.xml | 4 +-- scala-package/examples/pom.xml | 4 +-- .../init-native/linux-x86_64/pom.xml | 4 +-- scala-package/init-native/osx-x86_64/pom.xml | 4 +-- scala-package/init-native/pom.xml | 4 +-- scala-package/init/pom.xml | 4 +-- scala-package/macros/pom.xml | 4 +-- scala-package/native/linux-x86_64-cpu/pom.xml | 4 +-- scala-package/native/linux-x86_64-gpu/pom.xml | 4 +-- scala-package/native/osx-x86_64-cpu/pom.xml | 4 +-- scala-package/native/pom.xml | 4 +-- scala-package/pom.xml | 2 +- scala-package/spark/pom.xml | 4 +-- snapcraft.yaml | 2 +- 23 files changed, 71 insertions(+), 38 deletions(-) diff --git a/NEWS.md b/NEWS.md index 2557aadfed27..4fdd31430002 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,37 @@ MXNet Change Log ================ +## 0.11.0-rc0 +### - Major Features + - Apple Core ML model converter + - Support for Keras v1.2.2 + - For more information see [full release notes](https://cwiki.apache.org/confluence/display/MXNET/v0.11.0+Release+Notes) +### - API Changes + - Added `CachedOp`. You can now cache the operators that’s called frequently with the same set of arguments to reduce overhead. + - Added sample_multinomial for sampling from multinomial distributions. + - Added `trunc` operator for rounding towards zero. + - Added linalg_gemm, linalg_potrf, ... operators for lapack support. + - Added verbose option to Initializer for printing out initialization details. + - Added DeformableConvolution to contrib from the Deformable Convolutional Networks paper. + - Added float64 support for dot and batch_dot operator. + - `allow_extra` is added to Module.set_params to ignore extra parameters. + - Added `mod` operator for modulo. + - Added `multi_precision` option to SGD optimizer to improve training with float16. Resnet50 now achieves the same accuracy when trained with float16 and gives 50% speedup on Titan XP. +### - Performance Improvements + - ImageRecordIter now stores data in pinned memory to improve GPU memcopy speed. +### - Bugfixes + - Cython interface is fixed. `make cython` and `python setup.py install --with-cython` should install the cython interface and reduce overhead in applications that use imperative/bucketing. + - Fixed various bugs in Faster-RCNN example: https://github.com/dmlc/mxnet/pull/6486 + - Fixed various bugs in SSD example. + - Fixed `out` argument not working for `zeros`, `ones`, `full`, etc. + - `expand_dims` now supports backward shape inference. + - Fixed a bug in rnn. BucketingSentenceIter that causes incorrect layout handling on multi-GPU. + - Fixed context mismatch when loading optimizer states. + - Fixed a bug in ReLU activation when using MKL. + - Fixed a few race conditions that causes crashes on shutdown. +### - Refactors + - Refactored TShape/TBlob to use int64 dimensions and DLTensor as internal storage. Getting ready for migration to DLPack. As a result TBlob::dev_mask_ and TBlob::stride_ are removed. + + ## 0.10.0 - Overhauled documentation for commonly used Python APIs, Installation instructions, Tutorials, HowTos and MXNet Architecture. - Updated mxnet.io for improved readability. diff --git a/R-package/DESCRIPTION b/R-package/DESCRIPTION index fb57b4b7e31f..e0b435513718 100644 --- a/R-package/DESCRIPTION +++ b/R-package/DESCRIPTION @@ -1,7 +1,7 @@ Package: mxnet Type: Package Title: MXNet: A Flexible and Efficient Machine Learning Library for Heterogeneous Distributed Systems -Version: 0.10.1 +Version: 0.11.0 Date: 2017-06-27 Author: Tianqi Chen, Qiang Kou, Tong He Maintainer: Qiang Kou diff --git a/README.md b/README.md index 5027f6d3fdb6..4a354d9bef75 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ deep learning systems, and interesting insights of DL systems for hackers. What's New ---------- +* [Version 0.11.0-rc0 Release](https://github.com/dmlc/mxnet/releases/tag/v0.11.0-rc0) - MXNet 0.11.0-rc0 Release. * [Apache Incubator](http://incubator.apache.org/projects/mxnet.html) - We are now an Apache Incubator project. * [Version 0.10.0 Release](https://github.com/dmlc/mxnet/releases/tag/v0.10.0) - MXNet 0.10.0 Release. * [Version 0.9.3 Release](./docs/architecture/release_note_0_9.md) - First 0.9 official release. diff --git a/include/mxnet/base.h b/include/mxnet/base.h index 50642049b8f8..514bb0c8e54d 100644 --- a/include/mxnet/base.h +++ b/include/mxnet/base.h @@ -103,9 +103,9 @@ /*! \brief major version */ #define MXNET_MAJOR 0 /*! \brief minor version */ -#define MXNET_MINOR 10 +#define MXNET_MINOR 11 /*! \brief patch version */ -#define MXNET_PATCH 1 +#define MXNET_PATCH 0 /*! \brief mxnet version */ #define MXNET_VERSION (MXNET_MAJOR*10000 + MXNET_MINOR*100 + MXNET_PATCH) /*! \brief helper for making version number */ diff --git a/python/mxnet/libinfo.py b/python/mxnet/libinfo.py index d6521c558ac4..7da0dcfc8d2d 100644 --- a/python/mxnet/libinfo.py +++ b/python/mxnet/libinfo.py @@ -61,4 +61,4 @@ def find_lib_path(): # current version -__version__ = "0.10.1" +__version__ = "0.11.0" diff --git a/scala-package/assembly/linux-x86_64-cpu/pom.xml b/scala-package/assembly/linux-x86_64-cpu/pom.xml index 138c5c84304f..a838765a9b91 100644 --- a/scala-package/assembly/linux-x86_64-cpu/pom.xml +++ b/scala-package/assembly/linux-x86_64-cpu/pom.xml @@ -6,13 +6,13 @@ ml.dmlc.mxnet mxnet-full-parent_2.11 - 0.10.1-SNAPSHOT + 0.11.0-SNAPSHOT ../pom.xml ml.dmlc.mxnet mxnet-full_2.11-linux-x86_64-cpu - 0.10.1-SNAPSHOT + 0.11.0-SNAPSHOT MXNet Scala Package - Full Linux-x86_64 CPU-only jar diff --git a/scala-package/assembly/linux-x86_64-gpu/pom.xml b/scala-package/assembly/linux-x86_64-gpu/pom.xml index 7e818cb28123..e248c491ec8e 100644 --- a/scala-package/assembly/linux-x86_64-gpu/pom.xml +++ b/scala-package/assembly/linux-x86_64-gpu/pom.xml @@ -6,13 +6,13 @@ ml.dmlc.mxnet mxnet-full-parent_2.11 - 0.10.1-SNAPSHOT + 0.11.0-SNAPSHOT ../pom.xml ml.dmlc.mxnet mxnet-full_2.11-linux-x86_64-gpu - 0.10.1-SNAPSHOT + 0.11.0-SNAPSHOT MXNet Scala Package - Full Linux-x86_64 GPU jar diff --git a/scala-package/assembly/osx-x86_64-cpu/pom.xml b/scala-package/assembly/osx-x86_64-cpu/pom.xml index ead035668892..e6ef43fbd4c4 100644 --- a/scala-package/assembly/osx-x86_64-cpu/pom.xml +++ b/scala-package/assembly/osx-x86_64-cpu/pom.xml @@ -6,13 +6,13 @@ ml.dmlc.mxnet mxnet-full-parent_2.11 - 0.10.1-SNAPSHOT + 0.11.0-SNAPSHOT ../pom.xml ml.dmlc.mxnet mxnet-full_2.11-osx-x86_64-cpu - 0.10.1-SNAPSHOT + 0.11.0-SNAPSHOT MXNet Scala Package - Full OSX-x86_64 CPU-only jar diff --git a/scala-package/assembly/pom.xml b/scala-package/assembly/pom.xml index a1009ae6b08c..cad677feea3f 100644 --- a/scala-package/assembly/pom.xml +++ b/scala-package/assembly/pom.xml @@ -6,13 +6,13 @@ ml.dmlc.mxnet mxnet-parent_2.11 - 0.10.1-SNAPSHOT + 0.11.0-SNAPSHOT ../pom.xml ml.dmlc.mxnet mxnet-full-parent_2.11 - 0.10.1-SNAPSHOT + 0.11.0-SNAPSHOT MXNet Scala Package - Full Parent pom diff --git a/scala-package/core/pom.xml b/scala-package/core/pom.xml index 7f639b9a8d39..0c79d322d29b 100644 --- a/scala-package/core/pom.xml +++ b/scala-package/core/pom.xml @@ -6,13 +6,13 @@ ml.dmlc.mxnet mxnet-parent_2.11 - 0.10.1-SNAPSHOT + 0.11.0-SNAPSHOT ../pom.xml ml.dmlc.mxnet mxnet-core_2.11 - 0.10.1-SNAPSHOT + 0.11.0-SNAPSHOT MXNet Scala Package - Core diff --git a/scala-package/examples/pom.xml b/scala-package/examples/pom.xml index bda4fcdab5c4..84f406a53991 100644 --- a/scala-package/examples/pom.xml +++ b/scala-package/examples/pom.xml @@ -6,12 +6,12 @@ ml.dmlc.mxnet mxnet-parent_2.11 - 0.10.1-SNAPSHOT + 0.11.0-SNAPSHOT ../pom.xml mxnet-examples_2.11 - 0.10.1-SNAPSHOT + 0.11.0-SNAPSHOT MXNet Scala Package - Examples diff --git a/scala-package/init-native/linux-x86_64/pom.xml b/scala-package/init-native/linux-x86_64/pom.xml index 7e6c02aefd83..3d06bb10aa52 100644 --- a/scala-package/init-native/linux-x86_64/pom.xml +++ b/scala-package/init-native/linux-x86_64/pom.xml @@ -6,12 +6,12 @@ ml.dmlc.mxnet mxnet-scala-init-native-parent - 0.10.1-SNAPSHOT + 0.11.0-SNAPSHOT ../pom.xml libmxnet-init-scala-linux-x86_64 - 0.10.1-SNAPSHOT + 0.11.0-SNAPSHOT MXNet Scala Package - Initializer Native Linux-x86_64 http://maven.apache.org diff --git a/scala-package/init-native/osx-x86_64/pom.xml b/scala-package/init-native/osx-x86_64/pom.xml index 4f5125c06f15..dbc9fdd55887 100644 --- a/scala-package/init-native/osx-x86_64/pom.xml +++ b/scala-package/init-native/osx-x86_64/pom.xml @@ -6,12 +6,12 @@ ml.dmlc.mxnet mxnet-scala-init-native-parent - 0.10.1-SNAPSHOT + 0.11.0-SNAPSHOT ../pom.xml libmxnet-init-scala-osx-x86_64 - 0.10.1-SNAPSHOT + 0.11.0-SNAPSHOT MXNet Scala Package - Initializer Native OSX-x86_64 http://maven.apache.org diff --git a/scala-package/init-native/pom.xml b/scala-package/init-native/pom.xml index 3ce227a9b587..f5839837a250 100644 --- a/scala-package/init-native/pom.xml +++ b/scala-package/init-native/pom.xml @@ -6,12 +6,12 @@ ml.dmlc.mxnet mxnet-parent_2.11 - 0.10.1-SNAPSHOT + 0.11.0-SNAPSHOT ../pom.xml mxnet-scala-init-native-parent - 0.10.1-SNAPSHOT + 0.11.0-SNAPSHOT MXNet Scala Package - Initializer Native Parent pom diff --git a/scala-package/init/pom.xml b/scala-package/init/pom.xml index 9f079565874e..dd6f55b8151c 100644 --- a/scala-package/init/pom.xml +++ b/scala-package/init/pom.xml @@ -6,12 +6,12 @@ ml.dmlc.mxnet mxnet-parent_2.11 - 0.10.1-SNAPSHOT + 0.11.0-SNAPSHOT ../pom.xml mxnet-init_2.11 - 0.10.1-SNAPSHOT + 0.11.0-SNAPSHOT MXNet Scala Package - Initializer diff --git a/scala-package/macros/pom.xml b/scala-package/macros/pom.xml index fd7fe3e4ab7b..842abeef03b5 100644 --- a/scala-package/macros/pom.xml +++ b/scala-package/macros/pom.xml @@ -6,12 +6,12 @@ ml.dmlc.mxnet mxnet-parent_2.11 - 0.10.1-SNAPSHOT + 0.11.0-SNAPSHOT ../pom.xml mxnet-macros_2.11 - 0.10.1-SNAPSHOT + 0.11.0-SNAPSHOT MXNet Scala Package - Macros diff --git a/scala-package/native/linux-x86_64-cpu/pom.xml b/scala-package/native/linux-x86_64-cpu/pom.xml index b2cfa4263cda..4d1d18678232 100644 --- a/scala-package/native/linux-x86_64-cpu/pom.xml +++ b/scala-package/native/linux-x86_64-cpu/pom.xml @@ -6,13 +6,13 @@ ml.dmlc.mxnet mxnet-scala-native-parent - 0.10.1-SNAPSHOT + 0.11.0-SNAPSHOT ../pom.xml ml.dmlc.mxnet libmxnet-scala-linux-x86_64-cpu - 0.10.1-SNAPSHOT + 0.11.0-SNAPSHOT MXNet Scala Package - Native Linux-x86_64 CPU-only http://maven.apache.org diff --git a/scala-package/native/linux-x86_64-gpu/pom.xml b/scala-package/native/linux-x86_64-gpu/pom.xml index 27f9221c3bad..794beece66dd 100644 --- a/scala-package/native/linux-x86_64-gpu/pom.xml +++ b/scala-package/native/linux-x86_64-gpu/pom.xml @@ -6,13 +6,13 @@ ml.dmlc.mxnet mxnet-scala-native-parent - 0.10.1-SNAPSHOT + 0.11.0-SNAPSHOT ../pom.xml ml.dmlc.mxnet libmxnet-scala-linux-x86_64-gpu - 0.10.1-SNAPSHOT + 0.11.0-SNAPSHOT MXNet Scala Package - Native Linux-x86_64 GPU http://maven.apache.org diff --git a/scala-package/native/osx-x86_64-cpu/pom.xml b/scala-package/native/osx-x86_64-cpu/pom.xml index f924106a605c..a6c09f1677e1 100644 --- a/scala-package/native/osx-x86_64-cpu/pom.xml +++ b/scala-package/native/osx-x86_64-cpu/pom.xml @@ -6,12 +6,12 @@ ml.dmlc.mxnet mxnet-scala-native-parent - 0.10.1-SNAPSHOT + 0.11.0-SNAPSHOT ../pom.xml libmxnet-scala-osx-x86_64-cpu - 0.10.1-SNAPSHOT + 0.11.0-SNAPSHOT MXNet Scala Package - Native OSX-x86_64 CPU-only http://maven.apache.org diff --git a/scala-package/native/pom.xml b/scala-package/native/pom.xml index 0af9e087f906..4ccd42014cc0 100644 --- a/scala-package/native/pom.xml +++ b/scala-package/native/pom.xml @@ -6,12 +6,12 @@ ml.dmlc.mxnet mxnet-parent_2.11 - 0.10.1-SNAPSHOT + 0.11.0-SNAPSHOT ../pom.xml mxnet-scala-native-parent - 0.10.1-SNAPSHOT + 0.11.0-SNAPSHOT MXNet Scala Package - Native Parent pom diff --git a/scala-package/pom.xml b/scala-package/pom.xml index 86d8cfc16a43..69dcf1608803 100644 --- a/scala-package/pom.xml +++ b/scala-package/pom.xml @@ -5,7 +5,7 @@ 4.0.0 ml.dmlc.mxnet mxnet-parent_2.11 - 0.10.1-SNAPSHOT + 0.11.0-SNAPSHOT MXNet Scala Package - Parent https://github.com/dmlc/mxnet/tree/master/scala-package MXNet Scala Package diff --git a/scala-package/spark/pom.xml b/scala-package/spark/pom.xml index f35cbe45d9de..c0c699c30560 100644 --- a/scala-package/spark/pom.xml +++ b/scala-package/spark/pom.xml @@ -6,12 +6,12 @@ ml.dmlc.mxnet mxnet-parent_2.11 - 0.10.1-SNAPSHOT + 0.11.0-SNAPSHOT ../pom.xml mxnet-spark_2.11 - 0.10.1-SNAPSHOT + 0.11.0-SNAPSHOT MXNet Scala Package - Spark ML diff --git a/snapcraft.yaml b/snapcraft.yaml index b9329a0ccd41..27356c332a29 100644 --- a/snapcraft.yaml +++ b/snapcraft.yaml @@ -1,5 +1,5 @@ name: mxnet -version: '0.10.1' +version: '0.11.0' summary: MXNet is a deep learning framework designed for efficiency and flexibility. description: | MXNet is a deep learning framework designed for both efficiency and From 16c8f96e7cb961b6a7b0a25e780906098e8d9bff Mon Sep 17 00:00:00 2001 From: Pracheer Gupta Date: Fri, 11 Aug 2017 18:18:00 -0700 Subject: [PATCH 367/834] MXNet -> Apple CoreML converter (#7438) * added coreml test converter for mxnet * added prioritized todo items * Updating parameters for 0.4.0 coreml. This mainly required re-arranging the existing parameters. The current test output looks like this: ====START===== (coremltools) Pracheers-MacBook-Pro:core_ml pracheer$ python test_mxnet_converer.py test_conv_random (__main__.MXNetSingleLayerTest) ... test_mxnet_converer.py:21: DeprecationWarning: mxnet.model.FeedForward has been deprecated. Please use mxnet.mod.Module instead. model = mx.model.FeedForward(net, engine, arg_params = engine.arg_dict) 3 : conv_1_output, Convolution ok test_flatten (__main__.MXNetSingleLayerTest) ... 3 : conv_1, Convolution 4 : flatten1, Flatten 7 : fc1, FullyConnected 9 : softmax_output, SoftmaxOutput ok test_really_tiny_2_inner_product_ones_input (__main__.MXNetSingleLayerTest) ... 3 : fc1_output, FullyConnected ok test_really_tiny_conv_random_3d_input (__main__.MXNetSingleLayerTest) ... 3 : conv_1_output, Convolution ok test_really_tiny_conv_random_input (__main__.MXNetSingleLayerTest) ... 3 : conv_1_output, Convolution ok test_really_tiny_conv_random_input_multi_filter (__main__.MXNetSingleLayerTest) ... 3 : conv_1_output, Convolution ok test_really_tiny_inner_product_ones_input (__main__.MXNetSingleLayerTest) ... 3 : fc1_output, FullyConnected ok test_tiny_asym_conv_random_asym_input (__main__.MXNetSingleLayerTest) ... 3 : conv_1, Convolution 4 : tanh_output, Activation ok test_tiny_asym_conv_random_input (__main__.MXNetSingleLayerTest) ... 3 : conv_1_output, Convolution ok test_tiny_conv_ones_input (__main__.MXNetSingleLayerTest) ... 3 : conv_1_output, Convolution ok test_tiny_conv_pooling_random_input (__main__.MXNetSingleLayerTest) ... 3 : conv_1, Convolution 4 : pool_1_output, Pooling ok test_tiny_conv_random_3d_input (__main__.MXNetSingleLayerTest) ... 3 : conv_1_output, Convolution ok test_tiny_conv_random_input (__main__.MXNetSingleLayerTest) ... 3 : conv_1_output, Convolution ok test_tiny_conv_random_input_multi_filter (__main__.MXNetSingleLayerTest) ... 3 : conv_1_output, Convolution ok test_tiny_inner_product_ones_input (__main__.MXNetSingleLayerTest) ... 3 : fc1_output, FullyConnected ok test_tiny_inner_product_random_input (__main__.MXNetSingleLayerTest) ... 3 : fc1_output, FullyConnected ok test_tiny_inner_product_zero_input (__main__.MXNetSingleLayerTest) ... 3 : fc1_output, FullyConnected ok test_tiny_relu_activation_random_input (__main__.MXNetSingleLayerTest) ... 3 : fc1, FullyConnected 4 : relu1_output, Activation ok test_tiny_sigmoid_activation_random_input (__main__.MXNetSingleLayerTest) ... 3 : fc1, FullyConnected 4 : sigmoid1_output, Activation ok test_tiny_softmax_random_input (__main__.MXNetSingleLayerTest) ... 3 : fc1, FullyConnected 5 : softmax_output, SoftmaxOutput ok test_tiny_tanh_activation_random_input (__main__.MXNetSingleLayerTest) ... 3 : fc1, FullyConnected 4 : tanh1_output, Activation ok test_transpose (__main__.MXNetSingleLayerTest) ... 1 : transpose, transpose 4 : conv_1_output, Convolution ok ---------------------------------------------------------------------- Ran 22 tests in 2.167s OK ====END===== * Convert reshape operator into coreml equivalent. * Set pre-processing parameters on coreml model. * Adding synsets w/ unit test. class_labels only used when coreml model is used in classifier mode so had to re-juggle few parameters to make it work. * Minor documentation change for pre-processing args. * Adding Deconvolution layer. Currently target_shape has been compulsory since it is required by coreml. Also, we are not currently able to evaluate the affect of padding. If we try to xplicitly add padding to coreml model at the end we get an error. This is how we were adding the padding: pad = literal_eval(param['pad']) for i in range(len(pad)): convLayer.valid.paddingAmounts.borderAmounts[i].startEdgeSize = pad[i] convLayer.valid.paddingAmounts.borderAmounts[i].endEdgeSize = pad[i] Error: File "test_mxnet_converer.py", line 22, in _get_coreml_model spec = mxnet_converter.convert(model, class_labels=class_labels, mode=mode, **input_shape) File "_mxnet_converter.py", line 197, in convert converter_func(net, node, model, builder) File "_layers.py", line 515, in convert_deconvolution convLayer.valid.paddingAmounts.borderAmounts[i].startEdgeSize = pad[i] File "/Users/pracheer/miniconda3/envs/coremltools/lib/python2.7/site-packages/google/protobuf/internal/containers.py", line 204, in __getitem__ return self._values[key] IndexError: list index out of range Will fix the above two issues after discussions with coreml people. * Skip dropout layer while converting. * added quick todo * Enable padding > 1 by adding an extra layer for padding in coreml. Added unit tests. Caveat: Currently deconv layer with padding !- (0, 0) is not working: Model successfully converts but mxnet predictions from coreml ones. * Unit tests to convert entire model from zoo. * fix module / BN * refactor * minor fix * [BugFix] input-data as a dictionary. In our previous commit, input-data was assumed as an array which caused unit tests to fail. This change fixes that. Also, add a missing parameter to couple of the tests in models-test. * Use delta of 1e-3 instead of 1e-7 which was accidently pushed. * Test inception_v3 and remove tests that do only conversion. fwiw:On inception_v3, the predictions are off by more than delta. * update converter unittests with mxnet module * BatchNorm UnitTest+eps. * add image classification test * "Force" flag to force conversion of layers. This is needed for layers which don't have an exact one-to-one correspondence in CoreML. By default, the conversion should fail if it detects that CoreML doesn't support the layer as it is but this behavior can be overriden if anyone provides force flag while calling convert. Summary of changes: - Add "force" flag to all the layers in _layers. - For batchnorm conversion, don't throw the error if force flag is provided. - 2 unit tests: one tests that an exception is thrown for converting batch-norm layer with local batch stats; other that tests that "force" flag causes it to not throw the exception. * Minor: documentation fixes, fixing imports, etc. * ModelsUnitTests: Improved documentation, using force flag where reqd, ability to download the model files if they don't exist. * Minor: documentation update on KL divergence. * Minor: Removing unused variables. * Minor: documentation update for MXNetSingleLayerTest. * test_image: add force flag for resnet. * README; change name of classes; assert KLDivergence < 1e-4. * Updated README. * Minor: cosmetic changes to readme. * convert return coreml instead of protobuf * Minor: cosmetic changes to README. * Enable SingleLayerTest.test_tiny_synset_random_input. * Minor: fixing some formatting of README * testing readme formatting * Minor: Heading for TODOs in README. * ImagenetTest: fix shapes, add more models. * move test location, add mxnet_coreml_converter as Command Line Tool * add mxnet random seed in converter unittest * Updated README,fix vgg16 test, refactor deconv code. * refactor directory and moving .mlmodel files * Fixing README to have dimensions as 224. * Adding periods at the end of sentences. * Instead of commenting out a test, skip it. * remove force flag, add preprocessing_args * Updated README for pre-processing arguments. * Deconv w/ padding; pooling w/ pooling_convention. Earlier deconv w/ padding was giving incorrect predictions. We added crop layer which fixed the issues. As for the pooling_convention, the current coremltools doesn't provide the support so we added our own custom implementation (w/ help from Apple) to overcome the issue. * added coreml test converter for mxnet * added prioritized todo items * Updating parameters for 0.4.0 coreml. This mainly required re-arranging the existing parameters. The current test output looks like this: ====START===== (coremltools) Pracheers-MacBook-Pro:core_ml pracheer$ python test_mxnet_converer.py test_conv_random (__main__.MXNetSingleLayerTest) ... test_mxnet_converer.py:21: DeprecationWarning: mxnet.model.FeedForward has been deprecated. Please use mxnet.mod.Module instead. model = mx.model.FeedForward(net, engine, arg_params = engine.arg_dict) 3 : conv_1_output, Convolution ok test_flatten (__main__.MXNetSingleLayerTest) ... 3 : conv_1, Convolution 4 : flatten1, Flatten 7 : fc1, FullyConnected 9 : softmax_output, SoftmaxOutput ok test_really_tiny_2_inner_product_ones_input (__main__.MXNetSingleLayerTest) ... 3 : fc1_output, FullyConnected ok test_really_tiny_conv_random_3d_input (__main__.MXNetSingleLayerTest) ... 3 : conv_1_output, Convolution ok test_really_tiny_conv_random_input (__main__.MXNetSingleLayerTest) ... 3 : conv_1_output, Convolution ok test_really_tiny_conv_random_input_multi_filter (__main__.MXNetSingleLayerTest) ... 3 : conv_1_output, Convolution ok test_really_tiny_inner_product_ones_input (__main__.MXNetSingleLayerTest) ... 3 : fc1_output, FullyConnected ok test_tiny_asym_conv_random_asym_input (__main__.MXNetSingleLayerTest) ... 3 : conv_1, Convolution 4 : tanh_output, Activation ok test_tiny_asym_conv_random_input (__main__.MXNetSingleLayerTest) ... 3 : conv_1_output, Convolution ok test_tiny_conv_ones_input (__main__.MXNetSingleLayerTest) ... 3 : conv_1_output, Convolution ok test_tiny_conv_pooling_random_input (__main__.MXNetSingleLayerTest) ... 3 : conv_1, Convolution 4 : pool_1_output, Pooling ok test_tiny_conv_random_3d_input (__main__.MXNetSingleLayerTest) ... 3 : conv_1_output, Convolution ok test_tiny_conv_random_input (__main__.MXNetSingleLayerTest) ... 3 : conv_1_output, Convolution ok test_tiny_conv_random_input_multi_filter (__main__.MXNetSingleLayerTest) ... 3 : conv_1_output, Convolution ok test_tiny_inner_product_ones_input (__main__.MXNetSingleLayerTest) ... 3 : fc1_output, FullyConnected ok test_tiny_inner_product_random_input (__main__.MXNetSingleLayerTest) ... 3 : fc1_output, FullyConnected ok test_tiny_inner_product_zero_input (__main__.MXNetSingleLayerTest) ... 3 : fc1_output, FullyConnected ok test_tiny_relu_activation_random_input (__main__.MXNetSingleLayerTest) ... 3 : fc1, FullyConnected 4 : relu1_output, Activation ok test_tiny_sigmoid_activation_random_input (__main__.MXNetSingleLayerTest) ... 3 : fc1, FullyConnected 4 : sigmoid1_output, Activation ok test_tiny_softmax_random_input (__main__.MXNetSingleLayerTest) ... 3 : fc1, FullyConnected 5 : softmax_output, SoftmaxOutput ok test_tiny_tanh_activation_random_input (__main__.MXNetSingleLayerTest) ... 3 : fc1, FullyConnected 4 : tanh1_output, Activation ok test_transpose (__main__.MXNetSingleLayerTest) ... 1 : transpose, transpose 4 : conv_1_output, Convolution ok ---------------------------------------------------------------------- Ran 22 tests in 2.167s OK ====END===== * Convert reshape operator into coreml equivalent. * Adding Deconvolution layer. Currently target_shape has been compulsory since it is required by coreml. Also, we are not currently able to evaluate the affect of padding. If we try to xplicitly add padding to coreml model at the end we get an error. This is how we were adding the padding: pad = literal_eval(param['pad']) for i in range(len(pad)): convLayer.valid.paddingAmounts.borderAmounts[i].startEdgeSize = pad[i] convLayer.valid.paddingAmounts.borderAmounts[i].endEdgeSize = pad[i] Error: File "test_mxnet_converer.py", line 22, in _get_coreml_model spec = mxnet_converter.convert(model, class_labels=class_labels, mode=mode, **input_shape) File "_mxnet_converter.py", line 197, in convert converter_func(net, node, model, builder) File "_layers.py", line 515, in convert_deconvolution convLayer.valid.paddingAmounts.borderAmounts[i].startEdgeSize = pad[i] File "/Users/pracheer/miniconda3/envs/coremltools/lib/python2.7/site-packages/google/protobuf/internal/containers.py", line 204, in __getitem__ return self._values[key] IndexError: list index out of range Will fix the above two issues after discussions with coreml people. * Adding synsets w/ unit test. class_labels only used when coreml model is used in classifier mode so had to re-juggle few parameters to make it work. * Set pre-processing parameters on coreml model. * Minor documentation change for pre-processing args. * Skip dropout layer while converting. * added quick todo * Enable padding > 1 by adding an extra layer for padding in coreml. Added unit tests. Caveat: Currently deconv layer with padding !- (0, 0) is not working: Model successfully converts but mxnet predictions from coreml ones. * Unit tests to convert entire model from zoo. * fix module / BN * refactor * minor fix * [BugFix] input-data as a dictionary. In our previous commit, input-data was assumed as an array which caused unit tests to fail. This change fixes that. Also, add a missing parameter to couple of the tests in models-test. * Use delta of 1e-3 instead of 1e-7 which was accidently pushed. * Test inception_v3 and remove tests that do only conversion. fwiw:On inception_v3, the predictions are off by more than delta. * update converter unittests with mxnet module * BatchNorm UnitTest+eps. * add image classification test * "Force" flag to force conversion of layers. This is needed for layers which don't have an exact one-to-one correspondence in CoreML. By default, the conversion should fail if it detects that CoreML doesn't support the layer as it is but this behavior can be overriden if anyone provides force flag while calling convert. Summary of changes: - Add "force" flag to all the layers in _layers. - For batchnorm conversion, don't throw the error if force flag is provided. - 2 unit tests: one tests that an exception is thrown for converting batch-norm layer with local batch stats; other that tests that "force" flag causes it to not throw the exception. * Minor: documentation fixes, fixing imports, etc. * ModelsUnitTests: Improved documentation, using force flag where reqd, ability to download the model files if they don't exist. * Minor: documentation update on KL divergence. * Minor: Removing unused variables. * Minor: documentation update for MXNetSingleLayerTest. * test_image: add force flag for resnet. * README; change name of classes; assert KLDivergence < 1e-4. * Updated README. * Minor: cosmetic changes to readme. * convert return coreml instead of protobuf * Minor: cosmetic changes to README. * Enable SingleLayerTest.test_tiny_synset_random_input. * Minor: fixing some formatting of README * testing readme formatting * Minor: Heading for TODOs in README. * ImagenetTest: fix shapes, add more models. * move test location, add mxnet_coreml_converter as Command Line Tool * add mxnet random seed in converter unittest * Updated README,fix vgg16 test, refactor deconv code. * refactor directory and moving .mlmodel files * Fixing README to have dimensions as 224. * Adding periods at the end of sentences. * Instead of commenting out a test, skip it. * remove force flag, add preprocessing_args * Updated README for pre-processing arguments. * Deconv w/ padding; pooling w/ pooling_convention. Earlier deconv w/ padding was giving incorrect predictions. We added crop layer which fixed the issues. As for the pooling_convention, the current coremltools doesn't provide the support so we added our own custom implementation (w/ help from Apple) to overcome the issue. * Moving files from core_ml directory to coreml directory since Apple guys pushed their code to coreml and we don't want to lose their history. * Moved Apple files from core_ml to coreml directory since Apple pushed their code changes to coreml and we don't want to lose their history. This change also add Apache license to all the files. * Updated documentation for utils.py. * Fixing Batchnorm test with the right delta. * Updating README with information about mode/pre-processing-args/class-labels. --- tools/coreml/README.md | 95 ++ tools/coreml/_layers.py | 397 -------- tools/coreml/{ => converter}/__init__.py | 1 - tools/coreml/converter/_add_pooling.py | 118 +++ tools/coreml/converter/_layers.py | 569 +++++++++++ .../{ => converter}/_mxnet_converter.py | 77 +- tools/coreml/mxnet_coreml_converter.py | 114 +++ tools/coreml/test/test_mxnet_converter.py | 949 ++++++++++++++++++ tools/coreml/test/test_mxnet_image.py | 136 +++ tools/coreml/test/test_mxnet_models.py | 155 +++ tools/coreml/test_mxnet_converer.py | 477 --------- tools/coreml/utils.py | 77 ++ 12 files changed, 2262 insertions(+), 903 deletions(-) create mode 100644 tools/coreml/README.md delete mode 100644 tools/coreml/_layers.py rename tools/coreml/{ => converter}/__init__.py (96%) create mode 100644 tools/coreml/converter/_add_pooling.py create mode 100644 tools/coreml/converter/_layers.py rename tools/coreml/{ => converter}/_mxnet_converter.py (69%) create mode 100644 tools/coreml/mxnet_coreml_converter.py create mode 100644 tools/coreml/test/test_mxnet_converter.py create mode 100644 tools/coreml/test/test_mxnet_image.py create mode 100644 tools/coreml/test/test_mxnet_models.py delete mode 100644 tools/coreml/test_mxnet_converer.py create mode 100644 tools/coreml/utils.py diff --git a/tools/coreml/README.md b/tools/coreml/README.md new file mode 100644 index 000000000000..32cde339d3a9 --- /dev/null +++ b/tools/coreml/README.md @@ -0,0 +1,95 @@ +# Convert MXNet models into Apple CoreML format. + +This tool helps convert MXNet models into [Apple CoreML](https://developer.apple.com/documentation/coreml) format which can then be run on Apple devices. + +## Installation +In order to use this tool you need to have these installed: +* MacOS - High Sierra 10.13 +* Xcode 9 +* coremltools 0.5.0 or greater (pip install coremltools) +* mxnet 0.10.0 or greater. [Installation instructions](http://mxnet.io/get_started/install.html). +* yaml (pip install pyyaml) +* python 2.7 + +## How to use +Let's say you want to use your MXNet model in an iPhone App. For the purpose of this example, let's say you want to use squeezenet-v1.1. + +1. Download the model into the directory where this converter resides. Squeezenet can be downloaded from [here](http://data.mxnet.io/models/imagenet/squeezenet/). +2. Run this command: + + ```bash +python mxnet_coreml_converter.py --model-prefix='squeezenet_v1.1' --epoch=0 --input-shape='{"data":"3,227,227"}' --mode=classifier --pre-processing-arguments='{"image_input_names":"data"}' --class-labels classLabels.txt --output-file="squeezenetv11.mlmodel" +``` + + The above command will save the converted model into squeezenet-v11.mlmodel in CoreML format. Internally MXNet first loads the model and then we walk through the entire symbolic graph converting each operator into its CoreML equivalent. Some of the parameters are used by MXNet in order to load and generate the symbolic graph in memory while others are used by CoreML either to pre-process the input before the going through the neural network or to process the output in a particular way. + + In the command above: + + * _model-prefix_: refers to the MXNet model prefix (may include the directory path). + * _epoch_: refers to the suffix of the MXNet model file. + * _input-shape_: refers to the input shape information in a JSON string format where the key is the name of the input variable (="data") and the value is the shape of that variable. If the model takes multiple inputs, input-shape for all of them need to be provided. + * _mode_: refers to the coreml model mode. Can either be 'classifier', 'regressor' or None. In this case, we use 'classifier' since we want the resulting CoreML model to classify images into various categories. + * _pre-processing-arguments_: In the Apple world images have to be of type Image. By providing image_input_names as "data", we are saying that the input variable "data" is of type Image. + * _class-labels_: refers to the name of the file which contains the classification labels (a.k.a. synset file). +output-file: the file where the CoreML model will be dumped. + +3. The generated ".mlmodel" file can directly be integrated into your app. For more instructions on how to do this, please see [Apple CoreML's tutorial](https://developer.apple.com/documentation/coreml/integrating_a_core_ml_model_into_your_app). + + +### Providing class labels +You could provide a file containing class labels (as above) so that CoreML will return the predicted category the image belongs to. The file should have a label per line and labels can have any special characters. The line number of the label in the file should correspond with the index of softmax output. E.g. + +```bash +python mxnet_coreml_converter.py --model-prefix='squeezenet_v1.1' --epoch=0 --input-shape='{"data":"3,227,227"}' --mode=classifier --class-labels classLabels.txt --output-file="squeezenetv11.mlmodel" +``` + +### Providing label names +You may have to provide the label names of the MXNet model's outputs. For example, if you try to convert [vgg16](http://data.mxnet.io/models/imagenet/vgg/), you may have to provide label-name as "prob_label". By default "softmax_label" is assumed. + +```bash +python mxnet_coreml_converter.py --model-prefix='vgg16' --epoch=0 --input-shape='{"data":"3,224,224"}' --mode=classifier --pre-processing-arguments='{"image_input_names":"data"}' --class-labels classLabels.txt --output-file="vgg16.mlmodel" --label-names="prob_label" +``` + +### Adding a pre-processing to CoreML model. +You could ask CoreML to pre-process the images before passing them through the model. + +```bash +python mxnet_coreml_converter.py --model-prefix='squeezenet_v1.1' --epoch=0 --input-shape='{"data":"3,224,224"}' --pre-processing-arguments='{"red_bias":127,"blue_bias":117,"green_bias":103}' --output-file="squeezenet_v11.mlmodel" +``` + +If you are building an app for a model that takes image as an input, you will have to provide image_input_names as pre-processing arguments. This tells CoreML that a particular input variable is of type Image. E.g.: + +```bash +python mxnet_coreml_converter.py --model-prefix='squeezenet_v1.1' --epoch=0 --input-shape='{"data":"3,224,224"}' --pre-processing-arguments='{"red_bias":127,"blue_bias":117,"green_bias":103,"image_input_names":"data"}' --output-file="squeezenet_v11.mlmodel" +``` + +## Currently supported +### Models +This is a (growing) list of standard MXNet models that can be successfully converted using the converter. This means that any other model that uses similar operators as these models can also be successfully converted. + +1. Inception: [Inception-BN](http://data.mxnet.io/models/imagenet/inception-bn/), [Inception-V3](http://data.mxnet.io/models/imagenet/inception-v3.tar.gz) +2. [NiN](http://data.dmlc.ml/models/imagenet/nin/) +2. [Resnet](http://data.mxnet.io/models/imagenet/resnet/) +3. [Squeezenet](http://data.mxnet.io/models/imagenet/squeezenet/) +4. [Vgg](http://data.mxnet.io/models/imagenet/vgg/) + +### Layers +1. Activation +2. Batchnorm +3. Concat +4. Convolution +5. Deconvolution +6. Dense +7. Elementwise +8. Flatten +9. Pooling +10. Reshape +11. Softmax +12. Transpose + +## Known issues +Currently there are no known issues. + +## This tool has been tested on environment with: +* MacOS - High Sierra 10.13 Beta. +* Xcode 9 beta 5. diff --git a/tools/coreml/_layers.py b/tools/coreml/_layers.py deleted file mode 100644 index 51489849cd28..000000000000 --- a/tools/coreml/_layers.py +++ /dev/null @@ -1,397 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -import numpy as _np - -def _get_input_output_name(net, node, index = 0): - name = node['name'] - inputs = node['inputs'] - - if index == 'all': - input_name = [_get_node_name(net, inputs[id][0]) for id in range(len(inputs))] - elif type(index) == int: - input_name = _get_node_name(net, inputs[0][0]) - else: - input_name = [_get_node_name(net, inputs[id][0]) for id in index] - return input_name, name - -def _get_node_name(net, node_id): - return net['nodes'][node_id]['name'] - -def _get_node_shape(net, node_id): - return net['nodes'][node_id]['shape'] - -def convert_transpose(net, node, model, builder): - """Convert a transpose layer from mxnet to coreml. - - Parameters - ---------- - network: net - A mxnet network object. - - layer: node - Node to convert. - - model: model - An model for MXNet - - builder: NeuralNetworkBuilder - A neural network builder object. - """ - input_name, output_name = _get_input_output_name(net, node) - name = node['name'] - param = node['attr'] - from ast import literal_eval - axes = literal_eval(param['axes']) - builder.add_permute(name, input_name, output_name, axes) - -def convert_flatten(net, node, model, builder): - """Convert a flatten layer from mxnet to coreml. - - Parameters - ---------- - network: net - A mxnet network object. - - layer: node - Node to convert. - - model: model - An model for MXNet - - builder: NeuralNetworkBuilder - A neural network builder object. - """ - input_name, output_name = _get_input_output_name(net, node) - name = node['name'] - builder.add_flatten(0, name, input_name, output_name) - -def convert_softmax(net, node, model, builder): - """Convert a softmax layer from mxnet to coreml. - - Parameters - ---------- - network: net - A mxnet network object. - - layer: node - Node to convert. - - model: model - An model for MXNet - - builder: NeuralNetworkBuilder - A neural network builder object. - """ - input_name, output_name = _get_input_output_name(net, node) - name = node['name'] - builder.add_softmax(name = name, - input_name = input_name, - output_name = output_name) - -def convert_activation(net, node, model, builder): - """Convert an activation layer from mxnet to coreml. - - Parameters - ---------- - network: net - A mxnet network object. - - layer: node - Node to convert. - - model: model - An model for MXNet - - builder: NeuralNetworkBuilder - A neural network builder object. - """ - input_name, output_name = _get_input_output_name(net, node) - name = node['name'] - mx_non_linearity = node['attr']['act_type'] - if mx_non_linearity == 'relu': - non_linearity = 'RELU' - elif mx_non_linearity == 'tanh': - non_linearity = 'TANH' - elif mx_non_linearity == 'sigmoid': - non_linearity = 'SIGMOID' - else: - raise TypeError('Unknown activation type %s' % mx_non_linearity) - builder.add_activation(name = name, - non_linearity = non_linearity, - input_name = input_name, - output_name = output_name) - -def convert_elementwise_add(net, node, model, builder): - """Convert an elementwise add layer from mxnet to coreml. - - Parameters - ---------- - network: net - A mxnet network object. - - layer: node - Node to convert. - - model: model - An model for MXNet - - builder: NeuralNetworkBuilder - A neural network builder object. - """ - - input_names, output_name = _get_input_output_name(net, node,[0,1]) - name = node['name'] - - builder.add_elementwise(name, input_names, output_name, 'ADD') - -def convert_dense(net, node, model, builder): - """Convert a dense layer from mxnet to coreml. - - Parameters - ---------- - network: net - A mxnet network object. - - layer: node - Node to convert. - - model: model - An model for MXNet - - builder: NeuralNetworkBuilder - A neural network builder object. - """ - input_name, output_name = _get_input_output_name(net, node) - param = node['attr'] - has_bias = True - name = node['name'] - - inputs = node['inputs'] - outputs = node['outputs'] - args = model.arg_params - W = args[_get_node_name(net, inputs[1][0])].asnumpy() - if has_bias: - Wb = args[_get_node_name(net, inputs[2][0])].asnumpy() - else: - Wb = None - nC, nB = W.shape - - builder.add_inner_product(name = name, - W = W, - Wb = Wb, - nB = nB, - nC = nC, - has_bias = has_bias, - input_name = input_name, - output_name = output_name) - -def convert_convolution(net, node, model, builder): - """Convert a convolution layer from mxnet to coreml. - - Parameters - ---------- - network: net - A mxnet network object. - - layer: node - Node to convert. - - model: model - An model for MXNet - - builder: NeuralNetworkBuilder - A neural network builder object. - """ - input_name, output_name = _get_input_output_name(net, node) - name = node['name'] - param = node['attr'] - inputs = node['inputs'] - outputs = node['outputs'] - args = model.arg_params - - from ast import literal_eval - - if 'no_bias' in param.keys(): - has_bias = not literal_eval(param['no_bias']) - else: - has_bias = True - - border_mode = "same" if literal_eval(param['pad']) != (0, 0) else 'valid' - border_mode = "valid" - n_filters = int(param['num_filter']) - output_shape = None # (needed for de-conv) - - W = args[_get_node_name(net, inputs[1][0])].asnumpy() - if has_bias: - Wb = args[_get_node_name(net, inputs[2][0])].asnumpy() - else: - Wb = None - - n_filters, channels = W.shape[0:2] - stride_height, stride_width = literal_eval(param['stride']) - kernel_height, kernel_width = literal_eval(param['kernel']) - - W = W.transpose((2, 3, 1, 0)) - builder.add_convolution(name = name, - kernelChannels = channels, - outputChannels = n_filters, - height = kernel_height, - width = kernel_width, - stride_height = stride_height, - stride_width = stride_width, - borderMode = border_mode, - groups = 1, - W = W, - b = Wb, - has_bias = has_bias, - is_deconv = False, - output_shape = output_shape, - input_name = input_name, - output_name = output_name) - - # Add padding if there is any - convLayer = builder.nn_spec.layers[-1].convolution - pad = literal_eval(param['pad']) - for i in range(len(pad)): - convLayer.valid.paddingAmounts.borderAmounts[i].startEdgeSize = pad[i] - convLayer.valid.paddingAmounts.borderAmounts[i].endEdgeSize = pad[i] - -def convert_pooling(net, node, model, builder): - """Convert a pooling layer from mxnet to coreml. - - Parameters - ---------- - network: net - A mxnet network object. - - layer: node - Node to convert. - - model: model - An model for MXNet - - builder: NeuralNetworkBuilder - A neural network builder object. - """ - input_name, output_name = _get_input_output_name(net, node) - name = node['name'] - inputs = node['inputs'] - param = node['attr'] - outputs = node['outputs'] - args = model.arg_params - - layer_type_mx = param['pool_type'] - if layer_type_mx == 'max': - layer_type= 'MAX' - elif layer_type_mx == 'avg': - layer_type = 'AVERAGE' - else: - raise TypeError("Pooling type %s not supported" % layer_type_mx) - - from ast import literal_eval - stride_height, stride_width = literal_eval(param['stride']) - kernel_width, kernel_height = literal_eval(param['kernel']) - - padding_type = 'VALID' - if 'global_pool' in param.keys(): - is_global = literal_eval(param['global_pool']) - else: - is_global = False - builder.add_pooling(name = name, - height = kernel_height, - width = kernel_width, - stride_height = stride_height, - stride_width = stride_width, - layer_type = layer_type, - padding_type = padding_type, - exclude_pad_area = False, - is_global = is_global, - input_name = input_name, - output_name = output_name) - - # Add padding if there is any - poolingLayer = builder.nn_spec.layers[-1].pooling - pad = literal_eval(param['pad']) - for i in range(len(pad)): - poolingLayer.valid.paddingAmounts.borderAmounts[i].startEdgeSize = pad[i] - poolingLayer.valid.paddingAmounts.borderAmounts[i].endEdgeSize = pad[i] - -def convert_batchnorm(net, node, model, builder): - """Convert a transpose layer from mxnet to coreml. - - Parameters - ---------- - network: net - A mxnet network object. - - layer: node - Node to convert. - - model: model - An model for MXNet - - builder: NeuralNetworkBuilder - A neural network builder object. - """ - input_name, output_name = _get_input_output_name(net, node) - name = node['name'] - param = node['attr'] - inputs = node['inputs'] - outputs = node['outputs'] - args = model.arg_params - aux = model.aux_params - - gamma = args[_get_node_name(net, inputs[1][0])].asnumpy() - beta = args[_get_node_name(net, inputs[2][0])].asnumpy() - mean = aux[_get_node_name(net, inputs[3][0])].asnumpy() - variance = aux[_get_node_name(net, inputs[4][0])].asnumpy() - - nb_channels = gamma.shape[0] - - builder.add_batchnorm( - name = name, - channels = nb_channels, - gamma = gamma, - beta = beta, - mean = mean, - variance = variance, - input_name = input_name, - output_name = output_name) - -def convert_concat(net, node, model, builder): - """Convert concat layer from mxnet to coreml. - - Parameters - ---------- - network: net - A mxnet network object. - - layer: node - Node to convert. - - model: model - An model for MXNet - - builder: NeuralNetworkBuilder - A neural network builder object. - """ - # Get input and output names - input_names, output_name = _get_input_output_name(net, node, 'all') - name = node['name'] - mode = 'CONCAT' - builder.add_elementwise(name = name, input_names = input_names, - output_name = output_name, mode = mode) diff --git a/tools/coreml/__init__.py b/tools/coreml/converter/__init__.py similarity index 96% rename from tools/coreml/__init__.py rename to tools/coreml/converter/__init__.py index e56490a472cf..245692337bc3 100644 --- a/tools/coreml/__init__.py +++ b/tools/coreml/converter/__init__.py @@ -15,4 +15,3 @@ # specific language governing permissions and limitations # under the License. -from _mxnet_converter import * diff --git a/tools/coreml/converter/_add_pooling.py b/tools/coreml/converter/_add_pooling.py new file mode 100644 index 000000000000..51934f22190b --- /dev/null +++ b/tools/coreml/converter/_add_pooling.py @@ -0,0 +1,118 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from coremltools.proto import NeuralNetwork_pb2 as _NeuralNetwork_pb2 + + +def add_pooling_with_padding_types(builder, name, height, width, stride_height, stride_width, + layer_type, padding_type, input_name, output_name, + padding_top = 0, padding_bottom = 0, padding_left = 0, padding_right = 0, + same_padding_asymmetry_mode = 'BOTTOM_RIGHT_HEAVY', + exclude_pad_area = True, is_global = False): + """ + Add a pooling layer to the model. + + This is our own implementation of add_pooling since current CoreML's version (0.5.0) of builder + doesn't provide support for padding types apart from valid. This support will be added in the + next release of coremltools. When that happens, this can be removed. + + Parameters + + ---------- + builder: NeuralNetworkBuilder + A neural network builder object. + name: str + The name of this layer. + height: int + Height of pooling region. + width: int + Number of elements to be padded on the right side of the input blob. + stride_height: int + Stride along the height direction. + stride_width: int + Stride along the height direction. + layer_type: str + Type of pooling performed. Can either be 'MAX', 'AVERAGE' or 'L2'. + padding_type: str + Option for the output blob shape. Can be either 'VALID' , 'SAME' or 'INCLUDE_LAST_PIXEL'. Kindly look at NeuralNetwork.proto for details. + input_name: str + The input blob name of this layer. + output_name: str + The output blob name of this layer. + + padding_top, padding_bottom, padding_left, padding_right: int + values of height (top, bottom) and width (left, right) padding to be used if padding type is "VALID" or "INCLUDE_LAST_PIXEL" + + same_padding_asymmetry_mode : str. + Type of asymmetric padding to be used when padding_type = 'SAME'. Kindly look at NeuralNetwork.proto for details. Can be either 'BOTTOM_RIGHT_HEAVY' or 'TOP_LEFT_HEAVY'. + + exclude_pad_area: boolean + Whether to exclude padded area in the pooling operation. Defaults to True. + + - If True, the value of the padded area will be excluded. + - If False, the padded area will be included. + This flag is only used with average pooling. + is_global: boolean + Whether the pooling operation is global. Defaults to False. + + - If True, the pooling operation is global -- the pooling region is of the same size of the input blob. + Parameters height, width, stride_height, stride_width will be ignored. + + - If False, the pooling operation is not global. + + See Also + -------- + add_convolution, add_pooling, add_activation + """ + + spec = builder.spec + nn_spec = builder.nn_spec + + # Add a new layer + spec_layer = nn_spec.layers.add() + spec_layer.name = name + spec_layer.input.append(input_name) + spec_layer.output.append(output_name) + spec_layer_params = spec_layer.pooling + + # Set the parameters + spec_layer_params.type = \ + _NeuralNetwork_pb2.PoolingLayerParams.PoolingType.Value(layer_type) + + if padding_type == 'VALID': + height_border = spec_layer_params.valid.paddingAmounts.borderAmounts.add() + height_border.startEdgeSize = padding_top + height_border.endEdgeSize = padding_bottom + width_border = spec_layer_params.valid.paddingAmounts.borderAmounts.add() + width_border.startEdgeSize = padding_left + width_border.endEdgeSize = padding_right + elif padding_type == 'SAME': + if not (same_padding_asymmetry_mode == 'BOTTOM_RIGHT_HEAVY' or same_padding_asymmetry_mode == 'TOP_LEFT_HEAVY'): + raise ValueError("Invalid value %d of same_padding_asymmetry_mode parameter" % same_padding_asymmetry_mode) + spec_layer_params.same.asymmetryMode = _NeuralNetwork_pb2.SamePadding.SamePaddingMode.Value(same_padding_asymmetry_mode) + elif padding_type == 'INCLUDE_LAST_PIXEL': + if padding_top != padding_bottom or padding_left != padding_right: + raise ValueError("Only symmetric padding is supported with the INCLUDE_LAST_PIXEL padding type") + spec_layer_params.includeLastPixel.paddingAmounts.append(padding_top) + spec_layer_params.includeLastPixel.paddingAmounts.append(padding_left) + + spec_layer_params.kernelSize.append(height) + spec_layer_params.kernelSize.append(width) + spec_layer_params.stride.append(stride_height) + spec_layer_params.stride.append(stride_width) + spec_layer_params.avgPoolExcludePadding = exclude_pad_area + spec_layer_params.globalPooling = is_global diff --git a/tools/coreml/converter/_layers.py b/tools/coreml/converter/_layers.py new file mode 100644 index 000000000000..0a089949a1a6 --- /dev/null +++ b/tools/coreml/converter/_layers.py @@ -0,0 +1,569 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import _add_pooling +from ast import literal_eval + +def _get_input_output_name(net, node, index=0): + name = node['name'] + inputs = node['inputs'] + + if index == 'all': + input_name = [_get_node_name(net, inputs[idx][0]) for idx in range(len(inputs))] + elif type(index) == int: + input_name = _get_node_name(net, inputs[0][0]) + else: + input_name = [_get_node_name(net, inputs[idx][0]) for idx in index] + return input_name, name + + +def _get_node_name(net, node_id): + return net['nodes'][node_id]['name'] + + +def _get_node_shape(net, node_id): + return net['nodes'][node_id]['shape'] + + +# TODO These operators still need to be converted (listing in order of priority): +# High priority: +# mxnet.symbol.repeat -> builder.add_repeat to flatten and repeat the NDArray sequence +# mxnet.symbol.Crop -> builder.add_crop to crop image along spacial dimensions +# mxnet.symbol.Pad -> builder.add_padding putting 0's on height and width for tensor +# Low Priority: +# depthwise seperable convolution support through groups in builder.add_convolution +# add_optional -> for all RNNs defining what goes in and out (to define beam search or if input is streaming) +# mx.symbol.Embedding -> add_embedding takes indicies, word ids from dict that is outside coreml or +# in pipeline only if we have text mapping to indicies +# FusedRNNCell -> add_bidirlstm +# add_unilstm -> reverse_input param true as second and concat on outputs +# Do vanilla (0.9 mxnet) lstm, gru, vanilla_rnn + + +def convert_reshape(net, node, module, builder): + """Converts a reshape layer from mxnet to coreml. + + This doesn't currently handle the deprecated parameters for the reshape layer. + + Parameters + ---------- + network: net + An mxnet network object. + + layer: node + Node to convert. + + module: module + A module for MXNet + + builder: NeuralNetworkBuilder + A neural network builder object. + """ + input_name, output_name = _get_input_output_name(net, node) + name = node['name'] + target_shape = node['shape'] + + if any(item <= 0 for item in target_shape): + raise NotImplementedError('Special dimensional values less than or equal to 0 are not supported yet.' + 'Feel free to file an issue here: https://github.com/dmlc/mxnet/issues.') + + if 'reverse' in node and node['reverse'] == 'True': + raise NotImplementedError('"reverse" parameter is not supported by yet.' + 'Feel free to file an issue here: https://github.com/dmlc/mxnet/issues.') + + mode = 0 # CHANNEL_FIRST + builder.add_reshape(name, input_name, output_name, target_shape, mode) + + +def convert_transpose(net, node, module, builder): + """Convert a transpose layer from mxnet to coreml. + + Parameters + ---------- + network: net + A mxnet network object. + + layer: node + Node to convert. + + module: module + An module for MXNet + + builder: NeuralNetworkBuilder + A neural network builder object. + """ + input_name, output_name = _get_input_output_name(net, node) + name = node['name'] + param = node['attr'] + + axes = literal_eval(param['axes']) + builder.add_permute(name, axes, input_name, output_name) + + +def convert_flatten(net, node, module, builder): + """Convert a flatten layer from mxnet to coreml. + + Parameters + ---------- + network: net + A mxnet network object. + + layer: node + Node to convert. + + module: module + An module for MXNet + + builder: NeuralNetworkBuilder + A neural network builder object. + """ + input_name, output_name = _get_input_output_name(net, node) + name = node['name'] + mode = 0 # CHANNEL_FIRST + builder.add_flatten(name, mode, input_name, output_name) + + +def convert_softmax(net, node, module, builder): + """Convert a softmax layer from mxnet to coreml. + + Parameters + ---------- + network: net + A mxnet network object. + + layer: node + Node to convert. + + module: module + An module for MXNet + + builder: NeuralNetworkBuilder + A neural network builder object. + """ + input_name, output_name = _get_input_output_name(net, node) + name = node['name'] + builder.add_softmax(name=name, + input_name=input_name, + output_name=output_name) + + +def convert_activation(net, node, module, builder): + """Convert an activation layer from mxnet to coreml. + + Parameters + ---------- + network: net + A mxnet network object. + + layer: node + Node to convert. + + module: module + An module for MXNet + + builder: NeuralNetworkBuilder + A neural network builder object. + """ + input_name, output_name = _get_input_output_name(net, node) + name = node['name'] + mx_non_linearity = node['attr']['act_type'] + #TODO add SCALED_TANH, SOFTPLUS, SOFTSIGN, SIGMOID_HARD, LEAKYRELU, PRELU, ELU, PARAMETRICSOFTPLUS, THRESHOLDEDRELU, LINEAR + if mx_non_linearity == 'relu': + non_linearity = 'RELU' + elif mx_non_linearity == 'tanh': + non_linearity = 'TANH' + elif mx_non_linearity == 'sigmoid': + non_linearity = 'SIGMOID' + else: + raise TypeError('Unknown activation type %s' % mx_non_linearity) + builder.add_activation(name = name, + non_linearity = non_linearity, + input_name = input_name, + output_name = output_name) + + +def convert_elementwise_add(net, node, module, builder): + """Convert an elementwise add layer from mxnet to coreml. + + Parameters + ---------- + network: net + A mxnet network object. + + layer: node + Node to convert. + + module: module + An module for MXNet + + builder: NeuralNetworkBuilder + A neural network builder object. + """ + + input_names, output_name = _get_input_output_name(net, node, [0, 1]) + name = node['name'] + + builder.add_elementwise(name, input_names, output_name, 'ADD') + + +def convert_dense(net, node, module, builder): + """Convert a dense layer from mxnet to coreml. + + Parameters + ---------- + network: net + A mxnet network object. + + layer: node + Node to convert. + + module: module + An module for MXNet + + builder: NeuralNetworkBuilder + A neural network builder object. + """ + input_name, output_name = _get_input_output_name(net, node) + has_bias = True + name = node['name'] + + inputs = node['inputs'] + args, _ = module.get_params() + W = args[_get_node_name(net, inputs[1][0])].asnumpy() + if has_bias: + Wb = args[_get_node_name(net, inputs[2][0])].asnumpy() + else: + Wb = None + nC, nB = W.shape + + builder.add_inner_product( + name=name, + W=W, + b=Wb, + input_channels=nB, + output_channels=nC, + has_bias=has_bias, + input_name=input_name, + output_name=output_name + ) + + +def convert_convolution(net, node, module, builder): + """Convert a convolution layer from mxnet to coreml. + + Parameters + ---------- + network: net + A mxnet network object. + + layer: node + Node to convert. + + module: module + An module for MXNet + + builder: NeuralNetworkBuilder + A neural network builder object. + """ + input_name, output_name = _get_input_output_name(net, node) + name = node['name'] + param = node['attr'] + inputs = node['inputs'] + args, _ = module.get_params() + + if 'no_bias' in param.keys(): + has_bias = not literal_eval(param['no_bias']) + else: + has_bias = True + + if literal_eval(param['pad']) != (0, 0): + pad = literal_eval(param['pad']) + builder.add_padding( + name=name+"_pad", + left=pad[1], + right=pad[1], + top=pad[0], + bottom=pad[0], + value=0, + input_name=input_name, + output_name=name+"_pad_output") + input_name = name+"_pad_output" + + border_mode = "valid" + + n_filters = int(param['num_filter']) + + W = args[_get_node_name(net, inputs[1][0])].asnumpy() + if has_bias: + Wb = args[_get_node_name(net, inputs[2][0])].asnumpy() + else: + Wb = None + + channels = W.shape[1] + stride_height, stride_width = literal_eval(param['stride']) + kernel_height, kernel_width = literal_eval(param['kernel']) + + W = W.transpose((2, 3, 1, 0)) + builder.add_convolution( + name=name, + kernel_channels=channels, + output_channels=n_filters, + height=kernel_height, + width=kernel_width, + stride_height=stride_height, + stride_width=stride_width, + border_mode=border_mode, + groups=1, + W=W, + b=Wb, + has_bias=has_bias, + is_deconv=False, + output_shape=None, + input_name=input_name, + output_name=output_name) + + +def convert_pooling(net, node, module, builder): + """Convert a pooling layer from mxnet to coreml. + + Parameters + ---------- + network: net + A mxnet network object. + + layer: node + Node to convert. + + module: module + An module for MXNet + + builder: NeuralNetworkBuilder + A neural network builder object. + """ + input_name, output_name = _get_input_output_name(net, node) + name = node['name'] + param = node['attr'] + + layer_type_mx = param['pool_type'] + if layer_type_mx == 'max': + layer_type = 'MAX' + elif layer_type_mx == 'avg': + layer_type = 'AVERAGE' + else: + raise TypeError("Pooling type %s not supported" % layer_type_mx) + + # Add padding if there is any + if literal_eval(param['pad']) != (0, 0): + pad = literal_eval(param['pad']) + builder.add_padding( + name=name+"_pad", + left=pad[1], + right=pad[1], + top=pad[0], + bottom=pad[0], + value=0, + input_name=input_name, + output_name=name+"_pad_output") + input_name = name+"_pad_output" + + stride_height, stride_width = literal_eval(param['stride']) + kernel_width, kernel_height = literal_eval(param['kernel']) + + type_map = {'valid': 'VALID', 'full': 'INCLUDE_LAST_PIXEL'} + padding_type = param['pooling_convention'] if 'pooling_convention' in param else 'valid' + if padding_type not in type_map: + raise KeyError("%s type is not supported in this converter. It is a Github issue.") + padding_type = type_map[padding_type] + + if 'global_pool' in param.keys(): + is_global = literal_eval(param['global_pool']) + else: + is_global = False + + # For reasons why we are not using the standard builder but having our own implementation, + # see the function documentation. + _add_pooling.add_pooling_with_padding_types( + builder=builder, + name=name, + height=kernel_height, + width=kernel_width, + stride_height=stride_height, + stride_width=stride_width, + layer_type=layer_type, + padding_type=padding_type, + exclude_pad_area=False, + is_global=is_global, + input_name=input_name, + output_name=output_name + ) + + +def convert_batchnorm(net, node, module, builder): + """Convert a transpose layer from mxnet to coreml. + + Parameters + ---------- + network: net + A mxnet network object. + + layer: node + Node to convert. + + module: module + An module for MXNet + + builder: NeuralNetworkBuilder + A neural network builder object. + """ + input_name, output_name = _get_input_output_name(net, node) + name = node['name'] + inputs = node['inputs'] + + + eps = 1e-3 # Default value of eps for MXNet. + use_global_stats = False # Default value of use_global_stats for MXNet. + if 'attr' in node: + if 'eps' in node['attr']: + eps = literal_eval(node['attr']['eps']) + + args, aux = module.get_params() + gamma = args[_get_node_name(net, inputs[1][0])].asnumpy() + beta = args[_get_node_name(net, inputs[2][0])].asnumpy() + mean = aux[_get_node_name(net, inputs[3][0])].asnumpy() + variance = aux[_get_node_name(net, inputs[4][0])].asnumpy() + nb_channels = gamma.shape[0] + builder.add_batchnorm( + name=name, + channels=nb_channels, + gamma=gamma, + beta=beta, + mean=mean, + variance=variance, + input_name=input_name, + output_name=output_name, + epsilon=eps) + + +def convert_concat(net, node, module, builder): + """Convert concat layer from mxnet to coreml. + + Parameters + ---------- + network: net + A mxnet network object. + + layer: node + Node to convert. + + module: module + An module for MXNet + + builder: NeuralNetworkBuilder + A neural network builder object. + """ + # Get input and output names + input_names, output_name = _get_input_output_name(net, node, 'all') + name = node['name'] + mode = 'CONCAT' + builder.add_elementwise(name = name, input_names = input_names, + output_name = output_name, mode = mode) + + +def convert_deconvolution(net, node, module, builder): + """Convert a deconvolution layer from mxnet to coreml. + + Parameters + ---------- + network: net + A mxnet network object. + + layer: node + Node to convert. + + module: module + An module for MXNet + + builder: NeuralNetworkBuilder + A neural network builder object. + """ + input_name, output_name = _get_input_output_name(net, node) + name = node['name'] + param = node['attr'] + inputs = node['inputs'] + args, _ = module.get_params() + + if 'no_bias' in param.keys(): + has_bias = not literal_eval(param['no_bias']) + else: + has_bias = False + + border_mode = "valid" + + n_filters = int(param['num_filter']) + + output_shape = None + if 'target_shape' in param: + target_shape = literal_eval(param['target_shape']) + output_shape = (int(target_shape[0]), int(target_shape[1])) + + W = args[_get_node_name(net, inputs[1][0])].asnumpy() + + if has_bias: + Wb = args[_get_node_name(net, inputs[2][0])].asnumpy() + else: + Wb = None + + channels = W.shape[0] + stride_height, stride_width = literal_eval(param['stride']) + kernel_height, kernel_width = literal_eval(param['kernel']) + W = W.transpose((2, 3, 0, 1)) + + use_crop = False + if literal_eval(param['pad']) != (0, 0) and output_shape is None: + use_crop = True + + builder.add_convolution( + name=name, + kernel_channels=channels, + output_channels=n_filters, + height=kernel_height, + width=kernel_width, + stride_height=stride_height, + stride_width=stride_width, + border_mode=border_mode, + groups=1, + W=W, + b=Wb, + has_bias=has_bias, + is_deconv=True, + output_shape=output_shape, + input_name=input_name, + output_name=output_name+'before_pad' if use_crop else output_name + ) + + if use_crop: + pad = literal_eval(param['pad']) + builder.add_crop( + name=name+"_pad", + left=pad[1], + right=pad[1], + top=pad[0], + bottom=pad[0], + offset=0, + input_names=[output_name+'before_pad'], + output_name=output_name + ) diff --git a/tools/coreml/_mxnet_converter.py b/tools/coreml/converter/_mxnet_converter.py similarity index 69% rename from tools/coreml/_mxnet_converter.py rename to tools/coreml/converter/_mxnet_converter.py index 88a980c61c1b..a9ea0f4d7ad6 100644 --- a/tools/coreml/_mxnet_converter.py +++ b/tools/coreml/converter/_mxnet_converter.py @@ -35,10 +35,13 @@ 'Concat' : _layers.convert_concat, 'BatchNorm' : _layers.convert_batchnorm, 'elemwise_add' : _layers.convert_elementwise_add, + 'Reshape' : _layers.convert_reshape, + 'Deconvolution' : _layers.convert_deconvolution, } _MXNET_SKIP_LAYERS = [ '_MulScalar', + 'Dropout', ] def _mxnet_remove_batch(input_data): @@ -73,7 +76,6 @@ def check_error(model, path, shapes, output = 'softmax_output', verbose = True): def _set_input_output_layers(builder, input_names, output_names): input_layers_indices = [] output_layers_indices = [] - spec = builder.spec layers = builder.spec.neuralNetwork.layers for idx, l in enumerate(layers): if set(input_names).intersection(l.input): @@ -83,8 +85,8 @@ def _set_input_output_layers(builder, input_names, output_names): builder.input_layers_indices = input_layers_indices builder.output_layers_indices = output_layers_indices - builder.input_layers_is1d = [False for i in input_names] - builder.output_layers_is1d = [False for i in output_names] + builder.input_layers_is1d = [False for _ in input_names] + builder.output_layers_is1d = [False for _ in output_names] def _get_layer_converter_fn(layer): """Get the right converter function for MXNet @@ -94,8 +96,9 @@ def _get_layer_converter_fn(layer): else: raise TypeError("MXNet layer of type %s is not supported." % layer) -def convert(model, order = None, **kwargs): - """Convert a keras model to the protobuf spec. + +def convert(model, input_shape, order = None, class_labels = None, mode = None, preprocessor_args = None): + """Convert an MXNet model to the protobuf spec. Parameters ---------- @@ -104,33 +107,46 @@ def convert(model, order = None, **kwargs): order: Order of inputs + class_labels: A string or list of strings. + As a string it represents the name of the file which contains the classification labels (one per line). + As a list of strings it represents a list of categories that map the index of the output of a neural network to labels in a classifier. + + mode: str ('classifier', 'regressor' or None) + Mode of the converted coreml model. + When mode = 'classifier', a NeuralNetworkClassifier spec will be constructed. + When mode = 'regressor', a NeuralNetworkRegressor spec will be constructed. + **kwargs : - Provide keyword arguments of known shapes. + Provide keyword arguments for: + - input shapes. Supplied as a dictionary object with keyword "input_shape". + - pre-processing arguments: Supplied as a dictionary object with keyword "preprocessor_args". The parameters in the dictionary + tell the converted coreml model how to pre-process any input before an inference is run on it. + For the list of pre-processing arguments see + http://pythonhosted.org/coremltools/generated/coremltools.models.neural_network.html#coremltools.models.neural_network.NeuralNetworkBuilder.set_pre_processing_parameters Returns ------- - model_spec: An object of type ModelSpec_pb. - Protobuf representation of the model + model: A coreml model. """ - if not kwargs: - raise TypeError("Must provide input shape to be able to perform conversion") + if not isinstance(input_shape, dict): + raise TypeError("Must provide a dictionary for input shape. e.g input_shape={'data':(3,224,224)}") def remove_batch(dim): return dim[1:] if order is None: - input_names = kwargs.keys() - input_dims = map(remove_batch, kwargs.values()) + input_names = input_shape.keys() + input_dims = map(remove_batch, input_shape.values()) else: - names = kwargs.keys() - shapes = map(remove_batch, kwargs.values()) + names = input_shape.keys() + shapes = map(remove_batch, input_shape.values()) input_names = [names[i] for i in order] input_dims = [shapes[i] for i in order] net = model.symbol # Infer shapes and store in a dictionary - shapes = net.infer_shape(**kwargs) + shapes = net.infer_shape(**input_shape) arg_names = net.list_arguments() output_names = net.list_outputs() aux_names = net.list_auxiliary_states() @@ -142,7 +158,6 @@ def remove_batch(dim): for idx, op in enumerate(aux_names): shape_dict[op] = shapes[2][idx] - # Get the inputs and outputs output_dims = shapes[1] input_types = [_datatypes.Array(*dim) for dim in input_dims] @@ -151,11 +166,11 @@ def remove_batch(dim): # Make the builder input_features = zip(input_names, input_types) output_features = zip(output_names, output_types) - builder = _neural_network.NeuralNetworkBuilder(input_features, output_features) - + builder = _neural_network.NeuralNetworkBuilder(input_features, output_features, mode) # Get out the layers net = _json.loads(net.tojson()) nodes = net['nodes'] + for i, node in enumerate(nodes): node['id'] = i @@ -178,7 +193,7 @@ def remove_batch(dim): head_node['shape'] = shape_dict[head_node['name']] # For skipped layers, make sure nodes are modified - for iter, node in enumerate(nodes): + for node in nodes: op = node['op'] inputs = node['inputs'] outputs = node['outputs'] @@ -187,24 +202,30 @@ def remove_batch(dim): nodes[outputs[0][0]]['inputs'][0] = inputs[0] # Find the input and output names for this node - for iter, node in enumerate(nodes): + for idx, node in enumerate(nodes): op = node['op'] if op == 'null' or op in _MXNET_SKIP_LAYERS: continue name = node['name'] - print("%d : %s, %s" % (iter, name, op)) + print("%d : %s, %s" % (idx, name, op)) converter_func = _get_layer_converter_fn(op) converter_func(net, node, model, builder) - spec = builder.spec - layers = spec.neuralNetwork.layers - # Set the right inputs and outputs _set_input_output_layers(builder, input_names, output_names) builder.set_input(input_names, input_dims) builder.set_output(output_names, output_dims) + if preprocessor_args is not None: + builder.set_pre_processing_parameters(**preprocessor_args) + + if class_labels is not None: + if type(class_labels) is str: + labels = [l.strip() for l in open(class_labels).readlines()] + elif type(class_labels) is list: + labels = class_labels + else: + raise TypeError("synset variable of unknown type. Type found: %s. Expected either string or list of strings." % type(class_labels)) + builder.set_class_labels(class_labels = labels) - # Return the spec - spec = builder.spec - layers = spec.neuralNetwork.layers - return spec + # Return the model + return _coremltools.models.MLModel(builder.spec) \ No newline at end of file diff --git a/tools/coreml/mxnet_coreml_converter.py b/tools/coreml/mxnet_coreml_converter.py new file mode 100644 index 000000000000..502377eca864 --- /dev/null +++ b/tools/coreml/mxnet_coreml_converter.py @@ -0,0 +1,114 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import print_function +import argparse +from converter._mxnet_converter import convert +from utils import load_model +import yaml +from ast import literal_eval + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Converts an MXNet model to a CoreML model') + + parser.add_argument( + '--model-prefix', required=True, type=str, + help="Prefix of the existing model. The model is expected to be stored in the same directory from where " + "this tool is being run. E.g. --model-prefix=squeezenet_v1.1. Note that this can include entire " + "directory name too. E.g. --model-prefix=~/Downloads/squeezenet_v1.1." + ) + parser.add_argument( + '--epoch', required=True, type=int, + help="The suffix of the MXNet model name which usually indicate the number of epochs. E.g. --epoch=0" + ) + parser.add_argument( + '--output-file', required=True, type=str, + help="File where the resulting CoreML model will be saved. E.g. --output-file=\"squeezenet-v11.mlmodel\"" + ) + parser.add_argument( + '--input-shape', required=True, type=str, + help="Input shape information in a JSON string format. E.g. --input-shape='{\"data\":\"3,224,224\"}' where" + " 'data' is the name of the input variable of the MXNet model and '3,244,244' is its shape " + "(channel, height and weight) of the input image data." + ) + parser.add_argument( + '--label-names', required=False, type=str, default='softmax_label', + help="label-names of the MXNet model's output variables. E.g. --label-names=softmax_label. " + "(Usually this is the name of the last layer followed by suffix _label.)" + ) + parser.add_argument( + '--mode', required=False, type=str, default=None, + help="When mode='classifier', a CoreML NeuralNetworkClassifier will be constructed. " + "When mode='regressor', a CoreML NeuralNetworkRegressor will be constructed. " + "When mode=None (default), a CoreML NeuralNetwork will be constructed." + ) + parser.add_argument( + '--class-labels', required=False, type=str, default=None, + help="As a string it represents the name of the file which contains the classification labels (synset file)." + ) + parser.add_argument( + '--pre-processing-arguments', required=False, type=str, default=None, + help="The parameters in the dictionary tell the converted coreml model how to pre-process any input " + "before an inference is run on it. For the list of pre-processing arguments see https://goo.gl/GzFe86" + "e.g. --pre-processing-arguments='{\"red_bias\": 127, \"blue_bias\":117, \"green_bias\": 103}'" + ) + + # TODO + # We need to test how to use the order + # parser.add_argument( + # '--order', required=True, type=str, default=None, + # help="" + # ) + + args, unknown = parser.parse_known_args() + + model_name = args.model_prefix + epoch_num = args.epoch + output_file = args.output_file + mode = args.mode + class_labels=args.class_labels + + # parse the input data name/shape and label name/shape + input_shape = yaml.safe_load(args.input_shape) + data_shapes = [] + for key in input_shape: + # We prepend 1 because the coreml model only accept 1 input data at a time. + shape = (1,)+literal_eval(input_shape[key]) + input_shape[key] = shape + data_shapes.append((key, shape)) + + # if label name is not in input then do not use the label + label_names = [args.label_names,] if args.label_names in input_shape else None + + pre_processing_arguments = args.pre_processing_arguments + + mod = load_model( + model_name=model_name, + epoch_num=epoch_num, + data_shapes=data_shapes, + label_shapes=None, + label_names=label_names + ) + + kwargs = {'input_shape': input_shape} + if pre_processing_arguments is not None: + kwargs['preprocessor_args'] = yaml.safe_load(pre_processing_arguments) + + coreml_model = convert(model=mod, mode=mode, class_labels=class_labels, **kwargs) + coreml_model.save(output_file) + print("\nSUCCESS\nModel %s has been converted and saved at %s\n" % (model_name, output_file)) diff --git a/tools/coreml/test/test_mxnet_converter.py b/tools/coreml/test/test_mxnet_converter.py new file mode 100644 index 000000000000..6692b44ec370 --- /dev/null +++ b/tools/coreml/test/test_mxnet_converter.py @@ -0,0 +1,949 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import unittest +import mxnet as mx +import numpy as np +import sys +import os +current_working_directory = os.getcwd() +sys.path.append(current_working_directory + "/..") +sys.path.append(current_working_directory + "/../converter/") +import _mxnet_converter as mxnet_converter +from collections import namedtuple + + +def _mxnet_remove_batch(input_data): + for blob in input_data: + input_data[blob] = np.reshape(input_data[blob], input_data[blob].shape[1:]) + return input_data + + +def _get_mxnet_module(net, input_shape, mode, label_names, input_names=None): + """ Given a symbolic graph, input shape and the initialization mode, + returns an MXNet module. + """ + mx.random.seed(1993) + + mod = mx.mod.Module( + symbol=net, + context=mx.cpu(), + label_names=label_names + ) + mod.bind( + for_training=False, + data_shapes=[('data', input_shape)], + label_shapes=input_names + ) + if mode == 'random': + mod.init_params( + initializer=mx.init.Uniform(scale=.1) + ) + elif mode == 'zeros': + mod.init_params( + initializer=mx.init.Zero() + ) + elif mode == 'ones': + mod.init_params( + initializer=mx.init.One() + ) + else: + Exception(KeyError("%s is not a valid initialization mode" % mode)) + + return mod + + +class SingleLayerTest(unittest.TestCase): + """ + Unit test class for testing where converter is able to convert individual layers or not. + In order to do so, it converts model and generates preds on both CoreML and MXNet and check they are the same. + """ + def _test_mxnet_model(self, net, input_shape, mode, class_labels=None, coreml_mode=None, label_names=None, delta=1e-3, + pre_processing_args=None): + """ Helper method that convert the CoreML model into CoreML and compares the predictions over random data. + + Parameters + ---------- + net: MXNet Symbol Graph + The graph that we'll be converting into CoreML. + + input_shape: tuple of ints + The shape of input data. Generally of the format (batch-size, channels, height, width) + + mode: (random|zeros|ones) + The mode to use in order to set the parameters (weights and biases). + + label_names: list of strings + The names of the output labels. Default: None + + delta: float + The maximum difference b/w predictions of MXNet and CoreML that is tolerable. + """ + mod = _get_mxnet_module(net, input_shape, mode, label_names) + + # Generate some dummy data + input_data = {'data': np.random.uniform(-10., 10., input_shape)} + Batch = namedtuple('Batch', ['data']) + mod.forward(Batch([mx.nd.array(input_data['data'])])) + mxnet_preds = mod.get_outputs()[0].asnumpy().flatten() + + # Get predictions from coreml + coreml_model = mxnet_converter.convert( + model=mod, + class_labels=class_labels, + mode=coreml_mode, + input_shape={'data': input_shape}, + preprocessor_args=pre_processing_args + ) + coreml_preds = coreml_model.predict(_mxnet_remove_batch(input_data)).values()[0].flatten() + + # Check prediction accuracy + self.assertEquals(len(mxnet_preds), len(coreml_preds)) + for i in range(len(mxnet_preds)): + self.assertAlmostEquals(mxnet_preds[i], coreml_preds[i], delta = delta) + + def test_tiny_inner_product_zero_input(self): + np.random.seed(1988) + input_shape = (1, 10) + net = mx.sym.Variable('data') + net = mx.sym.FullyConnected(data=net, name='fc1', num_hidden=5) + self._test_mxnet_model(net, input_shape=input_shape, mode='zeros') + + def test_really_tiny_inner_product_ones_input(self): + np.random.seed(1988) + input_shape = (1, 1) + net = mx.sym.Variable('data') + net = mx.sym.FullyConnected(data=net, name='fc1', num_hidden=1) + self._test_mxnet_model(net, input_shape=input_shape, mode='ones') + + def test_really_tiny_2_inner_product_ones_input(self): + np.random.seed(1988) + input_shape = (1, 1) + net = mx.sym.Variable('data') + net = mx.sym.FullyConnected(data=net, name='fc1', num_hidden=5) + self._test_mxnet_model(net, input_shape=input_shape, mode='ones') + + def test_tiny_inner_product_ones_input(self): + np.random.seed(1988) + input_shape = (1, 10) + net = mx.sym.Variable('data') + net = mx.sym.FullyConnected(data=net, name='fc1', num_hidden=5) + self._test_mxnet_model(net, input_shape=input_shape, mode='ones') + + def test_tiny_inner_product_random_input(self): + np.random.seed(1988) + input_shape = (1, 10) + net = mx.sym.Variable('data') + net = mx.sym.FullyConnected(data=net, name='fc1', num_hidden=5) + self._test_mxnet_model(net, input_shape=input_shape, mode='random') + + def test_tiny_softmax_random_input(self): + np.random.seed(1988) + input_shape = (1, 10) + net = mx.sym.Variable('data') + net = mx.sym.FullyConnected(data=net, name='fc1', num_hidden=5) + net = mx.sym.SoftmaxOutput(net, name='softmax') + self._test_mxnet_model(net, input_shape=input_shape, mode='random', label_names=['softmax_label']) + + def test_tiny_relu_activation_random_input(self): + np.random.seed(1988) + input_shape = (1, 10) + net = mx.sym.Variable('data') + net = mx.sym.FullyConnected(data=net, name='fc1', num_hidden=5) + net = mx.sym.Activation(net, name='relu1', act_type="relu") + self._test_mxnet_model(net, input_shape=input_shape, mode='random') + + def test_tiny_sigmoid_activation_random_input(self): + np.random.seed(1988) + input_shape = (1, 10) + net = mx.sym.Variable('data') + net = mx.sym.FullyConnected(data=net, name='fc1', num_hidden=5) + net = mx.sym.Activation(net, name='sigmoid1', act_type="sigmoid") + self._test_mxnet_model(net, input_shape=input_shape, mode='random') + + def test_tiny_tanh_activation_random_input(self): + np.random.seed(1988) + input_shape = (1, 10) + + # Define a model + net = mx.sym.Variable('data') + net = mx.sym.FullyConnected(data=net, name='fc1', num_hidden=5) + net = mx.sym.Activation(net, name='tanh1', act_type="tanh") + self._test_mxnet_model(net, input_shape=input_shape, mode='random') + + def test_really_tiny_conv_random_input(self): + np.random.seed(1988) + input_shape = (1, 1, 10, 10) + num_filter = 1 + kernel = (1 ,1) + stride = (1, 1) + pad = (0, 0) + + # Define a model + net = mx.sym.Variable('data') + net = mx.symbol.Convolution( + data=net, + num_filter=num_filter, + kernel=kernel, + stride=stride, + pad=pad, + name='conv_1' + ) + self._test_mxnet_model(net, input_shape=input_shape, mode='random') + + def test_tiny_conv_ones_input(self): + np.random.seed(1988) + input_shape = (1, 1, 10, 10) + num_filter = 1 + kernel = (5, 5) + stride = (1, 1) + pad = (0, 0) + + # Define a model + net = mx.sym.Variable('data') + net = mx.symbol.Convolution( + data=net, + num_filter=num_filter, + kernel=kernel, + stride=stride, + pad=pad, + name='conv_1' + ) + self._test_mxnet_model(net, input_shape=input_shape, mode='ones') + + def test_tiny_conv_random_input(self): + np.random.seed(1988) + input_shape = (1, 1, 10, 10) + num_filter = 1 + kernel = (5, 5) + stride = (1, 1) + pad = (0, 0) + + # define a model + net = mx.sym.Variable('data') + net = mx.symbol.Convolution( + data=net, + num_filter=num_filter, + kernel=kernel, + stride=stride, + pad=pad, + name='conv_1' + ) + self._test_mxnet_model(net, input_shape=input_shape, mode='random') + + def test_tiny_asym_conv_random_input(self): + np.random.seed(1988) + input_shape = (1, 1, 10, 10) + num_filter = 1 + kernel = (5 ,3) + stride = (1, 1) + pad = (0, 0) + net = mx.sym.Variable('data') + net = mx.symbol.Convolution( + data=net, + num_filter=num_filter, + kernel=kernel, + stride=stride, + pad=pad, + name='conv_1' + ) + self._test_mxnet_model(net, input_shape=input_shape, mode='random') + + def test_tiny_asym_conv_random_asym_input(self): + np.random.seed(1988) + input_shape = (1, 1, 28, 18) + num_filter = 16 + kernel = (5, 3) + stride = (1, 1) + pad = (0, 0) + dilate = (1, 1) + # define a model + net = mx.sym.Variable('data') + net = mx.symbol.Convolution( + data=net, + num_filter=num_filter, + kernel=kernel, + stride=stride, + pad=pad, + name='conv_1', + dilate=dilate) + net = mx.sym.Activation(net, name='tanh', act_type="tanh") + self._test_mxnet_model(net, input_shape=input_shape, mode='random') + + def test_tiny_conv_valid_pooling_random_input(self): + np.random.seed(1988) + input_shape = (1, 1, 10, 10) + num_filter = 1 + kernel = (2, 2) + stride = (2, 2) + pad = (0, 0) + net = mx.sym.Variable('data') + net = mx.symbol.Convolution( + data=net, + num_filter=num_filter, + kernel=kernel, + stride=stride, + pad=pad, + name='conv_1' + ) + net = mx.symbol.Pooling( + data=net, + kernel=kernel, + stride=stride, + pad=pad, + name='pool_1', + pool_type='avg', + pooling_convention='valid' + ) + self._test_mxnet_model(net, input_shape=input_shape, mode='random') + + def test_tiny_conv_pooling_full_random_input(self): + np.random.seed(1988) + input_shape = (1, 1, 10, 10) + num_filter = 1 + kernel = (2, 2) + stride = (2, 2) + pad = (0, 0) + net = mx.sym.Variable('data') + net = mx.symbol.Convolution( + data=net, + num_filter=num_filter, + kernel=kernel, + stride=stride, + pad=pad, + name='conv_1' + ) + net = mx.symbol.Pooling( + data=net, + kernel=kernel, + stride=stride, + pad=pad, + name='pool_1', + pool_type='avg', + pooling_convention='full' + ) + self._test_mxnet_model(net, input_shape=input_shape, mode='random') + + def test_tiny_conv_pooling_full_random_input_with_padding(self): + np.random.seed(1988) + input_shape = (1, 3, 10, 10) + num_filter = 2 + kernel = (2, 2) + stride = (2, 2) + pad = (1, 1) + net = mx.sym.Variable('data') + net = mx.symbol.Convolution( + data=net, + num_filter=num_filter, + kernel=kernel, + stride=stride, + pad=pad, + name='conv_1' + ) + net = mx.symbol.Pooling( + data=net, + kernel=kernel, + stride=stride, + pad=pad, + name='pool_1', + pool_type='avg', + pooling_convention='full' + ) + self._test_mxnet_model(net, input_shape=input_shape, mode='random') + + def test_really_tiny_conv_random_3d_input(self): + np.random.seed(1988) + input_shape = (1, 3, 10, 10) + num_filter = 1 + kernel = (1, 1) + stride = (1, 1) + pad = (0, 0) + net = mx.sym.Variable('data') + net = mx.symbol.Convolution( + data=net, + num_filter=num_filter, + kernel=kernel, + stride=stride, + pad=pad, + name='conv_1' + ) + self._test_mxnet_model(net, input_shape=input_shape, mode='random') + + def test_really_tiny_conv_random_input_multi_filter(self): + np.random.seed(1988) + input_shape = (1, 1, 10, 10) + num_filter = 64 + kernel = (1, 1) + stride = (1, 1) + pad = (0, 0) + net = mx.sym.Variable('data') + net = mx.symbol.Convolution( + data=net, + num_filter=num_filter, + kernel=kernel, + stride=stride, + pad=pad, + name='conv_1' + ) + self._test_mxnet_model(net, input_shape=input_shape, mode='random') + + def test_tiny_conv_random_3d_input(self): + np.random.seed(1988) + input_shape = (1, 3, 10, 10) + num_filter = 1 + kernel = (5 ,5) + stride = (1, 1) + pad = (0, 0) + net = mx.sym.Variable('data') + net = mx.symbol.Convolution( + data=net, + num_filter=num_filter, + kernel=kernel, + stride=stride, + pad=pad, + name='conv_1' + ) + self._test_mxnet_model(net, input_shape=input_shape, mode='random') + + def test_tiny_conv_random_input_multi_filter(self): + np.random.seed(1988) + input_shape = (1, 1, 10, 10) + num_filter = 64 + kernel = (5, 5) + stride = (1, 1) + pad = (0, 0) + net = mx.sym.Variable('data') + net = mx.symbol.Convolution( + data=net, + num_filter=num_filter, + kernel=kernel, + stride=stride, + pad=pad, + name='conv_1' + ) + self._test_mxnet_model(net, input_shape=input_shape, mode='random') + + def test_conv_random(self): + np.random.seed(1988) + input_shape = (1, 3, 10, 10) + num_filter = 64 + kernel = (5, 5) + stride = (1, 1) + pad = (0, 0) + net = mx.sym.Variable('data') + net = mx.symbol.Convolution( + data=net, + num_filter=num_filter, + kernel=kernel, + stride=stride, + pad=pad, + name='conv_1' + ) + self._test_mxnet_model(net, input_shape=input_shape, mode='random') + + def test_flatten(self): + np.random.seed(1988) + input_shape = (1, 3, 10, 10) + num_filter = 64 + kernel = (5, 5) + stride = (1, 1) + pad = (0, 0) + + # define a model + net = mx.sym.Variable('data') + net = mx.symbol.Convolution( + data=net, + num_filter=num_filter, + kernel=kernel, + stride=stride, + pad=pad, + name='conv_1' + ) + net = mx.sym.Flatten(data=net, name='flatten1') + net = mx.sym.FullyConnected(data=net, name='fc1', num_hidden=5) + net = mx.sym.SoftmaxOutput(net, name='softmax') + self._test_mxnet_model(net, input_shape=input_shape, mode='random', label_names=['softmax_label']) + + def test_transpose(self): + np.random.seed(1988) + input_shape = (1, 3, 10, 10) + num_filter = 64 + kernel = (5, 5) + stride = (1, 1) + pad = (0, 0) + + net = mx.sym.Variable('data') + net = mx.sym.transpose(data=net, name='transpose', axes=(0, 1, 2, 3)) + net = mx.symbol.Convolution( + data=net, + num_filter=num_filter, + kernel=kernel, + stride=stride, + pad=pad, + name='conv_1' + ) + self._test_mxnet_model(net, input_shape=input_shape, mode='random') + + def test_reshape(self): + np.random.seed(1988) + input_shape = (1, 8) + net = mx.sym.Variable('data') + net = mx.sym.reshape(data=net, shape=(1, 2, 2, 2)) + self._test_mxnet_model(net, input_shape=input_shape, mode='random') + + def test_tiny_synset_random_input(self): + np.random.seed(1989) + input_shape = (1, 10) + net = mx.sym.Variable('data') + net = mx.sym.FullyConnected(data=net, name='fc1', num_hidden=5) + net = mx.sym.SoftmaxOutput(net, name='softmax') + mod = _get_mxnet_module(net, + input_shape=input_shape, + mode='random', + label_names=['softmax_label']) + + # Generate some dummy data + input_data = np.random.uniform(-0.1, 0.1, input_shape) + + Batch = namedtuple('Batch', ['data']) + mod.forward(Batch([mx.nd.array(input_data)])) + + kwargs = {'input_shape': {'data': input_shape}} + # Get predictions from coreml + coreml_model = mxnet_converter.convert( + model=mod, + class_labels=['Category1','Category2','Category3','Category4','Category5'], + mode='classifier', + **kwargs + ) + + prediction = coreml_model.predict(_mxnet_remove_batch({'data': input_data})) + self.assertEqual(prediction['classLabel'], 'Category3') + + def test_really_tiny_deconv_random_input(self): + np.random.seed(1988) + input_shape = (1, 1, 10, 10) + num_filter = 1 + kernel = (1, 1) + stride = (1, 1) + pad = (0, 0) + + # Define a model + net = mx.sym.Variable('data') + net = mx.symbol.Deconvolution( + data=net, + num_filter=num_filter, + kernel=kernel, + stride=stride, + pad=pad, + name='deconv_1' + ) + # Test the mxnet model + self._test_mxnet_model(net, input_shape=input_shape, mode='random') + + def test_tiny_deconv_ones_input(self): + np.random.seed(1988) + input_shape = (1, 1, 10, 10) + num_filter = 1 + kernel = (5, 5) + stride = (1, 1) + pad = (0, 0) + + # Define a model + net = mx.sym.Variable('data') + net = mx.symbol.Deconvolution( + data=net, + num_filter=num_filter, + kernel=kernel, + stride=stride, + pad=pad, + name='deconv_1' + ) + # Test the mxnet model + self._test_mxnet_model(net, input_shape=input_shape, mode='ones') + + def test_tiny_deconv_random_input(self): + np.random.seed(1988) + input_shape = (1, 1, 10, 10) + num_filter = 1 + kernel = (5, 5) + stride = (1, 1) + pad = (0, 0) + + # Define a model + net = mx.sym.Variable('data') + net = mx.symbol.Deconvolution( + data=net, + num_filter=num_filter, + kernel=kernel, + stride=stride, + pad=pad, + name='deconv_1' + ) + # Test the mxnet model + self._test_mxnet_model(net, input_shape=input_shape, mode='random') + + def test_tiny_asym_deconv_random_input(self): + np.random.seed(1988) + input_shape = (1, 1, 10, 10) + num_filter = 1 + kernel = (5, 3) + stride = (1, 1) + pad = (0, 0) + + # Define a model + net = mx.sym.Variable('data') + net = mx.symbol.Deconvolution( + data=net, + num_filter=num_filter, + kernel=kernel, + stride=stride, + pad=pad, + name='deconv_1' + ) + # Test the mxnet model + self._test_mxnet_model(net, input_shape=input_shape, mode='random') + + def test_tiny_asym_deconv_random_asym_input(self): + np.random.seed(1988) + input_shape = (1, 1, 28, 18) + num_filter = 16 + kernel = (5, 3) + stride = (1, 1) + pad = (0, 0) + dilate = (1, 1) + + # define a model + net = mx.sym.Variable('data') + net = mx.symbol.Deconvolution( + data=net, + num_filter=num_filter, + kernel=kernel, + stride=stride, + pad=pad, + dilate=dilate, + name='deconv_1' + ) + net = mx.sym.Activation(net, name = 'tanh', act_type = "tanh") + # Test the mxnet model + self._test_mxnet_model(net, input_shape=input_shape, mode='random') + + def test_tiny_deconv_pooling_random_input(self): + np.random.seed(1988) + input_shape = (1, 1, 10, 10) + num_filter = 1 + kernel = (5, 5) + stride = (1, 1) + pad = (0, 0) + + # define a model + net = mx.sym.Variable('data') + net = mx.symbol.Deconvolution( + data=net, + num_filter=num_filter, + kernel=kernel, + stride=stride, + pad=pad, + name='deconv_1' + ) + net = mx.symbol.Pooling( + data=net, + kernel=kernel, + stride=stride, + pad=pad, + name='pool_1', + pool_type='max' + ) + # Test the mxnet model + self._test_mxnet_model(net, input_shape=input_shape, mode='random') + + def test_really_tiny_deconv_random_3d_input(self): + np.random.seed(1988) + input_shape = (1, 3, 10, 10) + num_filter = 1 + kernel = (1, 1) + stride = (1, 1) + pad = (0, 0) + + # define a model + net = mx.sym.Variable('data') + net = mx.symbol.Deconvolution( + data=net, + num_filter=num_filter, + kernel=kernel, + stride=stride, + pad=pad, + name='deconv_1' + ) + # Test the mxnet model + self._test_mxnet_model(net, input_shape=input_shape, mode='random') + + def test_really_tiny_deconv_random_input_multi_filter(self): + np.random.seed(1988) + input_shape = (1, 1, 10, 10) + num_filter = 64 + kernel = (1, 1) + stride = (1, 1) + pad = (0, 0) + + # define a model + net = mx.sym.Variable('data') + net = mx.symbol.Deconvolution( + data=net, + num_filter=num_filter, + kernel=kernel, + stride=stride, + pad=pad, + name='deconv_1' + ) + # Test the mxnet model + self._test_mxnet_model(net, input_shape=input_shape, mode='random') + + def test_tiny_deconv_random_3d_input(self): + np.random.seed(1988) + input_shape = (1, 3, 10, 10) + num_filter = 1 + kernel = (5, 5) + stride = (1, 1) + pad = (0, 0) + + # define a model + net = mx.sym.Variable('data') + net = mx.symbol.Deconvolution( + data=net, + num_filter=num_filter, + kernel=kernel, + stride=stride, + pad=pad, + name='deconv_1' + ) + # Test the mxnet model + self._test_mxnet_model(net, input_shape=input_shape, mode='random') + + def test_tiny_deconv_random_input_multi_filter(self): + np.random.seed(1988) + input_shape = (1, 1, 10, 10) + num_filter = 64 + kernel = (5 ,5) + stride = (1, 1) + pad = (0, 0) + + # define a model + net = mx.sym.Variable('data') + net = mx.symbol.Deconvolution( + data=net, + num_filter=num_filter, + kernel=kernel, + stride=stride, + pad=pad, + name='deconv_1' + ) + # Test the mxnet model + self._test_mxnet_model(net, input_shape=input_shape, mode='random') + + def test_deconv_random(self): + np.random.seed(1988) + input_shape = (1, 10, 4, 4) + num_filter = 3 + kernel = (2, 2) + stride = (1, 1) + pad = (0, 0) + + # define a model + net = mx.sym.Variable('data') + net = mx.symbol.Deconvolution( + data=net, + num_filter=num_filter, + kernel=kernel, + stride=stride, + pad=pad, + no_bias=False, + name='deconv_1' + ) + # test the mxnet model + self._test_mxnet_model(net, input_shape=input_shape, mode='random') + + def test_deconv_random_output_shape(self): + np.random.seed(1988) + input_shape = (1, 10, 4, 4) + num_filter = 3 + kernel = (2, 2) + stride = (1, 1) + pad = (0, 0) + target_shape = (5, 5) + + # define a model + net = mx.sym.Variable('data') + net = mx.symbol.Deconvolution( + data=net, + num_filter=num_filter, + kernel=kernel, + stride=stride, + pad=pad, + no_bias=False, + target_shape=target_shape, + name='deconv_1' + ) + # test the mxnet model + self._test_mxnet_model(net, input_shape=input_shape, mode='random') + + def test_deconv_random_padding(self): + np.random.seed(1988) + input_shape = (1, 10, 9, 9) + num_filter = 3 + kernel = (3, 3) + stride = (3, 3) + pad = (2, 2) + + # define a model + net = mx.sym.Variable('data') + net = mx.symbol.Deconvolution( + data=net, + num_filter=num_filter, + kernel=kernel, + stride=stride, + pad=pad, + no_bias=False, + name='deconv_1') + # test the mxnet model + self._test_mxnet_model(net, input_shape=input_shape, mode='random') + + def test_conv_random_padding_odd(self): + np.random.seed(1988) + input_shape = (1, 10, 6, 6) + num_filter = 3 + kernel = (5, 5) + stride = (1, 1) + pad = (3, 3) + + # define a model + net = mx.sym.Variable('data') + net = mx.symbol.Convolution( + data=net, + num_filter=num_filter, + kernel=kernel, + stride=stride, + pad=pad, + no_bias=False, + name='conv_1' + ) + self._test_mxnet_model(net, input_shape=input_shape, mode='random') + + def test_conv_random_padding_even(self): + np.random.seed(1988) + input_shape = (1, 10, 6, 6) + num_filter = 3 + kernel = (5, 5) + stride = (1, 1) + pad = (2, 2) + + # define a model + net = mx.sym.Variable('data') + net = mx.symbol.Convolution( + data=net, + num_filter=num_filter, + kernel=kernel, + stride=stride, + pad=pad, + no_bias=False, + name='conv_1' + ) + self._test_mxnet_model(net, input_shape=input_shape, mode='random') + + def test_deconv_random_all_inputs(self): + np.random.seed(1988) + input_shape = (1, 10, 5, 5) + num_filter = 3 + kernel = (3, 3) + stride = (2, 2) + pad = (1, 1) + dilate = (1, 1) + target_shape = (11, 11) + + # define a model + net = mx.sym.Variable('data') + net = mx.symbol.Deconvolution( + data=net, + num_filter=num_filter, + kernel=kernel, + stride=stride, + pad=pad, + no_bias=False, + target_shape=target_shape, + dilate=dilate, + name='deconv_1' + ) + self._test_mxnet_model(net, input_shape=input_shape, mode='random') + + def test_batch_norm(self): + np.random.seed(1988) + input_shape = (1, 1, 2, 3) + + net = mx.sym.Variable('data') + gamma = mx.sym.Variable('gamma') + beta = mx.sym.Variable('beta') + moving_mean = mx.sym.Variable('moving_mean') + moving_var = mx.sym.Variable('moving_var') + net = mx.symbol.BatchNorm( + data=net, + gamma=gamma, + beta=beta, + moving_mean=moving_mean, + moving_var=moving_var, + use_global_stats=True, + name='batch_norm_1') + self._test_mxnet_model(net, input_shape=input_shape, mode='random', delta=1e-2) + + def test_batch_norm_no_global_stats(self): + """ This test should throw an exception since converter doesn't support + conversion of MXNet models that use local batch stats (i.e. + use_global_stats=False). The reason for this is CoreML doesn't support + local batch stats. + """ + np.random.seed(1988) + input_shape = (1, 1, 2, 3) + + net = mx.sym.Variable('data') + gamma = mx.sym.Variable('gamma') + beta = mx.sym.Variable('beta') + moving_mean = mx.sym.Variable('moving_mean') + moving_var = mx.sym.Variable('moving_var') + net = mx.symbol.BatchNorm( + data=net, + gamma=gamma, + beta=beta, + moving_mean=moving_mean, + moving_var=moving_var, + use_global_stats=False, + name='batch_norm_1') + self._test_mxnet_model(net, input_shape=input_shape, mode='random', delta=1e-2) + + def test_pre_processing_args(self): + np.random.seed(1988) + input_shape = (1, 10) + net = mx.sym.Variable('data') + net = mx.sym.FullyConnected(data=net, name='fc1', num_hidden=5) + net = mx.sym.SoftmaxOutput(net, name='softmax') + self._test_mxnet_model(net, input_shape=input_shape, mode='random', label_names=['softmax_label'], + pre_processing_args={'red_bias':0, 'blue_bias':0, 'green_bias':0, 'image_scale':1}) + + # TODO test_concat + + +if __name__ == '__main__': + suite = unittest.TestLoader().loadTestsFromTestCase(SingleLayerTest) + unittest.TextTestRunner(verbosity=2).run(suite) diff --git a/tools/coreml/test/test_mxnet_image.py b/tools/coreml/test/test_mxnet_image.py new file mode 100644 index 000000000000..ac30ac7f5ad9 --- /dev/null +++ b/tools/coreml/test/test_mxnet_image.py @@ -0,0 +1,136 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import mxnet as mx +import numpy as np +import unittest +import sys +import os +current_working_directory = os.getcwd() +sys.path.append(current_working_directory + "/..") +sys.path.append(current_working_directory + "/../converter/") +import _mxnet_converter as mxnet_converter +from utils import load_model + + +VAL_DATA = 'data/val-5k-256.rec' +URL = 'http://data.mxnet.io/data/val-5k-256.rec' + + +def download_data(): + return mx.test_utils.download(URL, VAL_DATA) + + +def read_image(data_val, label_name): + data = mx.io.ImageRecordIter( + path_imgrec=data_val, + label_width=1, + preprocess_threads=4, + batch_size=32, + data_shape=(3,224,224), + label_name=label_name, + rand_corp=False, + rand_mirror=False, + shuffle=True + ) + return data + + +def is_correct_top_one(predict, label): + assert isinstance(predict, np.ndarray) + assert isinstance(label, np.float32) + predicted_label = np.argmax(predict) + return predicted_label == label + + +def is_correct_top_five(predict, label): + assert isinstance(predict, np.ndarray) + assert isinstance(label, np.float32) + top_five_preds = set(predict.argsort()[-5:]) + return label in top_five_preds + + +class ImageNetTest(unittest.TestCase): + def _test_image_prediction(self, model_name, epoch, label_name): + try: + data = read_image(VAL_DATA, label_name=label_name) + except: + download_data() + data = read_image(VAL_DATA, label_name=label_name) + + mod = load_model( + model_name=model_name, + epoch_num=epoch, + data_shapes=data.provide_data, + label_shapes=data.provide_label, + label_names=[label_name,] + ) + + input_shape = (1, 3, 224, 224) + coreml_model = mxnet_converter.convert(mod, input_shape={'data': input_shape}) + + mxnet_acc = [] + mxnet_top_5_acc = [] + coreml_acc = [] + coreml_top_5_acc = [] + + num_batch = 0 + + for batch in data: + mod.forward(batch, is_train=False) + mxnet_preds = mod.get_outputs()[0].asnumpy() + data_numpy = batch.data[0].asnumpy() + label_numpy = batch.label[0].asnumpy() + for i in xrange(32): + input_data = {'data': data_numpy[i]} + coreml_predict = coreml_model.predict(input_data).values()[0].flatten() + mxnet_predict = mxnet_preds[i] + label = label_numpy[i] + mxnet_acc.append(is_correct_top_one(mxnet_predict, label)) + mxnet_top_5_acc.append(is_correct_top_five(mxnet_predict, label)) + coreml_acc.append(is_correct_top_one(coreml_predict, label)) + coreml_top_5_acc.append(is_correct_top_five(coreml_predict, label)) + num_batch += 1 + if (num_batch == 5): break # we only use a subset of the batches. + + print "MXNet acc %s" % np.mean(mxnet_acc) + print "Coreml acc %s" % np.mean(coreml_acc) + print "MXNet top 5 acc %s" % np.mean(mxnet_top_5_acc) + print "Coreml top 5 acc %s" % np.mean(coreml_top_5_acc) + self.assertAlmostEqual(np.mean(mxnet_acc), np.mean(coreml_acc), delta=1e-4) + self.assertAlmostEqual(np.mean(mxnet_top_5_acc), np.mean(coreml_top_5_acc), delta=1e-4) + + def test_squeezenet(self): + print "Testing Image Classification with Squeezenet" + self._test_image_prediction(model_name='squeezenet_v1.1', epoch=0, label_name='prob_label') + + def test_inception_with_batch_normalization(self): + print "Testing Image Classification with Inception/BatchNorm" + self._test_image_prediction(model_name='Inception-BN', epoch=126, label_name='softmax_label') + + def test_resnet18(self): + print "Testing Image Classification with ResNet18" + self._test_image_prediction(model_name='resnet-18', epoch=0, label_name='softmax_label') + + def test_vgg16(self): + print "Testing Image Classification with vgg16" + self._test_image_prediction(model_name='vgg16', epoch=0, label_name='prob_label') + + +if __name__ == '__main__': + suite = unittest.TestLoader().loadTestsFromTestCase(ImageNetTest) + unittest.TextTestRunner(verbosity=2).run(suite) \ No newline at end of file diff --git a/tools/coreml/test/test_mxnet_models.py b/tools/coreml/test/test_mxnet_models.py new file mode 100644 index 000000000000..1732fb833c5f --- /dev/null +++ b/tools/coreml/test/test_mxnet_models.py @@ -0,0 +1,155 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import unittest +import mxnet as mx +import numpy as np +import sys +import os +current_working_directory = os.getcwd() +sys.path.append(current_working_directory + "/..") +sys.path.append(current_working_directory + "/../converter/") +import _mxnet_converter as mxnet_converter +from collections import namedtuple + + +def _mxnet_remove_batch(input_data): + for blob in input_data: + input_data[blob] = np.reshape(input_data[blob], input_data[blob].shape[1:]) + return input_data + + +def _kl_divergence(distribution1, distribution2): + """ Calculates Kullback-Leibler Divergence b/w two distributions. + + Parameters + ---------- + distribution1: list of floats + distribution2: list of floats + """ + assert len(distribution1) == len(distribution2) + n = len(distribution1) + result = 1./n * sum(distribution1 * (np.log(distribution1) - np.log(distribution2))) + return result + + +class ModelsTest(unittest.TestCase): + """ + Unit test class that tests converter on entire MXNet models . + In order to test each unit test converts MXNet model into CoreML model using the converter, generate predictions + on both MXNet and CoreML and verifies that predictions are same (or similar). + """ + def _load_model(self, model_name, epoch_num, input_shape): + sym, arg_params, aux_params = mx.model.load_checkpoint(model_name, epoch_num) + mod = mx.mod.Module( + symbol=sym, + context=mx.cpu(), + label_names=None + ) + mod.bind( + for_training=False, + data_shapes=[('data', input_shape)], + label_shapes=mod._label_shapes + ) + mod.set_params( + arg_params=arg_params, + aux_params=aux_params, + allow_missing=True + ) + return mod + + def _test_model(self, model_name, epoch_num, input_shape=(1, 3, 224, 224), files=None): + """ Tests whether the converted CoreML model's preds are equal to MXNet preds for a given model or not. + + Parameters + ---------- + model_name: str + Prefix of the MXNet model name as stored on the local directory. + + epoch_num : int + Epoch number of model we would like to load. + + input_shape: tuple + The shape of the input data in the form of (batch_size, channels, height, width) + + files: list of strings + List of URLs pertaining to files that need to be downloaded in order to use the model. + """ + + if files is not None: + print("Downloading files from urls: %s" % (files)) + for url in files: + mx.test_utils.download(url) + print("Downloaded %s" % (url)) + + module = self._load_model( + model_name=model_name, + epoch_num=epoch_num, + input_shape=input_shape + ) + + coreml_model = mxnet_converter.convert(module, input_shape={'data': input_shape}) + + # Get predictions from MXNet and coreml + div=[] # For storing KL divergence for each input. + for _ in xrange(1): + np.random.seed(1993) + input_data = {'data': np.random.uniform(0, 1, input_shape).astype(np.float32)} + Batch = namedtuple('Batch', ['data']) + module.forward(Batch([mx.nd.array(input_data['data'])]), is_train=False) + mxnet_pred = module.get_outputs()[0].asnumpy().flatten() + coreml_pred = coreml_model.predict(_mxnet_remove_batch(input_data)).values()[0].flatten() + self.assertEqual(len(mxnet_pred), len(coreml_pred)) + div.append(_kl_divergence(mxnet_pred, coreml_pred)) + + print "Average KL divergence is % s" % np.mean(div) + self.assertTrue(np.mean(div) < 1e-4) + + def test_pred_inception_bn(self): + self._test_model(model_name='Inception-BN', epoch_num=126, + files=["http://data.mxnet.io/models/imagenet/inception-bn/Inception-BN-0126.params", + "http://data.mxnet.io/models/imagenet/inception-bn/Inception-BN-symbol.json"]) + + def test_pred_squeezenet_v11(self): + self._test_model(model_name='squeezenet_v1.1', epoch_num=0, + files=["http://data.mxnet.io/models/imagenet/squeezenet/squeezenet_v1.1-symbol.json", + "http://data.mxnet.io/models/imagenet/squeezenet/squeezenet_v1.1-0000.params"]) + + def test_pred_resnet_50(self): + self._test_model(model_name='resnet-50', epoch_num=0, + files=["http://data.mxnet.io/models/imagenet/resnet/50-layers/resnet-50-symbol.json", + "http://data.mxnet.io/models/imagenet/resnet/50-layers/resnet-50-0000.params"]) + + def test_pred_vgg16(self): + self._test_model(model_name='vgg16', epoch_num=0, + files=["http://data.mxnet.io/models/imagenet/vgg/vgg16-symbol.json", + "http://data.mxnet.io/models/imagenet/vgg/vgg16-0000.params"]) + + def test_pred_nin(self): + self._test_model(model_name='nin', epoch_num=0, + files=["http://data.dmlc.ml/models/imagenet/nin/nin-symbol.json", + "http://data.dmlc.ml/models/imagenet/nin/nin-0000.params"]) + + @unittest.skip("You need to download and unzip file: " + "http://data.mxnet.io/models/imagenet/inception-v3.tar.gz in order to run this test.") + def test_pred_inception_v3(self): + self._test_model(model_name='Inception-7', epoch_num=1, input_shape=(1, 3, 299, 299)) + + +if __name__ == '__main__': + suite = unittest.TestLoader().loadTestsFromTestCase(ModelsTest) + unittest.TextTestRunner(verbosity=2).run(suite) diff --git a/tools/coreml/test_mxnet_converer.py b/tools/coreml/test_mxnet_converer.py deleted file mode 100644 index 179d04a10930..000000000000 --- a/tools/coreml/test_mxnet_converer.py +++ /dev/null @@ -1,477 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import unittest -import mxnet as mx -import numpy as np -import tempfile -import os -import mxnet_converter -import coremltools - -def _mxnet_remove_batch(input_data): - for blob in input_data: - input_data[blob] = np.reshape(input_data[blob], input_data[blob].shape[1:]) - return input_data - -def _get_coreml_model(net, engine, model_path, input_shape, - input_names = ['data'], output_names = ['output']): - model = mx.model.FeedForward(net, engine, arg_params = engine.arg_dict) - spec = mxnet_converter.convert(model, **input_shape) - return coremltools.models.MLModel(spec) - -def set_weights(net, engine, mode = 'random'): - for arg in net.list_arguments(): - if mode == 'random': - engine.arg_dict[arg][:] = np.random.uniform(-0.1, 0.1, engine.arg_dict[arg].shape) - elif mode == 'zeros': - engine.arg_dict[arg][:] = np.zeros(engine.arg_dict[arg].shape) - elif mode == 'ones': - engine.arg_dict[arg][:] = np.ones(engine.arg_dict[arg].shape) - return net - -class MXNetSingleLayerTest(unittest.TestCase): - """ - Unit test class for testing mxnet converter. - """ - def _test_mxnet_model(self, net, engine, delta = 1e-3, **input_shape): - - # Generate some dummy data - input_data = {} - for ip in input_shape: - input_data[ip] = engine.arg_dict[ip].asnumpy() - output_blob = net.list_outputs()[0] - - # Make predictions from mxnet (only works on single output for now) - mxnet_preds = engine.forward()[0].asnumpy().flatten() - - # Get predictions from coreml - model_path = os.path.join(tempfile.mkdtemp(), 'mxnet.mlmodel') - model = _get_coreml_model(net, engine, model_path, input_shape, input_data.keys()) - coreml_preds = model.predict(_mxnet_remove_batch(input_data)).values()[0].flatten() - - # Check prediction accuracy - self.assertEquals(len(mxnet_preds), len(coreml_preds)) - for i in range(len(mxnet_preds)): - self.assertAlmostEquals(mxnet_preds[i], coreml_preds[i], delta = delta) - - def test_tiny_inner_product_zero_input(self): - np.random.seed(1988) - input_shape = (1, 10) - - # Define a model - net = mx.sym.Variable('data') - net = mx.sym.FullyConnected(data = net, name = 'fc1', num_hidden = 5) - engine = net.simple_bind(ctx=mx.cpu(), data=input_shape) - - # Set some random weights - set_weights(net, engine, mode = 'zeros') - - # Test the mxnet model - self._test_mxnet_model(net, engine, data = input_shape) - - def test_really_tiny_inner_product_ones_input(self): - np.random.seed(1988) - input_shape = (1, 1) - - # Define a model - net = mx.sym.Variable('data') - net = mx.sym.FullyConnected(data = net, name = 'fc1', num_hidden = 1) - engine = net.simple_bind(ctx=mx.cpu(), data=input_shape) - - # Set some random weights - set_weights(net, engine, mode = 'ones') - - # Test the mxnet model - self._test_mxnet_model(net, engine, data = input_shape) - - def test_really_tiny_2_inner_product_ones_input(self): - np.random.seed(1988) - input_shape = (1, 1) - - # Define a model - net = mx.sym.Variable('data') - net = mx.sym.FullyConnected(data = net, name = 'fc1', num_hidden = 5) - engine = net.simple_bind(ctx=mx.cpu(), data=input_shape) - - # Set some random weights - set_weights(net, engine, mode = 'ones') - - # Test the mxnet model - self._test_mxnet_model(net, engine, data = input_shape) - - def test_tiny_inner_product_ones_input(self): - np.random.seed(1988) - input_shape = (1, 10) - - # Define a model - net = mx.sym.Variable('data') - net = mx.sym.FullyConnected(data = net, name = 'fc1', num_hidden = 5) - engine = net.simple_bind(ctx=mx.cpu(), data=input_shape) - - # Set some random weights - set_weights(net, engine, mode = 'ones') - - # Test the mxnet model - self._test_mxnet_model(net, engine, data = input_shape) - - def test_tiny_inner_product_random_input(self): - np.random.seed(1988) - input_shape = (1, 10) - - # Define a model - net = mx.sym.Variable('data') - net = mx.sym.FullyConnected(data = net, name = 'fc1', num_hidden = 5) - engine = net.simple_bind(ctx=mx.cpu(), data=input_shape) - - # Set some random weights - set_weights(net, engine, mode = 'random') - - # Test the mxnet model - self._test_mxnet_model(net, engine, data = input_shape) - - def test_tiny_softmax_random_input(self): - np.random.seed(1988) - input_shape = (1, 10) - - # Define a model - net = mx.sym.Variable('data') - net = mx.sym.FullyConnected(data = net, name = 'fc1', num_hidden = 5) - net = mx.sym.SoftmaxOutput(net, name = 'softmax') - engine = net.simple_bind(ctx = mx.cpu(), data = input_shape) - - # Set some random weights - set_weights(net, engine, mode = 'random') - - # Test the mxnet model - self._test_mxnet_model(net, engine, data = input_shape) - - def test_tiny_relu_activation_random_input(self): - np.random.seed(1988) - input_shape = (1, 10) - - # Define a model - net = mx.sym.Variable('data') - net = mx.sym.FullyConnected(data = net, name = 'fc1', num_hidden = 5) - net = mx.sym.Activation(net, name = 'relu1', act_type = "relu") - engine = net.simple_bind(ctx = mx.cpu(), data = input_shape) - - # Set some random weights - set_weights(net, engine, mode = 'random') - - # Test the mxnet model - self._test_mxnet_model(net, engine, data = input_shape) - - def test_tiny_sigmoid_activation_random_input(self): - np.random.seed(1988) - input_shape = (1, 10) - - # Define a model - net = mx.sym.Variable('data') - net = mx.sym.FullyConnected(data = net, name = 'fc1', num_hidden = 5) - net = mx.sym.Activation(net, name = 'sigmoid1', act_type = "sigmoid") - engine = net.simple_bind(ctx = mx.cpu(), data = input_shape) - - # Set some random weights - set_weights(net, engine, mode = 'random') - - # Test the mxnet model - self._test_mxnet_model(net, engine, data = input_shape) - - def test_tiny_tanh_activation_random_input(self): - np.random.seed(1988) - input_shape = (1, 10) - - # Define a model - net = mx.sym.Variable('data') - net = mx.sym.FullyConnected(data = net, name = 'fc1', num_hidden = 5) - net = mx.sym.Activation(net, name = 'tanh1', act_type = "tanh") - engine = net.simple_bind(ctx = mx.cpu(), data = input_shape) - - # Set some random weights - set_weights(net, engine, mode = 'random') - - # Test the mxnet model - self._test_mxnet_model(net, engine, data = input_shape) - - def test_really_tiny_conv_random_input(self): - np.random.seed(1988) - input_shape = (1, 1, 10, 10) - num_filter = 1 - kernel = (1 ,1) - stride = (1, 1) - pad = (0, 0) - - # Define a model - net = mx.sym.Variable('data') - net = mx.symbol.Convolution(data = net, num_filter = num_filter, kernel=kernel, - stride = stride, pad = pad, name = 'conv_1') - engine = net.simple_bind(ctx = mx.cpu(), data = input_shape) - - # Set some random weights - set_weights(net, engine, mode = 'random') - - # Test the mxnet model - self._test_mxnet_model(net, engine, data = input_shape) - - def test_tiny_conv_ones_input(self): - np.random.seed(1988) - input_shape = (1, 1, 10, 10) - num_filter = 1 - kernel = (5 ,5) - stride = (1, 1) - pad = (0, 0) - - # Define a model - net = mx.sym.Variable('data') - net = mx.symbol.Convolution(data = net, num_filter = num_filter, kernel=kernel, - stride = stride, pad = pad, name = 'conv_1') - engine = net.simple_bind(ctx = mx.cpu(), data = input_shape) - - # Set some random weights - set_weights(net, engine, mode = 'ones') - - # Test the mxnet model - self._test_mxnet_model(net, engine, data = input_shape) - - def test_tiny_conv_random_input(self): - np.random.seed(1988) - input_shape = (1, 1, 10, 10) - num_filter = 1 - kernel = (5 ,5) - stride = (1, 1) - pad = (0, 0) - - # define a model - net = mx.sym.Variable('data') - net = mx.symbol.Convolution(data = net, num_filter = num_filter, kernel=kernel, - stride = stride, pad = pad, name = 'conv_1') - engine = net.simple_bind(ctx = mx.cpu(), data = input_shape) - - # set some random weights - set_weights(net, engine, mode = 'random') - - # test the mxnet model - self._test_mxnet_model(net, engine, data = input_shape) - - def test_tiny_asym_conv_random_input(self): - np.random.seed(1988) - input_shape = (1, 1, 10, 10) - num_filter = 1 - kernel = (5 ,3) - stride = (1, 1) - pad = (0, 0) - - # define a model - net = mx.sym.Variable('data') - net = mx.symbol.Convolution(data = net, num_filter = num_filter, kernel=kernel, - stride = stride, pad = pad, name = 'conv_1') - engine = net.simple_bind(ctx = mx.cpu(), data = input_shape) - - # set some random weights - set_weights(net, engine, mode = 'random') - - # test the mxnet model - self._test_mxnet_model(net, engine, data = input_shape) - - def test_tiny_asym_conv_random_asym_input(self): - np.random.seed(1988) - input_shape = (1, 1, 28, 18) - num_filter = 16 - kernel = (5 ,3) - stride = (1, 1) - pad = (0, 0) - dilate = (1, 1) - - # define a model - net = mx.sym.Variable('data') - net = mx.symbol.Convolution(data = net, num_filter = num_filter, kernel=kernel, - stride = stride, pad = pad, name = 'conv_1', dilate = dilate) - net = mx.sym.Activation(net, name = 'tanh', act_type = "tanh") - engine = net.simple_bind(ctx = mx.cpu(), data = input_shape) - - # set some random weights - set_weights(net, engine, mode = 'random') - - # test the mxnet model - self._test_mxnet_model(net, engine, data = input_shape) - - def test_tiny_conv_pooling_random_input(self): - np.random.seed(1988) - input_shape = (1, 1, 10, 10) - num_filter = 1 - kernel = (5 ,5) - stride = (1, 1) - pad = (0, 0) - - # define a model - net = mx.sym.Variable('data') - net = mx.symbol.Convolution(data = net, num_filter = num_filter, kernel=kernel, - stride = stride, pad = pad, name = 'conv_1') - net = mx.symbol.Pooling(data = net, kernel=kernel, - stride = stride, pad = pad, name = 'pool_1', pool_type = 'max') - engine = net.simple_bind(ctx = mx.cpu(), data = input_shape) - - # set some random weights - set_weights(net, engine, mode = 'random') - - # test the mxnet model - self._test_mxnet_model(net, engine, data = input_shape) - - def test_really_tiny_conv_random_3d_input(self): - np.random.seed(1988) - input_shape = (1, 3, 10, 10) - num_filter = 1 - kernel = (1 ,1) - stride = (1, 1) - pad = (0, 0) - - # define a model - net = mx.sym.Variable('data') - net = mx.symbol.Convolution(data = net, num_filter = num_filter, kernel=kernel, - stride = stride, pad = pad, name = 'conv_1') - engine = net.simple_bind(ctx = mx.cpu(), data = input_shape) - - # set some random weights - set_weights(net, engine, mode = 'random') - - # test the mxnet model - self._test_mxnet_model(net, engine, data = input_shape) - - def test_really_tiny_conv_random_input_multi_filter(self): - np.random.seed(1988) - input_shape = (1, 1, 10, 10) - num_filter = 64 - kernel = (1 ,1) - stride = (1, 1) - pad = (0, 0) - - # define a model - net = mx.sym.Variable('data') - net = mx.symbol.Convolution(data = net, num_filter = num_filter, kernel=kernel, - stride = stride, pad = pad, name = 'conv_1') - engine = net.simple_bind(ctx = mx.cpu(), data = input_shape) - - # set some random weights - set_weights(net, engine, mode = 'random') - - # test the mxnet model - self._test_mxnet_model(net, engine, data = input_shape) - - def test_tiny_conv_random_3d_input(self): - np.random.seed(1988) - input_shape = (1, 3, 10, 10) - num_filter = 1 - kernel = (5 ,5) - stride = (1, 1) - pad = (0, 0) - - # define a model - net = mx.sym.Variable('data') - net = mx.symbol.Convolution(data = net, num_filter = num_filter, kernel=kernel, - stride = stride, pad = pad, name = 'conv_1') - engine = net.simple_bind(ctx = mx.cpu(), data = input_shape) - - # set some random weights - set_weights(net, engine, mode = 'random') - - # test the mxnet model - self._test_mxnet_model(net, engine, data = input_shape) - - def test_tiny_conv_random_input_multi_filter(self): - np.random.seed(1988) - input_shape = (1, 1, 10, 10) - num_filter = 64 - kernel = (5 ,5) - stride = (1, 1) - pad = (0, 0) - - # define a model - net = mx.sym.Variable('data') - net = mx.symbol.Convolution(data = net, num_filter = num_filter, kernel=kernel, - stride = stride, pad = pad, name = 'conv_1') - engine = net.simple_bind(ctx = mx.cpu(), data = input_shape) - - # set some random weights - set_weights(net, engine, mode = 'random') - - # test the mxnet model - self._test_mxnet_model(net, engine, data = input_shape) - - def test_conv_random(self): - np.random.seed(1988) - input_shape = (1, 3, 10, 10) - num_filter = 64 - kernel = (5 ,5) - stride = (1, 1) - pad = (0, 0) - - # define a model - net = mx.sym.Variable('data') - net = mx.symbol.Convolution(data = net, num_filter = num_filter, kernel=kernel, - stride = stride, pad = pad, name = 'conv_1') - engine = net.simple_bind(ctx = mx.cpu(), data = input_shape) - - # set some random weights - set_weights(net, engine, mode = 'random') - - # test the mxnet model - self._test_mxnet_model(net, engine, data = input_shape) - - def test_flatten(self): - np.random.seed(1988) - input_shape = (1, 3, 10, 10) - num_filter = 64 - kernel = (5 ,5) - stride = (1, 1) - pad = (0, 0) - - # define a model - net = mx.sym.Variable('data') - net = mx.symbol.Convolution(data = net, num_filter = num_filter, kernel=kernel, - stride = stride, pad = pad, name = 'conv_1') - net = mx.sym.Flatten(data = net, name = 'flatten1') - net = mx.sym.FullyConnected(data = net, name = 'fc1', num_hidden = 5) - net = mx.sym.SoftmaxOutput(net, name = 'softmax') - engine = net.simple_bind(ctx = mx.cpu(), data = input_shape) - - # set some random weights - set_weights(net, engine, mode = 'random') - - # test the mxnet model - self._test_mxnet_model(net, engine, data = input_shape) - - def test_transpose(self): - np.random.seed(1988) - input_shape = (1, 3, 10, 10) - num_filter = 64 - kernel = (5 ,5) - stride = (1, 1) - pad = (0, 0) - - # define a model - net = mx.sym.Variable('data') - net = mx.sym.transpose(data = net, name = 'transpose', axes = (0, 1, 2, 3)) - net = mx.symbol.Convolution(data = net, num_filter = num_filter, kernel=kernel, - stride = stride, pad = pad, name = 'conv_1') - engine = net.simple_bind(ctx = mx.cpu(), data = input_shape) - - # set some random weights - set_weights(net, engine, mode = 'random') - - # test the mxnet model - self._test_mxnet_model(net, engine, data = input_shape) diff --git a/tools/coreml/utils.py b/tools/coreml/utils.py new file mode 100644 index 000000000000..1e4ff7a4d975 --- /dev/null +++ b/tools/coreml/utils.py @@ -0,0 +1,77 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import mxnet as mx + + +def load_model(model_name, epoch_num, data_shapes, label_shapes, label_names, gpus=''): + """Loads and returns a given MXNet model. + + Parameters + ---------- + model_name: str + Prefix of the MXNet model name as stored on the local directory. + + epoch_num : int + Epoch number of model we would like to load. + + input_shape: tuple + The shape of the input data in the form of (batch_size, channels, height, width) + + files: list of strings + List of URLs pertaining to files that need to be downloaded in order to use the model. + + data_shapes: list of tuples. + List of tuples where each tuple is a pair of input variable name and its shape. + + label_shapes: list of (str, tuple) + Typically is ``data_iter.provide_label``. + + label_names: list of str + Name of the output labels in the MXNet symbolic graph. + + gpus: str + Comma separated string of gpu ids on which inferences are executed. E.g. 3,5,6 would refer to GPUs 3, 5 and 6. + If empty, we use CPU. + + Returns + ------- + MXNet module + """ + sym, arg_params, aux_params = mx.model.load_checkpoint(model_name, epoch_num) + if gpus == '': + devices = mx.cpu() + else: + devices = [mx.gpu(int(i)) for i in gpus.split(',')] + mod = mx.mod.Module( + symbol=sym, + context=devices, + label_names=label_names + ) + mod.bind( + for_training=False, + data_shapes=data_shapes, + label_shapes=label_shapes + ) + mod.set_params( + arg_params=arg_params, + aux_params=aux_params, + allow_missing=True + ) + return mod + + From 606d3a924ec1a42f26ffbbc5f42c59f799b75a54 Mon Sep 17 00:00:00 2001 From: Yao Wang Date: Fri, 11 Aug 2017 23:58:17 -0700 Subject: [PATCH 368/834] Website fix (#7435) * Modify website * Small fix * Fix --- docs/_static/mxnet-theme/footer.html | 2 +- docs/_static/mxnet-theme/index.html | 4 ++-- docs/build_version_doc/AddVersion.py | 5 ++++- docs/build_version_doc/build_doc.sh | 8 +++----- docs/how_to/index.md | 2 ++ 5 files changed, 12 insertions(+), 9 deletions(-) mode change 100644 => 100755 docs/build_version_doc/AddVersion.py diff --git a/docs/_static/mxnet-theme/footer.html b/docs/_static/mxnet-theme/footer.html index f7eec1321724..45ba457a0722 100644 --- a/docs/_static/mxnet-theme/footer.html +++ b/docs/_static/mxnet-theme/footer.html @@ -1,5 +1,5 @@
    -

    © 2015-2017 DMLC. All rights reserved.

    +

    diff --git a/docs/_static/mxnet-theme/index.html b/docs/_static/mxnet-theme/index.html index b39e710d6155..e381428758c0 100644 --- a/docs/_static/mxnet-theme/index.html +++ b/docs/_static/mxnet-theme/index.html @@ -46,7 +46,7 @@

    MXNet Joining Apache

    Examples

    Explore projects from simple demos to state-of-the-art research

    - MXNet Examples + MXNet Examples
    @@ -54,7 +54,7 @@

    Examples

    Model Zoo

    Off the shelf pre-trained models

    - Model Zoo + Model Zoo
    diff --git a/docs/build_version_doc/AddVersion.py b/docs/build_version_doc/AddVersion.py old mode 100644 new mode 100755 index 38ce48f63c2f..34ba40e0f3a4 --- a/docs/build_version_doc/AddVersion.py +++ b/docs/build_version_doc/AddVersion.py @@ -25,7 +25,7 @@ help='file to be modified') parser.add_argument('--current_version', type=str, default='master', help='Current version') -parser.add_argument('--root_url', type=str, default='https://mxnet.io', +parser.add_argument('--root_url', type=str, default='https://mxnet.incubator.apache.org/', help='Root URL') if __name__ == '__main__': @@ -59,6 +59,9 @@ continue with open(os.path.join(path, name), 'r') as html_file: content = bs(html_file, 'html.parser') + if os.path.join(path, name) == args.file_path + 'index.html': + content.find(id='example-link')['href'] = \ + 'https://github.com/apache/incubator-mxnet/tree/%s/example' % (args.current_version) navbar = content.find(id="main-nav") navbar_mobile = content.find(id="burgerMenu") if navbar and navbar_mobile: diff --git a/docs/build_version_doc/build_doc.sh b/docs/build_version_doc/build_doc.sh index 5a4f15d33c9c..f98e1e0683dc 100755 --- a/docs/build_version_doc/build_doc.sh +++ b/docs/build_version_doc/build_doc.sh @@ -55,8 +55,7 @@ then make docs || exit 1 echo -e "$latest_tag\n$(cat $tag_list_file)" > "$tag_list_file" cat $tag_list_file - tests/ci_build/ci_build.sh doc python docs/build_version_doc/AddVersion.py --file_path "docs/_build/html/" \ - --current_version "$latest_tag" --root_url "http://mxnet.incubator.apache.org/" + tests/ci_build/ci_build.sh doc python docs/build_version_doc/AddVersion.py --file_path "docs/_build/html/" --current_version "$latest_tag" tests/ci_build/ci_build.sh doc python docs/build_version_doc/AddPackageLink.py \ --file_path "docs/_build/html/get_started/install.html" --current_version "$latest_tag" cp -a "docs/_build/html/." "$local_build" @@ -79,8 +78,7 @@ make docs || exit 1 rm -rfv "$web_folder/versions/master/*" cp -a "docs/_build/html/." "$web_folder/versions/master" -tests/ci_build/ci_build.sh doc python docs/build_version_doc/AddVersion.py --file_path "$web_folder/versions/master" \ - --root_url "http://mxnet.incubator.apache.org/" +tests/ci_build/ci_build.sh doc python docs/build_version_doc/AddVersion.py --file_path "$web_folder/versions/master" # Update version list for all previous version website if [ $latest_tag != ${tag_list[0]} ] @@ -89,6 +87,6 @@ then for (( i=0; i<=$(( $total -1 )); i++ )) do tests/ci_build/ci_build.sh doc python docs/build_version_doc/AddVersion.py --file_path "$web_folder/versions/${tag_list[$i]}" \ - --current_version "${tag_list[$i]}" --root_url "http://mxnet.incubator.apache.org/" + --current_version "${tag_list[$i]}" done fi diff --git a/docs/how_to/index.md b/docs/how_to/index.md index cc21aa0b8ae8..4920e1cd3f78 100644 --- a/docs/how_to/index.md +++ b/docs/how_to/index.md @@ -38,6 +38,8 @@ and full working examples, visit the [tutorials section](../tutorials/index.md). * [How do I run Keras 1.2.2 with mxnet backend?](https://github.com/dmlc/keras/wiki/Installation) +* [How to convert MXNet models to Apple CoreML format?](https://github.com/apache/incubator-mxnet/tree/master/tools/coreml) + ## Extend and Contribute to MXNet * [How do I join the MXNet development discussion?](http://mxnet.io/community/mxnet_channels.html) From 83078d7b21491936dfe552866556b40040dadf5b Mon Sep 17 00:00:00 2001 From: moin Date: Sat, 12 Aug 2017 21:12:44 +0200 Subject: [PATCH 369/834] cuda support for linalg-functions, restructuring of linalg interfaces (#7147) * cuda support for linalg-functions, restructuring of linalg interfaces * incorporate newest mshadow * adjustments to linalg operators --- CMakeLists.txt | 4 +- Makefile | 6 + include/mxnet/base.h | 7 + mshadow | 2 +- src/common/cuda_utils.h | 42 ++ src/io/inst_vector.h | 1 + .../mxnet => src/operator}/c_lapack_api.h | 53 +- src/operator/contrib/krprod.h | 2 +- src/operator/linalg.h | 118 ++++ src/operator/linalg_impl.h | 508 ++++++++++++++++++ src/operator/tensor/la_op.cc | 4 +- src/operator/tensor/la_op.cu | 77 +++ src/operator/tensor/la_op.h | 171 ++---- src/operator/tensor/la_op_inline.h | 373 ++++++------- tests/python/unittest/test_operator.py | 9 +- 15 files changed, 1008 insertions(+), 369 deletions(-) rename {include/mxnet => src/operator}/c_lapack_api.h (74%) create mode 100644 src/operator/linalg.h create mode 100644 src/operator/linalg_impl.h create mode 100644 src/operator/tensor/la_op.cu diff --git a/CMakeLists.txt b/CMakeLists.txt index ab29b6a7aaaf..dc9ca5f7bb0c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -353,8 +353,10 @@ if(USE_CUDA) list(APPEND mxnet_LINKER_LIBS ${CUDA_cuda_LIBRARY}) FIND_LIBRARY(CUDA_cufft_LIBRARY nvrtc "${CUDA_TOOLKIT_ROOT_DIR}/lib/x64" "${CUDA_TOOLKIT_ROOT_DIR}/lib/win32") list(APPEND mxnet_LINKER_LIBS "${CUDA_cufft_LIBRARY}/../cufft.lib") # For fft operator + FIND_LIBRARY(CUDA_cusolver_LIBRARY nvrtc "${CUDA_TOOLKIT_ROOT_DIR}/lib/x64" "${CUDA_TOOLKIT_ROOT_DIR}/lib/win32") + list(APPEND mxnet_LINKER_LIBS "${CUDA_cusolver_LIBRARY}/../cusolver.lib") # For cusolver else(MSVC) - list(APPEND mxnet_LINKER_LIBS nvrtc cuda cufft) + list(APPEND mxnet_LINKER_LIBS nvrtc cuda cufft cusolver) link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") endif() list(APPEND SOURCE ${cuda_objs} ${CUDA}) diff --git a/Makefile b/Makefile index 560b77a3e81e..33151e574ea7 100644 --- a/Makefile +++ b/Makefile @@ -23,6 +23,10 @@ ifndef DLPACK_PATH DLPACK_PATH = $(ROOTDIR)/dlpack endif +ifndef AMALGAMATION_PATH + AMALGAMATION_PATH = $(ROOTDIR)/amalgamation +endif + ifneq ($(USE_OPENMP), 1) export NO_OPENMP = 1 endif @@ -439,6 +443,7 @@ clean: cyclean $(EXTRA_PACKAGES_CLEAN) cd $(DMLC_CORE); $(MAKE) clean; cd - cd $(PS_PATH); $(MAKE) clean; cd - cd $(NNVM_PATH); $(MAKE) clean; cd - + cd $(AMALGAMATION_PATH); $(MAKE) clean; cd - $(RM) -r $(patsubst %, %/*.d, $(EXTRA_OPERATORS)) $(patsubst %, %/*/*.d, $(EXTRA_OPERATORS)) $(RM) -r $(patsubst %, %/*.o, $(EXTRA_OPERATORS)) $(patsubst %, %/*/*.o, $(EXTRA_OPERATORS)) else @@ -448,6 +453,7 @@ clean: cyclean testclean $(EXTRA_PACKAGES_CLEAN) cd $(DMLC_CORE); $(MAKE) clean; cd - cd $(PS_PATH); $(MAKE) clean; cd - cd $(NNVM_PATH); $(MAKE) clean; cd - + cd $(AMALGAMATION_PATH); $(MAKE) clean; cd - endif clean_all: clean diff --git a/include/mxnet/base.h b/include/mxnet/base.h index 514bb0c8e54d..695408380ec9 100644 --- a/include/mxnet/base.h +++ b/include/mxnet/base.h @@ -56,6 +56,13 @@ #define MXNET_USE_CUDNN MSHADOW_USE_CUDNN #endif +/*! + *\brief whether to use cusolver library + */ +#ifndef MXNET_USE_CUSOLVER +#define MXNET_USE_CUSOLVER MSHADOW_USE_CUSOLVER +#endif + /*! \brief Error message for using gpu when MXNET_USE_CUDA==0 */ #define MXNET_GPU_NOT_ENABLED_ERROR "GPU is not enabled" diff --git a/mshadow b/mshadow index d32b5dacf2bb..497eb9180b24 160000 --- a/mshadow +++ b/mshadow @@ -1 +1 @@ -Subproject commit d32b5dacf2bb5af4121df5fd60eb7775704f9131 +Subproject commit 497eb9180b24592b7332e7e08f2c053ec5346524 diff --git a/src/common/cuda_utils.h b/src/common/cuda_utils.h index 2879ab3cbec2..8897007207fb 100644 --- a/src/common/cuda_utils.h +++ b/src/common/cuda_utils.h @@ -87,6 +87,35 @@ inline const char* CublasGetErrorString(cublasStatus_t error) { return "Unknown cuBLAS status"; } +/*! + * \brief Get string representation of cuSOLVER errors. + * \param error The error. + * \return String representation. + */ +inline const char* CusolverGetErrorString(cusolverStatus_t error) { + switch (error) { + case CUSOLVER_STATUS_SUCCESS: + return "CUSOLVER_STATUS_SUCCESS"; + case CUSOLVER_STATUS_NOT_INITIALIZED: + return "CUSOLVER_STATUS_NOT_INITIALIZED"; + case CUSOLVER_STATUS_ALLOC_FAILED: + return "CUSOLVER_STATUS_ALLOC_FAILED"; + case CUSOLVER_STATUS_INVALID_VALUE: + return "CUSOLVER_STATUS_INVALID_VALUE"; + case CUSOLVER_STATUS_ARCH_MISMATCH: + return "CUSOLVER_STATUS_ARCH_MISMATCH"; + case CUSOLVER_STATUS_EXECUTION_FAILED: + return "CUSOLVER_STATUS_EXECUTION_FAILED"; + case CUSOLVER_STATUS_INTERNAL_ERROR: + return "CUSOLVER_STATUS_INTERNAL_ERROR"; + case CUSOLVER_STATUS_MATRIX_TYPE_NOT_SUPPORTED: + return "CUSOLVER_STATUS_MATRIX_TYPE_NOT_SUPPORTED"; + default: + break; + } + return "Unknown cuSOLVER status"; +} + /*! * \brief Get string representation of cuRAND errors. * \param status The status. @@ -164,6 +193,19 @@ inline const char* CurandGetErrorString(curandStatus_t status) { << "cuBLAS: " << common::cuda::CublasGetErrorString(e); \ } +/*! + * \brief Protected cuSolver call. + * \param func Expression to call. + * + * It checks for cuSolver errors after invocation of the expression. + */ +#define CUSOLVER_CALL(func) \ + { \ + cusolverStatus_t e = (func); \ + CHECK_EQ(e, CUSOLVER_STATUS_SUCCESS) \ + << "cuSolver: " << common::cuda::CusolverGetErrorString(e); \ + } + /*! * \brief Protected cuRAND call. * \param func Expression to call. diff --git a/src/io/inst_vector.h b/src/io/inst_vector.h index 4bc2a6c758ba..6dc7bdfd730a 100644 --- a/src/io/inst_vector.h +++ b/src/io/inst_vector.h @@ -30,6 +30,7 @@ #include #include #include +#include #include #include diff --git a/include/mxnet/c_lapack_api.h b/src/operator/c_lapack_api.h similarity index 74% rename from include/mxnet/c_lapack_api.h rename to src/operator/c_lapack_api.h index 1ae90a9396d5..96a9b3a23709 100644 --- a/include/mxnet/c_lapack_api.h +++ b/src/operator/c_lapack_api.h @@ -19,14 +19,24 @@ /*! * \file c_lapack_api.h - * \brief Unified interface for LAPACK calls from within mxnet. + * \brief Unified interface for CPU-based LAPACK calls. * Purpose is to hide the platform specific differences. */ -#ifndef MXNET_C_LAPACK_API_H_ -#define MXNET_C_LAPACK_API_H_ +#ifndef MXNET_OPERATOR_C_LAPACK_API_H_ +#define MXNET_OPERATOR_C_LAPACK_API_H_ // Manually maintained list of LAPACK interfaces that can be used // within MXNET. Conventions: +// - We should only import LAPACK-functions that are useful and +// ensure that we support them most efficiently on CPU/GPU. As an +// example take "potrs": It can be emulated by two calls to +// "trsm" (from BLAS3) so not really needed from functionality point +// of view. In addition, trsm on GPU supports batch-mode processing +// which is much more efficient for a bunch of smaller matrices while +// there is no such batch support for potrs. As a result, we may +// not support "potrs" internally and if we want to expose it to the user as +// a convenience operator at some time, then we may implement it internally +// as a sequence of trsm. // - Interfaces must be compliant with lapacke.h in terms of signature and // naming conventions so wrapping a function "foo" which has the // signature @@ -36,14 +46,21 @@ // Note that function signatures in lapacke.h will always have as first // argument the storage order (row/col-major). All wrappers have to support // that argument. The underlying fortran functions will always assume a -// column-major layout. It is the responsibility of the wrapper function -// to handle the (usual) case that it is called with data in row-major -// format, either by doing appropriate transpositions explicitly or using -// transposition options of the underlying fortran function. -// - It is ok to assume that matrices are stored in contiguous memory -// (which removes the need to do special handling for lda/ldb parameters -// and enables us to save additional matrix transpositions around -// the fortran calls). +// column-major layout. +// - In the (usual) case that a wrapper is called specifying row-major storage +// order of input/output data, there are two ways to handle this: +// 1) The wrapper may support this without allocating any additional memory +// for example by exploiting the fact that a matrix is symmetric and switching +// certain flags (upper/lower triangular) when calling the fortran code. +// 2) The wrapper may cause a runtime error. In that case it should be clearly +// documented that these functions do only support col-major layout. +// Rationale: This is a low level interface that is not expected to be called +// directly from many upstream functions. Usually all calls should go through +// the tensor-based interfaces in linalg.h which simplify calls to lapack further +// and are better suited to handle additional transpositions that may be necessary. +// Also we want to push allocation of temporary storage higher up in order to +// allow more efficient re-use of temporal storage. And don't want to plaster +// these interfaces here with additional requirements of providing buffers. // - It is desired to add some basic checking in the C++-wrappers in order // to catch simple mistakes when calling these wrappers. // - Must support compilation without lapack-package but issue runtime error in this case. @@ -54,9 +71,10 @@ using namespace mshadow; extern "C" { + // Fortran signatures #define MXNET_LAPACK_FSIGNATURE1(func, dtype) \ - void func##_(char* uplo, int* n, dtype* a, int* lda, int *info); + void func##_(char *uplo, int *n, dtype *a, int *lda, int *info); MXNET_LAPACK_FSIGNATURE1(spotrf, float) MXNET_LAPACK_FSIGNATURE1(dpotrf, double) @@ -73,9 +91,6 @@ extern "C" { #define MXNET_LAPACK_ROW_MAJOR 101 #define MXNET_LAPACK_COL_MAJOR 102 -#define CHECK_LAPACK_CONTIGUOUS(a, b) \ - CHECK_EQ(a, b) << "non contiguous memory for array in lapack call"; - #define CHECK_LAPACK_UPLO(a) \ CHECK(a == 'U' || a == 'L') << "neither L nor U specified as triangle in lapack call"; @@ -117,9 +132,9 @@ inline void flip(int m, int n, #if MXNET_USE_LAPACK + // These functions can be called with either row- or col-major format. #define MXNET_LAPACK_CWRAPPER1(func, dtype) \ - inline int MXNET_LAPACK_##func(int matrix_layout, char uplo, int n, dtype* a, int lda ) { \ - CHECK_LAPACK_CONTIGUOUS(n, lda); \ + inline int MXNET_LAPACK_##func(int matrix_layout, char uplo, int n, dtype *a, int lda) { \ CHECK_LAPACK_UPLO(uplo); \ char o(loup(uplo, (matrix_layout == MXNET_LAPACK_ROW_MAJOR))); \ int ret(0); \ @@ -172,7 +187,7 @@ inline void flip(int m, int n, // Define compilable stubs. #define MXNET_LAPACK_CWRAPPER1(func, dtype) \ - inline int MXNET_LAPACK_##func(int matrix_layout, char uplo, int n, dtype* a, int lda ) { \ + inline int MXNET_LAPACK_##func(int matrix_layout, char uplo, int n, dtype* a, int lda) { \ LOG(FATAL) << "MXNet build without lapack. Function " << #func << " is not available."; \ return 1; \ } @@ -209,4 +224,4 @@ inline int MXNET_LAPACK_posv(int matrix_layout, char uplo, int n, return mxnet_lapack_dposv(matrix_layout, uplo, n, nrhs, a, lda, b, ldb); } -#endif // MXNET_C_LAPACK_API_H_ +#endif // MXNET_OPERATOR_C_LAPACK_API_H_ diff --git a/src/operator/contrib/krprod.h b/src/operator/contrib/krprod.h index 6ce94c648d79..a54ece79e9d7 100644 --- a/src/operator/contrib/krprod.h +++ b/src/operator/contrib/krprod.h @@ -26,7 +26,7 @@ #define MXNET_OPERATOR_CONTRIB_KRPROD_H_ #include #include "mshadow/tensor.h" -#include "mxnet/c_lapack_api.h" +#include "../c_lapack_api.h" namespace mxnet { namespace op { diff --git a/src/operator/linalg.h b/src/operator/linalg.h new file mode 100644 index 000000000000..9284a5825d2c --- /dev/null +++ b/src/operator/linalg.h @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file linalg.h + * \brief Unified tensor interface for advanced linear algebra functions + * (specifically BLAS3/LAPACK) from within mxnet. + */ +#ifndef MXNET_OPERATOR_LINALG_H_ +#define MXNET_OPERATOR_LINALG_H_ + +#include +#include "./c_lapack_api.h" +using namespace mshadow; + +// The purpose of this header is to expose the interfaces of the advanced +// linear algebra functions without clutter by the implementations. In contrast +// to the implementations in linalg_inline.h, no macros are used to generate +// similar functions that just differ by name/type in order to improve readability. +// +// Guidelines for extensions: +// For any type of computation the following should be provided at minimum: +// - 1 templated function supporting cpu/gpu float/double in non-batch mode +// - 1 templated function supporting cpu/gpu float/double in batch mode +// Naming conventions: +// - linalg_() +// - linalg_batch_() +// Signatures of CPU/GPU versions should be equivalent whenever possible including +// that a stream is supplied to the cpu-versions as (optional) last argument. +// The batched versions all work on tensors with one more dimension as the +// non-batched ones and the first/highest dimension iterates over the elements +// within the batch. + +//////////////////////////////// GEMM //////////////////////////////////////////// + +// CPU/GPU-versions of BLAS3 function "gemm". Please refer to the BLAS3-documentation +// for further information about the function and its parameters. +// Note that this is C = gemm(A,B,C), so C is input and output parameter. +template +void linalg_gemm(const Tensor& A, const Tensor& B, + const Tensor& C, DType alpha, DType beta, + bool tA, bool tB, Stream *s = 0); + +template +void linalg_batch_gemm(const Tensor& A, const Tensor& B, + const Tensor& C, DType alpha, DType beta, + bool tA, bool tB, Stream *s = 0); + +//////////////////////////////// TRSM //////////////////////////////////////////// + +// CPU/GPU-versions of BLAS3 function "trsm". Please refer to the BLAS3-documentation +// for further information about the function and its parameters. +// Note that this is B = trsm(A,B), so B is input and output parameter. +template +void linalg_trsm(const Tensor& A, const Tensor& B, + DType alpha, bool rightside, bool lower, bool transpose, Stream *s = 0); + +template +inline void linalg_batch_trsm(const Tensor& A, const Tensor& B, + DType alpha, bool rightside, bool lower, bool transpose, Stream *s = 0); + +//////////////////////////////// TRMM //////////////////////////////////////////// + +// CPU/GPU-versions of BLAS3 function "trmm". Please refer to the BLAS3-documentation +// for further information about the function and its parameters. +// Note that this is B = trmm(A,B), so B is input and output parameter. + +template +void linalg_trmm(const Tensor& A, const Tensor& B, + DType alpha, bool rightside, bool lower, bool transpose, Stream *s = 0); + +template +void linalg_batch_trmm(const Tensor& A, const Tensor& B, + DType alpha, bool rightside, bool lower, bool transpose, Stream *s = 0); + +//////////////////////////////// POTRF //////////////////////////////////////////// + +// CPU/GPU-versions of LAPACK function "potrf". Please refer to the LAPACK-documentation +// for further information about the function and its parameters. +// Note that this is A = potrf(A), so A is input and output parameter. + +template +void linalg_potrf(const Tensor& A, bool lower, Stream *s = 0); + +template +void linalg_batch_potrf(const Tensor& A, bool lower, Stream *s = 0); + +//////////////////////////////// POTRI //////////////////////////////////////////// + +// CPU/GPU-versions of LAPACK function "potri". Please refer to the LAPACK-documentation +// for further information about the function and its parameters. +// Note that this is A = potri(A), so A is input and output parameter. + +template +void linalg_potri(const Tensor& A, bool lower, Stream *s = 0); + +template +void linalg_batch_potri(const Tensor& A, bool lower, Stream *s = 0); + +#include "linalg_impl.h" + +#endif // MXNET_OPERATOR_LINALG_H_ diff --git a/src/operator/linalg_impl.h b/src/operator/linalg_impl.h new file mode 100644 index 000000000000..affa7941640b --- /dev/null +++ b/src/operator/linalg_impl.h @@ -0,0 +1,508 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file linalg.h + * \brief Implementation of unified tensor interface for advanced linear algebra functions + * (specifically BLAS3/LAPACK) from within mxnet. + */ +#ifndef MXNET_OPERATOR_LINALG_IMPL_H_ +#define MXNET_OPERATOR_LINALG_IMPL_H_ + +#include + +// Convenience functions. +inline void linalg_check_batch_size(int A, int B, int C) { + CHECK_EQ(A, B) << "Inconsistent batch size between arguments to linear algebra operator"; + CHECK_EQ(A, C) << "Inconsistent batch size between arguments to linear algebra operator"; + CHECK_GT(A, 0) << "Zero batch size for arguments to linear algebra operator"; +} + +//////////////////////////////// GEMM //////////////////////////////////////////// + +// CPU/GPU-versions of BLAS3 function "gemm". Please refer to the BLAS3-documentation +// for further information about the function and its parameters. +// Note that this is C = gemm(A,B,C), so C is input and output parameter. + +template +inline void check_gemm(const Tensor& A, const Tensor& B, + const Tensor& C, DType alpha, DType beta, bool tA, bool tB) { + // Any checking that helps user debug potential problems. + CHECK_EQ((tA ? A.size(1) : A.size(0)), C.size(0)) + << "Non compatible matrix dimensions between inputs A and C for gemm"; + CHECK_EQ((tB ? B.size(0) : B.size(1)), C.size(1)) + << "Non compatible matrix dimensions between inputs B and C for gemm"; + CHECK_EQ((tA ? A.size(0) : A.size(1)), (tB ? B.size(1) : B.size(0))) + << "Non compatible matrix dimensions between inputs A and B for gemm"; +} + +#define LINALG_CPU_GEMM(fname, DType) \ +template<> inline \ +void linalg_gemm(const Tensor& A, const Tensor& B, \ + const Tensor& C, DType alpha, DType beta, \ + bool tA, bool tB, Stream *s) { \ + check_gemm(A, B, C, alpha, beta, tA, tB); \ + cblas_##fname(CblasRowMajor, (tA ? CblasTrans : CblasNoTrans), (tB ? CblasTrans : CblasNoTrans), \ + C.size(0), C.size(1), (tA ? A.size(0) : A.size(1)), alpha, \ + A.dptr_, A.stride_, B.dptr_, B.stride_, beta, C.dptr_, C.stride_); \ +} +LINALG_CPU_GEMM(sgemm, float) +LINALG_CPU_GEMM(dgemm, double) + +#define LINALG_CPU_BATCH_GEMM(DType) \ +template<> inline \ +void linalg_batch_gemm(const Tensor& A, const Tensor& B, \ + const Tensor& C, DType alpha, DType beta, \ + bool tA, bool tB, Stream *s) { \ + linalg_check_batch_size(A.size(0), B.size(0), C.size(0)); \ + for (index_t i = 0; i < A.size(0); ++i) { \ + linalg_gemm(A[i], B[i], C[i], alpha, beta, tA, tB); \ + } \ +} +LINALG_CPU_BATCH_GEMM(float) +LINALG_CPU_BATCH_GEMM(double) + +#ifdef __CUDACC__ + +template +__global__ void linalgCollectBatchOffsetsGPU(DType *a[], DType* b, int stride, int N) { + for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < N; i += blockDim.x * gridDim.x) { + a[i] = b + i * stride; + } +} + +// cublas col-major processing accounted for by switching first two operands + +#define LINALG_GPU_GEMM(fname, DType) \ +template<> inline \ +void linalg_gemm(const Tensor& A, const Tensor& B, \ + const Tensor& C, DType alpha, DType beta, \ + bool tA, bool tB, Stream *s) { \ + using namespace mxnet; \ + using mshadow::gpu; \ + CHECK_NOTNULL(s); \ + check_gemm(A, B, C, alpha, beta, tA, tB); \ + CUBLAS_CALL(cublas##fname(Stream::GetBlasHandle(s), \ + (tB ? CUBLAS_OP_T : CUBLAS_OP_N), \ + (tA ? CUBLAS_OP_T : CUBLAS_OP_N), \ + C.size(1), C.size(0), (tB ? B.size(1) : B.size(0)), \ + &alpha, B.dptr_, B.stride_, A.dptr_, A.stride_, \ + &beta, C.dptr_, C.stride_)) \ +} +LINALG_GPU_GEMM(Sgemm, float) +LINALG_GPU_GEMM(Dgemm, double) + +#define LINALG_GPU_BATCH_GEMM(fname, DType) \ +template<> inline \ +void linalg_batch_gemm(const Tensor& A, const Tensor& B, \ + const Tensor& C, DType alpha, DType beta, \ + bool tA, bool tB, Stream *s) { \ + using namespace mxnet; \ + using mshadow::gpu; \ + CHECK_NOTNULL(s); \ + linalg_check_batch_size(A.size(0), B.size(0), C.size(0)); \ + check_gemm(A[0], B[0], C[0], alpha, beta, tA, tB); \ + Storage::Handle offsetsA, offsetsB, offsetsC; \ + offsetsA = Storage::Get()->Alloc(sizeof(DType*)*A.size(0), Context::GPU()); \ + offsetsB = Storage::Get()->Alloc(sizeof(DType*)*B.size(0), Context::GPU()); \ + offsetsC = Storage::Get()->Alloc(sizeof(DType*)*C.size(0), Context::GPU()); \ + using namespace mshadow::cuda; \ + int ngrid = std::min(kMaxGridNum, \ + static_cast((A.size(0) + kBaseThreadNum - 1) / kBaseThreadNum)); \ + linalgCollectBatchOffsetsGPU<<::GetStream(s)>>> \ + (static_cast(offsetsA.dptr), A.dptr_, A.size(1)*A.stride_, A.size(0)); \ + linalgCollectBatchOffsetsGPU<<::GetStream(s)>>> \ + (static_cast(offsetsB.dptr), B.dptr_, B.size(1)*B.stride_, B.size(0)); \ + linalgCollectBatchOffsetsGPU<<::GetStream(s)>>> \ + (static_cast(offsetsC.dptr), C.dptr_, C.size(1)*C.stride_, C.size(0)); \ + CUBLAS_CALL(cublas##fname(Stream::GetBlasHandle(s), \ + (tB ? CUBLAS_OP_T : CUBLAS_OP_N), \ + (tA ? CUBLAS_OP_T : CUBLAS_OP_N), \ + C.size(2), C.size(1), (tB ? B.size(2) : B.size(1)), \ + &alpha, static_cast(offsetsB.dptr), B.stride_, \ + static_cast(offsetsA.dptr), A.stride_, \ + &beta, static_cast(offsetsC.dptr), C.stride_, A.size(0))) \ + Storage::Get()->Free(offsetsA); \ + Storage::Get()->Free(offsetsB); \ + Storage::Get()->Free(offsetsC); \ +} +LINALG_GPU_BATCH_GEMM(SgemmBatched, float) +LINALG_GPU_BATCH_GEMM(DgemmBatched, double) + +#endif + +//////////////////////////////// TRSM //////////////////////////////////////////// + +// CPU/GPU-versions of BLAS3 function "trsm". Please refer to the BLAS3-documentation +// for further information about the function and its parameters. +// Note that this is B = trsm(A,B), so B is input and output parameter. + +template +inline void check_trsm(const Tensor& A, const Tensor& B, + DType alpha, bool rightside, bool lower, bool transpose) { + // Any checking that helps user debug potential problems. + CHECK_EQ(A.size(0), A.size(1)) + << "First input of trsm is not a square matrix."; + CHECK(!rightside || (B.size(1) == A.size(0))) + << "Non compatible matrix dimensions between inputs A and B for trsm"; + CHECK(rightside || (B.size(0) == A.size(1))) + << "Non compatible matrix dimensions between inputs A and B for trsm"; +} + +#define LINALG_CPU_TRSM(fname, DType) \ +template<> inline \ +void linalg_trsm(const Tensor& A, const Tensor& B, \ + DType alpha, bool rightside, bool lower, bool transpose, Stream *s) { \ + check_trsm(A, B, alpha, rightside, lower, transpose); \ + cblas_##fname(CblasRowMajor, (rightside ? CblasRight : CblasLeft), \ + (lower ? CblasLower : CblasUpper), (transpose ? CblasTrans : CblasNoTrans), \ + CblasNonUnit, B.size(0), B.size(1), alpha, A.dptr_, \ + A.stride_, B.dptr_, B.stride_); \ +} +LINALG_CPU_TRSM(strsm, float) +LINALG_CPU_TRSM(dtrsm, double) + +#define LINALG_CPU_BATCH_TRSM(DType) \ +template<> inline \ +void linalg_batch_trsm(const Tensor& A, const Tensor& B, \ + DType alpha, bool rightside, bool lower, bool transpose, Stream *s) { \ + linalg_check_batch_size(A.size(0), B.size(0), B.size(0)); \ + for (index_t i = 0; i < A.size(0); ++i) { \ + linalg_trsm(A[i], B[i], alpha, rightside, lower, transpose); \ + } \ +} +LINALG_CPU_BATCH_TRSM(float) +LINALG_CPU_BATCH_TRSM(double) + +#ifdef __CUDACC__ + +// cublas col-major processing accounted for by switching sides and fill mode + +#define LINALG_GPU_TRSM(fname, DType) \ +template<> inline \ +void linalg_trsm(const Tensor& A, const Tensor& B, \ + DType alpha, bool rightside, bool lower, bool transpose, Stream *s) { \ + using namespace mxnet; \ + using mshadow::gpu; \ + CHECK_NOTNULL(s); \ + check_trsm(A, B, alpha, rightside, lower, transpose); \ + CUBLAS_CALL(cublas##fname(Stream::GetBlasHandle(s), \ + (rightside ? CUBLAS_SIDE_LEFT : CUBLAS_SIDE_RIGHT), \ + (lower ? CUBLAS_FILL_MODE_UPPER : CUBLAS_FILL_MODE_LOWER), \ + (transpose ? CUBLAS_OP_T : CUBLAS_OP_N), \ + CUBLAS_DIAG_NON_UNIT, B.size(1), B.size(0), &alpha, \ + A.dptr_, A.stride_, B.dptr_, B.stride_)); \ +} +LINALG_GPU_TRSM(Strsm, float) +LINALG_GPU_TRSM(Dtrsm, double) + +#define LINALG_GPU_BATCH_TRSM(fname, DType) \ +template<> inline \ +void linalg_batch_trsm(const Tensor& A, const Tensor& B, \ + DType alpha, bool rightside, bool lower, bool transpose, Stream *s) { \ + using namespace mxnet; \ + using mshadow::gpu; \ + CHECK_NOTNULL(s); \ + linalg_check_batch_size(A.size(0), B.size(0), B.size(0)); \ + check_trsm(A[0], B[0], alpha, rightside, lower, transpose); \ + Storage::Handle offsetsA, offsetsB; \ + offsetsA = Storage::Get()->Alloc(sizeof(DType*)*A.size(0), Context::GPU()); \ + offsetsB = Storage::Get()->Alloc(sizeof(DType*)*B.size(0), Context::GPU()); \ + using namespace mshadow::cuda; \ + int ngrid = std::min(kMaxGridNum, \ + static_cast((A.size(0) + kBaseThreadNum - 1) / kBaseThreadNum)); \ + linalgCollectBatchOffsetsGPU<<::GetStream(s)>>> \ + (static_cast(offsetsA.dptr), A.dptr_, A.size(1)*A.stride_, A.size(0)); \ + linalgCollectBatchOffsetsGPU<<::GetStream(s)>>> \ + (static_cast(offsetsB.dptr), B.dptr_, B.size(1)*B.stride_, A.size(0)); \ + CUBLAS_CALL(cublas##fname(Stream::GetBlasHandle(s), \ + (rightside ? CUBLAS_SIDE_LEFT : CUBLAS_SIDE_RIGHT), \ + (lower ? CUBLAS_FILL_MODE_UPPER : CUBLAS_FILL_MODE_LOWER), \ + (transpose ? CUBLAS_OP_T : CUBLAS_OP_N), \ + CUBLAS_DIAG_NON_UNIT, B.size(2), B.size(1), &alpha, \ + static_cast(offsetsA.dptr), A.stride_, \ + static_cast(offsetsB.dptr), B.stride_, A.size(0))); \ + Storage::Get()->Free(offsetsA); \ + Storage::Get()->Free(offsetsB); \ +} +LINALG_GPU_BATCH_TRSM(StrsmBatched, float) +LINALG_GPU_BATCH_TRSM(DtrsmBatched, double) + +#endif + +//////////////////////////////// TRMM //////////////////////////////////////////// + +// CPU/GPU-versions of BLAS3 function "trmm". Please refer to the BLAS3-documentation +// for further information about the function and its parameters. +// Note that this is B = trmm(A,B), so B is input and output parameter. + +template +inline void check_trmm(const Tensor& A, const Tensor& B, + DType alpha, bool rightside, bool lower, bool transpose) { + // Any checking that helps user debug potential problems. + CHECK_EQ(A.size(0), A.size(1)) + << "First input of trmm is not a square matrix."; + CHECK(!rightside || (B.size(1) == A.size(0))) + << "Non compatible matrix dimensions between inputs A and B for trmm"; + CHECK(rightside || (B.size(0) == A.size(1))) + << "Non compatible matrix dimensions between inputs A and B for trmm"; +} + +#define LINALG_CPU_TRMM(fname, DType) \ +template<> inline \ +void linalg_trmm(const Tensor& A, const Tensor& B, \ + DType alpha, bool rightside, bool lower, bool transpose, Stream *s) { \ + check_trmm(A, B, alpha, rightside, lower, transpose); \ + cblas_##fname(CblasRowMajor, (rightside ? CblasRight : CblasLeft), \ + (lower ? CblasLower : CblasUpper), (transpose ? CblasTrans : CblasNoTrans), \ + CblasNonUnit, B.size(0), B.size(1), alpha, A.dptr_, \ + A.stride_, B.dptr_, B.stride_); \ +} +LINALG_CPU_TRMM(strmm, float) +LINALG_CPU_TRMM(dtrmm, double) + +#define LINALG_XPU_BATCH_TRMM(xpu, DType) \ +template<> inline \ +void linalg_batch_trmm(const Tensor& A, const Tensor& B, \ + DType alpha, bool rightside, bool lower, bool transpose, Stream *s) { \ + linalg_check_batch_size(A.size(0), B.size(0), B.size(0)); \ + for (index_t i = 0; i < A.size(0); ++i) { \ + linalg_trmm(A[i], B[i], alpha, rightside, lower, transpose, s); \ + } \ +} +LINALG_XPU_BATCH_TRMM(cpu, float) +LINALG_XPU_BATCH_TRMM(cpu, double) + +#ifdef __CUDACC__ + +// cublas col-major processing accounted for by switching sides and fill mode +// doing in-place computation by supplying B as second and third matrix +#define LINALG_GPU_TRMM(fname, DType) \ +template<> inline \ +void linalg_trmm(const Tensor& A, const Tensor& B, \ + DType alpha, bool rightside, bool lower, bool transpose, Stream *s) { \ + using namespace mxnet; \ + using mshadow::gpu; \ + CHECK_NOTNULL(s); \ + check_trmm(A, B, alpha, rightside, lower, transpose); \ + CUBLAS_CALL(cublas##fname(Stream::GetBlasHandle(s), \ + (rightside ? CUBLAS_SIDE_LEFT : CUBLAS_SIDE_RIGHT), \ + (lower ? CUBLAS_FILL_MODE_UPPER : CUBLAS_FILL_MODE_LOWER), \ + (transpose ? CUBLAS_OP_T : CUBLAS_OP_N), \ + CUBLAS_DIAG_NON_UNIT, B.size(0), B.size(1), &alpha, \ + A.dptr_, A.stride_, B.dptr_, B.stride_, \ + B.dptr_, B.stride_)); \ +} +LINALG_GPU_TRMM(Strmm, float) +LINALG_GPU_TRMM(Dtrmm, double) + +LINALG_XPU_BATCH_TRMM(gpu, float) +LINALG_XPU_BATCH_TRMM(gpu, double) + +#endif + +//////////////////////////////// POTRF //////////////////////////////////////////// + +// CPU/GPU-versions of LAPACK function "potrf". Please refer to the LAPACK-documentation +// for further information about the function and its parameters. +// Note that this is A = potrf(A), so A is input and output parameter. + +template +inline void check_potrf(const Tensor& A, bool lower) { + // Any checking that helps user debug potential problems. + CHECK_EQ(A.size(0), A.size(1)) + << "No square matrix as input to potrf."; +} + +#define LINALG_CPU_POTRF(fname, DType) \ +template<> inline \ +void linalg_potrf(const Tensor& A, bool lower, Stream *s) { \ + check_potrf(A, lower); \ + int ret(MXNET_LAPACK_##fname(MXNET_LAPACK_ROW_MAJOR, (lower ? 'L' : 'U'), A.size(0), \ + A.dptr_ , A.stride_)); \ + CHECK_EQ(ret, 0) << #fname << " failed in lapack on cpu."; \ +} +LINALG_CPU_POTRF(spotrf, float) +LINALG_CPU_POTRF(dpotrf, double) + +#define LINALG_CPU_BATCH_POTRF(DType) \ +template<> inline \ +void linalg_batch_potrf(const Tensor& A, bool lower, Stream *s) { \ + for (index_t i = 0; i < A.size(0); ++i) { \ + linalg_potrf(A[i], lower); \ + } \ +} +LINALG_CPU_BATCH_POTRF(float) +LINALG_CPU_BATCH_POTRF(double) + +#if MXNET_USE_CUSOLVER == 1 + +#define LINALG_GPU_BUFFSIZE_POTRF(fname, DType) \ +inline int linalg_potrf_buffsize(const Tensor& A, bool lower, Stream *s) { \ + using namespace mxnet; \ + using mshadow::gpu; \ + CHECK_NOTNULL(s); \ + int buffsize(0); \ + CUSOLVER_CALL(cusolver##fname(Stream::GetSolverHandle(s), \ + (lower ? CUBLAS_FILL_MODE_UPPER : CUBLAS_FILL_MODE_LOWER), \ + A.size(0), A.dptr_, A.stride_, &buffsize)); \ + return buffsize; \ +} +LINALG_GPU_BUFFSIZE_POTRF(DnSpotrf_bufferSize, float) +LINALG_GPU_BUFFSIZE_POTRF(DnDpotrf_bufferSize, double) + +#define LINALG_GPU_POTRF(fname, DType) \ +template<> inline \ +void linalg_potrf(const Tensor& A, bool lower, Stream *s) { \ + using namespace mxnet; \ + using mshadow::gpu; \ + CHECK_NOTNULL(s); \ + check_potrf(A, lower); \ + int buffsize(linalg_potrf_buffsize(A, lower, s)); \ + Storage::Handle buffer = Storage::Get()->Alloc(sizeof(DType)*buffsize, Context::GPU()); \ + Storage::Handle info = Storage::Get()->Alloc(sizeof(int), Context::GPU()); \ + CUSOLVER_CALL(cusolver##fname(Stream::GetSolverHandle(s), \ + (lower ? CUBLAS_FILL_MODE_UPPER : CUBLAS_FILL_MODE_LOWER), \ + A.size(0), A.dptr_, A.stride_, static_cast(buffer.dptr), buffsize, \ + static_cast(info.dptr))); \ + Storage::Get()->Free(buffer); \ + Storage::Get()->Free(info); \ +} +LINALG_GPU_POTRF(DnSpotrf, float) +LINALG_GPU_POTRF(DnDpotrf, double) + +#define LINALG_GPU_BATCH_POTRF(fname, DType) \ +template<> inline \ +void linalg_batch_potrf(const Tensor& A, bool lower, Stream *s) { \ + using namespace mxnet; \ + using mshadow::gpu; \ + CHECK_NOTNULL(s); \ + CHECK_GT(A.size(0), 0); \ + check_potrf(A[0], lower); \ + int buffsize(linalg_potrf_buffsize(A[0], lower, s)); \ + Storage::Handle buffer = Storage::Get()->Alloc(sizeof(DType)*buffsize, Context::GPU()); \ + Storage::Handle info = Storage::Get()->Alloc(sizeof(int), Context::GPU()); \ + for (mshadow::index_t i = 0; i < A.size(0); ++i) { \ + CUSOLVER_CALL(cusolver##fname(Stream::GetSolverHandle(s), \ + (lower ? CUBLAS_FILL_MODE_UPPER : CUBLAS_FILL_MODE_LOWER), \ + A[i].size(0), A[i].dptr_, A[i].stride_, \ + static_cast(buffer.dptr), buffsize, static_cast(info.dptr))); \ + } \ + Storage::Get()->Free(buffer); \ + Storage::Get()->Free(info); \ +} +LINALG_GPU_BATCH_POTRF(DnSpotrf, float) +LINALG_GPU_BATCH_POTRF(DnDpotrf, double) + +#endif + +//////////////////////////////// POTRI //////////////////////////////////////////// + +// CPU/GPU-versions of LAPACK function "potri". Please refer to the LAPACK-documentation +// for further information about the function and its parameters. +// Note that this is A = potri(A), so A is input and output parameter. + +template +inline void check_potri(const Tensor& A, bool lower) { + // Any checking that helps user debug potential problems. + CHECK_EQ(A.size(0), A.size(1)) << "No square matrix as input to potri."; +} + +#define LINALG_CPU_POTRI(fname, DType) \ +template<> inline \ +void linalg_potri(const Tensor& A, bool lower, Stream *s) { \ + check_potri(A, lower); \ + int ret(MXNET_LAPACK_##fname(MXNET_LAPACK_ROW_MAJOR, (lower ? 'L' : 'U'), A.size(0), \ + A.dptr_ , A.stride_)); \ + CHECK_EQ(ret, 0) << #fname << " failed in lapack on cpu."; \ +} +LINALG_CPU_POTRI(spotri, float) +LINALG_CPU_POTRI(dpotri, double) + +#define LINALG_CPU_BATCH_POTRI(DType) \ +template<> inline \ +void linalg_batch_potri(const Tensor& A, bool lower, Stream *s) { \ + for (index_t i = 0; i < A.size(0); ++i) { \ + linalg_potri(A[i], lower); \ + } \ +} +LINALG_CPU_BATCH_POTRI(float) +LINALG_CPU_BATCH_POTRI(double) + +#ifdef __CUDACC__ + +// Initializes multiple identity matrices on the same vector. +template +__global__ void linalgInitIdentityGPU(DType *a, int stride, int lda, int N) { + for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < N; i += blockDim.x * gridDim.x) { + // index relative to the matrix. + int index(i % stride); + a[i] = (index / lda == index % lda ? DType(1.0) : DType(0)); + } +} + +// There is no direct support for potri in cuda. We emulate the function by two calls to trsm. +#define LINALG_GPU_POTRI(DType) \ +template<> inline \ +void linalg_potri(const Tensor& A, bool lower, Stream *s) { \ + using namespace mxnet; \ + CHECK_NOTNULL(s); \ + check_potri(A, lower); \ + Storage::Handle buffer = Storage::Get()->Alloc(sizeof(DType)*A.MSize(), Context::GPU()); \ + using namespace mshadow::cuda; \ + int ngrid = std::min(kMaxGridNum, \ + static_cast((A.MSize() + kBaseThreadNum - 1) / kBaseThreadNum)); \ + linalgInitIdentityGPU<<::GetStream(s)>>> \ + (static_cast(buffer.dptr), A.MSize(), A.stride_, A.MSize()); \ + Tensor B((DType *)buffer.dptr, A.shape_, A.stride_, s); \ + linalg_trsm(A, B, DType(1.0), false, lower, !lower, s); \ + linalg_trsm(A, B, DType(1.0), false, lower, lower, s); \ + Copy(A, B, s); \ + B.dptr_ = 0; \ + Storage::Get()->Free(buffer); \ +} +LINALG_GPU_POTRI(float) +LINALG_GPU_POTRI(double) + +#define LINALG_GPU_BATCH_POTRI(DType) \ +template<> inline \ +void linalg_batch_potri(const Tensor& A, bool lower, Stream *s) { \ + using namespace mxnet; \ + CHECK_NOTNULL(s); \ + CHECK_GT(A.size(0), 0); \ + check_potri(A[0], lower); \ + Storage::Handle buffer = Storage::Get()->Alloc(sizeof(DType)*A.MSize(), Context::GPU()); \ + using namespace mshadow::cuda; \ + int ngrid = std::min(kMaxGridNum, \ + static_cast((A.MSize() + kBaseThreadNum - 1) / kBaseThreadNum)); \ + linalgInitIdentityGPU<<::GetStream(s)>>> \ + (static_cast(buffer.dptr), A.size(1)*A.stride_, A.stride_, A.MSize()); \ + Tensor B((DType *)buffer.dptr, A.shape_, A.stride_, s); \ + linalg_batch_trsm(A, B, DType(1.0), false, lower, !lower, s); \ + linalg_batch_trsm(A, B, DType(1.0), false, lower, lower, s); \ + Copy(A, B, s); \ + B.dptr_ = 0; \ + Storage::Get()->Free(buffer); \ +} +LINALG_GPU_BATCH_POTRI(float) +LINALG_GPU_BATCH_POTRI(double) + +#endif + +#endif // MXNET_OPERATOR_LINALG_IMPL_H_ diff --git a/src/operator/tensor/la_op.cc b/src/operator/tensor/la_op.cc index 1b726ced906b..70d4f9b766ad 100644 --- a/src/operator/tensor/la_op.cc +++ b/src/operator/tensor/la_op.cc @@ -401,7 +401,7 @@ Examples:: { return std::vector{"A"}; } ) .set_attr("FInferShape", LaReduceShape<2>) .set_attr("FInferType", ElemwiseType<1, 1>) -.set_attr("FCompute", LaReduceForward) +.set_attr("FCompute", LaOpForward) .set_attr("FGradient", ElemwiseGradUseIn{"_backward_linalg_sumlogdiag"}) .add_argument("A", "NDArray-or-Symbol", "Tensor of square matrices"); @@ -411,7 +411,7 @@ NNVM_REGISTER_OP(_backward_linalg_sumlogdiag) .set_attr("FResourceRequest", [](const NodeAttrs& attrs) { return std::vector{ResourceRequest::kTempSpace}; }) .set_attr("TIsBackward", true) -.set_attr("FCompute", LaReduceBackward); +.set_attr("FCompute", LaOpBackward); } // namespace op } // namespace mxnet diff --git a/src/operator/tensor/la_op.cu b/src/operator/tensor/la_op.cu new file mode 100644 index 000000000000..a89d98fd7f82 --- /dev/null +++ b/src/operator/tensor/la_op.cu @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file la_op.cu + * \brief GPU-Operators for advanced linear algebra. + */ +#include "./la_op.h" +#include "./la_op_inline.h" + +namespace mxnet { +namespace op { + +NNVM_REGISTER_OP(linalg_gemm) +.set_attr("FCompute", LaOpForward); + +NNVM_REGISTER_OP(_backward_linalg_gemm) +.set_attr("FCompute", LaOpBackward); + +NNVM_REGISTER_OP(linalg_gemm2) +.set_attr("FCompute", LaOpForward); + +NNVM_REGISTER_OP(_backward_linalg_gemm2) +.set_attr("FCompute", LaOpBackward); + +NNVM_REGISTER_OP(linalg_trmm) +.set_attr("FCompute", LaOpForward); + +NNVM_REGISTER_OP(_backward_linalg_trmm) +.set_attr("FCompute", LaOpBackward); + +NNVM_REGISTER_OP(linalg_trsm) +.set_attr("FCompute", LaOpForward); + +NNVM_REGISTER_OP(_backward_linalg_trsm) +.set_attr("FCompute", LaOpBackward); + +NNVM_REGISTER_OP(linalg_sumlogdiag) +.set_attr("FCompute", LaOpForward); + +NNVM_REGISTER_OP(_backward_linalg_sumlogdiag) +.set_attr("FCompute", LaOpBackward); + +NNVM_REGISTER_OP(linalg_potri) +.set_attr("FCompute", LaOpForward); + +NNVM_REGISTER_OP(_backward_linalg_potri) +.set_attr("FCompute", LaOpBackward); + +#if MXNET_USE_CUSOLVER == 1 + +NNVM_REGISTER_OP(linalg_potrf) +.set_attr("FCompute", LaOpForward); + +NNVM_REGISTER_OP(_backward_linalg_potrf) +.set_attr("FCompute", LaOpBackward); + +#endif + +} // namespace op +} // namespace mxnet diff --git a/src/operator/tensor/la_op.h b/src/operator/tensor/la_op.h index 977998855263..dd5fab985e3c 100644 --- a/src/operator/tensor/la_op.h +++ b/src/operator/tensor/la_op.h @@ -91,9 +91,9 @@ struct LaTriangMatrixMultParam : public dmlc::Parameter }; // Common function for shape inference for matrix mult and matrix mac. -bool LaMatrixMultMacOpShape(const nnvm::NodeAttrs& attrs, - std::vector* in_attrs, - std::vector* out_attrs) { +inline bool LaMatrixMultMacOpShape(const nnvm::NodeAttrs& attrs, + std::vector* in_attrs, + std::vector* out_attrs) { CHECK_GE(in_attrs->size(), 2); CHECK_EQ(out_attrs->size(), 1); bool transpose_a(false), transpose_b(false); @@ -132,9 +132,9 @@ bool LaMatrixMultMacOpShape(const nnvm::NodeAttrs& attrs, return false; } -bool LaTriangMatrixMultOpShape(const nnvm::NodeAttrs& attrs, - std::vector* in_attrs, - std::vector* out_attrs) { +inline bool LaTriangMatrixMultOpShape(const nnvm::NodeAttrs& attrs, + std::vector* in_attrs, + std::vector* out_attrs) { const LaTriangMatrixMultParam& param = nnvm::get(attrs.parsed); CHECK_EQ(in_attrs->size(), 2); CHECK_EQ(out_attrs->size(), 1); @@ -192,9 +192,9 @@ bool LaTriangMatrixMultOpShape(const nnvm::NodeAttrs& attrs, } template -bool LaReduceShape(const nnvm::NodeAttrs& attrs, - std::vector* in_attrs, - std::vector* out_attrs) { +inline bool LaReduceShape(const nnvm::NodeAttrs& attrs, + std::vector* in_attrs, + std::vector* out_attrs) { // Shape for reduction of the dim lowest dimensions to a scalar. // Can only deduct in forward direction. CHECK_EQ(in_attrs->size(), 1); @@ -203,7 +203,8 @@ bool LaReduceShape(const nnvm::NodeAttrs& attrs, if ( ndim < dim ) { return false; } - std::vector oshape(std::max(1, ndim-dim), 1); + std::vector oshape(std::max(1, ndim-dim)); + oshape[0] = 1; for ( int i = 0; i < ndim - dim; ++i ) { oshape[i] = (*in_attrs)[0][i]; } @@ -218,7 +219,6 @@ template& inputs, const std::vector& outputs, - const int index, const nnvm::NodeAttrs& attrs, mshadow::Stream *s) { CHECK(false) << "no specialized LaOpCaller defined for template parameters"; @@ -228,86 +228,75 @@ template struct LaOpCaller { static void op(const std::vector& inputs, const std::vector& outputs, - const int index, const nnvm::NodeAttrs& attrs, mshadow::Stream *s) { - laop::op(inputs[0].FlatToKD(s)[index], - outputs[0].FlatToKD(s)[index], attrs); + laop::op(inputs[0].FlatToKD(s), + outputs[0].FlatToKD(s), s, attrs); } }; template struct LaOpCaller { static void op(const std::vector& inputs, const std::vector& outputs, - const int index, const nnvm::NodeAttrs& attrs, mshadow::Stream *s) { - laop::op(inputs[0].FlatToKD(s)[index], - inputs[1].FlatToKD(s)[index], - outputs[0].FlatToKD(s)[index], - attrs); + laop::op(inputs[0].FlatToKD(s), + inputs[1].FlatToKD(s), + outputs[0].FlatToKD(s), s, attrs); } }; template struct LaOpCaller { static void op(const std::vector& inputs, const std::vector& outputs, - const int index, const nnvm::NodeAttrs& attrs, mshadow::Stream *s) { - laop::op(inputs[0].FlatToKD(s)[index], - inputs[1].FlatToKD(s)[index], - inputs[2].FlatToKD(s)[index], - outputs[0].FlatToKD(s)[index], - attrs); + laop::op(inputs[0].FlatToKD(s), + inputs[1].FlatToKD(s), + inputs[2].FlatToKD(s), + outputs[0].FlatToKD(s), s, attrs); } }; template struct LaOpCaller { static void op(const std::vector& inputs, const std::vector& outputs, - const int index, const nnvm::NodeAttrs& attrs, mshadow::Stream *s) { - laop::op(inputs[0].FlatToKD(s)[index], - inputs[1].FlatToKD(s)[index], - inputs[2].FlatToKD(s)[index], - outputs[0].FlatToKD(s)[index], - outputs[1].FlatToKD(s)[index], - attrs); + laop::op(inputs[0].FlatToKD(s), + inputs[1].FlatToKD(s), + inputs[2].FlatToKD(s), + outputs[0].FlatToKD(s), + outputs[1].FlatToKD(s), s, attrs); } }; template struct LaOpCaller { static void op(const std::vector& inputs, const std::vector& outputs, - const int index, const nnvm::NodeAttrs& attrs, mshadow::Stream *s) { - laop::op(inputs[0].FlatToKD(s)[index], - inputs[1].FlatToKD(s)[index], - inputs[2].FlatToKD(s)[index], - inputs[3].FlatToKD(s)[index], - outputs[0].FlatToKD(s)[index], - outputs[1].FlatToKD(s)[index], - attrs); + laop::op(inputs[0].FlatToKD(s), + inputs[1].FlatToKD(s), + inputs[2].FlatToKD(s), + inputs[3].FlatToKD(s), + outputs[0].FlatToKD(s), + outputs[1].FlatToKD(s), s, attrs); } }; template struct LaOpCaller { static void op(const std::vector& inputs, const std::vector& outputs, - const int index, const nnvm::NodeAttrs& attrs, mshadow::Stream *s) { - laop::op(inputs[0].FlatToKD(s)[index], - inputs[1].FlatToKD(s)[index], - inputs[2].FlatToKD(s)[index], - inputs[3].FlatToKD(s)[index], - outputs[0].FlatToKD(s)[index], - outputs[1].FlatToKD(s)[index], - outputs[2].FlatToKD(s)[index], - attrs); + laop::op(inputs[0].FlatToKD(s), + inputs[1].FlatToKD(s), + inputs[2].FlatToKD(s), + inputs[3].FlatToKD(s), + outputs[0].FlatToKD(s), + outputs[1].FlatToKD(s), + outputs[2].FlatToKD(s), s, attrs); } }; @@ -322,24 +311,8 @@ void LaOpForward(const nnvm::NodeAttrs& attrs, Stream *s = ctx.get_stream(); CHECK_EQ(inputs.size(), inum); CHECK_EQ(outputs.size(), onum); - MSHADOW_REAL_TYPE_SWITCH(outputs[0].type_flag_, OType, { - int N(-1); - for ( int i = 0; i < inum; ++i ) { - CHECK_EQ(inputs[i].CheckContiguous(), true); - const int M(inputs[i].FlatToKD(s).size(0)); - CHECK_EQ((N == -1 || N == M), true); - N = M; - } - for ( int i = 0; i < onum; ++i ) { - CHECK_EQ(outputs[i].CheckContiguous(), true); - CHECK_EQ((req[i] == kWriteTo || req[i] == kWriteInplace), true); - const int M(outputs[i].FlatToKD(s).size(0)); - CHECK_EQ((N == -1 || N == M), true); - N = M; - } - for ( int i = 0; i < N; ++i ) { - LaOpCaller::op(inputs, outputs, i, attrs, s); - } + MSHADOW_SGL_DBL_TYPE_SWITCH(outputs[0].type_flag_, OType, { + LaOpCaller::op(inputs, outputs, attrs, s); }); } @@ -354,28 +327,15 @@ void LaOpBackward(const nnvm::NodeAttrs& attrs, Stream *s = ctx.get_stream(); CHECK_EQ(inputs.size(), inum); CHECK_EQ(outputs.size(), onum); - MSHADOW_REAL_TYPE_SWITCH(outputs[0].type_flag_, OType, { - int N(-1); - for ( int i = 0; i < inum; ++i ) { - CHECK_EQ(inputs[i].CheckContiguous(), true); - const int M(inputs[i].FlatToKD(s).size(0)); - CHECK_EQ((N == -1 || N == M), true); - N = M; - } + MSHADOW_SGL_DBL_TYPE_SWITCH(outputs[0].type_flag_, OType, { std::vector tspace(outputs); for ( int i = 0; i < onum; ++i ) { - CHECK_EQ(outputs[i].CheckContiguous(), true); - const int M(outputs[i].FlatToKD(s).size(0)); - CHECK_EQ((N == -1 || N == M), true); - N = M; if ( req[i] == kAddTo ) { tspace[i].dptr_ = ctx.requested[ResourceRequest::kTempSpace] .get_space_typed(Shape1(outputs[i].Size()), s).dptr_; } } - for ( int i = 0; i < N; ++i ) { - LaOpCaller::op(inputs, tspace, i, attrs, s); - } + LaOpCaller::op(inputs, tspace, attrs, s); for ( int i = 0; i < onum; ++i ) { if ( req[i] == kAddTo ) { Tensor out = outputs[i].FlatTo1D(s); @@ -385,53 +345,6 @@ void LaOpBackward(const nnvm::NodeAttrs& attrs, }); } -template -void LaReduceForward(const nnvm::NodeAttrs& attrs, - const OpContext& ctx, - const std::vector& inputs, - const std::vector& req, - const std::vector& outputs) { - using namespace mshadow; - Stream *s = ctx.get_stream(); - CHECK_EQ(inputs.size(), 1); - CHECK_EQ(outputs.size(), 1); - CHECK_EQ(inputs[0].CheckContiguous(), true); - CHECK_EQ(outputs[0].CheckContiguous(), true); - MSHADOW_REAL_TYPE_SWITCH(outputs[0].type_flag_, OType, { - Tensor in(inputs[0].FlatToKD(s)); - Tensor out(outputs[0].FlatTo1D(s)); - const int N(outputs[0].Size()); - CHECK_EQ(in.size(0), N); - for ( int i = 0; i < N; ++i ) { - laop::op(in[i], out[i], attrs); - } - }); -} - -template -void LaReduceBackward(const nnvm::NodeAttrs& attrs, - const OpContext& ctx, - const std::vector& inputs, - const std::vector& req, - const std::vector& outputs) { - using namespace mshadow; - Stream *s = ctx.get_stream(); - CHECK_EQ(inputs.size(), 2); - CHECK_EQ(outputs.size(), 1); - CHECK_EQ(inputs[0].CheckContiguous(), true); - CHECK_EQ(inputs[1].CheckContiguous(), true); - CHECK_EQ(outputs[0].CheckContiguous(), true); - MSHADOW_REAL_TYPE_SWITCH(outputs[0].type_flag_, OType, { - const int N(inputs[0].Size()); - Tensor in0(inputs[0].FlatTo1D(s)); - Tensor in1(inputs[1].FlatToKD(s)); - Tensor out(outputs[0].FlatToKD(s)); - for ( int i = 0; i < N; ++i ) { - laop::op(in0[i], in1[i], out[i], attrs, (req[i] == kAddTo)); - } - }); -} - } // namespace op } // namespace mxnet diff --git a/src/operator/tensor/la_op_inline.h b/src/operator/tensor/la_op_inline.h index a032988edb75..34fb441f53f7 100644 --- a/src/operator/tensor/la_op_inline.h +++ b/src/operator/tensor/la_op_inline.h @@ -24,244 +24,186 @@ #ifndef MXNET_OPERATOR_TENSOR_LA_OP_INLINE_H_ #define MXNET_OPERATOR_TENSOR_LA_OP_INLINE_H_ -#include +#include "../linalg.h" namespace mxnet { namespace op { using namespace mshadow; -#define LA_OP_NOT_AVAIL " operator can only be called with float/double data type." - -// Signature for single matrix operations (decomposition/inversion). -#define FUNC_SIGNATURE_1(fname, arg1) {CHECK_EQ(MXNET_LAPACK_##fname(MXNET_LAPACK_ROW_MAJOR, 'L', \ - arg1.size(0), arg1.dptr_, arg1.size(0)), 0) << "fname failed in lapack";} - -// Signature for matrix-matrix multiplications involving one diagonal matrix. -#define FUNC_SIGNATURE_2(fname, arg1, arg2) \ - { cblas_##fname(CblasRowMajor, (rightside ? CblasRight : CblasLeft), \ - CblasLower, (transpose ? CblasTrans : CblasNoTrans), \ - CblasNonUnit, arg2.size(0), arg2.size(1), alpha, arg1.dptr_, \ - (rightside ? arg2.size(1) : arg2.size(0)), arg2.dptr_, arg2.size(1)); } - - // Helper functions. -template -void CopyLowerToUpper(DType *dptr, int N) - { for (int i = 1; i < N; ++i ) for ( int j = 0; j < i; ++j ) dptr[j*N+i] = dptr[i*N+j]; } -template -void ZeroUpper(DType *dptr, int N) - { for (int i = 0; i < N; ++i ) for ( int j = i+1; j < N; ++j ) dptr[i*N+j] = 0; } +struct CopyLowerToUpper { + template + MSHADOW_XINLINE static void Map(int i, int matrix_size, int stride, DType* data) { + // Below computation works even when we are dealing with a batch of matrices. + const int row((i % matrix_size) / stride), col(i % stride); + if ( row > col ) data[i + (col - row) * (stride - 1)] = data[i]; + } +}; +struct ZeroUpper { + template + MSHADOW_XINLINE static void Map(int i, int matrix_size, int stride, DType* data) { + const int row((i % matrix_size) / stride), col(i % stride); + if ( row < col ) data[i] = 0; + } +}; +struct Scale { + template + MSHADOW_XINLINE static void Map(int i, DType scale, DType* data) { + data[i] *= scale; + } +}; -// Forward operators +// Forward computations (always using batched processing) // D = gemm(A,B,C) struct gemm { template - static void op(const Tensor& A, const Tensor& B, - const Tensor& C, DType alpha, DType beta, bool tA, bool tB) - { CHECK(false) << "gemm" << LA_OP_NOT_AVAIL; } + static void op(const Tensor& A, const Tensor& B, + const Tensor& C, DType alpha, DType beta, bool tA, bool tB, Stream *s) { + linalg_batch_gemm(A, B, C, alpha, beta, tA, tB, s); + } template - static void op(const Tensor& A, const Tensor& B, - const Tensor& C, const Tensor& D, - const nnvm::NodeAttrs& attrs) { - if ( C.dptr_ != D.dptr_ ) Copy(D, C); + static void op(const Tensor& A, const Tensor& B, + const Tensor& C, const Tensor& D, + Stream *s, const nnvm::NodeAttrs& attrs) { + if ( C.dptr_ != D.dptr_ ) Copy(D, C, s); const LaMatrixMacParam& param = nnvm::get(attrs.parsed); - gemm::op(A, B, D, DType(param.alpha), DType(param.beta), param.transpose_a, param.transpose_b); + gemm::op(A, B, D, DType(param.alpha), DType(param.beta), + param.transpose_a, param.transpose_b, s); } }; -template<> -void gemm::op(const Tensor& A, const Tensor& B, - const Tensor& C, - float alpha, float beta, bool tA, bool tB ) { - CHECK_EQ((tA ? A.size(1) : A.size(0)), C.size(0)) - << "Non compatible matrix dimensions between inputs A and C for gemm operator"; - CHECK_EQ((tB ? B.size(0) : B.size(1)), C.size(1)) - << "Non compatible matrix dimensions between inputs B and C for gemm operator"; - CHECK_EQ((tA ? A.size(0) : A.size(1)), (tB ? B.size(1) : B.size(0))) - << "Non compatible matrix dimensions between inputs A and B for gemm operator"; - cblas_sgemm(CblasRowMajor, (tA ? CblasTrans : CblasNoTrans), (tB ? CblasTrans : CblasNoTrans), - (tA ? A.size(1):A.size(0)), (tB ? B.size(0): B.size(1)), - (tA ? A.size(0):A.size(1)), alpha, A.dptr_, A.size(1), B.dptr_, B.size(1), - beta, C.dptr_, (tB ? B.size(0): B.size(1))); -} -template<> -void gemm::op(const Tensor& A, const Tensor& B, - const Tensor& C, - double alpha, double beta, bool tA, bool tB) { - CHECK_EQ((tA ? A.size(1) : A.size(0)), C.size(0)) - << "Non compatible matrix dimensions between inputs A and C for gemm operator"; - CHECK_EQ((tB ? B.size(0) : B.size(1)), C.size(1)) - << "Non compatible matrix dimensions between inputs B and C for gemm operator"; - CHECK_EQ((tA ? A.size(0) : A.size(1)), (tB ? B.size(1) : B.size(0))) - << "Non compatible matrix dimensions between inputs A and B for gemm operator"; - cblas_dgemm(CblasRowMajor, (tA ? CblasTrans : CblasNoTrans), (tB ? CblasTrans : CblasNoTrans), - (tA ? A.size(1):A.size(0)), (tB ? B.size(0): B.size(1)), - (tA ? A.size(0):A.size(1)), alpha, A.dptr_, A.size(1), B.dptr_, B.size(1), - beta, C.dptr_, (tB ? B.size(0): B.size(1))); -} // C = gemm2(A,B) struct gemm2 { template - static void op(const Tensor& A, const Tensor& B, - const Tensor& C, const nnvm::NodeAttrs& attrs) { + static void op(const Tensor& A, const Tensor& B, + const Tensor& C, Stream *s, const nnvm::NodeAttrs& attrs) { const LaMatrixMultParam& param = nnvm::get(attrs.parsed); - gemm::op(A, B, C, DType(param.alpha), DType(0), param.transpose_a, param.transpose_b); + gemm::op(A, B, C, DType(param.alpha), DType(0), param.transpose_a, param.transpose_b, s); } }; // L = potrf(A). struct potrf { template - static void op(const Tensor& A, const Tensor& L, - const nnvm::NodeAttrs& attrs) - { CHECK(false) << "potrf" << LA_OP_NOT_AVAIL; } + static void op(const Tensor& A, const Tensor& L, + Stream *s, const nnvm::NodeAttrs& attrs) { + if ( A.dptr_ != L.dptr_ ) Copy(L, A, s); + linalg_batch_potrf(L, true, s); + using namespace mxnet_op; + Kernel::Launch(s, L.MSize(), L.size(1)*L.stride_, L.stride_, L.dptr_); + } }; -template<> -void potrf::op(const Tensor& A, const Tensor& L, - const nnvm::NodeAttrs& attrs) { - if ( A.dptr_ != L.dptr_ ) Copy(L, A); - FUNC_SIGNATURE_1(spotrf, L); - ZeroUpper(L.dptr_, L.size(0)); -} -template<> -void potrf::op(const Tensor& A, const Tensor& L, - const nnvm::NodeAttrs& attrs) { - if ( A.dptr_ != L.dptr_ ) Copy(L, A); - FUNC_SIGNATURE_1(dpotrf, L); - ZeroUpper(L.dptr_, L.size(0)); -} // A = potri(L). struct potri { template - static void op(const Tensor& L, const Tensor& A, - const nnvm::NodeAttrs& attrs) - { CHECK(false) << "potri" << LA_OP_NOT_AVAIL; } + static void op(const Tensor& L, const Tensor& A, + Stream *s, const nnvm::NodeAttrs& attrs) { + if ( A.dptr_ != L.dptr_ ) Copy(A, L, s); + linalg_batch_potri(A, true, s); + using namespace mxnet_op; + Kernel::Launch(s, A.MSize(), A.size(1)*A.stride_, A.stride_, A.dptr_); + } }; -template<> -void potri::op(const Tensor& L, const Tensor& A, - const nnvm::NodeAttrs& attrs) { - if ( A.dptr_ != L.dptr_ ) Copy(A, L); - FUNC_SIGNATURE_1(spotri, A); - CopyLowerToUpper(A.dptr_, A.size(0)); -} -template<> -void potri::op(const Tensor& A, const Tensor& L, - const nnvm::NodeAttrs& attrs) { - if ( A.dptr_ != L.dptr_ ) Copy(A, L); - FUNC_SIGNATURE_1(dpotri, A); - CopyLowerToUpper(A.dptr_, A.size(0)); -} // B = trsm(L,A) struct trsm { template - static void op(const Tensor& L, const Tensor& B, - DType alpha, bool rightside, bool transpose) - { CHECK(false) << "trsm" << LA_OP_NOT_AVAIL; } + static void op(const Tensor& L, const Tensor& B, + DType alpha, bool rightside, bool transpose, Stream *s) { + linalg_batch_trsm(L, B, alpha, rightside, true, transpose, s); + } template - static void op(const Tensor& L, const Tensor& A, - const Tensor& B, const nnvm::NodeAttrs& attrs) { - if ( A.dptr_ != B.dptr_ ) Copy(B, A); + static void op(const Tensor& L, const Tensor& A, + const Tensor& B, + Stream *s, const nnvm::NodeAttrs& attrs) { + if ( A.dptr_ != B.dptr_ ) Copy(B, A, s); const LaTriangMatrixMultParam& param = nnvm::get(attrs.parsed); - op(L, B, DType(param.alpha), param.rightside, param.transpose); + op(L, B, DType(param.alpha), param.rightside, param.transpose, s); } }; -template<> -void trsm::op(const Tensor& L, const Tensor& B, - float alpha, bool rightside, bool transpose) { - FUNC_SIGNATURE_2(strsm, L, B); -} -template<> -void trsm::op(const Tensor& L, const Tensor& B, - double alpha, bool rightside, bool transpose) { - FUNC_SIGNATURE_2(dtrsm, L, B); -} // B = trmm(L,A) struct trmm { template - static void op(const Tensor& L, const Tensor& B, - DType alpha, bool rightside, bool transpose) - { CHECK(false) << "trmm" << LA_OP_NOT_AVAIL; } + static void op(const Tensor& L, const Tensor& B, + DType alpha, bool rightside, bool transpose, Stream *s) { + linalg_batch_trmm(L, B, alpha, rightside, true, transpose, s); + } template - static void op(const Tensor& L, const Tensor& A, - const Tensor& B, const nnvm::NodeAttrs& attrs) { - if ( A.dptr_ != B.dptr_ ) Copy(B, A); + static void op(const Tensor& L, const Tensor& A, + const Tensor& B, Stream *s, const nnvm::NodeAttrs& attrs) { + if ( A.dptr_ != B.dptr_ ) Copy(B, A, s); const LaTriangMatrixMultParam& param = nnvm::get(attrs.parsed); - op(L, B, DType(param.alpha), param.rightside, param.transpose); + op(L, B, DType(param.alpha), param.rightside, param.transpose, s); } }; -template<> -void trmm::op(const Tensor& L, const Tensor& B, - float alpha, bool rightside, bool transpose) { - FUNC_SIGNATURE_2(strmm, L, B); -} -template<> -void trmm::op(const Tensor& L, const Tensor& B, - double alpha, bool rightside, bool transpose) { - FUNC_SIGNATURE_2(dtrmm, L, B); -} // Useful operator that is not part of BLAS/LAPACK. -struct sumlogdiag { - template::value, int>::type = 0> - static void op(const Tensor& A, DType& L, const nnvm::NodeAttrs& attrs) - { CHECK(false) << "sumlogdiag operator can only be called with float/double data type."; } - template::value, int>::type = 0> - static void op(const Tensor& A, DType& B, const nnvm::NodeAttrs& attrs) { - CHECK_EQ(A.size(0), A.size(1)) << "sumlogdiag operator requires a NxN matrix as input."; - const int N(A.size(0)); +struct ForwardSumLogDiag { + template + MSHADOW_XINLINE static void Map(int i, int N, int stride, DType* A, DType* B) { DType sum(0); - DType *p(A.dptr_); - for ( int i = 0; i < N; ++i, p += N+1 ) { - sum += log(*p); + const int offset(i * N * stride); + for ( int j = 0; j < N; ++j ) { + sum += log(A[offset+j*(stride+1)]); } - B = sum; + B[i] = sum; + } +}; +struct sumlogdiag { + template + static void op(const Tensor& A, const Tensor& B, + Stream *s, const nnvm::NodeAttrs& attrs) { + CHECK_EQ(A.size(1), A.size(2)) << "sumlogdiag operator requires square matrices as input."; + using namespace mxnet_op; + Kernel::Launch(s, A.size(0), A.size(1), A.stride_, A.dptr_, B.dptr_); } }; -// Backward operators +// Backward operators (always using batch processing) struct gemm_backward { template - static void op(const Tensor& dD, const Tensor& A, - const Tensor& B, const Tensor& C, - const Tensor& dA, const Tensor& dB, - const Tensor& dC, const nnvm::NodeAttrs& attrs) { + static void op(const Tensor& dD, const Tensor& A, + const Tensor& B, const Tensor& C, + const Tensor& dA, const Tensor& dB, + const Tensor& dC, + Stream* s, const nnvm::NodeAttrs& attrs) { const LaMatrixMacParam& param = nnvm::get(attrs.parsed); - (param.transpose_a ? gemm::op(B, dD, dA, DType(param.alpha), DType(0), param.transpose_b, true) - : gemm::op(dD, B, dA, DType(param.alpha), DType(0), false, !param.transpose_b)); - (param.transpose_b ? gemm::op(dD, A, dB, DType(param.alpha), DType(0), true, param.transpose_a) - : gemm::op(A, dD, dB, DType(param.alpha), DType(0), !param.transpose_a, false)); - const int N(dC.size(0)*dC.size(1)); - for ( int i = 0; i < N; ++i ) { - dC.dptr_[i] = param.beta * dD.dptr_[i]; - } + bool tA(param.transpose_a), tB(param.transpose_b); + (tA ? gemm::op(B, dD, dA, DType(param.alpha), DType(0), tB, true, s) + : gemm::op(dD, B, dA, DType(param.alpha), DType(0), false, !tB, s)); + (tB ? gemm::op(dD, A, dB, DType(param.alpha), DType(0), true, tA, s) + : gemm::op(A, dD, dB, DType(param.alpha), DType(0), !tA, false, s)); + Copy(dC, dD, s); + using namespace mxnet_op; + Kernel::Launch(s, dC.MSize(), DType(param.beta), dC.dptr_); } }; struct gemm2_backward { template - static void op(const Tensor& dC, const Tensor& A, - const Tensor& B, const Tensor& dA, - const Tensor& dB, const nnvm::NodeAttrs& attrs) { + static void op(const Tensor& dC, const Tensor& A, + const Tensor& B, const Tensor& dA, + const Tensor& dB, + Stream* s, const nnvm::NodeAttrs& attrs) { const LaMatrixMultParam& param = nnvm::get(attrs.parsed); - (param.transpose_a ? gemm::op(B, dC, dA, DType(param.alpha), DType(0), param.transpose_b, true) - : gemm::op(dC, B, dA, DType(param.alpha), DType(0), false, !param.transpose_b)); - (param.transpose_b ? gemm::op(dC, A, dB, DType(param.alpha), DType(0), true, param.transpose_a) - : gemm::op(A, dC, dB, DType(param.alpha), DType(0), !param.transpose_a, false)); + bool tA(param.transpose_a), tB(param.transpose_b); + (tA ? gemm::op(B, dC, dA, DType(param.alpha), DType(0), tB, true, s) + : gemm::op(dC, B, dA, DType(param.alpha), DType(0), false, !tB, s)); + (tB ? gemm::op(dC, A, dB, DType(param.alpha), DType(0), true, tA, s) + : gemm::op(A, dC, dB, DType(param.alpha), DType(0), !tA, false, s)); } }; struct potrf_backward { template - static void op(const Tensor& dL, const Tensor& L, - const Tensor& dA, const nnvm::NodeAttrs& attrs) { + static void op(const Tensor& dL, const Tensor& L, + const Tensor& dA, + Stream* s, const nnvm::NodeAttrs& attrs) { // Backward of L = potrf(A). // dA = 0.5 * L**T * symm(L**T * dL # E) * L**(-1) where // '#' denotes Hadamard product @@ -269,81 +211,96 @@ struct potrf_backward { // symm(X) = 0.5 * (X + X**T) // Hadamard product and symm can be realized by a single copy from lower to upper triangle. if ( dL.dptr_ != dA.dptr_ ) { - Copy(dA, dL); + Copy(dA, dL, s); } - trmm::op(L, dA, DType(1.0), false, true); - CopyLowerToUpper(dA.dptr_, dA.size(0)); - trsm::op(L, dA, DType(1.0), false, true); - trsm::op(L, dA, DType(0.5), true, false); + trmm::op(L, dA, DType(1.0), false, true, s); + using namespace mxnet_op; + Kernel::Launch + (s, dA.MSize(), dA.size(1)*dA.stride_, dA.stride_, dA.dptr_); + trsm::op(L, dA, DType(1.0), false, true, s); + trsm::op(L, dA, DType(0.5), true, false, s); } }; struct potri_backward { template - static void op(const Tensor& dA, const Tensor& L, - const Tensor& A, const Tensor& dL, - const nnvm::NodeAttrs& attrs) { + static void op(const Tensor& dA, const Tensor& L, + const Tensor& A, const Tensor& dL, + Stream* s, const nnvm::NodeAttrs& attrs) { // Backward of A = potri(L). // dL = -2 * tril(A * dA * L**(-T)), where tril() extracts lower triangle and diagonal. - gemm::op(A, dA, dL, DType(1.0), DType(0), false, false); - trsm::op(L, dL, DType(-2.0), true, true); - ZeroUpper(dL.dptr_, dL.size(0)); + gemm::op(A, dA, dL, DType(1.0), DType(0), false, false, s); + trsm::op(L, dL, DType(-2.0), true, true, s); + using namespace mxnet_op; + Kernel::Launch(s, dL.MSize(), dL.size(1)*dL.stride_, dL.stride_, dL.dptr_); } }; struct trsm_backward { template - static void op(const Tensor& dB, const Tensor& L, - const Tensor& A, const Tensor& B, - const Tensor& dL, const Tensor& dA, - const nnvm::NodeAttrs& attrs) { + static void op(const Tensor& dB, const Tensor& L, + const Tensor& A, const Tensor& B, + const Tensor& dL, const Tensor& dA, + Stream* s, const nnvm::NodeAttrs& attrs) { // Backward of B = trsm(L,A). const LaTriangMatrixMultParam& param = nnvm::get(attrs.parsed); // Compute dA - if ( dA.dptr_ != dB.dptr_ ) Copy(dA, dB); - trsm::op(L, dA, DType(param.alpha), param.rightside, !param.transpose); + if ( dA.dptr_ != dB.dptr_ ) Copy(dA, dB, s); + trsm::op(L, dA, DType(param.alpha), param.rightside, !param.transpose, s); // Compute dL const bool da_left(param.rightside == param.transpose); - (da_left ? - gemm::op(dA, B, dL, DType(-1.0/param.alpha), DType(0), param.transpose, !param.transpose) - : gemm::op(B, dA, dL, DType(-1.0/param.alpha), DType(0), !param.transpose, param.transpose)); - ZeroUpper(dL.dptr_, dL.size(0)); + DType scale(-1.0/param.alpha); + (da_left ? gemm::op(dA, B, dL, scale, DType(0), param.transpose, !param.transpose, s) + : gemm::op(B, dA, dL, scale, DType(0), !param.transpose, param.transpose, s)); + using namespace mxnet_op; + Kernel::Launch(s, dL.MSize(), dL.size(1)*dL.stride_, dL.stride_, dL.dptr_); } }; struct trmm_backward { template - static void op(const Tensor& dB, const Tensor& L, - const Tensor& A, const Tensor& B, - const Tensor& dL, const Tensor& dA, - const nnvm::NodeAttrs& attrs) { + static void op(const Tensor& dB, const Tensor& L, + const Tensor& A, const Tensor& B, + const Tensor& dL, const Tensor& dA, + Stream* s, const nnvm::NodeAttrs& attrs) { // Backward of B = trmm(L,A). const LaTriangMatrixMultParam& param = nnvm::get(attrs.parsed); // Compute dL const bool db_left(param.rightside == param.transpose); - (db_left ? gemm::op(dB, A, dL, DType(param.alpha), DType(0), param.transpose, !param.transpose) - : gemm::op(A, dB, dL, DType(param.alpha), DType(0), !param.transpose, param.transpose)); - ZeroUpper(dL.dptr_, dL.size(0)); + DType scale(param.alpha); + (db_left ? gemm::op(dB, A, dL, scale, DType(0), param.transpose, !param.transpose, s) + : gemm::op(A, dB, dL, scale, DType(0), !param.transpose, param.transpose, s)); + using namespace mxnet_op; + Kernel::Launch(s, dL.MSize(), dL.size(1)*dL.stride_, dL.stride_, dL.dptr_); // Compute dA - if ( dA.dptr_ != dB.dptr_ ) Copy(dA, dB); - trmm::op(L, dA, DType(param.alpha), param.rightside, !param.transpose); + if ( dA.dptr_ != dB.dptr_ ) Copy(dA, dB, s); + trmm::op(L, dA, scale, param.rightside, !param.transpose, s); } }; +struct BackwardSumLogDiag { + template + MSHADOW_XINLINE static void Map(int i, int N, int stride, DType* dB, DType* A, DType* dA) { + const int offset(i * N * stride); + for ( int j = 0; j < N; ++j ) { + dA[offset+j*(stride+1)] = dB[i]/A[offset+j*(stride+1)]; + } + } +}; struct sumlogdiag_backward { template - static void op(const DType& dB, const Tensor& A, const Tensor& dA, - const nnvm::NodeAttrs& attrs, bool add) { + static void op(const Tensor& dB, const Tensor& A, + const Tensor& dA, + Stream* s, const nnvm::NodeAttrs& attrs) { // Backward of B = sumlogdiag(A). - const int N(A.size(0)); - if ( !add ) { - for ( int i = 0; i < N*N; ++i ) { - dA.dptr_[i] = 0; - } - } - for ( int i = 0; i < N; ++i ) { - dA.dptr_[i*(N+1)] += dB / A.dptr_[i*N+i]; - } + // dB is actually a 1-d tensor but we convert it to a 3-D one before calling + // this function as the LaOpCaller-adapters can only deal with a uniform + // dimension for all tensor inputs. This doesn't matter as we will interpret + // it correctly internally in this function. + using namespace mxnet_op; + Kernel::Launch(s, dA.MSize(), DType(0), dA.dptr_); + Kernel::Launch + (s, A.size(0), A.size(1), A.stride_, dB.dptr_, A.dptr_, dA.dptr_); } }; diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py index 718e3df04871..7d56b46e21a0 100644 --- a/tests/python/unittest/test_operator.py +++ b/tests/python/unittest/test_operator.py @@ -3412,15 +3412,8 @@ def test_deformable_psroipooling(): def test_laop(): - return - - # Currently no support for GPU. Will be added soon - # so keep these tests here in this file and activate - # gpu-testing when it is ready. - dev = default_context() - if dev.device_type == 'gpu': - return + # enable numerical checking of gradients grad_check = 1 data1 = mx.symbol.Variable('data1') From 5e622d113f5604d8f4296640a44217ef1287e3e1 Mon Sep 17 00:00:00 2001 From: Seth Hendrickson Date: Sat, 12 Aug 2017 15:15:10 -0700 Subject: [PATCH 370/834] Small doc cleanups (#7439) * clean up architecture docs * basic data --- docs/architecture/overview.md | 2 +- docs/architecture/program_model.md | 25 +++++++++++++------------ docs/tutorials/basic/data.md | 2 +- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index 361e0c91de63..a7632d4a61e8 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -48,7 +48,7 @@ The following API is the core interface for the execution engine: This API allows you to push a function (`exec_fun`), along with its context information and dependencies, to the engine. `exec_ctx` is the context information in which the `exec_fun` should be executed, -`const_vars` denotes the variables that the function reads from, +`const_vars` denotes the variables that the function reads from, and `mutate_vars` are the variables to be modified. The engine provides the following guarantee: diff --git a/docs/architecture/program_model.md b/docs/architecture/program_model.md index 380990e7019f..519a9a9024d8 100644 --- a/docs/architecture/program_model.md +++ b/docs/architecture/program_model.md @@ -92,7 +92,7 @@ are powerful DSLs that generate callable computation graphs for neural networks. Intuitively, you might say that imperative programs -are more *native* than symbolic programs. +are more *native* than symbolic programs. It's easier to use native language features. For example, it's straightforward to print out the values in the middle of computation or to use native control flow and loops @@ -269,7 +269,7 @@ Recall the *be prepared to encounter all possible demands* requirement of impera If you are creating an array library that supports automatic differentiation, you have to keep the grad closure along with the computation. This means that none of the history variables can be -garbage-collected because they are referenced by variable `d` by way of function closure. +garbage-collected because they are referenced by variable `d` by way of function closure. What if you want to compute only the value of `d`, and don't want the gradient value? @@ -305,7 +305,6 @@ For example, one solution to the preceding problem is to introduce a context variable. You can introduce a no-gradient context variable to turn gradient calculation off. - ```python with context.NoGradient(): @@ -315,6 +314,8 @@ to turn gradient calculation off. d = c + 1 ``` + + However, this example still must be prepared to encounter all possible demands, which means that you can't perform the in-place calculation to reuse memory in the forward pass (a trick commonly used to reduce GPU memory usage). @@ -380,7 +381,7 @@ It's usually easier to write parameter updates in an imperative style, especially when you need multiple updates that relate to each other. For symbolic programs, the update statement is also executed as you call it. So in that sense, most symbolic deep learning libraries -fall back on the imperative approach to perform updates, +fall back on the imperative approach to perform updates, while using the symbolic approach to perform gradient calculation. ### There Is No Strict Boundary @@ -388,7 +389,7 @@ while using the symbolic approach to perform gradient calculation. In comparing the two programming styles, some of our arguments might not be strictly true, i.e., it's possible to make an imperative program -more like a traditional symbolic program or vice versa. +more like a traditional symbolic program or vice versa. However, the two archetypes are useful abstractions, especially for understanding the differences between deep learning libraries. We might reasonably conclude that there is no clear boundary between programming styles. @@ -400,7 +401,7 @@ information held in symbolic programs. ## Big vs. Small Operations -When designing a deep learning library, another important programming model decision +When designing a deep learning library, another important programming model decision is precisely what operations to support. In general, there are two families of operations supported by most deep learning libraries: @@ -418,7 +419,7 @@ For example, the sigmoid unit can simply be composed of division, addition and a sigmoid(x) = 1.0 / (1.0 + exp(-x)) ``` Using smaller operations as building blocks, you can express nearly anything you want. -If you're more familiar with CXXNet- or Caffe-style layers, +If you're more familiar with CXXNet- or Caffe-style layers, note that these operations don't differ from a layer, except that they are smaller. ```python @@ -433,7 +434,7 @@ because you only need to compose the components. Directly composing sigmoid layers requires three layers of operation, instead of one. ```python - SigmoidLayer(x) = EWiseDivisionLayer(1.0, AddScalarLayer(ExpLayer(-x), 1.0)) + SigmoidLayer(x) = EWiseDivisionLayer(1.0, AddScalarLayer(ExpLayer(-x), 1.0)) ``` This code creates overhead for computation and memory (which could be optimized, with cost). @@ -467,7 +468,7 @@ these optimizations are crucial to performance. Because the operations are small, there are many sub-graph patterns that can be matched. Also, because the final, generated operations -might not enumerable, +might not be enumerable, an explicit recompilation of the kernels is required, as opposed to the fixed amount of precompiled kernels in the big operation libraries. @@ -476,7 +477,7 @@ that support small operations. Requiring compilation optimization also creates engineering overhead for the libraries that solely support smaller operations. -As in the case of symbolic vs imperative, +As in the case of symbolic vs. imperative, the bigger operation libraries "cheat" by asking you to provide restrictions (to the common layer), so that you actually perform the sub-graph matching. @@ -522,7 +523,7 @@ The more suitable programming style depends on the problem you are trying to sol For example, imperative programs are better for parameter updates, and symbolic programs for gradient calculation. -We advocate *mixing* the approaches. +We advocate *mixing* the approaches. Sometimes the part that we want to be flexible isn't crucial to performance. In these cases, it's okay to leave some efficiency on the table @@ -562,7 +563,7 @@ This is exactly like writing C++ programs and exposing them to Python, which we Because parameter memory resides on the GPU, you might not want to use NumPy as an imperative component. Supporting a GPU-compatible imperative library -that interacts with symbolic compiled functions +that interacts with symbolic compiled functions or provides a limited amount of updating syntax in the update statement in symbolic program execution might be a better choice. diff --git a/docs/tutorials/basic/data.md b/docs/tutorials/basic/data.md index 93a1db066a8c..d4db7d0de1b6 100644 --- a/docs/tutorials/basic/data.md +++ b/docs/tutorials/basic/data.md @@ -30,7 +30,7 @@ Iterators provide an abstract interface for traversing various types of iterable without needing to expose details about the underlying data source. In MXNet, data iterators return a batch of data as `DataBatch` on each call to `next`. -A `DataBatch` often contains *n* training examples and their corresponding labels. Here *n* is the `batch_size` of the iterator. At the end of the data stream when there is no more data to read, the iterator raises ``StopIteration`` exception like Python `iter`. +A `DataBatch` often contains *n* training examples and their corresponding labels. Here *n* is the `batch_size` of the iterator. At the end of the data stream when there is no more data to read, the iterator raises ``StopIteration`` exception like Python `iter`. The structure of `DataBatch` is defined [here](http://mxnet.io/api/python/io.html#mxnet.io.DataBatch). Information such as name, shape, type and layout on each training example and their corresponding label can be provided as `DataDesc` data descriptor objects via the `provide_data` and `provide_label` properties in `DataBatch`. From 8ad3c8a7a98dfa6bd6f5065cf9c3688f2414c3d4 Mon Sep 17 00:00:00 2001 From: Nan Zhu Date: Sat, 12 Aug 2017 16:00:27 -0700 Subject: [PATCH 371/834] clean up pom (#7440) --- scala-package/assembly/linux-x86_64-cpu/pom.xml | 4 +--- scala-package/assembly/linux-x86_64-gpu/pom.xml | 4 +--- scala-package/assembly/osx-x86_64-cpu/pom.xml | 4 +--- scala-package/assembly/pom.xml | 4 +--- scala-package/core/pom.xml | 4 +--- scala-package/examples/pom.xml | 3 +-- scala-package/init-native/linux-x86_64/pom.xml | 3 +-- scala-package/init-native/osx-x86_64/pom.xml | 3 +-- scala-package/init-native/pom.xml | 3 +-- scala-package/init/pom.xml | 3 +-- scala-package/macros/pom.xml | 3 +-- scala-package/native/linux-x86_64-cpu/pom.xml | 4 +--- scala-package/native/linux-x86_64-gpu/pom.xml | 4 +--- scala-package/native/osx-x86_64-cpu/pom.xml | 3 +-- scala-package/native/pom.xml | 3 +-- scala-package/pom.xml | 3 ++- scala-package/spark/pom.xml | 3 +-- 17 files changed, 18 insertions(+), 40 deletions(-) diff --git a/scala-package/assembly/linux-x86_64-cpu/pom.xml b/scala-package/assembly/linux-x86_64-cpu/pom.xml index a838765a9b91..2c25e6856fd3 100644 --- a/scala-package/assembly/linux-x86_64-cpu/pom.xml +++ b/scala-package/assembly/linux-x86_64-cpu/pom.xml @@ -6,13 +6,11 @@ ml.dmlc.mxnet mxnet-full-parent_2.11 - 0.11.0-SNAPSHOT + ${project.version} ../pom.xml - ml.dmlc.mxnet mxnet-full_2.11-linux-x86_64-cpu - 0.11.0-SNAPSHOT MXNet Scala Package - Full Linux-x86_64 CPU-only jar diff --git a/scala-package/assembly/linux-x86_64-gpu/pom.xml b/scala-package/assembly/linux-x86_64-gpu/pom.xml index e248c491ec8e..892851281655 100644 --- a/scala-package/assembly/linux-x86_64-gpu/pom.xml +++ b/scala-package/assembly/linux-x86_64-gpu/pom.xml @@ -6,13 +6,11 @@ ml.dmlc.mxnet mxnet-full-parent_2.11 - 0.11.0-SNAPSHOT + ${project.version} ../pom.xml - ml.dmlc.mxnet mxnet-full_2.11-linux-x86_64-gpu - 0.11.0-SNAPSHOT MXNet Scala Package - Full Linux-x86_64 GPU jar diff --git a/scala-package/assembly/osx-x86_64-cpu/pom.xml b/scala-package/assembly/osx-x86_64-cpu/pom.xml index e6ef43fbd4c4..e3f433f673e4 100644 --- a/scala-package/assembly/osx-x86_64-cpu/pom.xml +++ b/scala-package/assembly/osx-x86_64-cpu/pom.xml @@ -6,13 +6,11 @@ ml.dmlc.mxnet mxnet-full-parent_2.11 - 0.11.0-SNAPSHOT + ${project.version} ../pom.xml - ml.dmlc.mxnet mxnet-full_2.11-osx-x86_64-cpu - 0.11.0-SNAPSHOT MXNet Scala Package - Full OSX-x86_64 CPU-only jar diff --git a/scala-package/assembly/pom.xml b/scala-package/assembly/pom.xml index cad677feea3f..52a2cc42228f 100644 --- a/scala-package/assembly/pom.xml +++ b/scala-package/assembly/pom.xml @@ -6,13 +6,11 @@ ml.dmlc.mxnet mxnet-parent_2.11 - 0.11.0-SNAPSHOT + ${project.version} ../pom.xml - ml.dmlc.mxnet mxnet-full-parent_2.11 - 0.11.0-SNAPSHOT MXNet Scala Package - Full Parent pom diff --git a/scala-package/core/pom.xml b/scala-package/core/pom.xml index 0c79d322d29b..51e8a3596b1a 100644 --- a/scala-package/core/pom.xml +++ b/scala-package/core/pom.xml @@ -6,13 +6,11 @@ ml.dmlc.mxnet mxnet-parent_2.11 - 0.11.0-SNAPSHOT + ${project.version} ../pom.xml - ml.dmlc.mxnet mxnet-core_2.11 - 0.11.0-SNAPSHOT MXNet Scala Package - Core diff --git a/scala-package/examples/pom.xml b/scala-package/examples/pom.xml index 84f406a53991..356690cf0176 100644 --- a/scala-package/examples/pom.xml +++ b/scala-package/examples/pom.xml @@ -6,12 +6,11 @@ ml.dmlc.mxnet mxnet-parent_2.11 - 0.11.0-SNAPSHOT + ${project.version} ../pom.xml mxnet-examples_2.11 - 0.11.0-SNAPSHOT MXNet Scala Package - Examples diff --git a/scala-package/init-native/linux-x86_64/pom.xml b/scala-package/init-native/linux-x86_64/pom.xml index 3d06bb10aa52..9d784c471cb9 100644 --- a/scala-package/init-native/linux-x86_64/pom.xml +++ b/scala-package/init-native/linux-x86_64/pom.xml @@ -6,12 +6,11 @@ ml.dmlc.mxnet mxnet-scala-init-native-parent - 0.11.0-SNAPSHOT + ${project.version} ../pom.xml libmxnet-init-scala-linux-x86_64 - 0.11.0-SNAPSHOT MXNet Scala Package - Initializer Native Linux-x86_64 http://maven.apache.org diff --git a/scala-package/init-native/osx-x86_64/pom.xml b/scala-package/init-native/osx-x86_64/pom.xml index dbc9fdd55887..fb3748e5698f 100644 --- a/scala-package/init-native/osx-x86_64/pom.xml +++ b/scala-package/init-native/osx-x86_64/pom.xml @@ -6,12 +6,11 @@ ml.dmlc.mxnet mxnet-scala-init-native-parent - 0.11.0-SNAPSHOT + ${project.version} ../pom.xml libmxnet-init-scala-osx-x86_64 - 0.11.0-SNAPSHOT MXNet Scala Package - Initializer Native OSX-x86_64 http://maven.apache.org diff --git a/scala-package/init-native/pom.xml b/scala-package/init-native/pom.xml index f5839837a250..2b633169501d 100644 --- a/scala-package/init-native/pom.xml +++ b/scala-package/init-native/pom.xml @@ -6,12 +6,11 @@ ml.dmlc.mxnet mxnet-parent_2.11 - 0.11.0-SNAPSHOT + ${project.version} ../pom.xml mxnet-scala-init-native-parent - 0.11.0-SNAPSHOT MXNet Scala Package - Initializer Native Parent pom diff --git a/scala-package/init/pom.xml b/scala-package/init/pom.xml index dd6f55b8151c..04413e219429 100644 --- a/scala-package/init/pom.xml +++ b/scala-package/init/pom.xml @@ -6,12 +6,11 @@ ml.dmlc.mxnet mxnet-parent_2.11 - 0.11.0-SNAPSHOT + ${project.version} ../pom.xml mxnet-init_2.11 - 0.11.0-SNAPSHOT MXNet Scala Package - Initializer diff --git a/scala-package/macros/pom.xml b/scala-package/macros/pom.xml index 842abeef03b5..2a1498cb2639 100644 --- a/scala-package/macros/pom.xml +++ b/scala-package/macros/pom.xml @@ -6,12 +6,11 @@ ml.dmlc.mxnet mxnet-parent_2.11 - 0.11.0-SNAPSHOT + ${project.version} ../pom.xml mxnet-macros_2.11 - 0.11.0-SNAPSHOT MXNet Scala Package - Macros diff --git a/scala-package/native/linux-x86_64-cpu/pom.xml b/scala-package/native/linux-x86_64-cpu/pom.xml index 4d1d18678232..df45cd9e6c2e 100644 --- a/scala-package/native/linux-x86_64-cpu/pom.xml +++ b/scala-package/native/linux-x86_64-cpu/pom.xml @@ -6,13 +6,11 @@ ml.dmlc.mxnet mxnet-scala-native-parent - 0.11.0-SNAPSHOT + ${project.version} ../pom.xml - ml.dmlc.mxnet libmxnet-scala-linux-x86_64-cpu - 0.11.0-SNAPSHOT MXNet Scala Package - Native Linux-x86_64 CPU-only http://maven.apache.org diff --git a/scala-package/native/linux-x86_64-gpu/pom.xml b/scala-package/native/linux-x86_64-gpu/pom.xml index 794beece66dd..edc70e923ff7 100644 --- a/scala-package/native/linux-x86_64-gpu/pom.xml +++ b/scala-package/native/linux-x86_64-gpu/pom.xml @@ -6,13 +6,11 @@ ml.dmlc.mxnet mxnet-scala-native-parent - 0.11.0-SNAPSHOT + ${project.version} ../pom.xml - ml.dmlc.mxnet libmxnet-scala-linux-x86_64-gpu - 0.11.0-SNAPSHOT MXNet Scala Package - Native Linux-x86_64 GPU http://maven.apache.org diff --git a/scala-package/native/osx-x86_64-cpu/pom.xml b/scala-package/native/osx-x86_64-cpu/pom.xml index a6c09f1677e1..b6fb83f26a71 100644 --- a/scala-package/native/osx-x86_64-cpu/pom.xml +++ b/scala-package/native/osx-x86_64-cpu/pom.xml @@ -6,12 +6,11 @@ ml.dmlc.mxnet mxnet-scala-native-parent - 0.11.0-SNAPSHOT + ${project.version} ../pom.xml libmxnet-scala-osx-x86_64-cpu - 0.11.0-SNAPSHOT MXNet Scala Package - Native OSX-x86_64 CPU-only http://maven.apache.org diff --git a/scala-package/native/pom.xml b/scala-package/native/pom.xml index 4ccd42014cc0..e68ebb96666e 100644 --- a/scala-package/native/pom.xml +++ b/scala-package/native/pom.xml @@ -6,12 +6,11 @@ ml.dmlc.mxnet mxnet-parent_2.11 - 0.11.0-SNAPSHOT + ${project.version} ../pom.xml mxnet-scala-native-parent - 0.11.0-SNAPSHOT MXNet Scala Package - Native Parent pom diff --git a/scala-package/pom.xml b/scala-package/pom.xml index 69dcf1608803..7bfd8774de6b 100644 --- a/scala-package/pom.xml +++ b/scala-package/pom.xml @@ -5,7 +5,7 @@ 4.0.0 ml.dmlc.mxnet mxnet-parent_2.11 - 0.11.0-SNAPSHOT + ${project.version} MXNet Scala Package - Parent https://github.com/dmlc/mxnet/tree/master/scala-package MXNet Scala Package @@ -48,6 +48,7 @@ + 0.11.0-SNAPSHOT 2.11.8 2.11 diff --git a/scala-package/spark/pom.xml b/scala-package/spark/pom.xml index c0c699c30560..18170b95579b 100644 --- a/scala-package/spark/pom.xml +++ b/scala-package/spark/pom.xml @@ -6,12 +6,11 @@ ml.dmlc.mxnet mxnet-parent_2.11 - 0.11.0-SNAPSHOT + ${project.version} ../pom.xml mxnet-spark_2.11 - 0.11.0-SNAPSHOT MXNet Scala Package - Spark ML From 4b8235bdacd319843dda7b331f207808e4a90a93 Mon Sep 17 00:00:00 2001 From: Wei Wu Date: Sun, 13 Aug 2017 11:48:30 +0800 Subject: [PATCH 372/834] broken link in readme (#7441) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4a354d9bef75..841c6f1f62c2 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ deep learning systems, and interesting insights of DL systems for hackers. What's New ---------- -* [Version 0.11.0-rc0 Release](https://github.com/dmlc/mxnet/releases/tag/v0.11.0-rc0) - MXNet 0.11.0-rc0 Release. +* [Version 0.11.0-rc0 Release](https://github.com/apache/incubator-mxnet/releases/tag/0.11.0.rc0) - MXNet 0.11.0-rc0 Release. * [Apache Incubator](http://incubator.apache.org/projects/mxnet.html) - We are now an Apache Incubator project. * [Version 0.10.0 Release](https://github.com/dmlc/mxnet/releases/tag/v0.10.0) - MXNet 0.10.0 Release. * [Version 0.9.3 Release](./docs/architecture/release_note_0_9.md) - First 0.9 official release. From 0142ea0f330393499ece834b9d7b6d5d00fd8f3c Mon Sep 17 00:00:00 2001 From: "Qiang Kou (KK)" Date: Sun, 13 Aug 2017 21:11:12 +0000 Subject: [PATCH 373/834] [R] vignette update (#7437) --- ...ctionTutorial.Rmd => CallbackFunction.Rmd} | 19 +- R-package/vignettes/CatsDogsFinetune.Rmd | 12 +- R-package/vignettes/CharRnnModel.Rmd | 27 +- R-package/vignettes/CustomIterator.Rmd | 207 +++++++++++++++ R-package/vignettes/CustomLossFunction.Rmd | 20 +- .../classifyRealImageWithPretrainedModel.Rmd | 38 +-- .../vignettes/fiveMinutesNeuralNetwork.Rmd | 12 +- R-package/vignettes/mnistCompetition.Rmd | 82 +++--- R-package/vignettes/ndarray.Rmd | 148 +++++++++++ .../vignettes/ndarrayAndSymbolTutorial.Rmd | 242 ------------------ R-package/vignettes/symbol.Rmd | 103 ++++++++ 11 files changed, 563 insertions(+), 347 deletions(-) rename R-package/vignettes/{CallbackFunctionTutorial.Rmd => CallbackFunction.Rmd} (93%) create mode 100644 R-package/vignettes/CustomIterator.Rmd create mode 100644 R-package/vignettes/ndarray.Rmd delete mode 100644 R-package/vignettes/ndarrayAndSymbolTutorial.Rmd create mode 100644 R-package/vignettes/symbol.Rmd diff --git a/R-package/vignettes/CallbackFunctionTutorial.Rmd b/R-package/vignettes/CallbackFunction.Rmd similarity index 93% rename from R-package/vignettes/CallbackFunctionTutorial.Rmd rename to R-package/vignettes/CallbackFunction.Rmd index 91b4c096ec18..12b7e28247e9 100644 --- a/R-package/vignettes/CallbackFunctionTutorial.Rmd +++ b/R-package/vignettes/CallbackFunction.Rmd @@ -1,13 +1,9 @@ -MXNet R Tutorial for Callback Function -====================================== +# Customized callback function This vignette gives users a guideline for using and writing callback functions, which can be very useful in model training. -This tutorial is written in Rmarkdown. You can find the Rmarkdown source from [here](https://github.com/dmlc/mxnet/blob/master/R-package/vignettes/CallbackFunctionTutorial.Rmd) - -Model training example ----------- +## Model training example Let's begin from a small example. We can build and train a model using the following code: @@ -32,8 +28,8 @@ model <- mx.model.FeedForward.create( Besides, we provide two optional parameters, `batch.end.callback` and `epoch.end.callback`, which can provide great flexibility in model training. -How to use callback functions ---------- +## How to use callback functions + Two callback functions are provided in this package: @@ -76,8 +72,8 @@ head(logger$train) head(logger$eval) ``` -How to write your own callback functions ----------- +## How to write your own callback functions + You can find the source code for two callback functions from [here](https://github.com/dmlc/mxnet/blob/master/R-package/R/callback.R) and they can be used as your template: @@ -159,3 +155,6 @@ model <- mx.model.FeedForward.create( ``` You can see once the validation metric goes below the threshold we set, the training process will stop early. + + + diff --git a/R-package/vignettes/CatsDogsFinetune.Rmd b/R-package/vignettes/CatsDogsFinetune.Rmd index 95f90beec519..680b5a302498 100644 --- a/R-package/vignettes/CatsDogsFinetune.Rmd +++ b/R-package/vignettes/CatsDogsFinetune.Rmd @@ -1,5 +1,4 @@ -Dogs vs. Cats classification with mxnet and R -============================================= +# Dogs vs. Cats classification with mxnet and R ## Packages and prerequisites @@ -13,11 +12,11 @@ It is an end-to-end R solution for the dogs vs cats Kaggle competition (https:// and it can be used as an example for fine-tuning. All the code has been test on Ubuntu 16.04. - ```{r, echo=FALSE} knitr::opts_chunk$set(eval = FALSE) ``` + ```{r} library(imager) library(mxnet) @@ -106,7 +105,7 @@ Map(function(x, y) { ### Creating .rec files -```{r, eval = FALSE} +```{r} cat_files <- list.files("train_pad_224x224/cat/", recursive=TRUE) cat_files <- paste0("cat/", cat_files) @@ -169,6 +168,8 @@ and we need to replace the last fully connected layer with a new layer for 2 cla ```{r} +download.file('http://data.dmlc.ml/data/Inception.zip', destfile = 'Inception.zip') +unzip("Inception.zip") inception_bn <- mx.model.load("./Inception-BN", iteration = 126) symbol <- inception_bn$symbol @@ -266,3 +267,6 @@ probs <- t(do.call(cbind, probs)) preds <- data.frame(id = 1:12500, label = probs[, 2]) write.csv(preds, "subm.csv", row.names = FALSE, quote = FALSE) ``` + + + diff --git a/R-package/vignettes/CharRnnModel.Rmd b/R-package/vignettes/CharRnnModel.Rmd index 9dc00a39d96b..9d3fd5c14786 100644 --- a/R-package/vignettes/CharRnnModel.Rmd +++ b/R-package/vignettes/CharRnnModel.Rmd @@ -1,11 +1,7 @@ -Char RNN Example -============================================= +# Char RNN Example -This example aims to show how to use the LSTM model to build a char-level language model, and generate text from it. We use a tiny shakespeare text for demo purpose. Data can be found at [here](https://github.com/dmlc/web-data/tree/master/mxnet/tinyshakespeare). -This tutorial is written in Rmarkdown. You can find the Rmarkdown source from [here](https://github.com/dmlc/mxnet/blob/master/R-package/vignettes/CharRnnModel.Rmd). -Load Data ---------- +## Load Data First of all, load in the data and preprocess it. @@ -148,8 +144,8 @@ X.train <- list(data=X.train.data, label=X.train.label) X.val <- list(data=X.val.data, label=X.val.label) ``` -Training Model --------------- +## Training Model + In `mxnet`, we have a function called `mx.lstm` so that users can build a general lstm model. @@ -172,8 +168,8 @@ model <- mx.lstm(X.train, X.val, ``` -Inference from model --------------------- +## Inference from model + Some helper functions for random sample. @@ -258,14 +254,19 @@ for (i in (1:(seq.len-1))) { message(out) ``` The result: + ``` ah not a drobl greens Settled asing lately sistering sounted to their hight ``` -Other RNN models ----------------- +## Other RNN models + In `mxnet`, other RNN models like custom RNN and gru is also provided. - For **custom RNN model**, you can replace `mx.lstm` with `mx.rnn` to train rnn model. Also, you can replace `mx.lstm.inference` and `mx.lstm.forward` with `mx.rnn.inference` and `mx.rnn.forward` to inference from rnn model and get forward result from the inference model. -- For **GRU model**, you can replace `mx.lstm` with `mx.gru` to train gru model. Also, you can replace `mx.lstm.inference` and `mx.lstm.forward` with `mx.gru.inference` and `mx.gru.forward` to inference from gru model and get forward result from the inference model. \ No newline at end of file +- For **GRU model**, you can replace `mx.lstm` with `mx.gru` to train gru model. Also, you can replace `mx.lstm.inference` and `mx.lstm.forward` with `mx.gru.inference` and `mx.gru.forward` to inference from gru model and get forward result from the inference model. + + + + \ No newline at end of file diff --git a/R-package/vignettes/CustomIterator.Rmd b/R-package/vignettes/CustomIterator.Rmd new file mode 100644 index 000000000000..22ac90fe0400 --- /dev/null +++ b/R-package/vignettes/CustomIterator.Rmd @@ -0,0 +1,207 @@ +# Customized iterator + + +This tutorial provides a guideline on how to use and write custom iterators, which can very useful when having a dataset that does not fit into memory. + +## Getting the data + +The data we are going to use is the [MNIST dataset](http://yann.lecun.com/exdb/mnist/) in CSV format, which can be found from [here](https://www.kaggle.com/c/digit-recognizer/data). + +To download the data: + +```{r} +download.file('https://s3-us-west-2.amazonaws.com/apache-mxnet/R/data/mnist_csv.zip', + destfile = 'mnist_csv.zip') +unzip('mnist_csv.zip', exdir = '.') +``` + +You'll get two files, `mnist_train.csv` that contains 60.000 examples of hand written numbers and `mxnist_test.csv` that contains 10.000 examples. The first element of each line in the CSV is the label, which is a number between 0 and 9. The rest of the line are 784 numbers between 0 and 255, corresponding to the levels of grey of a matrix of 28x28. Therefore, each line contains an image of 28x28 pixels of a hand written number and its true label. + +## Custom CSV Iterator + +Next we are going to create a custom CSV Iterator based on the [C++ CSVIterator class](https://github.com/dmlc/mxnet/blob/master/src/io/iter_csv.cc). + +For that we are going to use the R function `mx.io.CSVIter` as a base class. This class has as parameters `data.csv, data.shape, batch.size` and two main functions, `iter.next()` that calls the iterator in the next batch of data and `value()` that returns the train data and the label. + +The R Custom Iterator needs to inherit from the C++ data iterator class, for that we used the class `Rcpp_MXArrayDataIter` extracted with RCPP. Also, it needs to have the same parameters: `data.csv, data.shape, batch.size`. Apart from that, we can also add the field `iter`, which is the CSV Iterator that we are going to expand. + +```{r, eval=FALSE} +CustomCSVIter <- setRefClass("CustomCSVIter", + fields=c("iter", "data.csv", "data.shape", "batch.size"), + contains = "Rcpp_MXArrayDataIter", + #... + ) +``` + +The next step is to initialize the class. For that we call the base `mx.io.CSVIter` and fill the rest of the fields. + +```{r, eval=FALSE} +CustomCSVIter <- setRefClass("CustomCSVIter", + fields=c("iter", "data.csv", "data.shape", "batch.size"), + contains = "Rcpp_MXArrayDataIter", + methods=list( + initialize=function(iter, data.csv, data.shape, batch.size){ + feature_len <- data.shape*data.shape + 1 + csv_iter <- mx.io.CSVIter(data.csv=data.csv, data.shape=c(feature_len), batch.size=batch.size) + .self$iter <- csv_iter + .self$data.csv <- data.csv + .self$data.shape <- data.shape + .self$batch.size <- batch.size + .self + }, + #... + ) + ) +``` + +So far there is no difference between the original class and the custom class. Let's implement the function `value()`. In this case what we are going to do is transform the data that comes from the original class as an array of 785 numbers into a matrix of 28x28 and a label. We will also normalize the training data to be between 0 and 1. + +```{r, eval=FALSE} +CustomCSVIter <- setRefClass("CustomCSVIter", + fields=c("iter", "data.csv", "data.shape", "batch.size"), + contains = "Rcpp_MXArrayDataIter", + methods=list( + initialize=function(iter, data.csv, data.shape, batch.size){ + feature_len <- data.shape*data.shape + 1 + csv_iter <- mx.io.CSVIter(data.csv=data.csv, data.shape=c(feature_len), batch.size=batch.size) + .self$iter <- csv_iter + .self$data.csv <- data.csv + .self$data.shape <- data.shape + .self$batch.size <- batch.size + .self + }, + value=function(){ + val <- as.array(.self$iter$value()$data) + val.x <- val[-1,] + val.y <- val[1,] + val.x <- val.x/255 + dim(val.x) <- c(data.shape, data.shape, 1, ncol(val.x)) + val.x <- mx.nd.array(val.x) + val.y <- mx.nd.array(val.y) + list(data=val.x, label=val.y) + }, + #... + ) + ) +``` +Finally we are going to add the rest of the functions needed for the training to work correctly. The final `CustomCSVIter` looks like this: + +```{r} +CustomCSVIter <- setRefClass("CustomCSVIter", + fields=c("iter", "data.csv", "data.shape", "batch.size"), + contains = "Rcpp_MXArrayDataIter", + methods=list( + initialize=function(iter, data.csv, data.shape, batch.size){ + feature_len <- data.shape*data.shape + 1 + csv_iter <- mx.io.CSVIter(data.csv=data.csv, data.shape=c(feature_len), batch.size=batch.size) + .self$iter <- csv_iter + .self$data.csv <- data.csv + .self$data.shape <- data.shape + .self$batch.size <- batch.size + .self + }, + value=function(){ + val <- as.array(.self$iter$value()$data) + val.x <- val[-1,] + val.y <- val[1,] + val.x <- val.x/255 + dim(val.x) <- c(data.shape, data.shape, 1, ncol(val.x)) + val.x <- mx.nd.array(val.x) + val.y <- mx.nd.array(val.y) + list(data=val.x, label=val.y) + }, + iter.next=function(){ + .self$iter$iter.next() + }, + reset=function(){ + .self$iter$reset() + }, + num.pad=function(){ + .self$iter$num.pad() + }, + finalize=function(){ + .self$iter$finalize() + } + ) + ) +``` + +To call the class we can just do: + +```{r} +batch.size <- 100 +train.iter <- CustomCSVIter$new(iter = NULL, data.csv = "mnist_train.csv", data.shape = 28, batch.size = batch.size) +``` + +## CNN Model + + +For this tutorial we are going to use the known LeNet architecture: + +```{r} +library(mxnet) +lenet.model <- function(){ + data <- mx.symbol.Variable('data') + conv1 <- mx.symbol.Convolution(data=data, kernel=c(5,5), num_filter=20) #first conv + tanh1 <- mx.symbol.Activation(data=conv1, act_type="tanh") + pool1 <- mx.symbol.Pooling(data=tanh1, pool_type="max", kernel=c(2,2), stride=c(2,2)) + conv2 <- mx.symbol.Convolution(data=pool1, kernel=c(5,5), num_filter=50)# second conv + tanh2 <- mx.symbol.Activation(data=conv2, act_type="tanh") + pool2 <- mx.symbol.Pooling(data=tanh2, pool_type="max", kernel=c(2,2), stride=c(2,2)) + flatten <- mx.symbol.Flatten(data=pool2) + fc1 <- mx.symbol.FullyConnected(data=flatten, num_hidden=100) # first fullc + tanh3 <- mx.symbol.Activation(data=fc1, act_type="tanh") + fc2 <- mx.symbol.FullyConnected(data=tanh3, num_hidden=10) # second fullc + network <- mx.symbol.SoftmaxOutput(data=fc2) # loss + network +} +network <- lenet.model() +``` + +## Training with the Custom Iterator + +Finally, we can directly add the custom iterator as the training data source. + +```{r, eval=FALSE} +model <- mx.model.FeedForward.create(symbol=network, + X=train.iter, + ctx=mx.gpu(0), + num.round=10, + array.batch.size=batch.size, + learning.rate=0.1, + momentum=0.9, + eval.metric=mx.metric.accuracy, + wd=0.00001, + batch.end.callback=mx.callback.log.speedometer(batch.size, frequency = 100) + ) +``` + +The last 2 iterations with a K80 GPU looks like this: + +``` +## [8] Train-accuracy=0.998866666666667 +## Batch [100] Speed: 15413.0104454713 samples/sec Train-accuracy=0.999 +## Batch [200] Speed: 16629.3412459049 samples/sec Train-accuracy=0.99935 +## Batch [300] Speed: 18412.6900509319 samples/sec Train-accuracy=0.9995 +## Batch [400] Speed: 16757.2882328335 samples/sec Train-accuracy=0.999425 +## Batch [500] Speed: 17116.6529207406 samples/sec Train-accuracy=0.99946 +## Batch [600] Speed: 19627.589505195 samples/sec Train-accuracy=0.99945 +## [9] Train-accuracy=0.9991 +## Batch [100] Speed: 18971.5745536982 samples/sec Train-accuracy=0.9992 +## Batch [200] Speed: 15554.8822435383 samples/sec Train-accuracy=0.99955 +## Batch [300] Speed: 18327.6950115053 samples/sec Train-accuracy=0.9997 +## Batch [400] Speed: 17103.0705411788 samples/sec Train-accuracy=0.9997 +## Batch [500] Speed: 15104.8656902394 samples/sec Train-accuracy=0.99974 +## Batch [600] Speed: 13818.7899518255 samples/sec Train-accuracy=0.99975 +## [10] Train-accuracy=0.99975 +``` + +## Conclusion + + +We have shown how to create a custom CSV Iterator by extending the class `mx.io.CSVIter`. In our class, we iteratively read from a CSV file a batch of data that will be transformed and then processed in the stochastic gradient descent optimization. That way, we are able to manage CSV files that are bigger than the memory of the machine we are using. + +Based of this custom iterator, we can also create data loaders that internally transform or expand the data, allowing to manage files of any size. + + + diff --git a/R-package/vignettes/CustomLossFunction.Rmd b/R-package/vignettes/CustomLossFunction.Rmd index 1817109e1387..85e882567f8e 100644 --- a/R-package/vignettes/CustomLossFunction.Rmd +++ b/R-package/vignettes/CustomLossFunction.Rmd @@ -1,18 +1,8 @@ ---- -title: "Customized loss function" -output: - md_document: - variant: markdown_github ---- - -```{r setup, include=FALSE} -knitr::opts_chunk$set(echo = TRUE) -``` +# Customized loss function This tutorial provides guidelines for using customized loss function in network construction. -Model Training Example ----------- +## Model Training Example Let's begin with a small regression example. We can build and train a regression model with the following code: @@ -56,8 +46,7 @@ Besides the `LinearRegressionOutput`, we also provide `LogisticRegressionOutput` However, this might not be enough for real-world models. You can provide your own loss function by using `mx.symbol.MakeLoss` when constructing the network. -How to Use Your Own Loss Function ---------- +## How to Use Your Own Loss Function We still use our previous example, but this time we use `mx.symbol.MakeLoss` to minimize the `(pred-label)^2` @@ -157,3 +146,6 @@ pred6 <- predict(model6, test.x) sum(abs(test.y - pred6[1,])) / length(test.y) ``` +We got the same result as expected. + + diff --git a/R-package/vignettes/classifyRealImageWithPretrainedModel.Rmd b/R-package/vignettes/classifyRealImageWithPretrainedModel.Rmd index f9d14d920b80..ff631e0f5ce9 100644 --- a/R-package/vignettes/classifyRealImageWithPretrainedModel.Rmd +++ b/R-package/vignettes/classifyRealImageWithPretrainedModel.Rmd @@ -1,5 +1,5 @@ -Classify Real-world Images with Pre-trained Model -================================================= +# Classify Real-world Images with Pre-trained Model + MXNet is a flexible and efficient deep learning framework. One of the cool things that a deep learning algorithm can do is to classify real world images. @@ -10,12 +10,7 @@ real world image. The network architecture is described in [1]. The pre-trained Inception-BatchNorm network can be downloaded from [this link](http://data.mxnet.io/mxnet/data/Inception.zip). This model gives the recent state-of-art prediction accuracy on the image net dataset. -Preface -------- -This tutorial is written in Rmarkdown. You can find the Rmarkdown source from [here](https://github.com/dmlc/mxnet/blob/master/R-package/vignettes/classifyRealImageWithPretrainedModel.Rmd) - -Package Loading ---------------- +## Package Loading To get started, we load the `mxnet` package first. @@ -29,13 +24,15 @@ In this example, we also need the imager package to load and preprocess the imag require(imager) ``` -Load the Pretrained Model -------------------------- +## Load the Pretrained Model + Make sure you unzip the pre-trained model in current folder. And we can use the model loading function to load the model into R. ```{r} +download.file('http://data.dmlc.ml/data/Inception.zip', destfile = 'Inception.zip') +unzip("Inception.zip") model <- mx.model.load("Inception/Inception_BN", iteration = 39) ``` @@ -45,8 +42,8 @@ We also need to load in the mean image, which is used for preprocessing using `` mean.img <- as.array(mx.nd.load("Inception/mean_224.nd")[["mean_img"]]) ``` -Load and Preprocess the Image ------------------------------ +## Load and Preprocess the Image + Now we are ready to classify a real image. In this example, we simply take the parrots image from imager package. But you can always change it to other images. @@ -91,8 +88,8 @@ We use the defined preprocessing function to get the normalized image. normed <- preproc.image(im, mean.img) ``` -Classify the Image ------------------- +## Classify the Image + Now we are ready to classify the image! We can use the predict function to get the probability over classes. @@ -105,6 +102,7 @@ As you can see ```prob``` is a 1 times 1000 array, which gives the probability over the 1000 image classes of the input. We can use the ```max.col``` on the transpose of prob. get the class index. + ```{r} max.idx <- max.col(t(prob)) max.idx @@ -126,8 +124,8 @@ print(paste0("Predicted Top-class: ", synsets[[max.idx]])) Actually I do not know what does the word mean when I saw it. So I searched on the web to check it out.. and hmm it does get the right answer :) -Extract features ------------------- +## Extract features + Besides the final classification results, we can also extract the internal features. We need to get feature layer symbol out of internals first. Here we use `global_pool_output` @@ -157,6 +155,10 @@ dim(global_pooling_feature) ``` -Reference ---------- +## Reference + + [1] Ioffe, Sergey, and Christian Szegedy. "Batch normalization: Accelerating deep network training by reducing internal covariate shift." arXiv preprint arXiv:1502.03167 (2015). + + + diff --git a/R-package/vignettes/fiveMinutesNeuralNetwork.Rmd b/R-package/vignettes/fiveMinutesNeuralNetwork.Rmd index bc45c9612e0f..fb023bb5435f 100644 --- a/R-package/vignettes/fiveMinutesNeuralNetwork.Rmd +++ b/R-package/vignettes/fiveMinutesNeuralNetwork.Rmd @@ -1,17 +1,9 @@ -Neural Network with MXNet in Five Minutes -============================================= +# Neural Network with MXNet in Five Minutes This is the first tutorial for new users of the R package `mxnet`. You will learn to construct a neural network to do regression in 5 minutes. We will show you how to do classification and regression tasks respectively. The data we use comes from the package `mlbench`. -## Preface - -This tutorial is written in Rmarkdown. - -- You can directly view the hosted version of the tutorial from [MXNet R Document](http://mxnet.readthedocs.io/en/latest/packages/r/fiveMinutesNeuralNetwork.html) -- You can find the download the Rmarkdown source from [here](https://github.com/dmlc/mxnet/blob/master/R-package/vignettes/fiveMinutesNeuralNetwork.Rmd) - ## Classification First of all, let us load in the data and preprocess it: @@ -177,3 +169,5 @@ dim(test.y) ``` Congratulations! Now you have learnt the basic for using `mxnet`. Please check the other tutorials for advanced features. + + diff --git a/R-package/vignettes/mnistCompetition.Rmd b/R-package/vignettes/mnistCompetition.Rmd index 74145415ec2a..988fd18e8b4d 100644 --- a/R-package/vignettes/mnistCompetition.Rmd +++ b/R-package/vignettes/mnistCompetition.Rmd @@ -1,19 +1,21 @@ -Handwritten Digits Classification Competition -============================================= +# Handwritten Digits Classification Competition [MNIST](http://yann.lecun.com/exdb/mnist/) is a handwritten digits image data set created by Yann LeCun. Every digit is represented by a 28x28 image. It has become a standard data set to test classifiers on simple image input. Neural network is no doubt a strong model for image classification tasks. There's a [long-term hosted competition](https://www.kaggle.com/c/digit-recognizer) on Kaggle using this data set. We will present the basic usage of [mxnet](https://github.com/dmlc/mxnet/tree/master/R-package) to compete in this challenge. -This tutorial is written in Rmarkdown. You can download the source [here](https://github.com/dmlc/mxnet/blob/master/R-package/vignettes/mnistCompetition.Rmd) and view a -hosted version of tutorial [here](http://mxnet.readthedocs.io/en/latest/packages/r/mnistCompetition.html). - ## Data Loading First, let us download the data from [here](https://www.kaggle.com/c/digit-recognizer/data), and put them under the `data/` folder in your working directory. Then we can read them in R and convert to matrices. -```{r, eval=FALSE} +```{r, echo=FALSE} +download.file('https://s3-us-west-2.amazonaws.com/apache-mxnet/R/data/mnist_csv.zip', destfile = 'mnist_csv.zip') +unzip('mnist_csv.zip', exdir = '.') +``` + + +```{r} require(mxnet) train <- read.csv("train.csv", header=TRUE) test <- read.csv("test.csv", header=TRUE) @@ -57,7 +59,7 @@ test.y <- load_label_file('mnist/t10k-labels-idx1-ubyte') Here every image is represented as a single row in train/test. The greyscale of each image falls in the range [0, 255], we can linearly transform it into [0,1] by -```{r, eval=FALSE} +```{r} train.x <- t(train.x/255) test <- t(test/255) ``` @@ -65,7 +67,7 @@ We also transpose the input matrix to npixel x nexamples, which is the column ma In the label part, we see the number of each digit is fairly even: -```{r, eval=FALSE} +```{r} table(train.y) ``` @@ -73,7 +75,7 @@ table(train.y) Now we have the data. The next step is to configure the structure of our network. -```{r, eval=FALSE} +```{r} data <- mx.symbol.Variable("data") fc1 <- mx.symbol.FullyConnected(data, name="fc1", num_hidden=128) act1 <- mx.symbol.Activation(fc1, name="relu1", act_type="relu") @@ -108,40 +110,42 @@ softmax <- mx.symbol.Variable("data") %>% We are almost ready for the training process. Before we start the computation, let's decide what device should we use. -```{r, eval=FALSE} +```{r} devices <- mx.cpu() ``` Here we assign CPU to `mxnet`. After all these preparation, you can run the following command to train the neural network! Note that `mx.set.seed` is the correct function to control the random process in `mxnet`. -```{r, eval=FALSE} +```{r} mx.set.seed(0) -model <- mx.model.FeedForward.create(softmax, X=train.x, y=train.y, - ctx=devices, num.round=10, array.batch.size=100, - learning.rate=0.07, momentum=0.9, eval.metric=mx.metric.accuracy, - initializer=mx.init.uniform(0.07), - batch.end.callback=mx.callback.log.train.metric(100)) +model <- mx.model.FeedForward.create(softmax, X = train.x, y = train.y, + ctx = devices, num.round = 5, + array.batch.size = 100, + learning.rate = 0.07, momentum = 0.9, + eval.metric = mx.metric.accuracy, + initializer = mx.init.uniform(0.07), + batch.end.callback = mx.callback.log.train.metric(100)) ``` ## Prediction and Submission To make prediction, we can simply write -```{r, eval=FALSE} +```{r} preds <- predict(model, test) dim(preds) ``` It is a matrix with 28000 rows and 10 cols, containing the desired classification probabilities from the output layer. To extract the maximum label for each row, we can use the `max.col` in R: -```{r, eval=FALSE} +```{r} pred.label <- max.col(t(preds)) - 1 table(pred.label) ``` With a little extra effort in the csv format, we can have our submission to the competition! -```{r, eval=FALSE} +```{r, eval = FALSE} submission <- data.frame(ImageId=1:ncol(test), Label=pred.label) write.csv(submission, file='submission.csv', row.names=FALSE, quote=FALSE) ``` @@ -179,16 +183,16 @@ lenet <- mx.symbol.SoftmaxOutput(data=fc2) Then let us reshape the matrices into arrays: -```{r, eval=FALSE} +```{r} train.array <- train.x dim(train.array) <- c(28, 28, 1, ncol(train.x)) -test.array <- test.x -dim(test.array) <- c(28, 28, 1, ncol(test.x)) +test.array <- test +dim(test.array) <- c(28, 28, 1, ncol(test)) ``` Next we are going to compare the training speed on different devices, so the definition of the devices goes first: -```{r, eval=FALSE} +```{r} n.gpu <- 1 device.cpu <- mx.cpu() device.gpu <- lapply(0:(n.gpu-1), function(i) { @@ -201,38 +205,42 @@ but since internal computation of cpu is already multi-threaded, there is less g We start by training on CPU first. Because it takes a bit time to do so, we will only run it for one iteration. -```{r, eval=FALSE} +```{r} mx.set.seed(0) tic <- proc.time() -model <- mx.model.FeedForward.create(lenet, X=train.array, y=train.y, - ctx=device.cpu, num.round=1, array.batch.size=100, - learning.rate=0.05, momentum=0.9, wd=0.00001, - eval.metric=mx.metric.accuracy, - batch.end.callback=mx.callback.log.train.metric(100)) +model <- mx.model.FeedForward.create(lenet, X = train.array, y = train.y, + ctx = device.cpu, num.round = 1, + array.batch.size = 100, + learning.rate = 0.05, momentum = 0.9, wd = 0.00001, + eval.metric = mx.metric.accuracy, + batch.end.callback = mx.callback.log.train.metric(100)) print(proc.time() - tic) ``` Training on GPU: -```{r, eval=FALSE} +```{r} mx.set.seed(0) tic <- proc.time() -model <- mx.model.FeedForward.create(lenet, X=train.array, y=train.y, - ctx=device.gpu, num.round=5, array.batch.size=100, - learning.rate=0.05, momentum=0.9, wd=0.00001, - eval.metric=mx.metric.accuracy, - batch.end.callback=mx.callback.log.train.metric(100)) +model <- mx.model.FeedForward.create(lenet, X = train.array, y = train.y, + ctx = device.gpu, num.round = 5, + array.batch.size = 100, + learning.rate = 0.05, momentum = 0.9, wd = 0.00001, + eval.metric = mx.metric.accuracy, + batch.end.callback = mx.callback.log.train.metric(100)) print(proc.time() - tic) ``` As you can see by using GPU, we can get a much faster speedup in training! Finally we can submit the result to Kaggle again to see the improvement of our ranking! -```{r, eval=FALSE} +```{r, eval = FALSE} preds <- predict(model, test.array) pred.label <- max.col(t(preds)) - 1 submission <- data.frame(ImageId=1:ncol(test), Label=pred.label) write.csv(submission, file='submission.csv', row.names=FALSE, quote=FALSE) ``` -![](../web-data/mxnet/knitr/mnistCompetition-kaggle-submission.png) +![](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/knitr/mnistCompetition-kaggle-submission.png) + + diff --git a/R-package/vignettes/ndarray.Rmd b/R-package/vignettes/ndarray.Rmd new file mode 100644 index 000000000000..08786b25fa86 --- /dev/null +++ b/R-package/vignettes/ndarray.Rmd @@ -0,0 +1,148 @@ +# NDArray: Vectorized Tensor Computations on CPUs and GPUs + +`NDArray` is the basic vectorized operation unit in MXNet for matrix and tensor computations. +Users can perform usual calculations as on an R"s array, but with two additional features: + +- Multiple devices: All operations can be run on various devices including +CPUs and GPUs. + +- Automatic parallelization: All operations are automatically executed in + parallel with each other. + +## Create and Initialize + +Let"s create `NDArray` on either a GPU or a CPU: + +```{r} +require(mxnet) +a <- mx.nd.zeros(c(2, 3)) # create a 2-by-3 matrix on cpu +b <- mx.nd.zeros(c(2, 3), mx.cpu()) # create a 2-by-3 matrix on cpu +c <- mx.nd.zeros(c(2, 3), mx.gpu(0)) # create a 2-by-3 matrix on gpu 0, if you have CUDA enabled. +``` + +Typically for CUDA-enabled devices, the device id of a GPU starts from 0. +That's why we passed in 0 to the GPU id. + +We can initialize an `NDArray` object in various ways: + + +```{r} +a <- mx.nd.ones(c(4, 4)) +b <- mx.rnorm(c(4, 5)) +c <- mx.nd.array(1:5) +``` + +To check the numbers in an `NDArray`, we can simply run: + + +```{r} +a <- mx.nd.ones(c(2, 3)) +b <- as.array(a) +class(b) +``` + +```{r} +b +``` + +## Performing Basic Operations + +### Elemental-wise Operations + +You can perform elemental-wise operations on `NDArray` objects, as follows: + + +```{r} +a <- mx.nd.ones(c(2, 4)) * 2 +b <- mx.nd.ones(c(2, 4)) / 8 +as.array(a) +``` + +```{r} +as.array(b) +``` + +```{r} +c <- a + b +as.array(c) +``` + +```{r} +d <- c / a - 5 +as.array(d) +``` + +If two `NDArray`s are located on different devices, we need to explicitly move them to the same one. For instance: + + +```{r} +a <- mx.nd.ones(c(2, 3)) * 2 +b <- mx.nd.ones(c(2, 3), mx.gpu()) / 8 +c <- mx.nd.copyto(a, mx.gpu()) * b +as.array(c) +``` + +### Loading and Saving + +You can save a list of `NDArray` object to your disk with `mx.nd.save`: + + +```{r} +a <- mx.nd.ones(c(2, 3)) +mx.nd.save(list(a), "temp.ndarray") +``` + +You can load it back easily: + + +```{r} +a <- mx.nd.load("temp.ndarray") +as.array(a[[1]]) +``` + +We can directly save data to and load it from a distributed file system, such as Amazon S3 and HDFS: + + +```{r, eval=FALSE} +mx.nd.save(list(a), "s3://mybucket/mydata.bin") +mx.nd.save(list(a), "hdfs///users/myname/mydata.bin") +``` + +## Automatic Parallelization + +`NDArray` can automatically execute operations in parallel. Automatic parallelization is useful when +using multiple resources, such as CPU cards, GPU cards, and CPU-to-GPU memory bandwidth. + +For example, if we write `a <- a + 1` followed by `b <- b + 1`, and `a` is on a CPU and +`b` is on a GPU, executing them in parallel improves +efficiency. Furthermore, because copying data between CPUs and GPUs are also expensive, running in parallel with other computations further increases efficiency. + +It's hard to find the code that can be executed in parallel by eye. In the +following example, `a <- a + 1` and `c <- c * 3` can be executed in parallel, but `a <- a + 1` and +`b <- b * 3` should be in sequential. + + +```{r} +a <- mx.nd.ones(c(2,3)) +b <- a +c <- mx.nd.copyto(a, mx.cpu()) +a <- a + 1 +b <- b * 3 +c <- c * 3 +``` + +Luckily, MXNet can automatically resolve the dependencies and +execute operations in parallel accurately. This allows us to write our program assuming there is only a single thread. MXNet will +automatically dispatch the program to multiple devices. + +MXNet achieves this with lazy evaluation. Each operation is issued to an +internal engine, and then returned. For example, if we run `a <- a + 1`, it +returns immediately after pushing the plus operator to the engine. This +asynchronous processing allows us to push more operators to the engine. It determines +the read and write dependencies and the best way to execute them in +parallel. + +The actual computations are finished, allowing us to copy the results someplace else, such as `as.array(a)` or `mx.nd.save(a, "temp.dat")`. To write highly parallelized codes, we only need to postpone when we need +the results. + + diff --git a/R-package/vignettes/ndarrayAndSymbolTutorial.Rmd b/R-package/vignettes/ndarrayAndSymbolTutorial.Rmd deleted file mode 100644 index a47147c9437a..000000000000 --- a/R-package/vignettes/ndarrayAndSymbolTutorial.Rmd +++ /dev/null @@ -1,242 +0,0 @@ -MXNet R Tutorial on NDArray and Symbol -====================================== - -This vignette gives a general overview of MXNet"s R package. MXNet contains a -mixed flavor of elements to bake flexible and efficient -applications. There are two major concepts introduced in this tutorial. - -* [NDArray](#ndarray-numpy-style-tensor-computations-on-cpus-and-gpus) - offers matrix and tensor computations on both CPU and GPU, with automatic - parallelization -* [Symbol](#symbol-and-automatic-differentiation) makes defining a neural - network extremely easy, and provides automatic differentiation. - -## NDArray: Vectorized tensor computations on CPUs and GPUs - -`NDArray` is the basic vectorized operation unit in MXNet for matrix and tensor computations. -Users can perform usual calculations as on R"s array, but with two additional features: - -1. **multiple devices**: all operations can be run on various devices including -CPU and GPU -2. **automatic parallelization**: all operations are automatically executed in - parallel with each other - -### Create and Initialization - -Let"s create `NDArray` on either GPU or CPU - -```{r} -require(mxnet) -a <- mx.nd.zeros(c(2, 3)) # create a 2-by-3 matrix on cpu -b <- mx.nd.zeros(c(2, 3), mx.cpu()) # create a 2-by-3 matrix on cpu -# c <- mx.nd.zeros(c(2, 3), mx.gpu(0)) # create a 2-by-3 matrix on gpu 0, if you have CUA enabled. -``` - -As a side note, normally for CUDA enabled devices, the device id of GPU starts from 0. -So that is why we passed in 0 to GPU id. We can also initialize an `NDArray` object in various ways: - -```{r} -a <- mx.nd.ones(c(4, 4)) -b <- mx.rnorm(c(4, 5)) -c <- mx.nd.array(1:5) -``` - -To check the numbers in an `NDArray`, we can simply run - -```{r} -a <- mx.nd.ones(c(2, 3)) -b <- as.array(a) -class(b) -b -``` - -### Basic Operations - -#### Elemental-wise operations - -You can perform elemental-wise operations on `NDArray` objects: - -```{r} -a <- mx.nd.ones(c(2, 4)) * 2 -b <- mx.nd.ones(c(2, 4)) / 8 -as.array(a) -as.array(b) -c <- a + b -as.array(c) -d <- c / a - 5 -as.array(d) -``` - -If two `NDArray`s sit on different devices, we need to explicitly move them -into the same one. For instance: - -```{r, eval=FALSE} -a <- mx.nd.ones(c(2, 3)) * 2 -b <- mx.nd.ones(c(2, 3), mx.gpu()) / 8 -c <- mx.nd.copyto(a, mx.gpu()) * b -as.array(c) -``` - -#### Load and Save - -You can save a list of `NDArray` object to your disk with `mx.nd.save`: - -```{r} -a <- mx.nd.ones(c(2, 3)) -mx.nd.save(a, "temp.ndarray") -``` - -You can also load it back easily: - -```{r} -a <- mx.nd.load("temp.ndarray") -as.array(a[[1]]) -``` - -In case you want to save data to the distributed file system such as S3 and HDFS, -we can directly save to and load from them. For example: - -```{r,eval=FALSE} -mx.nd.save(list(a), "s3://mybucket/mydata.bin") -mx.nd.save(list(a), "hdfs///users/myname/mydata.bin") -``` - -### Automatic Parallelization - -`NDArray` can automatically execute operations in parallel. It is desirable when we -use multiple resources such as CPU, GPU cards, and CPU-to-GPU memory bandwidth. - -For example, if we write `a <- a + 1` followed by `b <- b + 1`, and `a` is on CPU while -`b` is on GPU, then want to execute them in parallel to improve the -efficiency. Furthermore, data copy between CPU and GPU are also expensive, we -hope to run it parallel with other computations as well. - -However, finding the codes can be executed in parallel by eye is hard. In the -following example, `a <- a + 1` and `c <- c * 3` can be executed in parallel, but `a <- a + 1` and -`b <- b * 3` should be in sequential. - -```{r} -a <- mx.nd.ones(c(2,3)) -b <- a -c <- mx.nd.copyto(a, mx.cpu()) -a <- a + 1 -b <- b * 3 -c <- c * 3 -``` - -Luckily, MXNet can automatically resolve the dependencies and -execute operations in parallel with correctness guaranteed. In other words, we -can write program as by assuming there is only a single thread, while MXNet will -automatically dispatch it into multi-devices, such as multi GPU cards or multi -machines. - -It is achieved by lazy evaluation. Any operation we write down is issued into a -internal engine, and then returned. For example, if we run `a <- a + 1`, it -returns immediately after pushing the plus operator to the engine. This -asynchronous allows us to push more operators to the engine, so it can determine -the read and write dependency and find a best way to execute them in -parallel. - -The actual computations are finished if we want to copy the results into some -other place, such as `as.array(a)` or `mx.nd.save(a, "temp.dat")`. Therefore, if we -want to write highly parallelized codes, we only need to postpone when we need -the results. - -## Symbol and Automatic Differentiation - -WIth the computational unit `NDArray`, we need a way to construct neural networks. MXNet provides a symbolic interface named Symbol to do so. The symbol combines both flexibility and efficiency. - -### Basic Composition of Symbols - -The following codes create a two layer perceptrons network: - -```{r} -require(mxnet) -net <- mx.symbol.Variable("data") -net <- mx.symbol.FullyConnected(data=net, name="fc1", num_hidden=128) -net <- mx.symbol.Activation(data=net, name="relu1", act_type="relu") -net <- mx.symbol.FullyConnected(data=net, name="fc2", num_hidden=64) -net <- mx.symbol.SoftmaxOutput(data=net, name="out") -class(net) -``` - -Each symbol takes a (unique) string name. *Variable* often defines the inputs, -or free variables. Other symbols take a symbol as the input (*data*), -and may accept other hyper-parameters such as the number of hidden neurons (*num_hidden*) -or the activation type (*act_type*). - -The symbol can be simply viewed as a function taking several arguments, whose -names are automatically generated and can be get by - -```{r} -arguments(net) -``` - -As can be seen, these arguments are the parameters need by each symbol: - -- *data* : input data needed by the variable *data* -- *fc1_weight* and *fc1_bias* : the weight and bias for the first fully connected layer *fc1* -- *fc2_weight* and *fc2_bias* : the weight and bias for the second fully connected layer *fc2* -- *out_label* : the label needed by the loss - -We can also specify the automatic generated names explicitly: - -```{r} -data <- mx.symbol.Variable("data") -w <- mx.symbol.Variable("myweight") -net <- mx.symbol.FullyConnected(data=data, weight=w, name="fc1", num_hidden=128) -arguments(net) -``` - -### More Complicated Composition - -MXNet provides well-optimized symbols for -commonly used layers in deep learning. We can also easily define new operators -in python. The following example first performs an elementwise add between two -symbols, then feed them to the fully connected operator. - -```{r} -lhs <- mx.symbol.Variable("data1") -rhs <- mx.symbol.Variable("data2") -net <- mx.symbol.FullyConnected(data=lhs + rhs, name="fc1", num_hidden=128) -arguments(net) -``` - -We can also construct symbol in a more flexible way rather than the single -forward composition we addressed before. - -```{r} -net <- mx.symbol.Variable("data") -net <- mx.symbol.FullyConnected(data=net, name="fc1", num_hidden=128) -net2 <- mx.symbol.Variable("data2") -net2 <- mx.symbol.FullyConnected(data=net2, name="net2", num_hidden=128) -composed.net <- mx.apply(net, data=net2, name="compose") -arguments(composed.net) -``` - -In the above example, *net* is used a function to apply to an existing symbol -*net*, the resulting *composed.net* will replace the original argument *data* by -*net2* instead. - -### Training a Neural Net. - -The [model API](../../R-package/R/model.R) is a thin wrapper around the symbolic executors to support neural net training. - -You are also highly encouraged to read [Symbolic Configuration and Execution in Pictures for python package](../python/symbol_in_pictures.md), -which provides a detailed explanation of concepts in pictures. - -### How Efficient is Symbolic API - -In short, they are designed to be very efficient in both memory and runtime. - -The major reason for us to introduce Symbolic API, is to bring the efficient C++ -operations in powerful toolkits such as cxxnet and caffe together with the -flexible dynamic NArray operations. All the memory and computation resources are -allocated statically during Bind, to maximize the runtime performance and memory -utilization. - -The coarse grained operators are equivalent to cxxnet layers, which are -extremely efficient. We also provide fine grained operators for more flexible -composition. Because we are also doing more inplace memory allocation, mxnet can -be ***more memory efficient*** than cxxnet, and gets to same runtime, with -greater flexiblity. diff --git a/R-package/vignettes/symbol.Rmd b/R-package/vignettes/symbol.Rmd new file mode 100644 index 000000000000..228c6b26606c --- /dev/null +++ b/R-package/vignettes/symbol.Rmd @@ -0,0 +1,103 @@ +# Symbol and Automatic Differentiation + +The computational unit `NDArray` requires a way to construct neural networks. MXNet provides a symbolic interface, named Symbol, to do this. Symbol combines both flexibility and efficiency. + +## Basic Composition of Symbols + +The following code creates a two-layer perceptron network: + + +```{r} +require(mxnet) +net <- mx.symbol.Variable("data") +net <- mx.symbol.FullyConnected(data=net, name="fc1", num_hidden=128) +net <- mx.symbol.Activation(data=net, name="relu1", act_type="relu") +net <- mx.symbol.FullyConnected(data=net, name="fc2", num_hidden=64) +net <- mx.symbol.Softmax(data=net, name="out") +class(net) +``` + + +Each symbol takes a (unique) string name. *Variable* often defines the inputs, +or free variables. Other symbols take a symbol as the input (*data*), +and may accept other hyper parameters, such as the number of hidden neurons (*num_hidden*) +or the activation type (*act_type*). + +A symbol can be viewed as a function that takes several arguments, whose +names are automatically generated and can be retrieved with the following command: + + +```{r} +arguments(net) +``` + +The arguments are the parameters need by each symbol: + +- *data*: Input data needed by the variable *data* +- *fc1_weight* and *fc1_bias*: The weight and bias for the first fully connected layer, *fc1* +- *fc2_weight* and *fc2_bias*: The weight and bias for the second fully connected layer, *fc2* +- *out_label*: The label needed by the loss + +We can also specify the automatically generated names explicitly: + + +```{r} +data <- mx.symbol.Variable("data") +w <- mx.symbol.Variable("myweight") +net <- mx.symbol.FullyConnected(data=data, weight=w, name="fc1", num_hidden=128) +arguments(net) +``` + +## More Complicated Composition of Symbols + +MXNet provides well-optimized symbols for +commonly used layers in deep learning. You can also define new operators +in Python. The following example first performs an element-wise add between two +symbols, then feeds them to the fully connected operator: + + +```{r} +lhs <- mx.symbol.Variable("data1") +rhs <- mx.symbol.Variable("data2") +net <- mx.symbol.FullyConnected(data=lhs + rhs, name="fc1", num_hidden=128) +arguments(net) +``` + +We can construct a symbol more flexibly than by using the single +forward composition, for example: + +```{r} +net <- mx.symbol.Variable("data") +net <- mx.symbol.FullyConnected(data=net, name="fc1", num_hidden=128) +net2 <- mx.symbol.Variable("data2") +net2 <- mx.symbol.FullyConnected(data=net2, name="net2", num_hidden=128) +composed.net <- mx.apply(net, data=net2, name="compose") +arguments(composed.net) +``` + +In the example, *net* is used as a function to apply to an existing symbol +*net*. The resulting *composed.net* will replace the original argument *data* with +*net2* instead. + +## Training a Neural Net + +The [model API](../../../R-package/R/model.R) is a thin wrapper around the symbolic executors to support neural net training. + +We encourage you to read [Symbolic Configuration and Execution in Pictures for python package](../python/symbol_in_pictures.md)for a detailed explanation of concepts in pictures. + +## How Efficient Is the Symbolic API? + +The Symbolic API brings the efficient C++ +operations in powerful toolkits, such as CXXNet and Caffe, together with the +flexible dynamic NDArray operations. All of the memory and computation resources are +allocated statically during bind operations, to maximize runtime performance and memory +utilization. + +The coarse-grained operators are equivalent to CXXNet layers, which are +extremely efficient. We also provide fine-grained operators for more flexible +composition. Because MXNet does more in-place memory allocation, it can +be more memory efficient than CXXNet and gets to the same runtime with +greater flexibility. + + + \ No newline at end of file From 568b5a2d3e701768ff6f270238e5edccc2f35ff1 Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Sun, 13 Aug 2017 17:01:13 -0700 Subject: [PATCH 374/834] bce loss (#7304) --- python/mxnet/gluon/loss.py | 69 +++++++++++++++++++++++------- tests/python/unittest/test_loss.py | 30 +++++++++++++ 2 files changed, 84 insertions(+), 15 deletions(-) diff --git a/python/mxnet/gluon/loss.py b/python/mxnet/gluon/loss.py index 2b31840ad959..583910590868 100644 --- a/python/mxnet/gluon/loss.py +++ b/python/mxnet/gluon/loss.py @@ -20,7 +20,7 @@ """ losses for training neural networks """ from __future__ import absolute_import -from .. import symbol, ndarray +from .. import ndarray from ..base import numeric_types from .block import HybridBlock @@ -54,6 +54,11 @@ def _apply_weighting(F, loss, weight=None, sample_weight=None): return loss +def _reshape_label_as_output(F, output, label): + # for symbolic output.shape is not available so we reshape + # to empty shape and let it be inferred from output's shape + # via the '-' operator later. + return label.reshape(output.shape) if F is ndarray else label.reshape(()) class Loss(HybridBlock): """Base class for loss. @@ -113,13 +118,8 @@ def __init__(self, weight=1., batch_axis=0, **kwargs): super(L2Loss, self).__init__(weight, batch_axis, **kwargs) def hybrid_forward(self, F, output, label, sample_weight=None): - if F is ndarray: - loss = ndarray.square(output - label.reshape(output.shape)) - else: - # for symbolic output.shape is not available so we reshape - # to empty shape and let it be inferred from output's shape - # via the '-' operator later. - loss = symbol.square(output - label.reshape(())) + label = _reshape_label_as_output(F, output, label) + loss = F.square(output - label) loss = _apply_weighting(F, loss, self._weight/2, sample_weight) return F.mean(loss, axis=self._batch_axis, exclude=True) @@ -148,19 +148,56 @@ def __init__(self, weight=None, batch_axis=0, **kwargs): super(L1Loss, self).__init__(weight, batch_axis, **kwargs) def hybrid_forward(self, F, output, label, sample_weight=None): - if F is ndarray: - loss = ndarray.abs(output - label.reshape(output.shape)) + label = _reshape_label_as_output(F, output, label) + loss = F.abs(output - label) + loss = _apply_weighting(F, loss, self._weight, sample_weight) + return F.mean(loss, axis=self._batch_axis, exclude=True) + + +class SigmoidBinaryCrossEntropyLoss(Loss): + r"""The cross-entropy loss for binary classification. (alias: SigmoidBCELoss) + + BCE loss is useful when training logistic regression. + + .. math:: + loss(o, t) = - 1/n \sum_i (t[i] * log(o[i]) + (1 - t[i]) * log(1 - o[i])) + + + Parameters + ---------- + from_sigmoid : bool, default is `False` + Whether the input is from the output of sigmoid. Set this to false will make + the loss calculate sigmoid and then BCE, which is more numerically stable through + log-sum-exp trick. + weight : float or None + Global scalar weight for loss. + sample_weight : Symbol or None + Per sample weighting. Must be broadcastable to + the same shape as loss. For example, if loss has + shape (64, 10) and you want to weight each sample + in the batch, `sample_weight` should have shape (64, 1). + batch_axis : int, default 0 + The axis that represents mini-batch. + """ + def __init__(self, from_sigmoid=False, weight=None, batch_axis=0, **kwargs): + super(SigmoidBinaryCrossEntropyLoss, self).__init__(weight, batch_axis, **kwargs) + self._from_sigmoid = from_sigmoid + + def hybrid_forward(self, F, output, label, sample_weight=None): + label = _reshape_label_as_output(F, output, label) + if not self._from_sigmoid: + max_val = F.maximum(-output, 0) + loss = output - output*label + max_val + F.log(F.exp(-max_val)+F.exp(-output-max_val)) else: - # for symbolic output.shape is not available so we reshape - # to empty shape and let it be inferred from output's shape - # via the '-' operator later. - loss = symbol.abs(output - label.reshape(())) + loss = -(F.log(output+1e-8)*label + F.log(1.-output+1e-8)*(1.-label)) loss = _apply_weighting(F, loss, self._weight, sample_weight) return F.mean(loss, axis=self._batch_axis, exclude=True) +SigmoidBCELoss = SigmoidBinaryCrossEntropyLoss + class SoftmaxCrossEntropyLoss(Loss): - """Computes the softmax cross entropy loss. + """Computes the softmax cross entropy loss. (alias: SoftmaxCELoss) If `sparse_label` is `True`, label should contain integer category indicators: @@ -216,6 +253,8 @@ def hybrid_forward(self, F, output, label, sample_weight=None): loss = _apply_weighting(F, loss, self._weight, sample_weight) return F.mean(loss, axis=self._batch_axis, exclude=True) +SoftmaxCELoss = SoftmaxCrossEntropyLoss + class KLDivLoss(Loss): """The Kullback-Leibler divergence loss. diff --git a/tests/python/unittest/test_loss.py b/tests/python/unittest/test_loss.py index 8eced7bc00d1..714ea7562fdb 100644 --- a/tests/python/unittest/test_loss.py +++ b/tests/python/unittest/test_loss.py @@ -18,6 +18,7 @@ import mxnet as mx import numpy as np from mxnet import gluon +from mxnet.test_utils import assert_almost_equal def test_loss_ndarray(): @@ -81,6 +82,34 @@ def test_ce_loss(): assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.01 +def test_bce_loss(): + mx.random.seed(1234) + np.random.seed(1234) + N = 20 + data = mx.random.uniform(-1, 1, shape=(N, 20)) + label = mx.nd.array(np.random.randint(2, size=(N,)), dtype='float32') + data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label') + output = get_net(1) + fc2 = output.get_internals()['fc2_output'] + l = mx.symbol.Variable('label') + Loss = gluon.loss.SigmoidBinaryCrossEntropyLoss() + loss = Loss(output, l) + loss = mx.sym.make_loss(loss) + mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',)) + mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 1.}, + eval_metric=mx.metric.Loss()) + assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.01 + +def test_bce_equal_ce2(): + N = 100 + loss1 = gluon.loss.SigmoidBCELoss(from_sigmoid=True) + loss2 = gluon.loss.SoftmaxCELoss(from_logits=True) + out1 = mx.random.uniform(0, 1, shape=(N, 1)) + out2 = mx.nd.log(mx.nd.concat(1-out1, out1, dim=1) + 1e-8) + label = mx.nd.round(mx.random.uniform(0, 1, shape=(N, 1))) + assert_almost_equal(loss1(out1, label).asnumpy(), loss2(out2, label).asnumpy()) + + def test_kl_loss(): mx.random.seed(1234) np.random.seed(1234) @@ -117,6 +146,7 @@ def test_l2_loss(): eval_metric=mx.metric.Loss()) assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.05 + def test_l1_loss(): mx.random.seed(1234) np.random.seed(1234) From a13fa6fc1d70d6c23defc76efbfcbe5de88bef17 Mon Sep 17 00:00:00 2001 From: Pracheer Gupta Date: Mon, 14 Aug 2017 14:29:40 -0700 Subject: [PATCH 375/834] Updating CoreML readme file (#7459) * Fixing CoreML converter's README: typos/grammar/etc. * CoreML converter README update: Talk about layers first and then about models. * Providing examples on converting various standard models; calling out issues with InceptionV3. --- tools/coreml/README.md | 77 ++++++++++++++++++++++++++---------------- 1 file changed, 48 insertions(+), 29 deletions(-) diff --git a/tools/coreml/README.md b/tools/coreml/README.md index 32cde339d3a9..e29eebe84bc1 100644 --- a/tools/coreml/README.md +++ b/tools/coreml/README.md @@ -21,59 +21,45 @@ Let's say you want to use your MXNet model in an iPhone App. For the purpose of python mxnet_coreml_converter.py --model-prefix='squeezenet_v1.1' --epoch=0 --input-shape='{"data":"3,227,227"}' --mode=classifier --pre-processing-arguments='{"image_input_names":"data"}' --class-labels classLabels.txt --output-file="squeezenetv11.mlmodel" ``` - The above command will save the converted model into squeezenet-v11.mlmodel in CoreML format. Internally MXNet first loads the model and then we walk through the entire symbolic graph converting each operator into its CoreML equivalent. Some of the parameters are used by MXNet in order to load and generate the symbolic graph in memory while others are used by CoreML either to pre-process the input before the going through the neural network or to process the output in a particular way. + The above command will save the converted model in CoreML format to file squeezenet-v11.mlmodel. Internally, the model is first loaded by MXNet recreating the entire symbolic graph in memory. The converter walks through this symbolic graph converting each operator into its CoreML equivalent. Some of the supplied arguments to the converter are used by MXNet to generate the graph while others are used by CoreML either to pre-process the input (before passing it to the neural network) or to process the output of the neural network in a particular way. In the command above: - * _model-prefix_: refers to the MXNet model prefix (may include the directory path). - * _epoch_: refers to the suffix of the MXNet model file. - * _input-shape_: refers to the input shape information in a JSON string format where the key is the name of the input variable (="data") and the value is the shape of that variable. If the model takes multiple inputs, input-shape for all of them need to be provided. + * _model-prefix_: refers to the prefix of the file containing the MXNet model that needs to be converted (may include the directory path). E.g. for squeezenet model above the model files are squeezenet_v1.1-symbol.json and squeezenet_v1.1-0000.params and, therefore, model-prefix is "squeezenet_v1.1" (or "/squeezenet_v1.1") + * _epoch_: refers to the suffix of the MXNet model filename. For squeezenet model above, it'll be 0. + * _input-shape_: refers to the input shape information in a JSON string format where the key is the name of the input variable (i.e. "data") and the value is the shape of that variable. If the model takes multiple inputs, input-shape for all of them need to be provided. * _mode_: refers to the coreml model mode. Can either be 'classifier', 'regressor' or None. In this case, we use 'classifier' since we want the resulting CoreML model to classify images into various categories. - * _pre-processing-arguments_: In the Apple world images have to be of type Image. By providing image_input_names as "data", we are saying that the input variable "data" is of type Image. + * _pre-processing-arguments_: In the Apple world, images have to be of type "Image". By providing image_input_names as "data", the converter will assume that the input variable "data" is of type "Image". * _class-labels_: refers to the name of the file which contains the classification labels (a.k.a. synset file). -output-file: the file where the CoreML model will be dumped. + * _output-file_: the file where resulting CoreML model will be stored. 3. The generated ".mlmodel" file can directly be integrated into your app. For more instructions on how to do this, please see [Apple CoreML's tutorial](https://developer.apple.com/documentation/coreml/integrating_a_core_ml_model_into_your_app). ### Providing class labels -You could provide a file containing class labels (as above) so that CoreML will return the predicted category the image belongs to. The file should have a label per line and labels can have any special characters. The line number of the label in the file should correspond with the index of softmax output. E.g. +You could provide a file containing class labels (as above) so that CoreML will return the category a given image belongs to. The file should have a label per line and labels can have any special characters. The line number of the label in the file should correspond with the index of softmax output. E.g. ```bash python mxnet_coreml_converter.py --model-prefix='squeezenet_v1.1' --epoch=0 --input-shape='{"data":"3,227,227"}' --mode=classifier --class-labels classLabels.txt --output-file="squeezenetv11.mlmodel" ``` -### Providing label names -You may have to provide the label names of the MXNet model's outputs. For example, if you try to convert [vgg16](http://data.mxnet.io/models/imagenet/vgg/), you may have to provide label-name as "prob_label". By default "softmax_label" is assumed. - -```bash -python mxnet_coreml_converter.py --model-prefix='vgg16' --epoch=0 --input-shape='{"data":"3,224,224"}' --mode=classifier --pre-processing-arguments='{"image_input_names":"data"}' --class-labels classLabels.txt --output-file="vgg16.mlmodel" --label-names="prob_label" -``` - -### Adding a pre-processing to CoreML model. -You could ask CoreML to pre-process the images before passing them through the model. +### Adding a pre-processing layer to CoreML model. +You could ask CoreML to pre-process the images before passing them through the model. The following command provides image re-centering parameters for red, blue and green channel. ```bash python mxnet_coreml_converter.py --model-prefix='squeezenet_v1.1' --epoch=0 --input-shape='{"data":"3,224,224"}' --pre-processing-arguments='{"red_bias":127,"blue_bias":117,"green_bias":103}' --output-file="squeezenet_v11.mlmodel" ``` -If you are building an app for a model that takes image as an input, you will have to provide image_input_names as pre-processing arguments. This tells CoreML that a particular input variable is of type Image. E.g.: - +If you are building an app for a model that takes "Image" as an input, you will have to provide image_input_names as pre-processing arguments. This tells CoreML that a particular input variable is of type Image. E.g.: + ```bash python mxnet_coreml_converter.py --model-prefix='squeezenet_v1.1' --epoch=0 --input-shape='{"data":"3,224,224"}' --pre-processing-arguments='{"red_bias":127,"blue_bias":117,"green_bias":103,"image_input_names":"data"}' --output-file="squeezenet_v11.mlmodel" ``` ## Currently supported -### Models -This is a (growing) list of standard MXNet models that can be successfully converted using the converter. This means that any other model that uses similar operators as these models can also be successfully converted. - -1. Inception: [Inception-BN](http://data.mxnet.io/models/imagenet/inception-bn/), [Inception-V3](http://data.mxnet.io/models/imagenet/inception-v3.tar.gz) -2. [NiN](http://data.dmlc.ml/models/imagenet/nin/) -2. [Resnet](http://data.mxnet.io/models/imagenet/resnet/) -3. [Squeezenet](http://data.mxnet.io/models/imagenet/squeezenet/) -4. [Vgg](http://data.mxnet.io/models/imagenet/vgg/) - ### Layers +List of MXNet layers that can be converted into their CoreML equivalent: + 1. Activation 2. Batchnorm 3. Concat @@ -87,9 +73,42 @@ This is a (growing) list of standard MXNet models that can be successfully conve 11. Softmax 12. Transpose +### Models +Any MXNet model that uses the above operators can be converted easily. For instance, the following standard models can be converted: + +1. [Inception-BN](http://data.mxnet.io/models/imagenet/inception-bn/) + +```bash +python mxnet_coreml_converter.py --model-prefix='Inception-BN' --epoch=126 --input-shape='{"data":"3,224,224"}' --mode=classifier --pre-processing-arguments='{"image_input_names":"data"}' --class-labels classLabels.txt --output-file="InceptionBN.mlmodel" +``` + +2. [NiN](http://data.dmlc.ml/models/imagenet/nin/) + +```bash +python mxnet_coreml_converter.py --model-prefix='nin' --epoch=0 --input-shape='{"data":"3,224,224"}' --mode=classifier --pre-processing-arguments='{"image_input_names":"data"}' --class-labels classLabels.txt --output-file="nin.mlmodel" +``` + +3. [Resnet](http://data.mxnet.io/models/imagenet/resnet/) + +```bash +python mxnet_coreml_converter.py --model-prefix='resnet-50' --epoch=0 --input-shape='{"data":"3,224,224"}' --mode=classifier --pre-processing-arguments='{"image_input_names":"data"}' --class-labels classLabels.txt --output-file="resnet50.mlmodel" +``` + +4. [Squeezenet](http://data.mxnet.io/models/imagenet/squeezenet/) + +```bash +python mxnet_coreml_converter.py --model-prefix='squeezenet_v1.1' --epoch=0 --input-shape='{"data":"3,227,227"}' --mode=classifier --pre-processing-arguments='{"image_input_names":"data"}' --class-labels classLabels.txt --output-file="squeezenetv11.mlmodel" +``` + +5. [Vgg](http://data.mxnet.io/models/imagenet/vgg/) + +```bash +python mxnet_coreml_converter.py --model-prefix='vgg16' --epoch=0 --input-shape='{"data":"3,224,224"}' --mode=classifier --pre-processing-arguments='{"image_input_names":"data"}' --class-labels classLabels.txt --output-file="vgg16.mlmodel" +``` + ## Known issues -Currently there are no known issues. +* [Inception-V3](http://data.mxnet.io/models/imagenet/inception-v3.tar.gz) model can be converted into CoreML format but is unable to run on Xcode. -## This tool has been tested on environment with: +## This tool has been tested with: * MacOS - High Sierra 10.13 Beta. * Xcode 9 beta 5. From 1470bf614b9ee3da7f81919b526db6df18451587 Mon Sep 17 00:00:00 2001 From: Naveen Swamy Date: Mon, 14 Aug 2017 14:43:15 -0700 Subject: [PATCH 376/834] add Naveen's Code Signing Key (#7460) --- KEYS | 59 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/KEYS b/KEYS index 19ec1a3e5f15..070f38d4f78e 100644 --- a/KEYS +++ b/KEYS @@ -130,3 +130,62 @@ TZQhIRekaaV+bCQQxnwDOJ31bIUUpxaMdvygjq55Gri/5C75TsMNcgbhqYWLGKe2 kRsGTxyO+fQ6/Q== =FuXU -----END PGP PUBLIC KEY BLOCK----- +pub rsa4096 2017-08-14 [SC] + AA3EBCC3E65A768AE3D2A64B8EF47B8720E8C549 +uid [ultimate] Naveen Swamy (CODE SIGNING KEY) +sig 3 8EF47B8720E8C549 2017-08-14 Naveen Swamy (CODE SIGNING KEY) +sub rsa4096 2017-08-14 [E] +sig 8EF47B8720E8C549 2017-08-14 Naveen Swamy (CODE SIGNING KEY) + +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBFmSC4cBEADFOKHTd2QFZk94eCCh5kqDTcZk2zgu+tNb2PY0v/EVC/rEZN2O +IS+Y16gO7DQEnyreoPBe9QdwT85iCshhl80x6ojfRHztCcXADzNLPc0knhPNeRUt +feQOwbxtWmIyglQRPbeRkhQtZbceHMLT0tjpDdU2ogI1tt4OfFkCdXX2k9nxeCfQ +KKVMvK/vPFtkcLrTDPzG31XDvbJdHzKjHXVR1D88gVX23+YTZQX2ZFD4aWyix8xy +LcH1PE0oNY3Ja6YSXqgxPa+cvOslyd0HMO8EzJTfv65jEqf2CDJTxIER8ihfyjLa +GQAH8pNHZFrIDrOVNQXgNq0oG629rtFJVBb9MLTEi3zMf4aKddcE57j0aodEGXEs +eWWmULty4s/fhFb7DaEQ9TJpcMJYE89/zVP342nAMTjMAsPsW2RnaL7Q8uGDN3aT +O87ifl6LERp5CHJQxyZPm3no6WPEaI9WdoXPsz10EnzGP95zYRM/lsKEXu3ur0P3 +1xQXXfFyzvVeeor0Yyf7Oh63TJ76A+tTLiXMeFGd7xs65vh6yUHuhQZmqygFi0fI +zO8Wc1hr5LxEh0kFIKAngL0AL4ukf5Aii6wFvOj0kx6AxlsP8Jas4dQd3e1G3Apo +lij78wpeqLRPl04XTp8HNu5+wq5qj/GwNlx0SMwVT1h/2SC1cUaKi0DUuwARAQAB +tDNOYXZlZW4gU3dhbXkgKENPREUgU0lHTklORyBLRVkpIDxuc3dhbXlAYXBhY2hl +Lm9yZz6JAk4EEwEIADgCGwMCHgECF4AWIQSqPrzD5lp2iuPSpkuO9HuHIOjFSQUC +WZIMrAULCQgHAwUVCgkICwUWAgMBAAAKCRCO9HuHIOjFSRaoD/9P2ktLKFjEwm3j +sf/HDqmKd4jNHtCv/FUhzM0kb4F4gxXcnoFavDUdyLdTisEYx033Enkyv3jSBKB8 +bYxH4awmQ/47pexEPnpLPrw6Rpsbiuk8O2RLMWw2ObRATrNXg088YbBXgg4xrxXd +4tjpd8FB1TJJnsmvrAawScjwz8ZxPQTaCqxb7oyrkRJYgswPmVD2MrB4LAjxMbpW +pUkrQSxt6OEmteZXQd1Wn9UnD88YQEfaviCevo7cpsFrUHHXH9ihUI+fjihc+NpB +LW9O4gVXY0O9BOMIU4xqHvFMht0s7Tjj698xoANosvGtO7mV/OKCtEHuqQCKzP4/ +9QS9PJrci/msBd/UwYqtYggACFnAtijOT70a7PRp3zHK5um5lsIsxuGJWJutlXiB +cCrvgrdEaEXSUQsghygsUNzYzohAzYyV3FYuvaxuFwkLKewMzSOLW5DewPpZTTSa +pO+CsmiDL2RJYS2dbz84elq1FUlNZZevFmrZmtpKClOrQ/2A6lHvs/dH5Qs4Ews/ +Wl0Hwsk2ET1VbJEVjK+CZd9CwYXZBaW2ntLr88LfrbsbXg5HW9cowmMdbMq9Rb1L +4z/OaOUTp+M7nfQP9F5/6JmGICM/2RC2DYwkqrwQe+mvp6P6QNGe2z7OG19sHMyb +qDWc+N4+VcribZV3AQsdloX7Y6GscrkCDQRZkguHARAAustOuroA9Oieela+WUZP +0M9srwsH1XHpfKHgGgPAFXVQZ2YGXl9uxG73v4kat5kOdwPERPbuEYqOM/FyIs87 +8AxgQ+dh1YB7boDslubqUAbXPaxso4ZRyxDidmdR+XRi9ZZRNTYdiA+RhS7/Y3lp +Fb2Xr4xZWtqRzuNOTp1OQ51uOaFRAj/hDZJi7v73LNIocnrk8mFDCUGaHcNzUqxY +FvVkzi8fr8diM9Y1DJsTuQicJdYFQAIfFneddp2YyHTlB6IxbBLME3DJcN6pF6Eq +1pTP77Nss4voR/0RXgByZ4OeMgFudnuN+bz8mBVtr/ToWb/c8hhYBOrbBcegSXMg +gqPIk8FjYblmPqW1qUpI4fV66TIh2XT/bOoDZ8+FGRKznD2gWzeOOeq8vLG+rQN9 +ko0YMgrdqvtioD9vOd2CKpE5eZbalRjAttqC92mcURC2t/oVEB8kOdURenkOMzCN +T4MpMrzIL2x98tmiq8/wP7HDH+Yq4HSGnpHTK5INO9rmKpewiSKdLU1HKeCjF4mn +P9kfWCCz6U6bHO4vm6UQ0EgV8nM616laDWE49DFO/9WqoPzK3CanLp/Gy2pdK3CQ +R71OzB8XOMratmA5oL/c8hIZdF1i63KjLCSaQ7w6VR/j2gh61ftO0rtD8NmksphM +X25F37SwZ6ro8QQKONkhWncAEQEAAYkCNgQYAQgAIBYhBKo+vMPmWnaK49KmS470 +e4cg6MVJBQJZkguHAhsMAAoJEI70e4cg6MVJxZ0QAKCHbB2DgoED0JZ4xnADcc7t +o1Bz5SQgAWfh9eJD1Ou4cqhk9u2Bh5mX/z6UBc6ZeSsgI55NWxaZh0LiaeKqIufY +2+4a8PfuJPLQ1Q94NMMTAyA2tpIqsFk6V+5IB/heC94L3US8H3v9CvvlZyErhSsu +OVoIxM5S0f6W3vA3nX5iNUQHzRllAMkzoFmTET6ZzWskwOCjQ/qr/tasehpsYTaJ +pUWRZA7ExbIAIclnjuQM9FsMVzsaJcxqw2gbJFjVPumysz9NKOghAGzRH4JBnxpu +wAo/UH+668R1GpFDZpHFKwEdh3zXffo6Zq9lQmAJ5NTa7L5JUGuzlIF40asLG2MN +0ywDW9/oHuCDaM0tITSmRLn6v+QVApoGD89svQ6yCZ5MeqRfP+H6CSFf6fQ3E4Cu +kIoH1GBllwnRmoQrAKyR4a7OqTVm6B+LyA+jTaa79g5UjDN7qlbGQ8MR5rE/yutP +8PNCFmE/EsImQ7NREfRKqle0+mSAWqKkdg4pX5bJNbVQX2LOLgMF5LJdUtwq8ISJ +7/k9J/FTJyuqgwXvkUOq7eEehxUpvX85gzJ5tpMSN+jYgPeMWcd8mTvVgwWDd7Qu +TNxwR0b9K/mLKGh58n1vVT79QReQFQ4wWFyQkmFkL9ybG04wTKe00VDNP987nSBg +FuSamX64+S6T8IwAuP9U +=KRiV +-----END PGP PUBLIC KEY BLOCK----- From 507e8307ed75f22a06d61f73e5671d9152c3b618 Mon Sep 17 00:00:00 2001 From: Yao Wang Date: Mon, 14 Aug 2017 17:17:58 -0700 Subject: [PATCH 377/834] Fix toc (#7465) --- docs/_static/js/sidebar.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/_static/js/sidebar.js b/docs/_static/js/sidebar.js index 1df628c4089f..42607068e16e 100644 --- a/docs/_static/js/sidebar.js +++ b/docs/_static/js/sidebar.js @@ -42,7 +42,15 @@ function render_lefttoc() { for(var i = 0; i < TITLE_WITH_LANG.length; ++i) { var path = TITLE_WITH_LANG[i]; if (url.indexOf(path) != -1) { - var urlPath = 'http://' + window.location.host + path; + urlElem = url.split('/'); + version = ''; + for (var j = 0; j < urlElem.length; ++j) { + if(urlElem[j] == 'versions') { + version = '/versions/' + urlElem[j + 1]; + break; + } + } + var urlPath = 'https://' + window.location.host + version + path; $.get(urlPath + indexTrailing, null, function(data) { var currentText = $($.parseHTML(data)).find('.leftsidebar > .sphinxsidebarwrapper > ul.current > li.current > a').html(); if (isAPI) { From b51515f29667cd432e18c7b43f086a27f70948bf Mon Sep 17 00:00:00 2001 From: Yao Wang Date: Mon, 14 Aug 2017 17:21:09 -0700 Subject: [PATCH 378/834] Fix apache link (#7468) --- docs/build_version_doc/AddVersion.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/docs/build_version_doc/AddVersion.py b/docs/build_version_doc/AddVersion.py index 34ba40e0f3a4..c48c630565b7 100755 --- a/docs/build_version_doc/AddVersion.py +++ b/docs/build_version_doc/AddVersion.py @@ -64,6 +64,7 @@ 'https://github.com/apache/incubator-mxnet/tree/%s/example' % (args.current_version) navbar = content.find(id="main-nav") navbar_mobile = content.find(id="burgerMenu") + outstr = str(content) if navbar and navbar_mobile: version_tag = content.find(id="dropdown-menu-position-anchor-version") version_tag_mobile = content.find(id="dropdown-menu-position-anchor-version-mobile") @@ -74,5 +75,17 @@ navbar.append(version_str) navbar_mobile.append(version_str_mobile) outstr = str(content).replace('<', '<').replace('>', '>') - with open(os.path.join(path, name), "w") as outf: - outf.write(outstr) + # Fix link + if args.current_version == tag_list[0]: + print("Fixing" + os.path.join(path, name)) + outstr = outstr.replace('https://mxnet.io', 'https://mxnet.incubator.apache.org') + outstr = outstr.replace('http://mxnet.io', 'https://mxnet.incubator.apache.org') + else: + outstr = outstr.replace('https://mxnet.io', 'https://mxnet.incubator.apache.org/' + 'versions/%s' % (args.current_version)) + outstr = outstr.replace('http://mxnet.io', 'https://mxnet.incubator.apache.org/' + 'versions/%s' % (args.current_version)) + + with open(os.path.join(path, name), "w") as outf: + outf.write(outstr) + From 09303cb9453fe99302f24a22c671ed7e4a42fb2d Mon Sep 17 00:00:00 2001 From: Steffen Rochel Date: Mon, 14 Aug 2017 20:11:35 -0700 Subject: [PATCH 379/834] Update Jenkinsfile (#7466) added make clean before make docs --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index 632789ac194a..95115cf58920 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -422,6 +422,7 @@ try { ws('workspace/docs') { if (env.BRANCH_NAME == "master") { init_git() + sh "make clean" sh "make docs" } } From a21d3e0526588c1bbe7efcf8a93e9108dfb207b5 Mon Sep 17 00:00:00 2001 From: Sandeep Krishnamurthy Date: Tue, 15 Aug 2017 11:01:22 -0700 Subject: [PATCH 380/834] Fix more broken links (#7480) --- docs/get_started/windows_setup.md | 2 +- docs/model_zoo/index.md | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/get_started/windows_setup.md b/docs/get_started/windows_setup.md index 86104c6be5f3..d695c5993f0c 100755 --- a/docs/get_started/windows_setup.md +++ b/docs/get_started/windows_setup.md @@ -23,7 +23,7 @@ This produces a library called ```libmxnet.dll```. To build and install MXNet yourself, you need the following dependencies. Install the required dependencies: 1. If [Microsoft Visual Studio 2013](https://www.visualstudio.com/downloads/) is not already installed, download and install it. You can download and install the free community edition. -2. Install [Visual C++ Compiler Nov 2013 CTP](https://www.microsoft.com/en-us/download/details.aspx?id=41151). +2. Install [Visual C++ Compiler](http://landinghub.visualstudio.com/visual-cpp-build-tools). 3. Back up all of the files in the ```C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC``` folder to a different location. 4. Copy all of the files in the ```C:\Program Files (x86)\Microsoft Visual C++ Compiler Nov 2013 CTP``` folder (or the folder where you extracted the zip archive) to the ```C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC``` folder, and overwrite all existing files. 5. Download and install [OpenCV](http://sourceforge.net/projects/opencvlibrary/files/opencv-win/3.0.0/opencv-3.0.0.exe/download). diff --git a/docs/model_zoo/index.md b/docs/model_zoo/index.md index a5a2b327937a..19811f22552d 100644 --- a/docs/model_zoo/index.md +++ b/docs/model_zoo/index.md @@ -32,7 +32,7 @@ Convolutional neural networks are the state-of-art architecture for many image a * [Places2](http://places2.csail.mit.edu/download.html): There are 1.6 million train images from 365 scene categories in the Places365-Standard, which are used to train the Places365 CNNs. There are 50 images per category in the validation set and 900 images per category in the testing set. Compared to the train set of Places365-Standard, the train set of Places365-Challenge has 6.2 million extra images, leading to totally 8 million train images for the Places365 challenge 2016. The validation set and testing set are the same as the Places365-Standard. * [Multimedia Commons](https://aws.amazon.com/public-datasets/multimedia-commons/): YFCC100M (99.2 million images and 0.8 million videos from Flickr) and supplemental material (pre-extracted features, additional annotations). -For instructions on using these models, see [the python tutorial on using pre-trained ImageNet models](http://mxnet.io/tutorials/python/predict_imagenet.html). +For instructions on using these models, see [the python tutorial on using pre-trained ImageNet models](https://mxnet.incubator.apache.org/tutorials/python/predict_image.html). | Model Definition | Dataset | Model Weights | Research Basis | Contributors | | --- | --- | --- | --- | --- | @@ -53,19 +53,19 @@ For instructions on using these models, see [the python tutorial on using pre-tr ## Recurrent Neural Networks (RNNs) including LSTMs -MXNet supports many types of recurrent neural networks (RNNs), including Long Short-Term Memory ([LSTM](http://deeplearning.cs.cmu.edu/pdfs/Hochreiter97_lstm.pdf)) +MXNet supports many types of recurrent neural networks (RNNs), including Long Short-Term Memory ([LSTM](http://www.bioinf.jku.at/publications/older/2604.pdf)) and Gated Recurrent Units (GRU) networks. Some available datasets include: -* [Penn Treebank (PTB)](https://www.cis.upenn.edu/~treebank/): Text corpus with ~1 million words. Vocabulary is limited to 10,000 words. The task is predicting downstream words/characters. +* [Penn Treebank (PTB)](https://catalog.ldc.upenn.edu/LDC95T7): Text corpus with ~1 million words. Vocabulary is limited to 10,000 words. The task is predicting downstream words/characters. * [Shakespeare](http://cs.stanford.edu/people/karpathy/char-rnn/): Complete text from Shakespeare's works. -* [IMDB reviews](https://s3.amazonaws.com/text-datasets): 25,000 movie reviews, labeled as positive or negative +* [IMDB reviews](https://getsatisfaction.com/imdb/topics/imdb-data-now-available-in-amazon-s3): 25,000 movie reviews, labeled as positive or negative * [Facebook bAbI](https://research.facebook.com/researchers/1543934539189348): As a set of 20 question & answer tasks, each with 1,000 training examples. * [Flickr8k, COCO](http://mscoco.org/): Images with associated caption (sentences). Flickr8k consists of 8,092 images captioned by AmazonTurkers with ~40,000 captions. COCO has 328,000 images, each with 5 captions. The COCO images also come with labeled objects using segmentation algorithms. | Model Definition | Dataset | Model Weights | Research Basis | Contributors | | --- | --- | --- | --- | --- | -| LSTM - Image Captioning | Flickr8k, MS COCO | | [Vinyals et al.., 2015](https://arxiv.org/pdf/ 1411.4555v2.pdf) | @... | +| LSTM - Image Captioning | Flickr8k, MS COCO | | [Vinyals et al.., 2015](https://arxiv.org/pdf/1411.4555.pdf) | @... | | LSTM - Q&A System| bAbl | | [Weston et al.., 2015](https://arxiv.org/pdf/1502.05698v10.pdf) | | | LSTM - Sentiment Analysis| IMDB | | [Li et al.., 2015](http://arxiv.org/pdf/1503.00185v5.pdf) | | From 7d6385a515dbda096560271628abf34e91e9be63 Mon Sep 17 00:00:00 2001 From: Eric Junyuan Xie Date: Tue, 15 Aug 2017 12:24:35 -0700 Subject: [PATCH 381/834] fix autograd memory cost (#7478) * fix autograd memory cost * grad_req setting * fix * fix * fix tests --- nnvm | 2 +- python/mxnet/gluon/data/dataloader.py | 10 ++- python/mxnet/gluon/data/dataset.py | 7 +- python/mxnet/gluon/nn/basic_layers.py | 12 ++-- python/mxnet/gluon/parameter.py | 75 ++++++++++++++++++--- src/executor/attach_op_execs_pass.cc | 40 +++++++----- src/ndarray/autograd.cc | 87 +++++++++++++++++++++---- src/ndarray/autograd.h | 2 +- tests/python/gpu/test_operator_gpu.py | 11 ++++ tests/python/unittest/test_gluon.py | 13 +++- tests/python/unittest/test_gluon_rnn.py | 1 + 11 files changed, 211 insertions(+), 49 deletions(-) diff --git a/nnvm b/nnvm index 0a45136fae47..bcfbf903429d 160000 --- a/nnvm +++ b/nnvm @@ -1 +1 @@ -Subproject commit 0a45136fae475a8313dc66b6bebd87a722f20e7f +Subproject commit bcfbf903429d086f16b19b4d202788de06e45536 diff --git a/python/mxnet/gluon/data/dataloader.py b/python/mxnet/gluon/data/dataloader.py index 6497c7eb9be9..772209a6f2aa 100644 --- a/python/mxnet/gluon/data/dataloader.py +++ b/python/mxnet/gluon/data/dataloader.py @@ -51,6 +51,13 @@ class DataLoader(object): Whether to shuffle the samples. sampler : Sampler The sampler to use. Either specify sampler or shuffle, not both. + last_batch : {'keep', 'discard', 'rollover'} + How to handle the last batch if batch_size does not evenly divide + `len(dataset)`. + + keep - A batch with less samples than previous batches is returned. + discard - The last batch is discarded if its incomplete. + rollover - The remaining samples are rolled over to the next epoch. batch_sampler : Sampler A sampler that returns mini-batches. Do not specify batch_size, shuffle, sampler, and last_batch if batch_sampler is specified. @@ -71,7 +78,8 @@ def __init__(self, dataset, batch_size=None, shuffle=False, sampler=None, elif shuffle: raise ValueError("shuffle must not be specified if sampler is specified") - batch_sampler = _sampler.BatchSampler(sampler, batch_size, last_batch) + batch_sampler = _sampler.BatchSampler( + sampler, batch_size, last_batch if last_batch else 'keep') elif batch_size is not None or shuffle or sampler is not None or \ last_batch is not None: raise ValueError("batch_size, shuffle, sampler and last_batch must " \ diff --git a/python/mxnet/gluon/data/dataset.py b/python/mxnet/gluon/data/dataset.py index 37d103266d8f..2fa20ccc522f 100644 --- a/python/mxnet/gluon/data/dataset.py +++ b/python/mxnet/gluon/data/dataset.py @@ -20,7 +20,7 @@ """Dataset container.""" import os -from ... import recordio +from ... import recordio, ndarray class Dataset(object): """Abstract dataset class. All datasets should have this interface. @@ -52,7 +52,10 @@ class ArrayDataset(Dataset): def __init__(self, data, label): assert len(data) == len(label) self._data = data - self._label = label + if isinstance(label, ndarray.NDArray) and len(label.shape) == 1: + self._label = label.asnumpy() + else: + self._label = label def __getitem__(self, idx): return self._data[idx], self._label[idx] diff --git a/python/mxnet/gluon/nn/basic_layers.py b/python/mxnet/gluon/nn/basic_layers.py index 63411caf2be7..7901a7ae2350 100644 --- a/python/mxnet/gluon/nn/basic_layers.py +++ b/python/mxnet/gluon/nn/basic_layers.py @@ -308,18 +308,22 @@ def __init__(self, axis=1, momentum=0.9, epsilon=1e-5, center=True, scale=True, self.gamma = self.params.get('gamma', grad_req='write' if scale else 'null', shape=(in_channels,), init=gamma_initializer, - allow_deferred_init=True) + allow_deferred_init=True, + differentiable=scale) self.beta = self.params.get('beta', grad_req='write' if center else 'null', shape=(in_channels,), init=beta_initializer, - allow_deferred_init=True) + allow_deferred_init=True, + differentiable=center) self.running_mean = self.params.get('running_mean', grad_req='null', shape=(in_channels,), init=running_mean_initializer, - allow_deferred_init=True) + allow_deferred_init=True, + differentiable=False) self.running_var = self.params.get('running_var', grad_req='null', shape=(in_channels,), init=running_variance_initializer, - allow_deferred_init=True) + allow_deferred_init=True, + differentiable=False) def hybrid_forward(self, F, x, gamma, beta, running_mean, running_var): return F.BatchNorm(x, gamma, beta, running_mean, running_var, diff --git a/python/mxnet/gluon/parameter.py b/python/mxnet/gluon/parameter.py index 69f60388fe25..bef55d67e140 100644 --- a/python/mxnet/gluon/parameter.py +++ b/python/mxnet/gluon/parameter.py @@ -80,9 +80,22 @@ class Parameter(object): init : Initializer, default None Initializer of this parameter. Will use the global initializer by default. + Attributes + ---------- + grad_req : {'write', 'add', 'null'} + This can be set before or after initialization. Setting grad_req to null + with `x.grad_req = 'null'` saves memory and computation when you don't + need gradient w.r.t x. """ def __init__(self, name, grad_req='write', shape=None, dtype=mx_real_t, - lr_mult=1.0, wd_mult=1.0, init=None, allow_deferred_init=False): + lr_mult=1.0, wd_mult=1.0, init=None, allow_deferred_init=False, + differentiable=True): + self._var = None + self._data = None + self._grad = None + self._deferred_init = () + self._differentiable = differentiable + self._grad_req = None self.name = name self.shape = shape self.dtype = dtype @@ -91,15 +104,31 @@ def __init__(self, name, grad_req='write', shape=None, dtype=mx_real_t, self.grad_req = grad_req self.init = init self.allow_deferred_init = allow_deferred_init - self._var = None - self._data = None - self._grad = None - self._deferred_init = () def __repr__(self): s = 'Parameter {name} (shape={shape}, dtype={dtype})' return s.format(**self.__dict__) + @property + def grad_req(self): + return self._grad_req + + @grad_req.setter + def grad_req(self, req): + assert req in ['write', 'add', 'null'], \ + "grad_req must be one of write, add, or null, but got %s"%req + if not self._differentiable: + req = 'null' + if self._grad_req == req: + return + self._grad_req = req + if req == 'null' and self._grad is not None: + self._grad = None + for ctx in self._data: + self._data[ctx] = self._data[ctx].detach() + elif self._data is not None: + self._init_grad() + def _check_initialized(self, ctx=None): if self._data is not None: if ctx is not None and ctx not in self._data: @@ -172,13 +201,16 @@ def _init_impl(self, data, ctx): self._data = OrderedDict() for i in ctx: self._data[i] = data.copyto(i) + self._init_grad() + def _init_grad(self): + """Initialize grad buffers.""" if self.grad_req == 'null': self._grad = None return self._grad = OrderedDict() - for i in ctx: + for i in self._data: self._grad[i] = ndarray.zeros_like(self._data[i]) autograd.mark_variables(self.list_data(), self.list_grad(), self.grad_req) @@ -381,9 +413,6 @@ def __init__(self, prefix='', shared=None): self._params = OrderedDict() self._shared = shared - def __getitem__(self, key): - return self._params[key] - def __repr__(self): s = '{name}(\n{content}\n)' name = self._prefix+' ' if self._prefix else '' @@ -391,6 +420,12 @@ def __repr__(self): content='\n'.join([_indent(' {0}'.format(v), 2) for v in self.values()])) + def __getitem__(self, key): + return self._params[key] + + def __iter__(self): + return iter(self._params) + def items(self): return self._params.items() @@ -495,6 +530,28 @@ def reset_ctx(self, ctx): for i in self.values(): i.reset_ctx(ctx) + def setattr(self, name, value): + """Set an attribute to a new value for all Parameters. + + For example, set grad_req to null if you don't need gradient w.r.t a + model's Parameters:: + + model.collect_params().setattr('grad_req', 'null') + + or change the learning rate multiplier:: + + model.collect_params().setattr('lr_mult', 0.5) + + Parameters + ---------- + name : str + Name of the attribute. + value : valid type for attribute name + The new value for the attribute. + """ + for i in self.values(): + setattr(i, name, value) + def save(self, filename, strip_prefix=''): """Save parameters to file. diff --git a/src/executor/attach_op_execs_pass.cc b/src/executor/attach_op_execs_pass.cc index 046460b85900..47b74758d702 100644 --- a/src/executor/attach_op_execs_pass.cc +++ b/src/executor/attach_op_execs_pass.cc @@ -44,6 +44,17 @@ namespace exec { class StatefulComputeExecutor : public OpExecutor { public: void Run(RunContext rctx) override { + if (!init_) { + in_data_.clear(); + for (size_t i = 0; i < in_array.size(); ++i) { + in_data_.push_back(in_array[i].data()); + } + out_data_.clear(); + for (size_t i = 0; i < out_array.size(); ++i) { + out_data_.push_back(out_array[i].data()); + } + init_ = true; + } op_ctx.run_ctx = rctx; fcompute_(state_, op_ctx, in_data_, req, out_data_); #if MKL_EXPERIMENTAL == 1 @@ -53,14 +64,7 @@ class StatefulComputeExecutor : public OpExecutor { } void Setup() override { - in_data_.clear(); - for (size_t i = 0; i < in_array.size(); ++i) { - in_data_.push_back(in_array[i].data()); - } - out_data_.clear(); - for (size_t i = 0; i < out_array.size(); ++i) { - out_data_.push_back(out_array[i].data()); - } + init_ = false; } ExecType exec_type() const override { @@ -81,6 +85,7 @@ class StatefulComputeExecutor : public OpExecutor { OpStatePtr state_; FStatefulCompute fcompute_; ExecType exec_type_; + bool init_; std::vector in_data_, out_data_; }; @@ -120,6 +125,16 @@ class StatefulComputeExExecutor : public OpExecutor { class FComputeExecutor : public OpExecutor { public: void Run(RunContext rctx) override { + if (!init_) { + in_data_.resize(in_array.size()); + out_data_.resize(out_array.size()); + auto get_blob = [](const NDArray& nd) { + return nd.data(); + }; + std::transform(in_array.begin(), in_array.end(), in_data_.begin(), get_blob); + std::transform(out_array.begin(), out_array.end(), out_data_.begin(), get_blob); + init_ = true; + } op_ctx.run_ctx = rctx; fcompute_(attrs_, op_ctx, in_data_, req, out_data_); #if MKL_EXPERIMENTAL == 1 @@ -129,13 +144,7 @@ class FComputeExecutor : public OpExecutor { } void Setup() override { - in_data_.resize(in_array.size()); - out_data_.resize(out_array.size()); - auto get_blob = [](const NDArray& nd) { - return nd.data(); - }; - std::transform(in_array.begin(), in_array.end(), in_data_.begin(), get_blob); - std::transform(out_array.begin(), out_array.end(), out_data_.begin(), get_blob); + init_ = false; } ExecType exec_type() const override { @@ -151,6 +160,7 @@ class FComputeExecutor : public OpExecutor { NodeAttrs attrs_; FCompute fcompute_; ExecType exec_type_; + bool init_; std::vector in_data_, out_data_; }; diff --git a/src/ndarray/autograd.cc b/src/ndarray/autograd.cc index 78b98dabc661..5ecea5decf03 100644 --- a/src/ndarray/autograd.cc +++ b/src/ndarray/autograd.cc @@ -126,18 +126,66 @@ AutogradRuntime* AutogradRuntime::Get() { return ptr; } -AGNodePtr AutogradRuntime::RecordOp(const nnvm::Op* op, +void AutogradRuntime::RecordOp(const nnvm::Op* op, const nnvm::NodeAttrs& attrs, std::vector *p_inputs, std::vector *p_outputs, const OpStatePtr& state) { + static auto& fgradient = nnvm::Op::GetAttr("FGradient"); std::vector& inputs = *p_inputs; std::vector& outputs = *p_outputs; + for (uint32_t i = 0; i < outputs.size(); ++i) { + CHECK(outputs[i].entry_.is_none()) + << "Inplace operations (+=, -=, x[:]=, etc) are not supported when " + << "recording with autograd. " + << "Assigning to NDArrays that are already in a computational graph " + << "will cause undefined behavior when evaluating gradients. " + << "Please call backward first to clear the graph or do this out side of " + << "a record section. "; + } + if (!fgradient.count(attrs.op)) return; + bool need_grad = false; + for (const auto& i : inputs) { + if (!i.entry_.is_none()) { + need_grad = true; + break; + } + } + if (!need_grad) return; + NodePtr nn_node = Node::Create(); nn_node->attrs = attrs; nn_node->attrs.name = "node_" + std::to_string(node_count_++); + // Get backward dependency + std::vector save_inputs(inputs.size()), save_outputs(outputs.size()); + for (uint32_t i = 0; i < inputs.size(); ++i) { + nn_node->inputs.emplace_back(NodeEntry{nullptr, i, 0}); + } + std::vector ograd_entries; + for (uint32_t i = 0; i < outputs.size(); ++i) { + ograd_entries.emplace_back(NodeEntry{nullptr, i, 1}); + } + auto igrad_entries = fgradient[nn_node->op()](nn_node, ograd_entries); + for (const auto& i : igrad_entries) { + if (i.node == nullptr && i.version == 0) { + save_inputs[i.index] = true; + } else if (i.node == nn_node) { + save_outputs[i.index] = true; + } + } + DFSVisit(igrad_entries, [&](const NodePtr& node) { + if (!node || node == nn_node) return; + for (const auto& i : node->inputs) { + if (i.node == nullptr && i.version == 0) { + save_inputs[i.index] = true; + } else if (i.node == nn_node) { + save_outputs[i.index] = true; + } + } + }); + AGNodePtr ag_node = AGNode::Create(nn_node); ag_node->state = state; @@ -147,28 +195,35 @@ AGNodePtr AutogradRuntime::RecordOp(const nnvm::Op* op, AGNode::Create( nnvm::Symbol::CreateVariable( "null" + std::to_string(variable_count_++)).outputs[0].node), 0, 0}; - e.ag_node->outputs.emplace_back(inputs[i]); + if (save_inputs[i]) { + e.ag_node->outputs.emplace_back(inputs[i]); + } else { + // Put a dummy array here since it will not be used. + e.ag_node->outputs.emplace_back( + TBlob(nullptr, inputs[i].shape(), inputs[i].ctx().dev_mask(), + inputs[i].dtype()), inputs[i].ctx().dev_id); + } e.ag_node->out_grads.emplace_back(); inputs[i].entry_ = std::move(e); // assign last to prevent cyclic reference } - nn_node->inputs.push_back(inputs[i].entry_.nn_entry()); + nn_node->inputs[i] = inputs[i].entry_.nn_entry(); ag_node->inputs.push_back(inputs[i].entry_); + if (save_inputs[i]) { + inputs[i].entry_.ag_node->outputs[inputs[i].entry_.index] = inputs[i].Detach(); + } } for (uint32_t i = 0; i < outputs.size(); ++i) { - CHECK(outputs[i].entry_.is_none()) - << "Inplace operations (+=, -=, x[:]=, etc) are not supported when " - << "recording with autograd. " - << "Assigning to NDArrays that are already in a computational graph " - << "will cause undefined behavior when evaluating gradients. " - << "Please call backward first to clear the graph or do this out side of " - << "a record section. "; - outputs[i].entry_.clear(); - ag_node->outputs.push_back(outputs[i]); + if (save_outputs[i]) { + ag_node->outputs.emplace_back(outputs[i].Detach()); + } else { + // Put a dummy array here since it will not be used. + ag_node->outputs.emplace_back( + TBlob(nullptr, outputs[i].shape(), outputs[i].ctx().dev_mask(), + outputs[i].dtype()), outputs[i].ctx().dev_id); + } outputs[i].entry_ = AGNodeEntry{ag_node, i, 0}; } - - return ag_node; } void AutogradRuntime::ComputeGradient(const std::vector& outputs, @@ -257,6 +312,10 @@ void AutogradRuntime::ComputeGradient(const std::vector& outputs, } } + // std::stringstream os; + // exec->Print(os); + // LOG(INFO) << os.str(); + exec->Backward(head_grads, is_train); delete exec; } diff --git a/src/ndarray/autograd.h b/src/ndarray/autograd.h index 52e461d52c2d..199af350bf93 100644 --- a/src/ndarray/autograd.h +++ b/src/ndarray/autograd.h @@ -127,7 +127,7 @@ class AutogradRuntime { private: /*! \brief to record operator, return corresponding node. */ - AGNodePtr RecordOp(const nnvm::Op* op, + void RecordOp(const nnvm::Op* op, const nnvm::NodeAttrs& attrs, std::vector* p_inputs, std::vector* p_outputs, diff --git a/tests/python/gpu/test_operator_gpu.py b/tests/python/gpu/test_operator_gpu.py index cd8e85ac9157..866f6ad8abc0 100644 --- a/tests/python/gpu/test_operator_gpu.py +++ b/tests/python/gpu/test_operator_gpu.py @@ -1336,6 +1336,17 @@ def test_sequence_reverse(): check_sequence_reverse(mx.gpu(0)) +def test_autograd_save_memory(): + x = mx.nd.zeros((128, 1024, 1024), ctx=mx.gpu(0)) + x.attach_grad() + + with mx.autograd.record(): + for i in range(50): + x = x + 1 + x.wait_to_read() + x.backward() + + if __name__ == '__main__': import nose nose.runmodule() diff --git a/tests/python/unittest/test_gluon.py b/tests/python/unittest/test_gluon.py index 4fff23f2c5a4..cafa08bc04ca 100644 --- a/tests/python/unittest/test_gluon.py +++ b/tests/python/unittest/test_gluon.py @@ -84,6 +84,11 @@ def test_basic(): assert x.shape == (32, 32) x.wait_to_read() + model.collect_params().setattr('grad_req', 'null') + assert list(model.collect_params().values())[0]._grad is None + model.collect_params().setattr('grad_req', 'write') + assert list(model.collect_params().values())[0]._grad is not None + def test_symbol_block(): model = nn.HybridSequential() @@ -108,14 +113,18 @@ def test_symbol_block(): def check_layer_forward(layer, dshape): layer.collect_params().initialize() + x = mx.nd.ones(shape=dshape) + x.attach_grad() with mx.autograd.record(): - out = layer(mx.nd.ones(shape=dshape)) + out = layer(x) out.backward() layer.hybridize() + x = mx.nd.ones(shape=dshape) + x.attach_grad() with mx.autograd.record(): - out = layer(mx.nd.ones(shape=dshape)) + out = layer(x) out.backward() def test_conv(): diff --git a/tests/python/unittest/test_gluon_rnn.py b/tests/python/unittest/test_gluon_rnn.py index 40620136645a..5dcbdfa65d35 100644 --- a/tests/python/unittest/test_gluon_rnn.py +++ b/tests/python/unittest/test_gluon_rnn.py @@ -181,6 +181,7 @@ def test_zoneout(): def check_rnn_forward(layer, inputs): + inputs.attach_grad() layer.collect_params().initialize() with mx.autograd.record(): layer.unroll(3, inputs, merge_outputs=True)[0].backward() From bca9c4cf0b7c90374557170eec088a2b30b8bb72 Mon Sep 17 00:00:00 2001 From: "Joshua Z. Zhang" Date: Tue, 15 Aug 2017 19:22:06 -0700 Subject: [PATCH 382/834] add gluon resnet18_v2, resnet34_v2 models (#7484) --- python/mxnet/gluon/model_zoo/model_store.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/mxnet/gluon/model_zoo/model_store.py b/python/mxnet/gluon/model_zoo/model_store.py index 67ba572deb8c..e524f215416d 100644 --- a/python/mxnet/gluon/model_zoo/model_store.py +++ b/python/mxnet/gluon/model_zoo/model_store.py @@ -36,6 +36,8 @@ ('38d6d423c22828718ec3397924b8e116a03e6ac0', 'resnet18_v1'), ('4dc2c2390a7c7990e0ca1e53aeebb1d1a08592d1', 'resnet34_v1'), ('2a903ab21260c85673a78fe65037819a843a1f43', 'resnet50_v1'), + ('8aacf80ff4014c1efa2362a963ac5ec82cf92d5b', 'resnet18_v2'), + ('0ed3cd06da41932c03dea1de7bc2506ef3fb97b3', 'resnet34_v2'), ('264ba4970a0cc87a4f15c96e25246a1307caf523', 'squeezenet1.0'), ('33ba0f93753c83d86e1eb397f38a667eaf2e9376', 'squeezenet1.1'), ('dd221b160977f36a53f464cb54648d227c707a05', 'vgg11'), From d634bea6720e8df63f865a93dfad7f4468b43232 Mon Sep 17 00:00:00 2001 From: Yao Wang Date: Wed, 16 Aug 2017 10:31:34 -0700 Subject: [PATCH 383/834] Fix more links (#7485) --- docs/get_started/windows_setup.md | 1 - src/operator/contrib/deformable_psroi_pooling.cc | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/get_started/windows_setup.md b/docs/get_started/windows_setup.md index d695c5993f0c..f9067732d11a 100755 --- a/docs/get_started/windows_setup.md +++ b/docs/get_started/windows_setup.md @@ -9,7 +9,6 @@ You can either use a prebuilt binary package or build from source to build the M MXNet provides a prebuilt package for Windows. The prebuilt package includes the MXNet library, all of the dependent third-party libraries, a sample C++ solution for Visual Studio, and the Python installation script. To install the prebuilt package: 1. Download the latest prebuilt package from the [Releases](https://github.com/dmlc/mxnet/releases) tab of MXNet. - There are two versions. One with GPU support (using CUDA and CUDNN v3), and one without GPU support. Choose the version that suits your hardware configuration. For more information on which version works on each hardware configuration, see [Requirements for GPU](http://mxnet.io/get_started/setup.html#requirements-for-using-gpus). 2. Unpack the package into a folder, with an appropriate name, such as ```D:\MXNet```. 3. Open the folder, and install the package by double-clicking ```setupenv.cmd```. This sets up all of the environment variables required by MXNet. 4. Test the installation by opening the provided sample C++ Visual Studio solution and building it. diff --git a/src/operator/contrib/deformable_psroi_pooling.cc b/src/operator/contrib/deformable_psroi_pooling.cc index 93bb64d2113c..47f369a32d24 100644 --- a/src/operator/contrib/deformable_psroi_pooling.cc +++ b/src/operator/contrib/deformable_psroi_pooling.cc @@ -102,7 +102,7 @@ namespace op { MXNET_REGISTER_OP_PROPERTY(_contrib_DeformablePSROIPooling, DeformablePSROIPoolingProp) .describe("Performs deformable position-sensitive region-of-interest pooling on inputs." - "The DeformablePSROIPooling operation is described in https://arxiv.org/abs/1703.06211." + "The DeformablePSROIPooling operation is described in https://arxiv.org/abs/1703.06211 ." "batch_size will change to the number of region bounding boxes after DeformablePSROIPooling") .add_argument("data", "Symbol", "Input data to the pooling operator, a 4D Feature maps") .add_argument("rois", "Symbol", "Bounding box coordinates, a 2D array of " From d7d31b2d9ce2eee98a3b9f41bc8c526a3125ce78 Mon Sep 17 00:00:00 2001 From: "shuqian.qu" Date: Thu, 17 Aug 2017 01:32:14 +0800 Subject: [PATCH 384/834] add depthwise convolution's gpu version optimization (#7393) * add depthwise convolution's gpu version optimization * add more config for test_depthwise_convolution * remove CUDA_1D_KERNEL_LOOP * fix windows compiling error * add support for kAddTo when cal input's backward * remove depthwise_conv_off params * Update convolution.cu * Update test_operator.py --- src/common/cuda_utils.h | 28 +- src/operator/convolution.cu | 14 + src/operator/depthwise_convolution-inl.h | 349 +++++++++++ src/operator/depthwise_convolution_tf.cuh | 703 ++++++++++++++++++++++ tests/python/unittest/test_operator.py | 38 ++ 5 files changed, 1131 insertions(+), 1 deletion(-) create mode 100644 src/operator/depthwise_convolution-inl.h create mode 100644 src/operator/depthwise_convolution_tf.cuh diff --git a/src/common/cuda_utils.h b/src/common/cuda_utils.h index 8897007207fb..483390fc9bea 100644 --- a/src/common/cuda_utils.h +++ b/src/common/cuda_utils.h @@ -153,6 +153,16 @@ inline const char* CurandGetErrorString(curandStatus_t status) { return "Unknown cuRAND status"; } +template +inline DType __device__ CudaMax(DType a, DType b) { + return a > b ? a : b; +} + +template +inline DType __device__ CudaMin(DType a, DType b) { + return a < b ? a : b; +} + } // namespace cuda } // namespace common } // namespace mxnet @@ -219,6 +229,14 @@ inline const char* CurandGetErrorString(curandStatus_t status) { << "cuRAND: " << common::cuda::CurandGetErrorString(e); \ } +#if !defined(_MSC_VER) +#define CUDA_UNROLL _Pragma("unroll") +#define CUDA_NOUNROLL _Pragma("nounroll") +#else +#define CUDA_UNROLL +#define CUDA_NOUNROLL +#endif + /*! * \brief Determine major version number of the gpu's cuda compute architecture. * \param device_id The device index of the cuda-capable gpu of interest. @@ -291,7 +309,6 @@ inline bool GetEnvAllowTensorCore() { return dmlc::GetEnv("MXNET_CUDA_ALLOW_TENSOR_CORE", dmlc::optional(default_value)).value(); } - #endif // MXNET_USE_CUDA #if MXNET_USE_CUDNN @@ -401,6 +418,15 @@ static inline __device__ void atomicAdd(mshadow::half::half_t *address, old = atomicCAS(address_as_ui, assumed, old); } while (assumed != old); } + +template +__device__ inline DType ldg(const DType* address) { +#if __CUDA_ARCH__ >= 350 + return __ldg(address); +#else + return *address; +#endif +} #endif #endif // MXNET_COMMON_CUDA_UTILS_H_ diff --git a/src/operator/convolution.cu b/src/operator/convolution.cu index ab354849600a..f5777c1714a4 100644 --- a/src/operator/convolution.cu +++ b/src/operator/convolution.cu @@ -29,6 +29,8 @@ #include "./cudnn_convolution-inl.h" #endif // MXNET_USE_CUDNN +#include "./depthwise_convolution-inl.h" + namespace mxnet { namespace op { @@ -45,6 +47,18 @@ Operator* CreateOp(ConvolutionParam param, int dtype, }) return op; } + + // depth wise conv + if (param.num_filter == param.num_group && + param.layout.value() == mshadow::kNCHW && + param.num_filter == (*in_shape)[conv::kData][1] && + param.kernel.ndim() == 2 && + param.dilate == mshadow::Shape2(1, 1) && + dtype == mshadow::kFloat32) { + op = new DepthwiseConvolutionOp(param, *in_shape, *out_shape); + return op; + } + #if MXNET_USE_CUDNN == 1 // The NVIDIA Pascal architecture was the first to include 16-bit ALUs. // Thus, when the framework is compiled with MSHADOW_USE_PASCAL == 1, we diff --git a/src/operator/depthwise_convolution-inl.h b/src/operator/depthwise_convolution-inl.h new file mode 100644 index 000000000000..5beea4595f7a --- /dev/null +++ b/src/operator/depthwise_convolution-inl.h @@ -0,0 +1,349 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file depthwise_convolution-inl.h + * \brief CUDA depthwise convolution code + * \author shuqian.qu@hobot.cc +*/ +#ifndef MXNET_OPERATOR_DEPTHWISE_CONVOLUTION_INL_H_ +#define MXNET_OPERATOR_DEPTHWISE_CONVOLUTION_INL_H_ +#include +#include +#include "./convolution-inl.h" +#include "../common/cuda_utils.h" + +#if MXNET_USE_CUDA +#include +#include "./depthwise_convolution_tf.cuh" + +namespace mxnet { +namespace op { +using namespace tf::depthwise_conv; +template +class DepthwiseConvolutionOp : public Operator { + public: + explicit DepthwiseConvolutionOp(const ConvolutionParam& param, + const std::vector& in_shape, + const std::vector& out_shape) { + args_.batch = in_shape[conv::kData][0]; + args_.in_channel = in_shape[conv::kData][1]; + args_.in_height = in_shape[conv::kData][2]; + args_.in_width = in_shape[conv::kData][3]; + args_.filter_height = in_shape[conv::kWeight][2]; + args_.filter_width = in_shape[conv::kWeight][3]; + args_.stride_height = param.stride[0]; + args_.stride_width = param.stride[1]; + args_.pad_height = param.pad[0]; + args_.pad_width = param.pad[1]; + args_.out_channel = out_shape[conv::kOut][1]; + args_.out_height = out_shape[conv::kOut][2]; + args_.out_width = out_shape[conv::kOut][3]; + bias_term_ = !param.no_bias; + } + + ~DepthwiseConvolutionOp() {} + + virtual void Forward(const OpContext &ctx, + const std::vector &in_data, + const std::vector &req, + const std::vector &out_data, + const std::vector &aux_args); + + virtual void Backward(const OpContext &ctx, + const std::vector &out_grad, + const std::vector &in_data, + const std::vector &out_data, + const std::vector &req, + const std::vector &in_grad, + const std::vector &aux_args); + + private: + DepthwiseArgs args_; + bool bias_term_; +}; // class DepthwiseConvolutionOp + +namespace depthwise_conv { +namespace cuda { +template +__global__ void __launch_bounds__(1024, 2) +DepthwiseConv2dBackwardFilterKernel(const DepthwiseArgs args, + const DType* out_grad, + const DType* input, + DType* filter_grad) { + const int in_height = args.in_height; + const int in_width = args.in_width; + const int channel = args.in_channel; + const int filter_height = kFilterHeight > 0 ? kFilterHeight : args.filter_height; + const int filter_width = kFilterWidth > 0 ? kFilterWidth : args.filter_width; + const int stride_height = args.stride_height; + const int stride_width = args.stride_width; + const int pad_height = args.pad_height; + const int pad_width = args.pad_width; + const int out_height = args.out_height; + const int out_width = args.out_width; + + const int filter_pixels = filter_width * filter_height; + const int out_pixels = out_height * out_width; + const int in_pixels = in_height * in_width; + const int batch_channel_num = channel * args.batch; + const int candidate_reduce_thread_num = out_pixels % blockDim.x; + + for (int b = blockIdx.x; b < batch_channel_num; b += gridDim.x) { + const int local_batch = b / channel; + const int local_channel = b % channel; + const int filter_offset_temp = local_channel * filter_pixels; + const int out_grad_offset_temp = (local_batch * channel * out_pixels) + + (local_channel * out_pixels); + + for (int out_id = threadIdx.x; out_id < out_pixels; out_id += blockDim.x) { + const int reduce_thread_num = ((out_pixels - out_id) > candidate_reduce_thread_num) ? + blockDim.x : candidate_reduce_thread_num; + + const int out_w = out_id % out_width; + const int out_h = (out_id / out_width) % out_height; + const int out_grad_offset = out_grad_offset_temp + (out_h * out_width) + (out_w); + const DType out_g = ldg(out_grad + out_grad_offset); + + const int in_h_start = out_h * stride_height - pad_height; + const int in_w_start = out_w * stride_width - pad_width; + CUDA_UNROLL for (int f_h = 0; f_h < filter_height; ++f_h) { + const int in_h = in_h_start + f_h; + const int input_offset_temp = (local_batch * channel * in_pixels) + + (local_channel * in_pixels) + (in_h * in_width); + const int filter_offset_h = filter_width * f_h; + + CUDA_UNROLL for (int f_w = 0; f_w < filter_width; ++f_w) { + const int in_w = in_w_start + f_w; + DType partial_grad = DType(0.0f); + if (in_h >= 0 && in_h < in_height && in_w >= 0 && in_w < in_width) { + const int input_offset = input_offset_temp + in_w; + partial_grad = ldg(input + input_offset) * out_g; + } + // reduce all valid partial grad in a block + typedef cub::BlockReduce BlockReduceT; + __shared__ typename BlockReduceT::TempStorage temp_storage_reduce; + DType aggregate = BlockReduceT(temp_storage_reduce).Sum(partial_grad, reduce_thread_num); + if (threadIdx.x == 0) { + DType* addr = filter_grad + f_w + filter_offset_h + filter_offset_temp; + atomicAdd(addr, aggregate); + } + __syncthreads(); + } // for filter_width + } // for filter_height + } // for out_pixels + __syncthreads(); + } // for batch_channel_num +} +} // namespace cuda + +template +void DepthwiseConv2dForwardGpu(mshadow::Stream *stream, + const DepthwiseArgs& args, + const std::vector &in_data, + const std::vector &out_data) { + using namespace mshadow; + using namespace mshadow::expr; + using namespace tf::depthwise_conv; + using namespace tf::depthwise_conv::cuda; + Tensor data = in_data[conv::kData].get(stream); + Tensor weight = in_data[conv::kWeight].get(stream); + Tensor out = out_data[conv::kOut].get(stream); + + // select kernel + if (CanLaunchDepthwiseConv2dGPUSmall(args)) { + LaunchDepthwiseConv2dGPUSmall( + stream, + args, + data.dptr_, + weight.dptr_, + out.dptr_); + } else { + int num_output = out_data[conv::kOut].shape_.Size(); + int block_num = std::min(num_output/mshadow::cuda::kBaseThreadNum + 1, + mshadow::cuda::kMaxGridNum); + auto s = mshadow::Stream::GetStream(stream); + if (args.filter_height == 3 && args.filter_width == 3) { + DepthwiseConv2dForwardKernel + <<>>(data.dptr_, + weight.dptr_, + args, + num_output, + out.dptr_); + } else { + DepthwiseConv2dForwardKernel + <<>>(data.dptr_, + weight.dptr_, + args, + num_output, + out.dptr_); + } + MSHADOW_CUDA_POST_KERNEL_CHECK(DepthwiseConv2dForwardKernel); + } +} + +template +void DepthwiseConv2dBackwardDataGpu(mshadow::Stream *stream, + const DepthwiseArgs& args, + const std::vector &out_grad, + const std::vector &in_data, + const std::vector &in_grad) { + using namespace mshadow; + using namespace mshadow::expr; + using namespace tf::depthwise_conv; + using namespace tf::depthwise_conv::cuda; + Tensor out_g = out_grad[conv::kOut].get(stream); + Tensor weight = in_data[conv::kWeight].get(stream); + Tensor in_data_g = in_grad[conv::kData].get(stream); + // select kernel + if (CanLaunchDepthwiseConv2dGPUSmall(args)) { + LaunchDepthwiseConv2dGPUSmall( + stream, + args, + out_g.dptr_, + weight.dptr_, + in_data_g.dptr_); + } else { + int num_in_grad = in_grad[conv::kData].shape_.Size(); + auto s = mshadow::Stream::GetStream(stream); + int block_num = std::min(num_in_grad/mshadow::cuda::kBaseThreadNum + 1, + mshadow::cuda::kMaxGridNum); + DepthwiseConv2dBackwardDataKernel + <<>>(args, + out_g.dptr_, + weight.dptr_, + in_data_g.dptr_, + num_in_grad); + MSHADOW_CUDA_POST_KERNEL_CHECK(DepthwiseConv2dBackwardDataKernel); + } +} + +template +void DepthwiseConv2dBackwardFilterGpu(mshadow::Stream *stream, + const DepthwiseArgs& args, + const std::vector &out_grad, + const std::vector &in_data, + const std::vector &in_grad) { + using namespace mshadow; + using namespace mshadow::expr; + using namespace tf::depthwise_conv; + Tensor out_g = out_grad[conv::kOut].get(stream); + Tensor in_d = in_data[conv::kData].get(stream); + Tensor weight_grad = in_grad[conv::kWeight].get(stream); + // select kernel + if (TryLaunchDepthwiseConv2dBackwardFilterGPUSmall(stream, args, + out_g.dptr_, + in_d.dptr_, + weight_grad.dptr_)) { + return; + } else { + int num_out_grad = out_grad[conv::kOut].shape_.Size(); + auto s = mshadow::Stream::GetStream(stream); + int block_num = std::min(args.out_channel * args.batch, mshadow::cuda::kMaxGridNum); + if (args.filter_width == 3 && args.filter_height == 3) { + cuda::DepthwiseConv2dBackwardFilterKernel + <<>>(args, + out_g.dptr_, + in_d.dptr_, + weight_grad.dptr_); + } else { + cuda::DepthwiseConv2dBackwardFilterKernel + <<>>(args, + out_g.dptr_, + in_d.dptr_, + weight_grad.dptr_); + } + MSHADOW_CUDA_POST_KERNEL_CHECK(DepthwiseConv2dBackwardFilterKernel); + } +} +} // namespace depthwise_conv + +template +void DepthwiseConvolutionOp::Forward(const OpContext &ctx, + const std::vector &in_data, + const std::vector &req, + const std::vector &out_data, + const std::vector &aux_states) { + using namespace mshadow; + using namespace mshadow::expr; + auto stream = ctx.get_stream(); + CHECK_EQ(req[conv::kOut], kWriteTo); + // output forward + depthwise_conv::DepthwiseConv2dForwardGpu(stream, args_, in_data, out_data); + + // bias forward + if (bias_term_) { + Tensor bias = in_data[conv::kBias].get(stream); + Tensor output_3d = out_data[conv::kOut].get_with_shape( + Shape3(args_.batch, args_.out_channel, args_.out_height * args_.out_width), stream); + // has bias term, broadcast it to the same shape of output_3d in channel dim + output_3d += mshadow::expr::broadcast<1>(bias, output_3d.shape_); + } +} + +template +void DepthwiseConvolutionOp::Backward(const OpContext &ctx, + const std::vector &out_grad, + const std::vector &in_data, + const std::vector &out_data, + const std::vector &req, + const std::vector &in_grad, + const std::vector &aux_states) { + using namespace mshadow; + using namespace mshadow::expr; + auto stream = ctx.get_stream(); + // backward data + if (req[conv::kData] != kNullOp) { + if (req[conv::kData] != kAddTo) { + mshadow::Tensor igrad = in_grad[conv::kData].get(stream); + igrad = 0.0f; + } + depthwise_conv::DepthwiseConv2dBackwardDataGpu(stream, + args_, + out_grad, + in_data, + in_grad); + } + + // backward filter + if (req[conv::kWeight] != kNullOp) { + if (req[conv::kWeight] != kAddTo) { + mshadow::Tensor wgrad = in_grad[conv::kWeight].get(stream); + wgrad = 0.0f; + } + depthwise_conv::DepthwiseConv2dBackwardFilterGpu(stream, + args_, + out_grad, + in_data, + in_grad); + } + + // backward bias + if (bias_term_) { + Tensor dbias = in_grad[conv::kBias].get(stream); + Tensor dout = out_grad[conv::kOut].get_with_shape( + Shape3(args_.batch, args_.out_channel, args_.out_height * args_.out_width), stream); + ASSIGN_DISPATCH(dbias, req[conv::kBias], sumall_except_dim<1>(dout)); + } +} +} // namespace op +} // namespace mxnet +#endif + +#endif // MXNET_OPERATOR_DEPTHWISE_CONVOLUTION_INL_H_ diff --git a/src/operator/depthwise_convolution_tf.cuh b/src/operator/depthwise_convolution_tf.cuh new file mode 100644 index 000000000000..a1538b68a7d0 --- /dev/null +++ b/src/operator/depthwise_convolution_tf.cuh @@ -0,0 +1,703 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file depthwise_convolution_tf.cuh + * \brief some depthwise convolution CUDA kernel code. The main logic comes + * from tensorflow, but the filter's layerout and many argument names + * are different with origin version. + * \author shuqian.qu@hobot.cc +*/ +#ifndef MXNET_OPERATOR_DEPTHWISE_CONVOLUTION_TF_CUH_ +#define MXNET_OPERATOR_DEPTHWISE_CONVOLUTION_TF_CUH_ +#include "../common/cuda_utils.h" +#include "./mxnet_op.h" + +namespace tf { +namespace depthwise_conv { +struct DepthwiseArgs { + // Input layer dimensions + int batch; + int in_height; + int in_width; + int in_channel; + int filter_height; + int filter_width; + int stride_height; + int stride_width; + int pad_height; + int pad_width; + + // Output layer dimensions + int out_height; + int out_width; + int out_channel; +}; + +namespace cuda { +template +__global__ void __launch_bounds__(1024, 2) +DepthwiseConv2dForwardKernel(const DType* input, + const DType* filter, + const DepthwiseArgs args, + int num_outputs, + DType* output) { + const int in_channel = args.in_channel; + const int in_height = args.in_height; + const int in_width = args.in_width; + const int filter_height = kFilterHeight > 0 ? kFilterHeight : args.filter_height; + const int filter_width = kFilterWidth > 0 ? kFilterWidth : args.filter_width; + const int stride_height = args.stride_height; + const int stride_width = args.stride_width; + const int pad_height = args.pad_height; + const int pad_width = args.pad_width; + const int out_channel = args.out_channel; + const int out_height = args.out_height; + const int out_width = args.out_width; + + CUDA_KERNEL_LOOP(thread_id, num_outputs) { + // Compute the indexes of this thread in the output. + // + // We want coalesced reads so we make sure that each warp reads + // a contiguous chunk of memory. + // + // THIS IS PROBABLY WRONG, we are not doing coalesced reads + // into the input, because of the depth multiplier division... + const int out_w = thread_id % out_width; + const int out_h = (thread_id / out_width) % out_height; + const int out_c = (thread_id / out_width / out_height) % out_channel; + const int out_b = thread_id / out_width / out_height / out_channel; + const int in_c = out_c; + + // Data is stored in the following format (let's assume we + // flatten the height and width into one contiguous dimension + // called "P". + // + // B1C1P1 B1C1P2 ..... B1C2P1 B1C2P2 .... + // B2C1P1 B2C1P2 ..... B2C2P1 B2C2P2 .... + // + // Each row contains in_channel * in_height * in_width values + // for each sample in the batch. + // + // We can further flatten it into: + // + // B1C1P1 B1C1P2 ..... + // B1C2P1 B1C2P2 .... + // B2C1P1 B2C1P2 ..... + // B2C2P1 B2C2P2 .... + // + // where each row is a contiguous array of all of the spatial + // pixels for a given batch and input depth. The following + // loop unrolls across the filter dimensions for a given thread, + // indexing into the filter value and the corresponding input + // patch. + // + // We can compute the index into the patch once right here. + const int input_offset_temp = (out_b * in_channel + in_c) * (in_height * in_width); + const int filter_offset_temp = in_c * filter_height * filter_width; + + // Finally, we can iterate over the spatial dimensions and perform the + // convolution, writing into the output at the end. + // + // We perform an additional optimization, where we can determine + // whether the patch fits within the image indices statically, and + // avoid boundary checking within the loop. + const int input_h_start = out_h * stride_height - pad_height; + const int input_w_start = out_w * stride_width - pad_width; + const int input_h_end = input_h_start + filter_height; + const int input_w_end = input_w_start + filter_width; + + DType sum = 0; + if (input_h_start >= 0 && input_w_start >= 0 && + input_h_end < in_height && input_w_end < in_width) { + // Loop that doesn't need to check for boundary conditions. + CUDA_UNROLL for (int f_h = 0; f_h < filter_height; ++f_h) { + const int in_h = input_h_start + f_h; + const int filter_offset_h = filter_width * f_h; + CUDA_UNROLL for (int f_w = 0; f_w < filter_width; ++f_w) { + const int in_w = input_w_start + f_w; + const int input_offset = (input_offset_temp) + (in_h * in_width) + in_w; + const int filter_offset = filter_offset_temp + filter_offset_h + f_w; + sum += ldg(input + input_offset) * ldg(filter + filter_offset); + } + } + } else { + // Loop that needs to check for boundary conditions. + CUDA_UNROLL for (int f_h = 0; f_h < filter_height; ++f_h) { + const int in_h = input_h_start + f_h; + const int filter_offset_h = filter_width * f_h; + CUDA_UNROLL for (int f_w = 0; f_w < filter_width; ++f_w) { + const int in_w = input_w_start + f_w; + // TODO(vrv): the in_h check can be done outside of this loop; + // benchmark both methods to determine the better decision. + if (in_h >= 0 && in_h < in_height && in_w >= 0 && in_w < in_width) { + const int in_w = input_w_start + f_w; + const int input_offset = input_offset_temp + (in_h * in_width) + in_w; + const int filter_offset = filter_offset_temp + filter_offset_h + f_w; + sum += ldg(input + input_offset) * ldg(filter + filter_offset); + } + } + } + } + output[thread_id] = sum; + } +} + +// The DepthwiseConv2dKernelSmall perform either forward or backward input +// convolution depending on a template argument of this enum. +enum DepthwiseConv2dDirection { DIRECTION_FORWARD, DIRECTION_BACKWARD }; + +// CUDA kernel to compute the depthwise convolution forward pass in NCHW format, +// tailored for small images up to 32x32. Only use this kernel if +// CanLaunchDepthwiseConv2dGPUSmall(args) returns true. +// Tiles of the input and filter tensors are loaded into shared memory before +// performing the convolution. Each thread handles two elements per iteration, +// one each in the lower and upper half of a tile. +// Backward input direction is the same as forward direction with the filter +// rotated by 180°. +template +__global__ __launch_bounds__(1024, 2) void DepthwiseConv2dKernelSmall( + const DepthwiseArgs args, const DType* input, const DType* filter, DType* output) { + extern __shared__ __align__(sizeof(DType)) unsigned char shared_memory[]; + DType* const shared_data = reinterpret_cast(shared_memory); + + const int in_height = args.in_height; + const int in_width = args.in_width; + const int in_channel = args.in_channel; + const int filter_height = kFilterHeight > 0 ? kFilterHeight : args.filter_height; + const int filter_width = kFilterWidth > 0 ? kFilterWidth : args.filter_width; + const int pad_height = args.pad_height; + const int pad_width = args.pad_width; + + // Fixed blockDim.z, tailored for maximum grid size for images of size 16x16. + const int block_height = blockDim.y; + + // These values are the same for all threads and could + // be precomputed on the CPU. + const int block_pixels = in_width * block_height; + const int block_size = block_pixels * kBlockSlices; + const int in_pixels = in_width * in_height; + const int in_increment = in_width - 1; + const int filter_pixels = filter_height * filter_width; + const int tile_width = in_width + filter_width - 1; + const int even_height = kEvenHeight || (1 & ~in_height); + const int tile_height = in_height + filter_height - even_height; + const int tile_pixels = tile_width * tile_height; + const int tile_size = tile_pixels * kBlockSlices; + const int tile_offset = block_height * tile_width; + const int pad_offset = pad_height * tile_width + pad_width; + const int in_slices = in_channel * args.batch; + const int in_blocks = (in_slices + kBlockSlices - 1) / kBlockSlices; + + const int thread_width = threadIdx.x; + const int thread_height = threadIdx.y; + const int thread_channel = threadIdx.z; + + // Position in block. + const int thread_pix = thread_height * in_width + thread_width; + const int thread_idx = thread_channel * block_pixels + thread_pix; + + // Initialize tile, in particular the padding. + for (int i = thread_idx; i < tile_size; i += block_size) { + shared_data[i] = DType(0); + } + __syncthreads(); + + // Position in tensors. + const int tensor_idx = thread_channel * in_pixels + thread_pix; + + // Position in (padded) shared memory. + const int data_pix = thread_height * tile_width + thread_width; + const int data_idx = thread_channel * tile_pixels + data_pix; + + // Position in shared memory, offset by pad_height / pad_width. + const int tile_idx = data_idx + pad_offset; + + const int filter_pix = thread_pix; + const int filter_channel = thread_channel; + const int filter_idx = filter_pixels * filter_channel + filter_pix; + + const int max_slice = in_slices - thread_channel; + const int filter_write_offset = filter_pix < filter_pixels ? tile_size + filter_idx : 0; + const int filter_read_offset = tile_size + + (kDirection == DIRECTION_FORWARD ? + filter_pixels * filter_channel : filter_pixels * (filter_channel + 1)); + const bool skip_second = !kEvenHeight && thread_height + (in_height & 1) == block_height; + + for (int b = blockIdx.x; b < in_blocks; b += gridDim.x) { + const int slice = b * kBlockSlices; + + const int inout_offset = slice * in_pixels + tensor_idx; + const bool slice_in_range = slice < max_slice; + + if (slice_in_range) { + const DType* const in_ptr = inout_offset + input; + DType* const tile_ptr = tile_idx + shared_data; + tile_ptr[0] = ldg(in_ptr); + if (!skip_second) { + tile_ptr[tile_offset] = ldg(block_pixels + in_ptr); + } + } + + if (filter_write_offset != 0) { + const int filter_offset = ((slice + filter_channel) % in_channel)* filter_pixels + filter_pix; + shared_data[filter_write_offset] = ldg(filter_offset + filter); + } + + // Note: the condition to reach this is uniform across the entire block. + __syncthreads(); + + if (slice_in_range) { + DType sum1 = 0; + DType sum2 = 0; + int shared_offset = data_idx; + const DType* filter_ptr = filter_read_offset + shared_data; + CUDA_UNROLL for (int r = 0; r < filter_height; ++r) { + CUDA_UNROLL for (int c = 0; c < filter_width; ++c) { + if (kDirection == DIRECTION_BACKWARD) { + filter_ptr--; + } + const DType filter_value = *filter_ptr; + const DType* const tile_ptr = shared_offset + shared_data; + sum1 += filter_value * tile_ptr[0]; + sum2 += filter_value * tile_ptr[tile_offset]; + ++shared_offset; + if (kDirection == DIRECTION_FORWARD) { + filter_ptr++; + } + } + shared_offset += in_increment; + } + DType* const out_ptr = inout_offset + output; + if (kDirection == DIRECTION_FORWARD) { + out_ptr[0] = sum1; + if (!skip_second) { + out_ptr[block_pixels] = sum2; + } + } else { + out_ptr[0] += sum1; + if (!skip_second) { + out_ptr[block_pixels] += sum2; + } + } + } + + // Note: the condition to reach this is uniform across the entire block. + __syncthreads(); + } +} + +template +__global__ void __launch_bounds__(640, 2) +DepthwiseConv2dBackwardDataKernel(const DepthwiseArgs args, + const DType* out_grad, + const DType* filter, DType* in_grad, + int num_in_grad) { + const int channel = args.in_channel; + const int in_height = args.in_height; + const int in_width = args.in_width; + const int filter_height = args.filter_height; + const int filter_width = args.filter_width; + const int stride_height = args.stride_height; + const int stride_width = args.stride_width; + const int pad_height = args.pad_height; + const int pad_width = args.pad_width; + const int out_height = args.out_height; + const int out_width = args.out_width; + + const int in_pixels = in_height * in_width; + const int out_pixels = out_height * out_width; + + CUDA_KERNEL_LOOP(thread_id, num_in_grad) { + // Compute the indexes of this thread in the input. + const int in_w = thread_id % in_width; + const int in_h = (thread_id / in_width) % in_height; + const int channel_idx = (thread_id / in_width / in_height) % channel; + const int batch_idx = thread_id / channel / in_width / in_height; + DType sum = 0.0f; + + const int out_h_start = mxnet::common::cuda::CudaMax( + 0, (in_h - filter_height + pad_height + stride_height) / stride_height); + const int out_h_end = mxnet::common::cuda::CudaMin( + out_height - 1, (in_h + pad_height) / stride_height); + const int out_w_start = mxnet::common::cuda::CudaMax( + 0, (in_w - filter_width + pad_width + stride_width) / stride_width); + const int out_w_end = mxnet::common::cuda::CudaMin( + out_width - 1, (in_w + pad_width) / stride_width); + + const int filter_offset_temp = channel_idx * filter_height * filter_width; + const int out_grad_offset_temp = (batch_idx * channel * out_pixels) + + (channel_idx * out_pixels); + + for (int out_h = out_h_start; out_h <= out_h_end; ++out_h) { + const int f_h = in_h + pad_height - out_h * stride_height; + const int filter_offset_h = filter_offset_temp + f_h * filter_width; + const int out_grad_offset_h = out_grad_offset_temp + out_h * out_width; + for (int out_w = out_w_start; out_w <= out_w_end; ++out_w) { + const int f_w = in_w + pad_width - out_w * stride_width; + const int filter_offset = filter_offset_h + f_w; + const int out_grad_offset = out_grad_offset_h + out_w; + sum += ldg(out_grad + out_grad_offset) * ldg(filter + filter_offset); + } + } + const int in_grad_offset = (batch_idx * channel * in_pixels) + + (channel_idx * in_pixels) + (in_h * in_width) + (in_w); + in_grad[in_grad_offset] += sum; + } +} + +// CUDA kernel to compute the depthwise convolution backward w.r.t. filter in +// NCHW format, tailored for small images up to 32x32. Only use this kernel if +// CanLaunchDepthwiseConv2dGPUSmall(args) returns true. +// Tiles of the input tensor are loaded into shared memory before performing the +// convolution. Per iteration and filter element, each thread first performs +// a partial convolution for two elements, one each in the lower and upper half +// of a tile. The intermediate result of all pixels of a warp are then +// accumulated and written to shared memory. Finally, the values in shared +// memory are warp-accumulated (in chunks of kAccumPixels elements) and summed +// up in global memory using atomics. +// Requirements: threads per block must be multiple of 32 and <= launch_bounds, +// kAccumPixels * 64 >= args.in_height * args.in_width * kBlockSlices. +template +__global__ +__launch_bounds__(1024, 2) void DepthwiseConv2dBackwardFilterKernelSmall( + const DepthwiseArgs args, const DType* output, const DType* input, DType* filter) { + extern __shared__ __align__(sizeof(DType)) unsigned char shared_memory[]; + DType* const shared_data = reinterpret_cast(shared_memory); + + const int in_height = args.in_height; + const int in_width = blockDim.x; // slower (see b/62280718): args.in_width; + const int in_channel = args.in_channel; + const int filter_height = kFilterHeight > 0 ? kFilterHeight : args.filter_height; + const int filter_width = kFilterWidth > 0 ? kFilterWidth : args.filter_width; + const int pad_height = args.pad_height; + const int pad_width = args.pad_width; + + const int block_height = blockDim.y; + + // These values are the same for all threads and could + // be precomputed on the CPU. + const int block_pixels = in_width * block_height; + const int block_size = block_pixels * kBlockSlices; + assert((block_size & 31) == 0); + const int in_pixels = in_width * in_height; + const int in_increment = in_width - 1; + const int filter_pixels = filter_height * filter_width; + const int tile_width = in_width + filter_width - 1; + const int tile_height = 2 * block_height + filter_height - 1; + const int tile_pixels = tile_width * tile_height; + const int tile_size = tile_pixels * kBlockSlices; + const int tile_offset = block_height * tile_width; + const int pad_offset = pad_height * tile_width + pad_width; + const int in_slices = in_channel * args.batch; + const int in_blocks = (in_slices + kBlockSlices - 1) / kBlockSlices; + // The accumulator has a fixed number of pixels that can be reduced by one + // warp. Pixels beyond ceil(in_pixels * kBlockSlices / 64) are never written. + assert(kAccumPixels * 64 >= in_height * in_width * kBlockSlices); + const int accum_increment = kAccumPixels * kBlockSlices; + const int accum_size = filter_pixels * accum_increment; + + const int thread_width = threadIdx.x; + const int thread_height = threadIdx.y; + const int thread_channel = threadIdx.z; + + // Position in block. + const int thread_pix = thread_height * in_width + thread_width; + const int thread_idx = thread_channel * block_pixels + thread_pix; + + // Initialize tile, in particular the padding and accumulator. + for (int i = thread_idx; i < tile_size + accum_size; i += block_size) { + shared_data[i] = DType(0); + } + __syncthreads(); + + // Position in tensors. + const int tensor_idx = thread_channel * in_pixels + thread_pix; + + // Position in (padded) shared memory. + const int data_pix = thread_height * tile_width + thread_width; + const int data_idx = thread_channel * tile_pixels + data_pix; + + // Position in shared memory, offset by pad_height / pad_width. + const int tile_idx = data_idx + pad_offset; + + // Position in accumulator (kBlockSlices per warp, depth major). + const int accum_pix = thread_pix / (32 / kBlockSlices); + const int accum_idx = thread_channel * kAccumPixels + accum_pix; + + const int max_slice = in_slices - thread_channel; + const int accum_offset = tile_size + accum_idx; + const bool skip_second = block_height + thread_height >= in_height; + + for (int b = blockIdx.x; b < in_blocks; b += gridDim.x) { + const int slice = b * kBlockSlices; + + const int inout_offset = slice * in_pixels + tensor_idx; + const bool slice_in_range = slice < max_slice; + + if (slice_in_range) { + const DType* const in_ptr = inout_offset + input; + DType* const tile_ptr = tile_idx + shared_data; + tile_ptr[0] = ldg(in_ptr); + if (!skip_second) { + tile_ptr[tile_offset] = ldg(block_pixels + in_ptr); + } + } + + // Note: the condition to reach this is uniform across the entire block. + __syncthreads(); + + if (slice_in_range) { + const DType* const out_ptr = inout_offset + output; + const DType out1 = ldg(out_ptr); + const DType out2 = skip_second ? DType(0) : ldg(block_pixels + out_ptr); + int shared_offset = data_idx; + DType* accum_ptr = accum_offset + shared_data; + CUDA_UNROLL for (int r = 0; r < filter_height; ++r) { + CUDA_UNROLL for (int c = 0; c < filter_width; ++c) { + const DType* const tile_ptr = shared_offset + shared_data; + DType val = out1 * tile_ptr[0] + out2 * tile_ptr[tile_offset]; + // Warp-accumulate pixels of the same depth and write to accumulator. + for (int delta = 16 / kBlockSlices; delta > 0; delta /= 2) { + val += __shfl_down(val, delta); + } + if (!(thread_idx & 32 / kBlockSlices - 1)) { + *accum_ptr = val; + } + ++shared_offset; + accum_ptr += accum_increment; + } + shared_offset += in_increment; + } + } + + // Note: the condition to reach this is uniform across the entire block. + __syncthreads(); + + const DType* const accum_data = tile_size + shared_data; + for (int i = thread_idx; i < accum_size; i += block_size) { + const int filter_idx = i / kAccumPixels; + const int filter_pix = filter_idx / kBlockSlices; + const int filter_channel = (slice + filter_idx % kBlockSlices) % in_channel; + // convert to CHW + const int filter_offset = filter_channel * filter_pixels + + (filter_pix/filter_width) * filter_height + filter_pix % filter_width; + + if (filter_channel < in_channel) { + DType val = accum_data[i]; + // Warp-accumulate pixels of the same depth from the accumulator. + for (int delta = kAccumPixels / 2; delta > 0; delta /= 2) { + val += __shfl_down(val, delta); + } + if (!(thread_idx & kAccumPixels - 1)) { + atomicAdd(filter_offset + filter, val); + } + } + } + } +} + + +} // namespace cuda + +// Returns whether depthwise convolution forward or backward input pass can be +// performed using the faster ('Small') variant of the kernel. +bool CanLaunchDepthwiseConv2dGPUSmall(const DepthwiseArgs& args) { + return args.stride_height == 1 && args.stride_width == 1 && args.in_height <= 32 && + args.in_width <= 32 && args.in_height == args.out_height && + args.in_width == args.out_width && args.pad_height >= 0 && + args.pad_height < args.filter_height && args.pad_width >= 0 && + args.pad_width < args.filter_width && + args.filter_height * args.filter_width <= (args.in_height + 1) / 2 * args.in_width; +} + +// Returns whether depthwise convolution backward filter pass can be performed +// using the faster ('Small') variant of the kernel. +bool CanLaunchDepthwiseConv2dBackwardFilterGPUSmall(const DepthwiseArgs args, + const int block_height) { + return args.stride_height == 1 && args.stride_width == 1 && args.in_height <= 32 && + args.in_width <= 32 && args.in_height == args.out_height && + args.in_width == args.out_width && args.pad_height >= 0 && + args.pad_height < args.filter_height && args.pad_width >= 0 && + args.pad_width < args.filter_width && block_height <= args.in_height && + args.filter_height * args.filter_width <= block_height * args.in_width; +} + +template +void LaunchDepthwiseConv2dGPUSmall(mshadow::Stream *stream, + const DepthwiseArgs args, + const DType* input, const DType* filter, DType* output) { + const int block_height = (args.in_height + 1) / 2; + dim3 block_dim = dim3(args.in_width, block_height, kBlockSlices); + + const int tile_width = args.in_width + args.filter_width - 1; + const int tile_height = block_height * 2 + args.filter_height - 1; + const int tile_pixels = tile_height * tile_width; + const int filter_pixels = args.filter_height * args.filter_width; + const int shared_memory_size = + kBlockSlices * (tile_pixels + filter_pixels) * sizeof(DType); + const int num_outputs = + args.batch * args.out_height * args.out_width * args.out_channel; + int block_count = std::min(num_outputs/(block_dim.x * block_dim.y * block_dim.z) + 1, + (unsigned)mshadow::cuda::kMaxGridNum); + auto s = mshadow::Stream::GetStream(stream); + if (args.filter_height == 3 && args.filter_width == 3) { + cuda::DepthwiseConv2dKernelSmall + <<>>(args, input, filter, output); + } else { + cuda::DepthwiseConv2dKernelSmall + <<>>(args, input, filter, output); + } + MSHADOW_CUDA_POST_KERNEL_CHECK(DepthwiseConv2dKernelSmall); +} + +template +void LaunchDepthwiseConv2dGPUSmall(mshadow::Stream *stream, + const DepthwiseArgs args, + const DType* input, const DType* filter, DType* output) { + if (args.in_height & 1) { + LaunchDepthwiseConv2dGPUSmall( + stream, args, input, filter, output); + } else { + LaunchDepthwiseConv2dGPUSmall( + stream, args, input, filter, output); + } +} + +template +void LaunchDepthwiseConv2dGPUSmall(mshadow::Stream *stream, + const DepthwiseArgs args, + const DType* input, const DType* filter, DType* output) { + // Maximize (power of two) kBlockSlices while keeping a block within 1024 + // threads (2 pixels per thread). + const int block_pixels = (args.in_height + 1) / 2 * args.in_width; + if (block_pixels > 256) { + LaunchDepthwiseConv2dGPUSmall(stream, args, input, filter, output); + } else if (block_pixels > 128) { + LaunchDepthwiseConv2dGPUSmall(stream, args, input, filter, output); + } else { + LaunchDepthwiseConv2dGPUSmall(stream, args, input, filter, output); + } +} + +template +bool TryLaunchDepthwiseConv2dBackwardFilterGPUSmall(mshadow::Stream *stream, + const DepthwiseArgs args, + const int block_height, + const DType* out_grad, + const DType* input, + DType* filter_grad) { + const int tile_width = args.in_width + args.filter_width - 1; + const int tile_height = block_height * 2 + args.filter_height - 1; + const int tile_pixels = tile_height * tile_width; + const int filter_pixels = args.filter_height * args.filter_width; + const int shared_memory_size = + kBlockSlices * (tile_pixels + filter_pixels * kAccumPixels) * sizeof(DType); + if (shared_memory_size > 46 * 1024) { + return false; + } + + dim3 block_dim = dim3(args.in_width, block_height, kBlockSlices); + const int num_out_grad = + args.batch * args.out_height * args.out_width * args.out_channel; + int block_count = num_out_grad/(block_dim.x * block_dim.y * block_dim.z) + 1; + auto s = mshadow::Stream::GetStream(stream); + if (args.filter_height == 3 && args.filter_width == 3) { + cuda::DepthwiseConv2dBackwardFilterKernelSmall + <<>>( + args, out_grad, input, filter_grad); + } else { + cuda::DepthwiseConv2dBackwardFilterKernelSmall + <<>>( + args, out_grad, input, filter_grad); + } + MSHADOW_CUDA_POST_KERNEL_CHECK(DepthwiseConv2dBackwardFilterKernelSmall); + return true; +} + +template +bool TryLaunchDepthwiseConv2dBackwardFilterGPUSmall(mshadow::Stream *stream, + const DepthwiseArgs args, + const int block_height, + const DType* out_grad, + const DType* input, + DType* filter_grad) { + // Minimize (power of two) kAccumPixels, while satisfying + // kAccumPixels * 32 >= block_height * in_width * kBlockSlices. + const int block_pixels = block_height * args.in_width * kBlockSlices; + if (block_pixels > 512) { + return TryLaunchDepthwiseConv2dBackwardFilterGPUSmall( + stream, args, block_height, out_grad, input, filter_grad); + } else if (block_pixels > 256) { + return TryLaunchDepthwiseConv2dBackwardFilterGPUSmall( + stream, args, block_height, out_grad, input, filter_grad); + } else { + return TryLaunchDepthwiseConv2dBackwardFilterGPUSmall( + stream, args, block_height, out_grad, input, filter_grad); + } +} + +template +bool TryLaunchDepthwiseConv2dBackwardFilterGPUSmall(mshadow::Stream *stream, + const DepthwiseArgs args, + const DType* out_grad, + const DType* input, + DType* filter_grad) { + // Maximize (power of two) kBlockSlices while keeping a block within 1024 + // threads (2 pixels per thread). + int block_slices = 8; + int block_height = (args.in_height + 1) / 2; + int round_mask = 1; + for (; block_slices > 1; block_slices /= 2) { + // args.in_width * block_height * kBlockSlices must be multiple of 32. + for (; block_height * args.in_width * block_slices & 31; + round_mask = round_mask * 2 + 1) { + block_height = block_height + round_mask & ~round_mask; + } + int block_size = block_height * args.in_width * block_slices; + if (block_size <= 1024) { + break; + } + } + + if (!CanLaunchDepthwiseConv2dBackwardFilterGPUSmall(args, block_height)) { + return false; + } + + switch (block_slices) { + case 8: + return TryLaunchDepthwiseConv2dBackwardFilterGPUSmall( + stream, args, block_height, out_grad, input, filter_grad); + case 4: + return TryLaunchDepthwiseConv2dBackwardFilterGPUSmall( + stream, args, block_height, out_grad, input, filter_grad); + case 2: + return TryLaunchDepthwiseConv2dBackwardFilterGPUSmall( + stream, args, block_height, out_grad, input, filter_grad); + default: + return false; + } +} + +} // namespace depthwise_conv +} // namespace tf + +#endif // MXNET_OPERATOR_DEPTHWISE_CONVOLUTION_TF_CUH_ diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py index 7d56b46e21a0..a33cb039c849 100644 --- a/tests/python/unittest/test_operator.py +++ b/tests/python/unittest/test_operator.py @@ -955,6 +955,44 @@ def test_convolution_grouping(): for arr1, arr2 in zip(exe1.outputs + exe1.grad_arrays, exe2.outputs + exe2.grad_arrays): np.testing.assert_allclose(arr1.asnumpy(), arr2.asnumpy(), rtol=1e-3, atol=1e-4) + +def test_depthwise_convolution(): + for num_base in [32, 64]: + for kernel in [(3,3), (5,5)]: + for stride in [(1,1), (2,2)]: + for pad in [(0,0), (1,1)]: + num_filter = num_base + num_group = num_base + shape = (2, num_base, 32, 32) + + x = mx.sym.Variable('x') + w = mx.sym.Variable('w') + b = mx.sym.Variable('b') + y1 = mx.sym.Convolution(data=x, weight=w, bias=b, num_filter=num_filter, num_group=num_group, + kernel=kernel, stride=stride, pad=pad) + xslice = mx.sym.SliceChannel(data=x, num_outputs=num_group, axis=1) + wslice = mx.sym.SliceChannel(data=w, num_outputs=num_group, axis=0) + bslice = mx.sym.SliceChannel(data=b, num_outputs=num_group, axis=0) + y2 = mx.sym.Concat(*[mx.sym.Convolution(data=xslice[i], weight=wslice[i], bias=bslice[i], + num_filter=num_filter//num_group, kernel=kernel, + stride=stride, pad=pad) + for i in range(num_group)]) + + dev = default_context() + exe1 = y1.simple_bind(dev, x=shape) + exe2 = y2.simple_bind(mx.cpu(), x=shape, w=(num_filter, shape[1]//num_group, kernel[0], kernel[1]), + b=(num_filter,)) + for arr1, arr2 in zip(exe1.arg_arrays, exe2.arg_arrays): + arr1[:] = np.random.normal(size=arr1.shape) + arr2[:] = arr1 + exe1.forward(is_train=True) + exe1.backward(exe1.outputs[0]) + exe2.forward(is_train=True) + exe2.backward(exe2.outputs[0]) + + for arr1, arr2 in zip(exe1.outputs + exe1.grad_arrays, exe2.outputs + exe2.grad_arrays): + np.testing.assert_allclose(arr1.asnumpy(), arr2.asnumpy(), rtol=1e-3, atol=1e-4) + def gen_broadcast_data(idx): # Manually set test cases binary_op_data_shape = np.array( From 75ee5976bc1a7ec24746d764b529d0ae8e6e9583 Mon Sep 17 00:00:00 2001 From: zjjxsjh Date: Wed, 16 Aug 2017 10:37:48 -0700 Subject: [PATCH 385/834] fix a formula typo in doc (#7434) * fix a formula typo in doc * change num_channel to channel --- src/operator/convolution.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/operator/convolution.cc b/src/operator/convolution.cc index 35ab5f01afa1..55cfe4e085dc 100644 --- a/src/operator/convolution.cc +++ b/src/operator/convolution.cc @@ -101,7 +101,7 @@ channel, height, width)*, the output is computed by .. math:: - out[n,i,:,:] = bias[i] + \sum_{j=0}^{num\_filter} data[n,j,:,:] \star + out[n,i,:,:] = bias[i] + \sum_{j=0}^{channel} data[n,j,:,:] \star weight[i,j,:,:] where :math:`\star` is the 2-D cross-correlation operator. From c79d4e50cb9561bcfd69dd866b8ededa47c64d76 Mon Sep 17 00:00:00 2001 From: Mu Li Date: Wed, 16 Aug 2017 13:26:28 -0700 Subject: [PATCH 386/834] remove WaitToRead in dist-kvstore (#7489) * remove waittoread in dist-kv * update --- src/kvstore/kvstore_dist.h | 40 ++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/src/kvstore/kvstore_dist.h b/src/kvstore/kvstore_dist.h index 52c7c132cb5c..b64d7c6369bc 100644 --- a/src/kvstore/kvstore_dist.h +++ b/src/kvstore/kvstore_dist.h @@ -118,23 +118,22 @@ class KVStoreDist : public KVStoreLocal { if (recv_buf.is_none()) { // it may happen for the first time a no-rank-0 worker pull the weight. recv_buf = NDArray( - grouped_vals[i][0]->shape(), pinned_ctx_, false, grouped_vals[i][0]->dtype()); + grouped_vals[i][0]->shape(), pinned_ctx_, true, grouped_vals[i][0]->dtype()); } -#if MKL_EXPERIMENTAL == 1 - mkl_set_tblob_eager_mode(recv_buf.data()); -#endif - real_t* data = static_cast(recv_buf.data().dptr_); - size_t size = recv_buf.shape().Size(); - - auto pull_from_servers = [this, key, data, size]( + auto pull_from_servers = [this, key, recv_buf]( RunContext rctx, Engine::CallbackOnComplete cb) { // convert to ps keys + size_t size = recv_buf.shape().Size(); PSKV& pskv = EncodeKey(key, size); - - // issue pull, false means no delete +#if MKL_EXPERIMENTAL == 1 + mkl_set_tblob_eager_mode(recv_buf.data()); +#endif + real_t* data = static_cast(recv_buf.data().dptr_); + // false means not to delete data when SArray is deleted auto vals = new ps::SArray(data, size, false); + // issue pull CHECK_NOTNULL(ps_worker_)->ZPull( - pskv.keys, vals, &pskv.lens, 0, [vals, cb](){ delete vals; cb(); }); + pskv.keys, vals, &pskv.lens, 0, [vals, cb](){ delete vals; cb(); }); }; CHECK_NOTNULL(Engine::Get())->PushAsync( @@ -226,27 +225,26 @@ class KVStoreDist : public KVStoreLocal { send_buf = merged; // avoid memory copy } else { if (send_buf.is_none()) { - send_buf = NDArray(merged.shape(), pinned_ctx_, false, merged.dtype()); + send_buf = NDArray(merged.shape(), pinned_ctx_, true, merged.dtype()); } CopyFromTo(merged, &send_buf); } // push to servers - send_buf.WaitToRead(); - size_t size = send_buf.shape().Size(); -#if MKL_EXPERIMENTAL == 1 - mkl_set_tblob_eager_mode(send_buf.data()); -#endif - real_t* data = static_cast(send_buf.data().dptr_); auto push_to_servers = - [this, key, data, size](RunContext rctx, Engine::CallbackOnComplete cb) { - // convert to ps keys + [this, key, send_buf](RunContext rctx, Engine::CallbackOnComplete cb) { + // convert to ps keys + size_t size = send_buf.shape().Size(); PSKV& pskv = EncodeKey(key, size); +#if MKL_EXPERIMENTAL == 1 + mkl_set_tblob_eager_mode(send_buf.data()); +#endif + real_t* data = static_cast(send_buf.data().dptr_); // do push. false means no delete ps::SArray vals(data, size, false); CHECK_NOTNULL(ps_worker_)->ZPush( - pskv.keys, vals, pskv.lens, 0, [cb]() { cb(); }); + pskv.keys, vals, pskv.lens, 0, [cb]() { cb(); }); }; Engine::Get()->PushAsync( push_to_servers, From 1286809a1fc76c0b808b988084fc0950300f40d4 Mon Sep 17 00:00:00 2001 From: Yao Wang Date: Wed, 16 Aug 2017 18:11:01 -0700 Subject: [PATCH 387/834] Change git clone to specific tag for installation (#7502) --- docs/build_version_doc/AddVersion.py | 9 +++++++++ docs/get_started/install.md | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/build_version_doc/AddVersion.py b/docs/build_version_doc/AddVersion.py index c48c630565b7..44959445edc0 100755 --- a/docs/build_version_doc/AddVersion.py +++ b/docs/build_version_doc/AddVersion.py @@ -86,6 +86,15 @@ outstr = outstr.replace('http://mxnet.io', 'https://mxnet.incubator.apache.org/' 'versions/%s' % (args.current_version)) + # Fix git clone to specific tag + if args.current_version == 'master': + outstr = outstr.replace('git clone --recursive https://github.com/dmlc/mxnet', + 'git clone --recursive https://github.com/apache/incubator-mxnet.git') + else: + outstr = outstr.replace('git clone --recursive https://github.com/dmlc/mxnet', + 'git clone --recursive https://github.com/apache/incubator-mxnet.git ' + '--branch %s' % (args.current_version)) + with open(os.path.join(path, name), "w") as outf: outf.write(outstr) diff --git a/docs/get_started/install.md b/docs/get_started/install.md index 0e88a0d2a2ee..2ab771d4cfef 100644 --- a/docs/get_started/install.md +++ b/docs/get_started/install.md @@ -2,7 +2,7 @@ Indicate your preferred configuration. Then, follow the customized commands to install *MXNet*. - + From 462dee7a1547367c8ab1aaa786d5b59f210788de Mon Sep 17 00:00:00 2001 From: Peiyun Hu Date: Thu, 17 Aug 2017 15:43:47 -0400 Subject: [PATCH 388/834] Fix description of argument parser (#7507) --- example/image-classification/train_imagenet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/image-classification/train_imagenet.py b/example/image-classification/train_imagenet.py index 5760a9af3782..f465fbc5f469 100644 --- a/example/image-classification/train_imagenet.py +++ b/example/image-classification/train_imagenet.py @@ -25,7 +25,7 @@ if __name__ == '__main__': # parse args - parser = argparse.ArgumentParser(description="train cifar10", + parser = argparse.ArgumentParser(description="train imagenet-1k", formatter_class=argparse.ArgumentDefaultsHelpFormatter) fit.add_fit_args(parser) data.add_data_args(parser) From 56eae588c097f12035356333b742489d8cf0eaae Mon Sep 17 00:00:00 2001 From: Dick Carter Date: Thu, 17 Aug 2017 21:14:18 -0700 Subject: [PATCH 389/834] Fixed Makefile so a null CUDA_ARCH is treated like an unset one. (#7515) --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 33151e574ea7..b6c5834a566b 100644 --- a/Makefile +++ b/Makefile @@ -166,8 +166,8 @@ endif # Sets 'CUDA_ARCH', which determines the GPU architectures supported # by the compiled kernels. Users can edit the KNOWN_CUDA_ARCHS list below -# to remove archs they don't wish to support to speed compilation, or they -# can pre-set the CUDA_ARCH args in config.mk for full control. +# to remove archs they don't wish to support to speed compilation, or they can +# pre-set the CUDA_ARCH args in config.mk to a non-null value for full control. # # For archs in this list, nvcc will create a fat-binary that will include # the binaries (SASS) for all architectures supported by the installed version @@ -175,7 +175,7 @@ endif # If these kernels are then run on a newer-architecture GPU, the binary will # be JIT-compiled by the updated driver from the included PTX. ifeq ($(USE_CUDA), 1) -ifeq ($(origin CUDA_ARCH), undefined) +ifeq ($(CUDA_ARCH),) KNOWN_CUDA_ARCHS := 30 35 50 52 60 61 70 # Run nvcc on a zero-length file to check architecture-level support. # Create args to include SASS in the fat binary for supported levels. From ff21e1fd41f118dbbaf55d8f02a9669842ef565f Mon Sep 17 00:00:00 2001 From: Dick Carter Date: Thu, 17 Aug 2017 21:16:51 -0700 Subject: [PATCH 390/834] Changed FullyConnected to use new linalg gemm, plus TensorCore if fp16 I/O. (#7505) * Converted FullyConnected to use new linalg gemm, plus TensorCore if fp16 I/O. * Simplified linalg_gemm interface to ease integration. * Correcting code in response to comments. * Removing Transpose(), leaving trailing req arg with default of kWriteTo. --- src/common/cuda_utils.h | 34 ++++++++--- src/operator/fully_connected-inl.h | 14 +++-- src/operator/linalg.h | 10 ++++ src/operator/linalg_impl.h | 86 +++++++++++++++++++++++++++ tests/python/gpu/test_operator_gpu.py | 5 ++ 5 files changed, 138 insertions(+), 11 deletions(-) diff --git a/src/common/cuda_utils.h b/src/common/cuda_utils.h index 483390fc9bea..0213c73177b3 100644 --- a/src/common/cuda_utils.h +++ b/src/common/cuda_utils.h @@ -200,7 +200,7 @@ inline DType __device__ CudaMin(DType a, DType b) { { \ cublasStatus_t e = (func); \ CHECK_EQ(e, CUBLAS_STATUS_SUCCESS) \ - << "cuBLAS: " << common::cuda::CublasGetErrorString(e); \ + << "cuBLAS: " << mxnet::common::cuda::CublasGetErrorString(e); \ } /*! @@ -213,7 +213,7 @@ inline DType __device__ CudaMin(DType a, DType b) { { \ cusolverStatus_t e = (func); \ CHECK_EQ(e, CUSOLVER_STATUS_SUCCESS) \ - << "cuSolver: " << common::cuda::CusolverGetErrorString(e); \ + << "cuSolver: " << mxnet::common::cuda::CusolverGetErrorString(e); \ } /*! @@ -226,7 +226,7 @@ inline DType __device__ CudaMin(DType a, DType b) { { \ curandStatus_t e = (func); \ CHECK_EQ(e, CURAND_STATUS_SUCCESS) \ - << "cuRAND: " << common::cuda::CurandGetErrorString(e); \ + << "cuRAND: " << mxnet::common::cuda::CurandGetErrorString(e); \ } #if !defined(_MSC_VER) @@ -304,11 +304,31 @@ inline bool SupportsTensorCore(int device_id) { * \return whether to allow TensorCore algo (if not specified by the Operator locally). */ inline bool GetEnvAllowTensorCore() { - // Use of optional here permits: "0", "1", "true" and "false" to all be legal. - bool default_value = MXNET_CUDA_ALLOW_TENSOR_CORE_DEFAULT; - return dmlc::GetEnv("MXNET_CUDA_ALLOW_TENSOR_CORE", - dmlc::optional(default_value)).value(); + // Since these statics are in the '.h' file, they will exist and will be set + // separately in each compilation unit. Not ideal, but cleaner than creating a + // cuda_utils.cc solely to have a single instance and initialization. + static bool allow_tensor_core = false; + static bool is_set = false; + if (!is_set) { + // Use of optional here permits: "0", "1", "true" and "false" to all be legal. + bool default_value = MXNET_CUDA_ALLOW_TENSOR_CORE_DEFAULT; + allow_tensor_core = dmlc::GetEnv("MXNET_CUDA_ALLOW_TENSOR_CORE", + dmlc::optional(default_value)).value(); + is_set = true; + } + return allow_tensor_core; +} + +#if CUDA_VERSION >= 9000 +// Sets the cuBLAS math mode that determines the 'allow TensorCore' policy. Returns previous. +inline cublasMath_t SetCublasMathMode(cublasHandle_t blas_handle, cublasMath_t new_math_type) { + auto handle_math_mode = CUBLAS_DEFAULT_MATH; + CUBLAS_CALL(cublasGetMathMode(blas_handle, &handle_math_mode)); + CUBLAS_CALL(cublasSetMathMode(blas_handle, new_math_type)); + return handle_math_mode; } +#endif + #endif // MXNET_USE_CUDA #if MXNET_USE_CUDNN diff --git a/src/operator/fully_connected-inl.h b/src/operator/fully_connected-inl.h index e2fab9f1f7dd..cf13655d9c97 100644 --- a/src/operator/fully_connected-inl.h +++ b/src/operator/fully_connected-inl.h @@ -33,7 +33,7 @@ #include #include "./operator_common.h" #include "./elemwise_op_common.h" - +#include "linalg.h" namespace mxnet { namespace op { @@ -96,7 +96,9 @@ class FullyConnectedOp : public Operator { Tensor wmat = in_data[fullc::kWeight].get(s); Tensor out = out_data[fullc::kOut].get_with_shape( Shape2(oshape[0], oshape.ProdShape(1, oshape.ndim())), s); - out = dot(data, wmat.T()); + // Legacy approach shown here for comparison: + // out = dot(data, wmat.T()); + linalg_gemm(data, wmat, out, false, true, s); if (!param_.no_bias) { Tensor bias = in_data[fullc::kBias].get(s); out += repmat(bias, data.size(0)); @@ -136,7 +138,9 @@ class FullyConnectedOp : public Operator { CHECK_NE(req[fullc::kWeight], kWriteInplace) << "cannot write weight inplace"; // gradient of weight Tensor gwmat = in_grad[fullc::kWeight].get(s); - Assign(gwmat, req[fullc::kWeight], dot(grad.T(), data)); + // Legacy approach shown here for comparison: + // Assign(gwmat, req[fullc::kWeight], dot(grad.T(), data)); + linalg_gemm(grad, data, gwmat, true, false, s, req[fullc::kWeight]); // gradient of bias if (!param_.no_bias) { Tensor gbias = in_grad[fullc::kBias].get(s); @@ -145,7 +149,9 @@ class FullyConnectedOp : public Operator { // gradient of data Tensor gdata = in_grad[fullc::kData].get_with_shape( Shape2(ishape[0], ishape.ProdShape(1, ishape.ndim())), s); - Assign(gdata, req[fullc::kData], dot(grad, wmat)); + // Legacy approach shown here for comparison: + // Assign(gdata, req[fullc::kData], dot(grad, wmat)); + linalg_gemm(grad, wmat, gdata, false, false, s, req[fullc::kData]); } private: diff --git a/src/operator/linalg.h b/src/operator/linalg.h index 9284a5825d2c..76acf7b98f41 100644 --- a/src/operator/linalg.h +++ b/src/operator/linalg.h @@ -26,6 +26,8 @@ #define MXNET_OPERATOR_LINALG_H_ #include +#include + #include "./c_lapack_api.h" using namespace mshadow; @@ -62,6 +64,14 @@ void linalg_batch_gemm(const Tensor& A, const Tensor& C, DType alpha, DType beta, bool tA, bool tB, Stream *s = 0); +template +inline void linalg_gemm(const Tensor& A, + const Tensor& B, + const Tensor& C, + bool tA, bool tB, + Stream *s = 0, + mxnet::OpReqType req = mxnet::kWriteTo); + //////////////////////////////// TRSM //////////////////////////////////////////// // CPU/GPU-versions of BLAS3 function "trsm". Please refer to the BLAS3-documentation diff --git a/src/operator/linalg_impl.h b/src/operator/linalg_impl.h index affa7941640b..1e3b0e66e641 100644 --- a/src/operator/linalg_impl.h +++ b/src/operator/linalg_impl.h @@ -25,8 +25,12 @@ #ifndef MXNET_OPERATOR_LINALG_IMPL_H_ #define MXNET_OPERATOR_LINALG_IMPL_H_ +#include + #include +#include "../common/cuda_utils.h" + // Convenience functions. inline void linalg_check_batch_size(int A, int B, int C) { CHECK_EQ(A, B) << "Inconsistent batch size between arguments to linear algebra operator"; @@ -108,6 +112,55 @@ void linalg_gemm(const Tensor& A, const Tensor for DType=mshadow::half::half_t. +template<> inline +void linalg_gemm(const Tensor& A, + const Tensor& B, + const Tensor& C, + mshadow::half::half_t alpha, + mshadow::half::half_t beta, + bool tA, bool tB, Stream *s) { + using namespace mxnet; + using mshadow::gpu; + CHECK_NOTNULL(s); + check_gemm(A, B, C, alpha, beta, tA, tB); + +#if CUDA_VERSION >= 7050 + auto blas_handle = Stream::GetBlasHandle(s); +#if CUDA_VERSION >= 9000 + auto cublas_math_mode = GetEnvAllowTensorCore() ? CUBLAS_TENSOR_OP_MATH + : CUBLAS_DEFAULT_MATH; + auto previous_math_mode = SetCublasMathMode(blas_handle, cublas_math_mode); +#endif + + // pseudo-fp16 (fp32 math with fp16 I/O) + float alpha_f = float(alpha); // NOLINT(*) + float beta_f = float(beta); // NOLINT(*) + + // As of cuda8, cublas adopted the cuda datatype, rather than maintaining its own datatype. +#if CUDA_VERSION >= 8000 + cudaDataType_t half_datatype = CUDA_R_16F; +#else + cublasDataType_t half_datatype = CUBLAS_DATA_HALF; +#endif + CUBLAS_CALL(cublasSgemmEx(blas_handle, + (tB ? CUBLAS_OP_T : CUBLAS_OP_N), + (tA ? CUBLAS_OP_T : CUBLAS_OP_N), + C.size(1), C.size(0), (tB ? B.size(1) : B.size(0)), + &alpha_f, + B.dptr_, half_datatype, B.stride_, + A.dptr_, half_datatype, A.stride_, + &beta_f, + C.dptr_, half_datatype, C.stride_)); +#if CUDA_VERSION >= 9000 + SetCublasMathMode(blas_handle, previous_math_mode); +#endif +#else + LOG(FATAL) << "FP16 gemm requires CUDA version >= 7.5!"; +#endif // CUDA_VERSION >= 7050 +} + + #define LINALG_GPU_BATCH_GEMM(fname, DType) \ template<> inline \ void linalg_batch_gemm(const Tensor& A, const Tensor& B, \ @@ -246,6 +299,39 @@ LINALG_GPU_BATCH_TRSM(DtrsmBatched, double) #endif +/*! + * \brief Performs gemm, setting alpha and beta as appropriate for `req`. + * + * \param A the first operand of the gemm + * \param B the second operand of the gemm + * \param C the data to be assigned + * \tparam tA whether the `A` operand should be transposed first. + * \tparam tB whether the `B` operand should be transposed first. + * \tparam s the stream to perform the operation + * \param req the assignment request + */ +template +inline void linalg_gemm(const Tensor& A, + const Tensor& B, + const Tensor& C, + bool tA, bool tB, Stream *s, + mxnet::OpReqType req) { + using namespace mxnet; + switch (req) { + case kNullOp: + break; + case kWriteTo: + case kWriteInplace: + linalg_gemm(A, B, C, DType(1.0), DType(0.0), tA, tB, s); + break; + case kAddTo: + linalg_gemm(A, B, C, DType(1.0), DType(1.0), tA, tB, s); + break; + default: + LOG(FATAL) << "not reached"; + } +} + //////////////////////////////// TRMM //////////////////////////////////////////// // CPU/GPU-versions of BLAS3 function "trmm". Please refer to the BLAS3-documentation diff --git a/tests/python/gpu/test_operator_gpu.py b/tests/python/gpu/test_operator_gpu.py index 866f6ad8abc0..81492fe6bbdb 100644 --- a/tests/python/gpu/test_operator_gpu.py +++ b/tests/python/gpu/test_operator_gpu.py @@ -926,6 +926,11 @@ def test_fullyconnected_with_type(): {'ctx': mx.cpu(0), 'inner_data': (2, 10), 'type_dict': {'inner_data': np.float64}}, {'ctx': mx.cpu(0), 'inner_data': (2, 10), 'type_dict': {'inner_data': np.float32}}] check_consistency(sym, ctx_list) + # Sizes are divisible by 8 to test TensorCore on Volta GPU. + sym = mx.sym.FullyConnected(num_hidden=8, name='inner') + ctx_list = [{'ctx': mx.gpu(0), 'inner_data': (16, 24), 'type_dict': {'inner_data': np.float16}}, + {'ctx': mx.cpu(0), 'inner_data': (16, 24), 'type_dict': {'inner_data': np.float32}}] + check_consistency(sym, ctx_list) def test_activation_with_type(): From 6004e529320a3230607ff535d2ec16190130959d Mon Sep 17 00:00:00 2001 From: Yao Wang Date: Thu, 17 Aug 2017 21:20:29 -0700 Subject: [PATCH 391/834] Modify pip install to specific tag (#7514) * Modify pip installation to specific tag * Fix --- docs/build_version_doc/AddVersion.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/build_version_doc/AddVersion.py b/docs/build_version_doc/AddVersion.py index 44959445edc0..e5569a0fdaac 100755 --- a/docs/build_version_doc/AddVersion.py +++ b/docs/build_version_doc/AddVersion.py @@ -86,14 +86,25 @@ outstr = outstr.replace('http://mxnet.io', 'https://mxnet.incubator.apache.org/' 'versions/%s' % (args.current_version)) - # Fix git clone to specific tag + # Fix git clone and pip installation to specific tag + pip_pattern = ['', '-cu80', '-cu75', '-cu80mkl', '-cu75mkl', '-mkl'] if args.current_version == 'master': outstr = outstr.replace('git clone --recursive https://github.com/dmlc/mxnet', 'git clone --recursive https://github.com/apache/incubator-mxnet.git') + for trail in pip_pattern: + outstr = outstr.replace('pip install mxnet%s<' % (trail), + 'pip install mxnet%s --pre<' % (trail)) + outstr = outstr.replace('pip install mxnet%s\n<' % (trail), + 'pip install mxnet%s --pre\n<' % (trail)) else: outstr = outstr.replace('git clone --recursive https://github.com/dmlc/mxnet', 'git clone --recursive https://github.com/apache/incubator-mxnet.git ' '--branch %s' % (args.current_version)) + for trail in pip_pattern: + outstr = outstr.replace('pip install mxnet%s<' % (trail), + 'pip install mxnet%s==%s<' % (trail, args.current_version)) + outstr = outstr.replace('pip install mxnet%s\n<' % (trail), + 'pip install mxnet%s==%s\n<' % (trail, args.current_version)) with open(os.path.join(path, name), "w") as outf: outf.write(outstr) From d2dbffe194f4eed728c883927ce639919cb2078f Mon Sep 17 00:00:00 2001 From: Nan Zhu Date: Fri, 18 Aug 2017 02:27:31 -0700 Subject: [PATCH 392/834] [scala-package][spark] fix example script (#7411) * temp * temp * fix example script * update indent --- .gitignore | 2 + scala-package/spark/bin/run-mnist-example.sh | 59 ++++++++++++-------- scala-package/spark/pom.xml | 2 +- 3 files changed, 40 insertions(+), 23 deletions(-) diff --git a/.gitignore b/.gitignore index 82d2e560237d..4e4ff78d3489 100644 --- a/.gitignore +++ b/.gitignore @@ -146,3 +146,5 @@ bld target bin/im2rec + +model/ \ No newline at end of file diff --git a/scala-package/spark/bin/run-mnist-example.sh b/scala-package/spark/bin/run-mnist-example.sh index cae19386a8ee..57b8a1803363 100755 --- a/scala-package/spark/bin/run-mnist-example.sh +++ b/scala-package/spark/bin/run-mnist-example.sh @@ -18,47 +18,62 @@ # under the License. CURR_DIR=$(cd `dirname $0`; pwd) -MODULE_DIR=$(cd $CURR_DIR/../; pwd) -ROOT_DIR=$(cd $CURR_DIR/../../; pwd) +SPARK_MODULE_DIR=$(cd $CURR_DIR/../; pwd) +SCALA_PKG_DIR=$(cd $CURR_DIR/../../; pwd) +OS="" -LIB_DIR=${MODULE_DIR}/target/classes/lib -JAR=${MODULE_DIR}/target/mxnet-spark_2.10-0.1.2-SNAPSHOT.jar +if [ "$(uname)" == "Darwin" ]; then + # Do something under Mac OS X platform + OS='osx-x86_64-cpu' +elif [ "$(expr substr $(uname -s) 1 5)" == "Linux" ]; then + OS='linux-x86_64-cpu' +fi -LIBS=${ROOT_DIR}/assembly/linux-x86_64-cpu/target/mxnet-full_2.10-linux-x86_64-cpu-0.1.2-SNAPSHOT.jar -LIBS="${LIBS},${LIB_DIR}/args4j-2.0.29.jar,${LIB_DIR}/scala-library-2.10.4.jar,${JAR}" +LIB_DIR=${SPARK_MODULE_DIR}/target/classes/lib +SPARK_JAR=`find ${SPARK_MODULE_DIR}/target -name "*.jar" -type f -exec ls "{}" + | grep -v -E '(javadoc|sources)'` +SCALA_JAR=`find ${SCALA_PKG_DIR}/assembly/$OS/target -maxdepth 1 -name "*.jar" -type f -exec ls "{}" + | grep -v -E '(javadoc|sources)'` -SPARK_OPTS+=" --name mxnet" +SPARK_OPTS+=" --name mxnet-spark-mnist" SPARK_OPTS+=" --driver-memory 1g" SPARK_OPTS+=" --executor-memory 1g" SPARK_OPTS+=" --num-executors 2" SPARK_OPTS+=" --executor-cores 1" -SPARK_OPTS+=" --jars ${LIBS}" +SPARK_OPTS+=" --jars ${SCALA_JAR}" -# You can download these two files as training & validation set. -# They were converted from the MNIST dataset, -# in which each sample was simply flatterned to an array of floats. -# https://s3-us-west-2.amazonaws.com/mxnet.liuyz/data/mnist/train.txt -# https://s3-us-west-2.amazonaws.com/mxnet.liuyz/data/mnist/val.txt +# Download training and test set +if [ ! -f ./train.txt ]; then + wget https://s3-us-west-2.amazonaws.com/mxnet.liuyz/data/mnist/train.txt +fi + +if [ ! -f ./val.txt ]; then + wget https://s3-us-west-2.amazonaws.com/mxnet.liuyz/data/mnist/val.txt +fi # running opts -RUN_OPTS+=" --input ${INPUT_TRAIN}" -RUN_OPTS+=" --input-val ${INPUT_VAL}" -RUN_OPTS+=" --output ${OUTPUT}" +RUN_OPTS+=" --input train.txt" +RUN_OPTS+=" --input-val val.txt" +RUN_OPTS+=" --output ./" # These jars are required by the KVStores at runtime. # They will be uploaded and distributed to each node automatically. -RUN_OPTS+=" --jars ${LIBS}" +RUN_OPTS+=" --jars $SCALA_JAR,$SPARK_JAR" RUN_OPTS+=" --num-server 1" RUN_OPTS+=" --num-worker 2" -RUN_OPTS+=" --java /usr/local/jdk1.8.0_60/bin/java" +RUN_OPTS+=" --java $JAVA_HOME/bin/java" RUN_OPTS+=" --model mlp" RUN_OPTS+=" --cpus 0,1" RUN_OPTS+=" --num-epoch 5" -${SPARK_HOME}/bin/spark-submit --master spark://localhost:7077 \ - --conf spark.dynamicAllocation.enabled=false \ - --conf spark.speculation=false \ +# check if SPARK_HOME is set +if [ -z "$SPARK_HOME" ]; then + echo "SPARK_HOME is unset"; + exit 1 +fi + +HOST=`hostname` + +$SPARK_HOME/bin/spark-submit --master spark://$HOST:7077 \ --class ml.dmlc.mxnet.spark.example.ClassificationExample \ ${SPARK_OPTS} \ - ${JAR} \ + ${SPARK_JAR} \ ${RUN_OPTS} diff --git a/scala-package/spark/pom.xml b/scala-package/spark/pom.xml index 18170b95579b..c59662f6debc 100644 --- a/scala-package/spark/pom.xml +++ b/scala-package/spark/pom.xml @@ -14,7 +14,7 @@ MXNet Scala Package - Spark ML - 1.6.1 + 1.6.3 From 406bc198538c904d6105ffafe4cc230d0c858545 Mon Sep 17 00:00:00 2001 From: buryang <419494197@163.com> Date: Sat, 19 Aug 2017 00:41:06 +0800 Subject: [PATCH 393/834] V0.11.0 (#7518) * Update NOTICE & README * New code signing key & README file changes (#7464) * add Naveen's Code Signing Key (#7460) * Updating CoreML readme file (#7459) * Fixing CoreML converter's README: typos/grammar/etc. * CoreML converter README update: Talk about layers first and then about models. * Providing examples on converting various standard models; calling out issues with InceptionV3. * Change RC version in NEWS (#7467) * add Naveen's Code Signing Key (#7460) * Updating CoreML readme file (#7459) * Fixing CoreML converter's README: typos/grammar/etc. * CoreML converter README update: Talk about layers first and then about models. * Providing examples on converting various standard models; calling out issues with InceptionV3. * Update NEWS * Update NEWS * update to rc2 --- NEWS.md | 2 +- NOTICE | 7 +++++-- README.md | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/NEWS.md b/NEWS.md index 4fdd31430002..4f1ecd15689c 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,6 +1,6 @@ MXNet Change Log ================ -## 0.11.0-rc0 +## 0.11.0-rc2 ### - Major Features - Apple Core ML model converter - Support for Keras v1.2.2 diff --git a/NOTICE b/NOTICE index 2051e3c00d53..03695607e3e9 100644 --- a/NOTICE +++ b/NOTICE @@ -1,2 +1,5 @@ -MXNet -Copyright (c) 2015-2016 by Contributors +Apache MXNET (incubating) +Copyright [2015-2017] The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). diff --git a/README.md b/README.md index 841c6f1f62c2..a11780aa019b 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ deep learning systems, and interesting insights of DL systems for hackers. What's New ---------- -* [Version 0.11.0-rc0 Release](https://github.com/apache/incubator-mxnet/releases/tag/0.11.0.rc0) - MXNet 0.11.0-rc0 Release. +* [Version 0.11.0-rc2 Release](https://github.com/apache/incubator-mxnet/releases/tag/0.11.0.rc2) - MXNet 0.11.0-rc2 Release. * [Apache Incubator](http://incubator.apache.org/projects/mxnet.html) - We are now an Apache Incubator project. * [Version 0.10.0 Release](https://github.com/dmlc/mxnet/releases/tag/v0.10.0) - MXNet 0.10.0 Release. * [Version 0.9.3 Release](./docs/architecture/release_note_0_9.md) - First 0.9 official release. From 5ee1cfe96852eb695f933dfe083e0429d97b704e Mon Sep 17 00:00:00 2001 From: Nan Zhu Date: Fri, 18 Aug 2017 20:35:52 -0700 Subject: [PATCH 394/834] Update README.md for easy copy and paste --- scala-package/spark/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scala-package/spark/README.md b/scala-package/spark/README.md index 974691650ff4..06106648c059 100644 --- a/scala-package/spark/README.md +++ b/scala-package/spark/README.md @@ -11,7 +11,7 @@ The MXNet on Spark is still in *experimental stage*. Any suggestion or contribut Build ------------ -Checkout the [Installation Guide](http://mxnet.io/get_started/setup.html) contains instructions to install mxnet. Remember to enable the distributed training, i.e., set `USE_DIST_KVSTORE = 1`. +Checkout the [Installation Guide](http://mxnet.io/get_started/setup.html) contains instructions to install mxnet. Remember to enable the distributed training, i.e., set `USE_DIST_KVSTORE=1`. Compile the Scala Package by From 0efc326e2243625d622a43287bf15c62e6afd1b0 Mon Sep 17 00:00:00 2001 From: Ziyue Huang Date: Mon, 21 Aug 2017 05:43:02 +0800 Subject: [PATCH 395/834] Fix a bug in SequentialRNNCell.reset() (#7449) * remove self-implemented speedometer * fix bug in SequentialRNNCell.reset * Revert "remove self-implemented speedometer" This reverts commit 17aa4c0887c099f22c4769de079ef0130ed5f3e8. * fix lint * fix * fix reset in origin rnn and gluon rnn * fix origin rnn --- python/mxnet/gluon/rnn/rnn_cell.py | 2 ++ python/mxnet/rnn/rnn_cell.py | 3 +++ 2 files changed, 5 insertions(+) diff --git a/python/mxnet/gluon/rnn/rnn_cell.py b/python/mxnet/gluon/rnn/rnn_cell.py index c9186fd3ce09..eb67fd73db7d 100644 --- a/python/mxnet/gluon/rnn/rnn_cell.py +++ b/python/mxnet/gluon/rnn/rnn_cell.py @@ -121,6 +121,8 @@ def reset(self): """Reset before re-using the cell for another graph.""" self._init_counter = -1 self._counter = -1 + for cell in self._children: + cell.reset() def state_info(self, batch_size=0): """shape and layout information of states""" diff --git a/python/mxnet/rnn/rnn_cell.py b/python/mxnet/rnn/rnn_cell.py index 1c3452041494..b2bf107c38ca 100644 --- a/python/mxnet/rnn/rnn_cell.py +++ b/python/mxnet/rnn/rnn_cell.py @@ -134,6 +134,9 @@ def reset(self): """Reset before re-using the cell for another graph.""" self._init_counter = -1 self._counter = -1 + if hasattr(self, '_cells'): + for cell in self._cells: + cell.reset() def __call__(self, inputs, states): """Unroll the RNN for one time step. From 686153e59a6c87f33f8f93297c8326516e74c964 Mon Sep 17 00:00:00 2001 From: Xingjian Shi Date: Tue, 22 Aug 2017 02:00:54 +0800 Subject: [PATCH 396/834] Fix argsort + Update MShadow (#7535) * fix the topk related operators to use int for indexing and use CUB for sorting. * Update MShadow * fix bug * fix lint --- mshadow | 2 +- src/operator/tensor/init_op.h | 7 +-- src/operator/tensor/ordering_op-inl.h | 76 ++++++++++++++++----------- 3 files changed, 50 insertions(+), 35 deletions(-) diff --git a/mshadow b/mshadow index 497eb9180b24..6d75df228978 160000 --- a/mshadow +++ b/mshadow @@ -1 +1 @@ -Subproject commit 497eb9180b24592b7332e7e08f2c053ec5346524 +Subproject commit 6d75df228978ca5f182dd707578ef704099ab5ee diff --git a/src/operator/tensor/init_op.h b/src/operator/tensor/init_op.h index bdc74d332491..30a5a3a3af1b 100644 --- a/src/operator/tensor/init_op.h +++ b/src/operator/tensor/init_op.h @@ -186,9 +186,10 @@ inline bool RangeShape(const nnvm::NodeAttrs& attrs, << "(" << param.start << "," << param.stop.value() << "," << param.step << ")"; } SHAPE_ASSIGN_CHECK(*out_attrs, 0, - mshadow::Shape1(param.repeat * - ceil((param.stop.value() - - param.start) / param.step))); + mshadow::Shape1(mshadow::expr::RangeOutSize(param.start, + param.stop.value(), + param.step, + param.repeat))); return true; } diff --git a/src/operator/tensor/ordering_op-inl.h b/src/operator/tensor/ordering_op-inl.h index eb28b010cbd3..560554151a19 100644 --- a/src/operator/tensor/ordering_op-inl.h +++ b/src/operator/tensor/ordering_op-inl.h @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include "../mshadow_op.h" @@ -175,8 +176,10 @@ void TopKImpl(RunContext ctx, } // 1. Parse and initialize information Stream *s = ctx.get_stream(); - Tensor workspace; - Tensor sorted_dat, indices, batch_id, sel_indices; + Tensor workspace; + Tensor temp_workspace; + Tensor sorted_dat; + Tensor indices, batch_id, sel_indices; Tensor mask_val; int batch_size, element_num; // number of batches + the size of each batch int axis = 0; @@ -187,49 +190,58 @@ void TopKImpl(RunContext ctx, ParseTopKParam(src.shape_, param, &target_shape, &batch_size, &element_num, &axis, &k, &do_transpose, &is_ascend); Tensor dat = src.FlatTo3D(axis, axis, s); + size_t temp_size = mxnet::op::SortByKeyWorkspaceSize(src.Size()); + temp_size = std::max(temp_size, mxnet::op::SortByKeyWorkspaceSize(src.Size())); + temp_size = std::max(temp_size, mxnet::op::SortByKeyWorkspaceSize(src.Size())); + size_t workspace_size = temp_size + sizeof(real_t) * src.Size() + sizeof(int) * src.Size() * 2; if (param.ret_typ == topk_enum::kReturnMask) { - workspace = - resource.get_space_typed(Shape1(src.Size() * 3 + 2 * batch_size * k), s); - } else { - workspace = resource.get_space_typed(mshadow::Shape1(src.Size() * 3), s); + workspace_size += sizeof(int) * batch_size * k + sizeof(real_t) * batch_size * k; } - sorted_dat = Tensor(workspace.dptr_, + workspace = resource.get_space_typed(Shape1(workspace_size), s); + char* workspace_curr_ptr = workspace.dptr_; + sorted_dat = Tensor(reinterpret_cast(workspace_curr_ptr), Shape1(src.Size()), s); // contain sorted dat - indices = Tensor(workspace.dptr_ + src.Size(), - Shape1(src.Size()), s); // indices in the original matrix - batch_id = Tensor(workspace.dptr_ + 2 * src.Size(), - Shape1(src.Size()), s); // batch id in the original matrix + workspace_curr_ptr += sizeof(real_t) * src.Size(); + indices = Tensor(reinterpret_cast(workspace_curr_ptr), + Shape1(src.Size()), s); // indices in the original matrix + workspace_curr_ptr += sizeof(int) * src.Size(); + batch_id = Tensor(reinterpret_cast(workspace_curr_ptr), + Shape1(src.Size()), s); // batch id in the original matrix + workspace_curr_ptr += sizeof(int) * src.Size(); if (do_transpose) { sorted_dat = reshape(transpose(dat, Shape3(0, 2, 1)), Shape1(src.Size())); } else { sorted_dat = reshape(dat, Shape1(src.Size())); } - indices = range(0, batch_size * element_num); + indices = range(0, batch_size * element_num); CHECK_EQ(sorted_dat.CheckContiguous(), true); CHECK_EQ(indices.CheckContiguous(), true); if (param.ret_typ == topk_enum::kReturnMask) { - sel_indices = Tensor(workspace.dptr_ + 3 * src.Size(), - Shape1(batch_size * k), s); - mask_val = Tensor(workspace.dptr_ + 3 * src.Size() + batch_size * k, + sel_indices = Tensor(reinterpret_cast(workspace_curr_ptr), + Shape1(batch_size * k), s); + workspace_curr_ptr += sizeof(int) * batch_size * k; + mask_val = Tensor(reinterpret_cast(workspace_curr_ptr), Shape2(batch_size * k, 1), s); + workspace_curr_ptr += sizeof(real_t) * batch_size * k; mask_val = scalar(1); CHECK_EQ(sel_indices.CheckContiguous(), true); CHECK_EQ(mask_val.CheckContiguous(), true); } - + temp_workspace = Tensor(workspace_curr_ptr, Shape1(temp_size), s); // temp space + workspace_curr_ptr += temp_size; // 2. Perform inplace batch sort using the `SortByKey` in MShadow // After sorting, each batch in `sorted_dat` will be sorted in the corresponding order // and the `indices` will contain the corresponding index in `sorted_dat` // Sort the data and keep record of the correspondence to global indices. - mxnet::op::SortByKey(sorted_dat, indices, is_ascend); + mxnet::op::SortByKey(sorted_dat, indices, is_ascend, &temp_workspace); // Calculate the corresponding batch indices of the elements - batch_id = F(indices / static_cast(element_num)); + batch_id = indices / element_num; // Since the SortByKey performs stable sort, the second SortByKey will reorder // the sorted_dat based on the order of the batch_id - mxnet::op::SortByKey(batch_id, sorted_dat, true); + mxnet::op::SortByKey(batch_id, sorted_dat, true, &temp_workspace); // Reorder the indices - batch_id = F(indices / static_cast(element_num)); - mxnet::op::SortByKey(batch_id, indices, true); + batch_id = indices / element_num; + mxnet::op::SortByKey(batch_id, indices, true, &temp_workspace); // 3. Assign results to the ret blob if (param.ret_typ == topk_enum::kReturnMask) { @@ -239,8 +251,8 @@ void TopKImpl(RunContext ctx, sel_indices = reshape(slice<1>( inplace_reshape(indices, Shape2(batch_size, - element_num)), 0, k), - Shape1(batch_size * k)); + element_num)), 0, k), + Shape1(batch_size * k)); if (do_transpose) { TShape src_shape = src.shape_.FlatTo3D(axis); CHECK_EQ(sel_indices.CheckContiguous(), true); @@ -249,23 +261,24 @@ void TopKImpl(RunContext ctx, } IndexFill(ret_mask, sel_indices, mask_val); } else if (param.ret_typ == topk_enum::kReturnIndices) { - indices -= batch_id * static_cast(element_num); + indices -= batch_id * element_num; if (do_transpose) { Tensor ret_indices = ret[0].FlatTo3D(axis, axis, s); - ret_indices = transpose( + ret_indices = tcast(transpose( slice<2>(inplace_reshape(indices, Shape3(ret_indices.shape_[0], ret_indices.shape_[2], element_num)), 0, k), - Shape3(0, 2, 1)); + Shape3(0, 2, 1))); } else { Tensor ret_indices = ret[0].get_with_shape(Shape2(batch_size, k), s); - ret_indices = slice<1>(inplace_reshape(indices, Shape2(batch_size, element_num)), 0, k); + ret_indices = tcast(slice<1>( + inplace_reshape(indices, Shape2(batch_size, element_num)), 0, k)); } } else { - indices -= batch_id * static_cast(element_num); + indices -= batch_id * element_num; if (do_transpose) { Tensor ret_value = ret[0].FlatTo3D(axis, axis, s); Tensor ret_indices = ret[1].FlatTo3D(axis, axis, s); @@ -274,20 +287,21 @@ void TopKImpl(RunContext ctx, Shape3(ret_value.shape_[0], ret_value.shape_[2], element_num)), 0, k), Shape3(0, 2, 1)); - ret_indices = transpose( + ret_indices = tcast(transpose( slice<2>(inplace_reshape(indices, Shape3(ret_indices.shape_[0], ret_indices.shape_[2], element_num)), 0, k), - Shape3(0, 2, 1)); + Shape3(0, 2, 1))); } else { Tensor ret_value = ret[0].get_with_shape(Shape2(batch_size, k), s); Tensor ret_indices = ret[1].get_with_shape(Shape2(batch_size, k), s); ret_value = slice<1>(inplace_reshape(sorted_dat, Shape2(batch_size, element_num)), 0, k); - ret_indices = slice<1>(inplace_reshape(indices, Shape2(batch_size, element_num)), 0, k); + ret_indices = tcast(slice<1>( + inplace_reshape(indices, Shape2(batch_size, element_num)), 0, k)); } } } From 32fc60b29d00201abe6d55ce0b751b25398b1d46 Mon Sep 17 00:00:00 2001 From: Eric Junyuan Xie Date: Mon, 21 Aug 2017 11:44:52 -0700 Subject: [PATCH 397/834] Update Jenkinsfile (#7541) --- Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 95115cf58920..bf237a589c99 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -154,7 +154,8 @@ try { node('mxnetlinux') { ws('workspace/amalgamation') { init_git() - make('cpu', '-C amalgamation/ USE_BLAS=openblas MIN=1') + make('cpu', '-C amalgamation/ clean') + make('cpu', '-C amalgamation/ USE_BLAS=openblas') } } }, From ab1486704a7e0fdf1c7b0619306f4899a6f1e8de Mon Sep 17 00:00:00 2001 From: Przemyslaw Tredak Date: Mon, 21 Aug 2017 16:15:55 -0700 Subject: [PATCH 398/834] Move usage of persistent BN to cuDNN 7.0.3 (#7543) --- src/operator/cudnn_batch_norm-inl.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/operator/cudnn_batch_norm-inl.h b/src/operator/cudnn_batch_norm-inl.h index b0c5f43157d0..258bed5ea326 100644 --- a/src/operator/cudnn_batch_norm-inl.h +++ b/src/operator/cudnn_batch_norm-inl.h @@ -112,7 +112,7 @@ class CuDNNBatchNormOp : public Operator { Tensor y = out_data[cudnnbatchnorm::kOut].get_with_shape(shape_, s); -#if CUDNN_VERSION >= 7000 +#if CUDNN_VERSION >= 7003 auto mode = CUDNN_BATCHNORM_SPATIAL_PERSISTENT; #else auto mode = CUDNN_BATCHNORM_SPATIAL; @@ -201,7 +201,7 @@ class CuDNNBatchNormOp : public Operator { out_grad[cudnnbatchnorm::kOut].get_with_shape(shape_, s); #if CUDNN_VERSION >= 4007 -#if CUDNN_VERSION >= 7000 +#if CUDNN_VERSION >= 7003 auto mode = CUDNN_BATCHNORM_SPATIAL_PERSISTENT; #else auto mode = CUDNN_BATCHNORM_SPATIAL; From 9796134077891de3d13e6774adb4db81a0a2ecb6 Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Mon, 21 Aug 2017 16:17:26 -0700 Subject: [PATCH 399/834] 0.11.1 (#7542) --- R-package/DESCRIPTION | 2 +- include/mxnet/base.h | 2 +- python/mxnet/libinfo.py | 2 +- scala-package/pom.xml | 2 +- snapcraft.yaml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/R-package/DESCRIPTION b/R-package/DESCRIPTION index e0b435513718..9d2951c0090c 100644 --- a/R-package/DESCRIPTION +++ b/R-package/DESCRIPTION @@ -1,7 +1,7 @@ Package: mxnet Type: Package Title: MXNet: A Flexible and Efficient Machine Learning Library for Heterogeneous Distributed Systems -Version: 0.11.0 +Version: 0.11.1 Date: 2017-06-27 Author: Tianqi Chen, Qiang Kou, Tong He Maintainer: Qiang Kou diff --git a/include/mxnet/base.h b/include/mxnet/base.h index 695408380ec9..61d105a5bc48 100644 --- a/include/mxnet/base.h +++ b/include/mxnet/base.h @@ -112,7 +112,7 @@ /*! \brief minor version */ #define MXNET_MINOR 11 /*! \brief patch version */ -#define MXNET_PATCH 0 +#define MXNET_PATCH 1 /*! \brief mxnet version */ #define MXNET_VERSION (MXNET_MAJOR*10000 + MXNET_MINOR*100 + MXNET_PATCH) /*! \brief helper for making version number */ diff --git a/python/mxnet/libinfo.py b/python/mxnet/libinfo.py index 7da0dcfc8d2d..f0838d7028c4 100644 --- a/python/mxnet/libinfo.py +++ b/python/mxnet/libinfo.py @@ -61,4 +61,4 @@ def find_lib_path(): # current version -__version__ = "0.11.0" +__version__ = "0.11.1" diff --git a/scala-package/pom.xml b/scala-package/pom.xml index 7bfd8774de6b..d6be0996bad3 100644 --- a/scala-package/pom.xml +++ b/scala-package/pom.xml @@ -48,7 +48,7 @@ - 0.11.0-SNAPSHOT + 0.11.1-SNAPSHOT 2.11.8 2.11 diff --git a/snapcraft.yaml b/snapcraft.yaml index 27356c332a29..6d45746aff74 100644 --- a/snapcraft.yaml +++ b/snapcraft.yaml @@ -1,5 +1,5 @@ name: mxnet -version: '0.11.0' +version: '0.11.1' summary: MXNet is a deep learning framework designed for efficiency and flexibility. description: | MXNet is a deep learning framework designed for both efficiency and From e0607bcfeeb872eca4b66aea192e904ddd3ce061 Mon Sep 17 00:00:00 2001 From: mbaijal <30911248+mbaijal@users.noreply.github.com> Date: Mon, 21 Aug 2017 18:38:20 -0700 Subject: [PATCH 400/834] remove MXNet License from rcnn license (#7549) --- example/rcnn/LICENSE | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/example/rcnn/LICENSE b/example/rcnn/LICENSE index 749e0a16b363..ac015288d18c 100644 --- a/example/rcnn/LICENSE +++ b/example/rcnn/LICENSE @@ -15,23 +15,6 @@ See the License for the specific language governing permissions and limitations under the License. -MXNet - -Copyright (c) 2015-2016 by Contributors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - - Fast R-CNN Copyright (c) Microsoft Corporation From f517d9d406525c4e109dd6fe10d9d175ed75d0db Mon Sep 17 00:00:00 2001 From: Chris Olivier Date: Tue, 22 Aug 2017 09:27:24 -0700 Subject: [PATCH 401/834] Fix optimizer parms in fit.py + Don't repeatedly call slow prepare_mkl.sh script (#7545) (#7547) * Only call MKL script once * Fix 'momentum' and 'multi_precision' optimizer args * fix working --- Makefile | 4 ++-- example/image-classification/common/fit.py | 13 ++++++++++--- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index b6c5834a566b..300f901bd662 100644 --- a/Makefile +++ b/Makefile @@ -36,8 +36,8 @@ include $(config) ifeq ($(USE_MKL2017), 1) # must run ./prepare_mkl before including mshadow.mk - RETURN_STRING = $(shell ./prepare_mkl.sh $(MKLML_ROOT)) - MKLROOT = $(firstword $(RETURN_STRING)) + RETURN_STRING := $(shell ./prepare_mkl.sh $(MKLML_ROOT)) + MKLROOT := $(firstword $(RETURN_STRING)) export USE_MKLML = $(lastword $(RETURN_STRING)) endif diff --git a/example/image-classification/common/fit.py b/example/image-classification/common/fit.py index 73235fc2e4ef..dfec2a886b80 100755 --- a/example/image-classification/common/fit.py +++ b/example/image-classification/common/fit.py @@ -163,10 +163,17 @@ def fit(args, network, data_loader, **kwargs): lr_scheduler = lr_scheduler optimizer_params = { 'learning_rate': lr, - 'momentum' : args.mom, 'wd' : args.wd, - 'lr_scheduler': lr_scheduler, - 'multi_precision': True} + 'lr_scheduler': lr_scheduler} + + # Add 'multi_precision' parameter only for SGD optimizer + if args.optimizer == 'sgd': + optimizer_params['multi_precision'] = True + + # Only a limited number of optimizers have 'momentum' property + has_momentum = {'sgd', 'dcasgd'} + if args.optimizer in has_momentum: + optimizer_params['momentum'] = args.mom monitor = mx.mon.Monitor(args.monitor, pattern=".*") if args.monitor > 0 else None From 0b1363116c84dcefa751a925749b2da04c3f2614 Mon Sep 17 00:00:00 2001 From: Haibin Lin Date: Tue, 22 Aug 2017 14:56:33 -0700 Subject: [PATCH 402/834] Sparse Tensor: request for reviews (#7082) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [WIP] Sparse Tensor (#5800) * squash merge with 38f7c5584016e92ba1e0ee1b00ea6632740f67ce compiles on GPU update check alloc: Checkpoint. Pass elem-sum gpu test bug fix for copyfromto. sparse sgd test pass on gpu inefficient implementation for csr copy update submodule fix lint Simple bind with infer storage type (#32) * Symbol binding for sparse tensor development. (#31) * Initial checkin * Add init functions for simple bind in graph_executor * Add simple_bind c_api * Add simple bind c-api * Assign zeros to in_args, arg_grads, and aux_states * Add simple_bind2 python interface * Fix python interface bugs * Interface changes * Fix * Fix core dump * Add bind_ith_exec c_api * Change simple_bind2 * Fix seg fault * Finish simple_bind * Change _bind_ith_exec * Refactor simple_bind initialization flow for bind * Consolidate bind and simple_bind graph init flow * Fix bug * Clean up * Add comments * Clean up * Clean up * Minor correction * Rename APIs in graph executor * Refactor * Rebase * Delete deprecated functions * Move more front-end work to backend * Bug fix * Fix failed tests * Minor fix * Fix lint * Fix lint * Revert unnecessary changes * Revert * Revert * Clean up * Fix lint Conflicts: python/mxnet/symbol.py src/executor/graph_executor.cc * Add inferstorage to graph executor * re-enable tests for sparse embedding with simple_bind * type switch fix in sparse embedding" ; change `default` to `default_storage` for cast storage op (#33) * change default to default_storage * disable cpp test build temporarily attempt to fix windows build error, and fix lint (#34) update nnvm submodule (#37) Scipy build (#38) * update nnvm submodule * add scipy pip install for dockerfile Python3 unit tests (#39) * change xrange to range for python3 compatiblity" * remove more xrange from tests replace long with int for python3 (#40) fix the rest of TShape constructor errors (#41) fix lint (#42) fix wrong usage of mshadow::Shape1" (#43) implementation for Csr slice on cpu (#36) * CPU implementation for CSR remove seg_len from csr slice add some docs for slice csr change indptr, values, etc to be private member bug fix in sparse embedding update nnvm submoduel fix lint update unit test for sparse nd" * add const for SliceCsrIndPtr kernel Fix sparse dot according to the new RSP definition (#35) * Fix csr dot dns * Fix sparse dot * Add fallback and test cases for dot(csr, dns)=dns * Add int type switch * Fix * Fix * Fix update mshadow submodule (#44) Fix dns to rsp (#46) fix lint (#47) add runtime storage fallback detection" (#48) * add runtime storage fallback detection" * replace cast storage ex with cast storage impl Fm example (#45) * update csr slice logic to avoid confusion. add more exmaples. * add hint to module.update * more testcases(fallback) for sparse_nd * add to_csr() and to_rsp() method. More unit test (fallback now) * add fm test. fix lint * register sparse sgd under Optim.SGD * update dmlc-core submoduel * change indptr to _indptr temporarily. add const ref to fname fix lint fix lint; (#51) Guard gpu cast storage (#50) * Clean up * Fix typo Rearrange unit test files (#52) fix lint. add scipy for python_test. fix scipy.sparse import error. fix truediv for python3 fix travis test (#54) * remove pyc files * add verbose for travis nosetests cleanup some testing code and enums (#57) * update Makefile * refactor test_sparse_operator * change `default_storage` back to `default` * remove unused cpp tests port libsvm parser to mxnet as libsvm iter (#55) * copied csv iter to libsvm iter test libsvm iter draft handle round batch == false for csr batch loader code refactoring add get stype, shape interface to iiter separate class for sparse iter add missing file fix mem corruption' rename variables add comments also read label from libsvm add test. update docs. update submodule Conflicts: python/mxnet/sparse_ndarray.py * update submodule * fix lint * update test * revert naming change add benchmark scritp for dot (#59) * add benchmark scritp for dot add gpu option for bench add get_data funciton for benchmark print t_sparse, too; add comment change nnz to dnesity add backward * add comment update fm test (#62) introduce CSRNDarray and rowsparseNDarray to python frontend api (#58) * introduce CSRNDarray and rowsparseNDarray to python frontend api * temporarily disable fm_module test fix lint (#64) fix typo. disable libsvm io test (#65) Improve dot (#61) * Init checkin * Fix * Adjust dot parallelization methods * Set num_omp_threads for benchmark from command line * Fix omp thread number * Clean up * Add scipy as dot baseline * Fix format sparse_retain op (#66) * Initial checkin * Fix bugs * Add unit test for sparse_retain * Add example and modify test add storage cast for outputs that have non-default storage (#67) fix gpu build (#69) Fix test_sparse_retain python3 issue (#68) revert nnvm version * draft for sgd rsp rsp (#75) support sgd(rsp, rsp) support dot(csr, rsp) when rsp is full add ref to const ndarray params support sparse embedding with rsp weight' fix lint modify embedding backward to produce dense grad remove invalid_rid for rsp->dns remove previous embedding op changes pass sparse embedding test add STORAGE_TYPE_ASSIGN_CHECK remove backward storage infer * fix lint (#78) * fix lint (#79) * serial elemwise sum impl (#80) update module kvstore interface add other missing params and functions revert some interface changes revert some more changes reomve explicit casting for gradients on kvstore update Comm interface update fm example Conflicts: python/mxnet/model.py python/mxnet/ndarray.py * bug fix for initializing module with row_sparse weight (#81) * bug fix for initializing module with row_sparse weight * update log message * Sparse ndarray serialization and deserialization (#77) * Initial checkin * Add unit tests * Fix lint * Fix lint (#84) * Sgd with row_sparse weight, dns gradient (#83) * sgd rsp dns draft * support sgd_mom(rsp, dns, rsp) * update doc * remove cast storage for kv updater * code refactoring * update mshadow version (#88) * csr slice bug fix (#90) * benchmark dot code refactor (#87) * q^x6x add some code in benchmark * refactor * minor fixes * fix * lint fix * Add unit test (#91) * add unittest * minor fix * remove commented lines * change test func name * add test rsp * kvstore push row sparse (#93) * Add multi-thread cpu elemwise sum for rsps * Minor fix * Add flag to switch between serial and multi-thread kvstore push * Fix lint in sparse_ndarray.py * Revert "Fix lint in sparse_ndarray.py" This reverts commit d7225ec267a1e8c0c3c8074d25af5844ed39a14d. * Fix ndarray init in copy(ctx) * Add env var to control the flow of serial/parallel reduce * Refactor * Fix copy ndarray bug * Fix lint * Refactor * Fix windows openmp build failure (#94) * update mshadow submoduel (#95) * Revert "update mshadow submoduel (#95)" (#96) This reverts commit 1a129e4cc39514a6c7b3aa1189949969b818aec3. * Refactor sparse tensor code (#99) * Initial checkin test_sparse_ndarray passes * Fix test failure * Clean up * Clean up * Move init backend op to ndarray_utils * Fix lint * Eliminate circular dependency on headers * More refactor * Fix gpu build and consolidate Slice for dense and sparse * Clean up * More refactor * Clean up * Fix gpu build * Fix comment * fix pylint (#100) * Fix refactor sparse gpu test (#104) * Fix gpu build * Fix * Fix gpu test failure * change idx types from int32 to int64 (#101) Conflicts: python/mxnet/test_utils.py tests/python/unittest/test_sparse_operator.py update mshadow submodule fix extra quotes in test script change indptr type to int64 better err message for rsp" * revert LOG(DEBUG) change (#105) * fix undefined zeros in optimizer.py (#106) * move init dns zeros to init_op.h for kvstore to use (#107) * Refactor cast storage (#109) * Refactor cast_storage * Add cast_storage cc and cu files * Remove redundant comments * Replace std::accumulate with ParallelAccumulate * Clean up * Fix windows build * Rowsparse kv (#111) * update kvstore unit test Conflicts: tests/python/unittest/test_kvstore.py update model/module.py Conflicts: python/mxnet/model.py python/mxnet/module/module.py fix lint resolve conflict remove int keys in kvstore update cast to str function * fix failed dist_sync_kv test * bug fix in comm to ensure merged gradient is of the right type bug fix in comm * row sparse dist kvstore draft (push only) row_sparse pull * add ndarray row sparse shared mem constructor * code refactoring * add test for row_sparse weight bug fix for kv server slicing add async support rsolve race condition in kvstore * resolve error after reb ase * fix lint (#113) * rename some python funciton (#114) * _to_rsp * _to_csr. raise NotImplementedError * todense * fix lint (#115) enable libsvm uniit test (#6839) remove shared mem slice for csr add csr ndarray iter test make osx nose test verbose disable libsvm iter test Move InferAttr to mxnet from nnvm (#6830) * Move InferAttr to mxnet from nnvm Replace nnvm infer attr functions in c_api Initial checkin Clean up Remove nnvm namespace for FInferShape, FInferType, and FInferStorageType Add new interface for InferStorageType Revert "Remove nnvm namespace for FInferShape, FInferType, and FInferStorageType" This reverts commit 8aedf054bfe29b076c6fcb6f54d996fd2752e4de. Fix and clean up Fix lint Add nnvm changes Change infer function interface to accept only rvalue reference of graph Clean up Flush commits to show up in PR Add error handling for storage type inference failure Update nnvm * Fix pylint Change idx type switch for aux data (#6860) * Change idx type switch for aux data * Add mshadow commit Sparse dot enhancement (#6842) * Initial checkin Initial checkin Fix sparse dot test Fix unitest and add fallback for sparse dot * Add benchmark code * Revert "Add benchmark code" This reverts commit be009fe4c5a2a321aa92e99ac6e9cc511198c742. * Fix bug * Fix storage shape * Remove unnecessary test code * Use idx type switch Implement dot(csr, rsp)=dns and dot(csr.T, rsp)=rsp and refactor (#6902) * Initial checkin Add dot(csr.T, rsp)=rsp2 Add infer storage for dot(csr, rsp)=dns and dot(csr.T, rsp)=rsp2 * Fix comments * Replace std::lower_bound with own impl for gpu use too * Add time profiling * Revert "Add time profiling" This reverts commit 8f5bb982867731df0305148b1b150b05661f8529. * Move dot and batch_dot to a single file * Move dot gpu impl to a .cuh file * More refactor * Fix include error LibsvmIter fix (#6898) * fix bug in libsvm iter which causes mem corruption * add test for news dataset * fix wrong path in test * fix import error for urllib * update url * replace bz command with bz module Optimized gpu dot kernels (#6937) * pulled update to mshadow * mshadow update * added optimized gpu kernels for dot(csr,dns)=dns and dot(csr.T,dns)=dns, and unit test * added __syncwarp to vector kernel and reduced number of writes to shared memory Refactor sparse tensor code (#6955) * Save stype in frontend to avoid c-api call for stype * Change storage_type to stype * Revert "Change storage_type to stype" This reverts commit 90db7d18b624f3ee4ffd37bf5680205e77ca2763. * Revert "Revert "Change storage_type to stype"" This reverts commit 09328382e926b92a42ba5b3df6f169f825975d88. Move ndarray.py, sparse_ndarray.py, ndarray_utils.py, and _ndarray_internal to ndarrary folder More refactor Move elementwise sum for rsp to ndarray_function.cc Remove unnecessary import in ndarray module Fix pylint Remove redundant code Remove _stype from slots Fix cpp-package build error caused by the change to imperative invoke interface Use relative import Remove print line Rename _ndarray_internal.py to _internal.py * Relaunch test... minor bug fix in warp synchronous code (#7029) * move storage type vector from nnvm to mxnet (#7054) * move storage type vector from nnvm to mxnet * update nnvm * update nnvm * Improve copy sparse tensors (#7003) * Use cast_storage when copying ndarrays of different stypes on same context * Relaunch test * fix failed tests. add back 64bit support for dot fix lint * bug fix for IdentityComputeRsp * fix lint fix lint fix lint * add data partition for libsvm iter (#7027) * remove sparse embedding (#7165) * fix ndarray namespace * remove untested gpu operators (#7172) * skip sparse dot gpu tset. add sparse_nd_zeros gpu test * remove sparse_retain gpu Conflicts: tests/python/gpu/test_operator_gpu.py * Fix ndarray aux data issue (#7098) * Fix getting sparse ndarray data/aux_data issues * Add tests for func csr and row_sparse * Make get/set data/aux_data thread safe * Fix a bug * Fix typo and comment * More comments * Correct comment Conflicts: tests/python/gpu/test_operator_gpu.py * Support K-dimensional row-sparse tensor (#7179) * remove check for k dimensional rowsparse tensor * change var name for rsp sgd operator * add checks for sparse dot * bug fix for kdim rowsparse cast storage cpu * update IdentityLikeRhsComputeEx interface * remove set_storage_shape from ndarray. support elemwise_add with kdim row_sparse tensor * use get_with_shape instead of reshape * update according to comments Conflicts: src/operator/tensor/elemwise_unary_op.h * Improve sparse ndarray error message (#7181) * add test for broadcast_to * add comments Conflicts: python/mxnet/base.py * construct row_sparse ndarray for dist-async fix bug in rsp add rsp sync push race condition for push fix bug in rsp pull. refactor test cleanup comments refactor dist server fix lint fix storage shape issue with the new ndarray constructor data sharding draft; fix lint. add comment add support for zeros gradients use std::upper_bound/lower_bound remove special init function for rowsparse dist kvstore temporary support for inplace operators for sparse add test. fix return type store kRowSparseNDArray in kv server remove fcomp_ex sgd with dns weight and rsp gradient bug fix in sparse retain sparse pull c_api revise rowsparse pull api use engine to compute unique to ensure thread safety add rowsparse pull to dist-kv fix lint add example for rsp_pull remove name2idx; add sparse_pull_dict param to module fix unit test and c rowid conversion support str key type in kvstore (#6765) * update kvstore unit test * update model/module.py * fix lint * remove int keys in kvstore * update cast to str function * remove _cast_to_str_keys * fix lint * always cast to str Conflicts: include/mxnet/c_api.h include/mxnet/kvstore.h python/mxnet/kvstore.py python/mxnet/model.py python/mxnet/module/module.py src/c_api/c_api.cc src/kvstore/kvstore_local.h tests/python/unittest/test_kvstore.py update module API for other submodules update stypes in kvstore after refactoring change type of size from size_t to int64_t add sparse linear regression example remove sparse_pull_dict from module fix init_optim for seq_module. update sparse example resolve conflict for binary add rsp rsp Conflicts: python/mxnet/kvstore.py tests/python/unittest/test_kvstore.py * fix DotCsrRspRspImpl error message (#7191) * GPU implementation of cast_storage (dense to csr) (#7081) * Added gpu implementation for cast_storage dense to csr, unit tests, and benchmark. Additionally, cast_storage interface change to accommodate the need of temporary storage in cuda kernels. * fixed whitespace * minor unittest update * removed whitespace * add cast storage benchmark params info Conflicts: tests/python/gpu/test_operator_gpu.py * Sparse square sum (#7206) * Add square_sum op * Add unit test and fix check_numeric_gradient * Add .cu file and example * Fix lint * Remove gpu registration * Use square_sum in test_module_fm * Modify and Add documentation for mx.nd.zeros (#7197) * Modify and Add documentation for mx.nd.zeros * Change context to cpu * Change stype to optional * Change ordering and remove optional for _zeros_sparse_ndarray * Expose kWriteInplace for imperative execution (fcompute_ex and fstatefulcompute_ex) (#133) * expose kWriteInplace to FComputeEx and FStatefulComputeEx * refactor ccode * remove duplicated test * Operator add_n for row sparse ndarrays (#7244) * Add add_n op for row-sparse ndarrays and identity FComputeEx * Fix bug in square_sum * Remove test_cast_storage_ex from gpu test since it's not implemented yet * Fix according to the cr Conflicts: src/operator/tensor/elemwise_sum.cc src/operator/tensor/elemwise_unary_op.cc tests/python/gpu/test_operator_gpu.py resolve conflict * GPU implementation of cast_storage (dense to rsp) (#7223) * CastStorageDnsRsp GPU Implementation * updating function doc and some variable types and names * adding cuda_get_device_prop() util function * added rand_shape function for n-dimensional tensors * updated cast storage unit test * added dns_to_rsp to cast storage benchmark script * removing redundant unit test * fix lint * minor change in benchmark script * fix lint * correct function description * change storage_type to stype * changed scope of using namespaces * changed variable types from index_t to dim_t * resolve merge conflict in ndarray.load * Improve StatefulOp/FCompute storage fallback (#134) * test for fcomp fallback add storage fallback test and optimize fallback logic rename function, add comments use std size() * add autograd test with sparse inputs * update sparse ndarray api (#139) * support mx.nd.empty for sparse ndarray Change SparseNDArray to BaseSparseNDArray support mx.nd.array with BaseSparseNDArray inputs. Update documentation with explicit subclasses of NDArrays Conflicts: python/mxnet/ndarray/__init__.py python/mxnet/ndarray/ndarray.py python/mxnet/ndarray/sparse_ndarray.py tests/python/unittest/test_sparse_ndarray.py * fix print msg in test * Handle ograd_stype='row_sparse' for square_sum backward (#143) * Add one kernel for square_sum backward pass to take rsp ograd * Add kNullOp and change to use type_assign in infer stype fallback * Sparse retain improvement (#138) * Add one more kernel for sparse retain * Fix compile * Change STORAGE_TYPE_ASSIGN_CHECK to type_assign for fallback * Fix * Add gpu compile * ignoring variables in SimpleBind that is used on python's sparse branch for now. (#135) * add bias term to fm test (#145) * update ndarray.nd, remove `invoke` from excluded members (#137) remove __weakref__ from SparseNDArray add data indice to doc revert dlpack update revert mxdoc changes move methods from BaseSparseNDarray to csrndarray and rwosparse ndarray * support storage fallback with mutable inputs (#147) * include mutatable inputs in storage fallback. refactor executor add fallback test for rms prop and adam fix lint fix lint fix test in optimizer * update according to comments * fix unit tests * fix gpu compilation err * Code changes based on reviews (#144) * code changes according to review comments remove executor debug. add doc to optimizer update sparse sgd test add dtype option to rand_sparse_ndarray * overhauled reqs for sparse operators * patch FCompExFallback with mutable inputs. update test_optimizer with more fallback cases * change executor debug macro to env var * add comment * update doc * change ndarray.aux_shape() to return const reference * remove todense to_rsp to_csr. replace with tostype * replace manual calls to cast_storage with tostype * disable gpu fallback test for optimizer * fix lint * add backward pass for cast_storage. refactor cast_storage test * rand_sparse_ndarray bug fix * fix cast_storage for gpu * disable csr test for fp16 * update row sparse ndarray doc * update doc * small edits according to reviews (#151) * fix lint (#152) * add license to all new files in sparse brnach (#154) * Allocate temp data on the fly for some casting operations (#149) * fix utf8 encoding in sparse ndarray * Extending the GPU dot operator (#7226) * Added GPU DotCsrRspDnsImpl declaration and TODOs * cleaning up function doc, variable types, and code-style * minor bug fixes * enable GPU dot(csr,rsp)=dns unit test * extend sparse dot unit test * adding GPU impl of DotCsrRspDns and its kernels * add TODO * changed variable types from index_t to dim_t * fix function description * added DotCsrRspRspImpl and its kernels (baseline, functionality) * added DotCsrDnsRspImpl and its kernels (baseline, functionality); plus code documentation * refactored dot benchmark * optimized DotCsrTransDnsRsp GPU kernel * change of dot impl interface to include OpContext, for temp storage * removing __device__ flag from CPU kernels * minor fixes and changing variable data types * minor fixes based on code reviews Conflicts: benchmark/python/sparse_op.py tests/python/gpu/test_operator_gpu.py tests/python/unittest/test_sparse_operator.py * Add get_synthetic_dataset function to util (#146) * Add get_synthetic_datasets * Move to test_utils * Remove _get_uniform_dataset * Move validation to its own function * Refactor the validation code for csr generation * Make test_powerlaw a nested function * Change SparseNDArray to CSRNDArray * Merge with dtype specific changes in test_utils * temporary fix for batch norm storage fallback (#156) * support random_uniform/normal/gamma with row_sparse output (#155) * add support for initilazer with rowsparse output * add scalar assignment to row_sparse * add setitem test to gpu * Revert "add scalar assignment to row_sparse" This reverts commit 8aef7a56c44038f67bbec93811977ea2f9fa3c30. * Revert "add setitem test to gpu" This reverts commit 3b969ac0980e8d7166a1cf46878ed2bd457986ed. * Square sum backward support one more case (#161) * Add documentation for sparse ops (#148) * draft doc for sparse op * add more stype doc for operators * add doc for cast_storage * see also cast_storage. remove base sparse ndarray. fix aux_types comemtn * grammar / spelling fix * A few fixes (#163) * fix batch norm gpu kernel. register random operators on gpu * register sparse random op on gpu, too * Minor fixes sparse ops (#160) * change CPU kernel inline directives, data types, and function doc * update dot dtype switch to use 32 and 64bit floating point only * use type_assign instead of STORAGE_TYPE_ASSIGN_CHECK * added tensor_util-inl.cuh file for common tensor operator GPU kernels * sparse Adam optimizer (#164) * add sparse adam * register gpu op * add comments * cr comments * kvstore.row_sparse_pull for GPU and end-to-end benchmark: CPU vs. multi-GPUs (#150) * Add gpu support for BroadcastRowSparse * Fix bugs * Add benchmark script * Increase output dim size * Update weight on CPU using single GPU for sparse tensors * More fix * Optimize sparse_retain for special case * Change row sparse pull locations * Avoid sparse retain on cpu if possible * Use acc for metric * Fix misc * fix bug in adam update (#167) fix a bug in adam update * change sparse example from regression to classification (#165) * fix python import (#166) * Add waitall to sparse_end2end.py (#169) * Add waitall() * Add dummy metric option * Add header license * Dot script changes (#159) * Add get_synthetic_datasets * Move to test_utils * Remove _get_uniform_dataset * Move validation to its own function * Refactor the validation code for csr generation * Make test_powerlaw a nested function * Change SparseNDArray to CSRNDArray * Refactoring changes to dot.py * Fix mxnet test_utils changes * Remove pdb statement * Add distribution parameter * Refactor benchmarking script * Remove unused code * Make style changes and remove unused code * Change typo in comment * Add transpose support * Change typo * 4 decimal points needed for density * Add rsp support for real datasets * Correct variable name mini_file_name * Move wait_to_read outside if * Seperate out scipy and mxnet logic in bench_dot * Fix lhs_trans issue * Move transpose outside measure_cost * Compute transpose inside measure_cost * Remove unused variables * Transpose only if trans_lhs (#171) * fix default val for distribution (#172) * fix lint (#175) * avoid cast_storage in dist-kvstore-server (#174) * avoid cast_storage in dist-kvstore-server * add stream arg to mshadow;;copy * fix copy order * Add sparse namespace to ndarray and symbol (#177) * Register dot, cast_storage, and sparse_retain under mxnet.ndarray.sparse * Add sparse to symbol namespace * Delete commented code * mv sparse_ndarray.py sparse.py * Clean up * Change docstring * changes based on code reviews (#176) * remove scipy dependency * move kvstore checks to backned * add const to lambda * temp fix to ndarray.md (#178) * Fix sparse namespace pylint (#179) * add comments and error msg (#181) * add clarification for csr (#182) * add clarification for csr * cr comments * revert change in test util (#183) * fix amalgamation (#184) * fix lint --- benchmark/python/sparse/cast_storage.py | 99 ++ benchmark/python/sparse/dot.py | 445 ++++++++ benchmark/python/sparse/sparse_end2end.py | 249 ++++ benchmark/python/sparse/sparse_op.py | 245 ++++ benchmark/python/sparse/util.py | 50 + docs/api/python/ndarray.md | 63 +- example/sparse/get_data.py | 32 + example/sparse/linear_classification.py | 185 +++ include/mxnet/c_api.h | 240 +++- include/mxnet/executor.h | 1 + include/mxnet/graph_attr_types.h | 48 + include/mxnet/kvstore.h | 24 + include/mxnet/ndarray.h | 502 +++++++- include/mxnet/op_attr_types.h | 18 +- include/mxnet/storage.h | 4 +- perl-package/AI-MXNetCAPI/mxnet.i | 6 + perl-package/AI-MXNetCAPI/mxnet_typemaps.i | 11 + python/mxnet/__init__.py | 3 +- python/mxnet/_ctypes/ndarray.py | 39 +- python/mxnet/base.py | 14 + python/mxnet/contrib/autograd.py | 1 + python/mxnet/executor.py | 5 +- python/mxnet/image/detection.py | 2 +- python/mxnet/image/image.py | 6 +- python/mxnet/io.py | 5 +- python/mxnet/kvstore.py | 67 +- python/mxnet/model.py | 8 +- python/mxnet/module/base_module.py | 3 +- python/mxnet/module/module.py | 8 +- python/mxnet/ndarray/__init__.py | 25 + .../_internal.py} | 0 python/mxnet/{ => ndarray}/ndarray.py | 499 +++----- python/mxnet/ndarray/op.py | 209 ++++ python/mxnet/ndarray/sparse.py | 923 +++++++++++++++ python/mxnet/ndarray/utils.py | 240 ++++ python/mxnet/optimizer.py | 23 +- python/mxnet/random.py | 14 +- python/mxnet/symbol/__init__.py | 23 + .../_internal.py} | 0 python/mxnet/symbol/op.py | 242 ++++ python/mxnet/symbol/sparse.py | 18 + python/mxnet/{ => symbol}/symbol.py | 275 +---- python/mxnet/test_utils.py | 227 +++- src/c_api/c_api.cc | 116 ++ src/c_api/c_api_common.h | 2 + src/c_api/c_api_executor.cc | 32 +- src/c_api/c_api_ndarray.cc | 237 +++- src/c_api/c_api_symbolic.cc | 5 +- src/c_api/c_predict_api.cc | 3 +- src/common/utils.cc | 39 + src/common/utils.cu | 39 + src/common/utils.h | 167 ++- src/executor/attach_op_execs_pass.cc | 176 ++- src/executor/exec_pass.h | 52 +- src/executor/graph_executor.cc | 326 ++++-- src/executor/graph_executor.h | 9 +- src/executor/infer_graph_attr_pass.cc | 356 ++++++ src/executor/inplace_addto_detect_pass.cc | 2 + src/io/iter_batchloader.h | 17 +- src/io/iter_libsvm.cc | 288 +++++ src/io/iter_prefetcher.h | 32 +- src/io/iter_sparse.h | 45 + src/io/iter_sparse_batchloader.h | 203 ++++ src/io/iter_sparse_prefetcher.h | 153 +++ src/kvstore/comm.h | 304 ++++- src/kvstore/kvstore_dist.h | 252 ++++- src/kvstore/kvstore_dist_server.h | 249 +++- src/kvstore/kvstore_local.h | 151 ++- src/ndarray/ndarray.cc | 450 +++++++- src/ndarray/ndarray_function-inl.h | 61 +- src/ndarray/ndarray_function.cc | 134 +++ src/ndarray/ndarray_function.h | 9 + src/nnvm/legacy_op_util.cc | 34 +- src/operator/batch_norm.cc | 2 +- src/operator/batch_norm.cu | 4 +- src/operator/deconvolution-inl.h | 2 +- src/operator/elemwise_op_common.h | 48 + src/operator/leaky_relu-inl.h | 5 +- src/operator/mxnet_op.h | 24 +- src/operator/operator_common.h | 82 ++ src/operator/optimizer_op-inl.h | 493 ++++++++ src/operator/optimizer_op.cc | 9 + src/operator/optimizer_op.cu | 9 +- src/operator/random/sample_op.cc | 9 +- src/operator/random/sample_op.cu | 60 +- src/operator/random/sample_op.h | 109 +- src/operator/tensor/cast_storage-inl.cuh | 589 ++++++++++ src/operator/tensor/cast_storage-inl.h | 392 +++++++ src/operator/tensor/cast_storage.cc | 87 ++ src/operator/tensor/cast_storage.cu | 35 + src/operator/tensor/dot-inl.cuh | 883 +++++++++++++++ src/operator/tensor/dot-inl.h | 1007 +++++++++++++++++ src/operator/tensor/dot.cc | 141 +++ src/operator/tensor/dot.cu | 45 + .../elemwise_binary_broadcast_op_basic.cc | 1 + src/operator/tensor/elemwise_binary_op.h | 169 ++- .../tensor/elemwise_binary_op_basic.cc | 20 +- src/operator/tensor/elemwise_sum.cc | 66 +- src/operator/tensor/elemwise_unary_op.cc | 9 +- src/operator/tensor/elemwise_unary_op.cu | 7 +- src/operator/tensor/elemwise_unary_op.h | 113 +- src/operator/tensor/indexing_op.cc | 1 - src/operator/tensor/indexing_op.h | 3 + src/operator/tensor/init_op.cc | 1 + src/operator/tensor/init_op.cu | 3 +- src/operator/tensor/init_op.h | 88 +- src/operator/tensor/matrix_op-inl.h | 449 ++------ src/operator/tensor/matrix_op.cc | 94 +- src/operator/tensor/matrix_op.cu | 12 - src/operator/tensor/sparse_retain-inl.h | 396 +++++++ src/operator/tensor/sparse_retain.cc | 80 ++ src/operator/tensor/sparse_retain.cu | 36 + src/operator/tensor/square_sum-inl.h | 456 ++++++++ src/operator/tensor/square_sum.cc | 52 + src/operator/tensor/util/tensor_util-inl.cuh | 240 ++++ .../ci_build/install/ubuntu_install_python.sh | 4 +- tests/cpp/operator/batchnorm_test.cc | 6 +- tests/nightly/dist_sync_kvstore.py | 166 ++- tests/python/gpu/test_kvstore_gpu.py | 68 ++ tests/python/gpu/test_operator_gpu.py | 3 + tests/python/unittest/test_autograd.py | 75 +- tests/python/unittest/test_infer_shape.py | 20 +- tests/python/unittest/test_io.py | 106 ++ tests/python/unittest/test_kvstore.py | 134 ++- tests/python/unittest/test_module.py | 105 +- .../python/unittest/test_multi_device_exec.py | 27 + tests/python/unittest/test_ndarray.py | 1 + tests/python/unittest/test_operator.py | 113 +- tests/python/unittest/test_optimizer.py | 182 ++- tests/python/unittest/test_sparse_ndarray.py | 524 +++++++++ tests/python/unittest/test_sparse_operator.py | 373 ++++++ tests/travis/run_test.sh | 20 +- tests/travis/setup.sh | 4 +- 133 files changed, 15525 insertions(+), 1781 deletions(-) create mode 100644 benchmark/python/sparse/cast_storage.py create mode 100644 benchmark/python/sparse/dot.py create mode 100644 benchmark/python/sparse/sparse_end2end.py create mode 100644 benchmark/python/sparse/sparse_op.py create mode 100644 benchmark/python/sparse/util.py create mode 100644 example/sparse/get_data.py create mode 100644 example/sparse/linear_classification.py create mode 100644 include/mxnet/graph_attr_types.h create mode 100644 python/mxnet/ndarray/__init__.py rename python/mxnet/{_ndarray_internal.py => ndarray/_internal.py} (100%) rename python/mxnet/{ => ndarray}/ndarray.py (87%) create mode 100644 python/mxnet/ndarray/op.py create mode 100644 python/mxnet/ndarray/sparse.py create mode 100644 python/mxnet/ndarray/utils.py create mode 100644 python/mxnet/symbol/__init__.py rename python/mxnet/{_symbol_internal.py => symbol/_internal.py} (100%) create mode 100644 python/mxnet/symbol/op.py create mode 100644 python/mxnet/symbol/sparse.py rename python/mxnet/{ => symbol}/symbol.py (90%) create mode 100644 src/common/utils.cc create mode 100644 src/common/utils.cu create mode 100644 src/executor/infer_graph_attr_pass.cc create mode 100644 src/io/iter_libsvm.cc create mode 100644 src/io/iter_sparse.h create mode 100644 src/io/iter_sparse_batchloader.h create mode 100644 src/io/iter_sparse_prefetcher.h create mode 100644 src/operator/tensor/cast_storage-inl.cuh create mode 100644 src/operator/tensor/cast_storage-inl.h create mode 100644 src/operator/tensor/cast_storage.cc create mode 100644 src/operator/tensor/cast_storage.cu create mode 100644 src/operator/tensor/dot-inl.cuh create mode 100644 src/operator/tensor/dot-inl.h create mode 100644 src/operator/tensor/dot.cc create mode 100644 src/operator/tensor/dot.cu create mode 100644 src/operator/tensor/sparse_retain-inl.h create mode 100644 src/operator/tensor/sparse_retain.cc create mode 100644 src/operator/tensor/sparse_retain.cu create mode 100644 src/operator/tensor/square_sum-inl.h create mode 100644 src/operator/tensor/square_sum.cc create mode 100644 src/operator/tensor/util/tensor_util-inl.cuh create mode 100644 tests/python/gpu/test_kvstore_gpu.py create mode 100644 tests/python/unittest/test_sparse_ndarray.py create mode 100644 tests/python/unittest/test_sparse_operator.py diff --git a/benchmark/python/sparse/cast_storage.py b/benchmark/python/sparse/cast_storage.py new file mode 100644 index 000000000000..7ae537398c42 --- /dev/null +++ b/benchmark/python/sparse/cast_storage.py @@ -0,0 +1,99 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import ctypes + +from mxnet.test_utils import * +import os +import time +import argparse + +from mxnet.base import check_call, _LIB + +parser = argparse.ArgumentParser(description="Benchmark cast storage operators", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument('--num-omp-threads', type=int, default=1, help='number of omp threads to set in MXNet') +args = parser.parse_args() + +def measure_cost(repeat, f, *args, **kwargs): + start = time.time() + results = [] + for i in range(repeat): + (f(*args, **kwargs)).wait_to_read() + end = time.time() + diff = end - start + return diff / repeat + + +def run_cast_storage_synthetic(): + def dense_to_sparse(m, n, density, ctx, repeat, stype): + set_default_context(ctx) + data_shape = (m, n) + dns_data = rand_ndarray(data_shape, stype, density).tostype('default') + dns_data.wait_to_read() + + # do one warm up run, verify correctness + assert same(mx.nd.cast_storage(dns_data, stype).asnumpy(), dns_data.asnumpy()) + + # start benchmarking + cost = measure_cost(repeat, mx.nd.cast_storage, dns_data, stype) + results = '{:10.1f} {:>10} {:8d} {:8d} {:10.2f}'.format(density*100, str(ctx), m, n, cost*1000) + print(results) + + check_call(_LIB.MXSetNumOMPThreads(ctypes.c_int(args.num_omp_threads))) + + # params + # m number of rows + # n number of columns + # density density of the matrix + # num_repeat number of benchmark runs to average over + # contexts mx.cpu(), mx.gpu() + # note: benchmark different contexts separately; to benchmark cpu, compile without CUDA + # benchmarks dns_to_csr, dns_to_rsp + m = [ 512, 512] + n = [50000, 100000] + density = [1.00, 0.80, 0.60, 0.40, 0.20, 0.10, 0.05, 0.02, 0.01] + num_repeat = 10 + contexts = [mx.gpu()] + benchmarks = ["dns_to_csr", "dns_to_rsp"] + + # run benchmark + for b in benchmarks: + stype = '' + print("==================================================") + if b is "dns_to_csr": + stype = 'csr' + print(" cast_storage benchmark: dense to csr, size m x n ") + elif b is "dns_to_rsp": + stype = 'row_sparse' + print(" cast_storage benchmark: dense to rsp, size m x n ") + else: + print("invalid benchmark: %s" %b) + continue + print("==================================================") + headline = '{:>10} {:>10} {:>8} {:>8} {:>10}'.format('density(%)', 'context', 'm', 'n', 'time(ms)') + print(headline) + for i in range(len(n)): + for ctx in contexts: + for den in density: + dense_to_sparse(m[i], n[i], den, ctx, num_repeat, stype) + print("") + print("") + + +if __name__ == "__main__": + run_cast_storage_synthetic() diff --git a/benchmark/python/sparse/dot.py b/benchmark/python/sparse/dot.py new file mode 100644 index 000000000000..fe322821a09f --- /dev/null +++ b/benchmark/python/sparse/dot.py @@ -0,0 +1,445 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import ctypes + +import os +import time +import argparse +import subprocess +import scipy.sparse as sp + +import mxnet as mx +import numpy as np +import numpy.random as rnd +from mxnet.test_utils import rand_ndarray, set_default_context, assert_almost_equal +from mxnet.base import check_call, _LIB +from util import get_data, estimate_density + +PARSER = argparse.ArgumentParser(description="Benchmark sparse operators", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) +PARSER.add_argument('--num-omp-threads', type=int, + default=1, help='number of omp threads to set in MXNet') +PARSER.add_argument('--gpu', action='store_true', + help="to be run on gpu") +# TODO: Use logging later +PARSER.add_argument('--verbose', action='store_true', + help="Verbose output") +ARGS = PARSER.parse_args() + +# some data information +KDDA = { + 'data_mini': 'kdda.t.mini', + 'data_name': 'kdda.t', + 'data_origin_name': 'kdda.t.bz2', + 'url': "https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary/kdda.t.bz2", + 'feature_dim': 20216830, + 'm': [1, 8, 32], + 'batch_size': [64], + 'default_index': {'batch_size': 0, + 'output_dim': 2}, + 'num_batches': 10 +} + +AVAZU = { + 'data_mini': 'avazu-app.t.mini', + 'data_name': 'avazu-app.t', + 'data_origin_name': 'avazu-app.t.bz2', + 'url': "https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary/avazu-app.t.bz2", + 'feature_dim': 1000000, + 'm': [1, 1000, 2000], + 'batch_size': [128, 256], + 'default_index': {'batch_size': 0, + 'output_dim': 1}, + 'num_batches': 10 +} + +CRITEO = { + 'data_mini': 'criteo.t.mini', + 'data_name': 'criteo.t', + 'data_origin_name': 'criteo.t.bz2', + 'url' : "https://s3-us-west-2.amazonaws.com/sparse-dataset/criteo.t.bz2", + 'feature_dim': 8388621, + 'm': [1, 8, 16, 32, 64], + 'batch_size': [64, 128], + 'default_index': {'batch_size': 1, + 'output_dim': 3}, + 'num_batches': 10 +} + +SYNTHETIC1 = { + 'feature_dim': [1000000], + 'm': [256, 1000], + 'density': [0.001, 0.005, 0.01, 0.02, 0.05, + 0.1, 0.2, 0.5, 0.65], + 'batch_size': [64, 128], + 'default_index': {'batch_size': 1, + 'density': 2, + 'output_dim': 1, + 'feature_dim': 0}, + 'num_repeat': 10 +} + +SYNTHETIC2 = { + 'feature_dim': [8000000, 16000000], + 'm': [1, 32], + 'density': [0.001, 0.005, 0.01, 0.02, 0.05, + 0.1, 0.2, 0.5, 0.65], + 'batch_size': [64, 128], + 'default_index': {'batch_size': 1, + 'density': 2, + 'output_dim': 1, + 'feature_dim': 0}, + 'num_repeat': 10 +} + +def measure_cost(repeat, scipy_trans_lhs, scipy_dns_lhs, func_name, *args, **kwargs): + """Measure time cost of running a function + """ + mx.nd.waitall() + args_list = [] + for arg in args: + args_list.append(arg) + start = time.time() + if scipy_trans_lhs: + args_list[0] = np.transpose(args_list[0]) if scipy_dns_lhs else sp.spmatrix.transpose(args_list[0]) + for _ in range(repeat): + func_name(*args_list, **kwargs) + mx.nd.waitall() + end = time.time() + diff = end - start + return diff / repeat + + +def _get_iter(path, data_shape, batch_size): + data_train = mx.io.LibSVMIter(data_libsvm=path, + data_shape=data_shape, + batch_size=batch_size) + data_iter = iter(data_train) + return data_iter + + +def _line_count(path): + return int(subprocess.check_output('wc -l {}'.format(path), shell=True).split()[0]) + + +def _compare_sparse_dense(data_dir, file_name, mini_file_name, feature_dim, + output_dim, density, batch_size, num_batches=3, num_repeat=5, transpose=False, + rsp=False): + + def create_mini_path(mini_path, path, num_batches): + """Samples batches of size: batch_size, total number: num_batches + from the dataset files for running benchmarks""" + if not os.path.exists(mini_path): + last = _line_count(path) - num_batches * batch_size + last = last if last >= 1 else 1 + start = int(rnd.uniform(1, last)) + os.system("sed -n '%d,%dp' %r > %r" + %(start, start + num_batches * batch_size, path, mini_path)) + assert os.path.exists(mini_path) + + + def run_benchmark(mini_path): + """Run benchmarks + """ + data_shape = (feature_dim, ) + train_iter = _get_iter(mini_path, data_shape, batch_size) + weight_row_dim = batch_size if transpose else feature_dim + weight_shape = (weight_row_dim, output_dim) + if not rsp: + weight = mx.nd.random_uniform(low=0, high=1, shape=weight_shape) + else: + weight = rand_ndarray(weight_shape, "row_sparse", density=0.05, distribution="uniform") + total_cost = {} + average_cost = {} + count = 0 + total_cost["sparse"] = 0. + total_cost["dense"] = 0. + for _ in train_iter: + csr_data = train_iter.getdata() + dns_data = csr_data.tostype('default') + cost_sparse = measure_cost(num_repeat, False, False, mx.nd.dot, csr_data, weight, transpose_a=transpose) + cost_dense = measure_cost(num_repeat, False, False, mx.nd.dot, dns_data, weight, transpose_a=transpose) + total_cost["sparse"] += cost_sparse + total_cost["dense"] += cost_dense + count = count + 1 + average_cost["sparse"] = total_cost["sparse"] / count + average_cost["dense"] = total_cost["dense"] / count + return (average_cost["sparse"], average_cost["dense"]) + + + def print_result(average_cost_sparse, average_cost_dense): + """Print result of comparison between sparse and dense + """ + ratio = average_cost_dense / average_cost_sparse + fmt = '{:15.4f} {:10d} {:10d} {:10d} {:20.2f} {:15.2f} {:15.2f} {:10} {:10}' + print(fmt.format(density * 100, batch_size, output_dim, feature_dim, + ratio, average_cost_dense*1000, average_cost_sparse*1000, + transpose, rsp)) + + mini_path = os.path.join(data_dir, mini_file_name) + path = os.path.join(data_dir, file_name) + create_mini_path(mini_path, path, num_batches) + average_cost_sparse, average_cost_dense = run_benchmark(mini_path) + print_result(average_cost_sparse, average_cost_dense) + + +def test_dot_real(data_dict): + """Dot operator testing with real datasets""" + data_dir = os.path.join(os.getcwd(), 'data') + + path = os.path.join(data_dir, data_dict['data_name']) + if not os.path.exists(path): + get_data( + data_dir, + data_dict['data_name'], + data_dict['url'], + data_dict['data_origin_name'] + ) + assert os.path.exists(path) + + k = data_dict['feature_dim'] + m = data_dict['m'] + batch_size_list = data_dict['batch_size'] + + default_output_index = data_dict['default_index']['output_dim'] + default_batch_size_index = data_dict['default_index']['batch_size'] + density = estimate_density(path, data_dict['feature_dim']) + num_batches = data_dict['num_batches'] + + assert default_batch_size_index < len(batch_size_list) + assert default_output_index < len(m) + if ARGS.verbose: + print("Running Benchmarking on %r data") % data_dict['data_mini'] + print('{:>15} {:>10} {:>10} {:>10} {:>20} {:>15} {:>15} {:>10} {:>10}'.format('density(%)', + 'n', + 'm', + 'k', + 't_dense/t_sparse', + 't_dense(ms)', + 't_sparse(ms)', + 'is_transpose', + 'rhs_rsp')) + + + for output_dim in m: + _compare_sparse_dense(data_dir, data_dict['data_name'], data_dict['data_mini'], + k, output_dim, density, + batch_size_list[default_batch_size_index], num_batches) + _compare_sparse_dense(data_dir, data_dict['data_name'], data_dict['data_mini'], + k, output_dim, density, + batch_size_list[default_batch_size_index], num_batches, + transpose=True) + _compare_sparse_dense(data_dir, data_dict['data_name'], data_dict['data_mini'], + k, output_dim, density, + batch_size_list[default_batch_size_index], num_batches, rsp=True) + + for batch_size in batch_size_list: + _compare_sparse_dense(data_dir, data_dict['data_name'], data_dict['data_mini'], + k, m[default_output_index], density, batch_size, num_batches) + _compare_sparse_dense(data_dir, data_dict['data_name'], data_dict['data_mini'], + k, m[default_output_index], density, batch_size, num_batches, + transpose=True) + _compare_sparse_dense(data_dir, data_dict['data_name'], data_dict['data_mini'], + k, output_dim, density, + batch_size_list[default_batch_size_index], num_batches, rsp=True) + + +def test_dot_synthetic(data_dict): + """benchmark sparse mxnet dot and scipy dot operator with matrices of given density. + `t_sparse` is the runtime of the invoked sparse dot operator in ms, while `t_dense` is the + runtime of dot(dns, dns), with the same matrices except that they are in default storage type. + """ + # Benchmark MXNet and Scipys dot operator + def bench_dot(lhs_shape, rhs_shape, lhs_stype, rhs_stype, + lhs_den, rhs_den, trans_lhs, ctx, num_repeat=10, fw="mxnet", distribution="uniform"): + set_default_context(ctx) + assert fw == "mxnet" or fw == "scipy" + # Set funcs + dot_func_sparse = mx.nd.dot if fw == "mxnet" else sp.spmatrix.dot + dot_func_dense = mx.nd.dot if fw == "mxnet" else np.dot + # Create matrix instances + lhs_nd = rand_ndarray(lhs_shape, lhs_stype, density=lhs_den, distribution=distribution) + # only uniform distribution supported for rhs + rhs_nd = rand_ndarray(rhs_shape, rhs_stype, density=rhs_den, distribution="uniform") + lhs_dns = None + rhs_dns = None + dense_cost = None + sparse_cost = None + + if fw == "mxnet": + lhs_dns = lhs_nd if lhs_stype == 'default' else lhs_nd.tostype('default') + rhs_dns = rhs_nd if rhs_stype == 'default' else rhs_nd.tostype('default') + # One warm up run, verify correctness + out = dot_func_sparse(lhs_nd, rhs_dns, trans_lhs) + out_expected = dot_func_dense(lhs_dns, rhs_dns, trans_lhs) + assert_almost_equal(out.asnumpy(), out_expected.asnumpy(), rtol=1e-1, atol=1e-1) + sparse_cost = measure_cost(num_repeat, False, False, dot_func_sparse, lhs_nd, rhs_nd, trans_lhs) + dense_cost = measure_cost(num_repeat, False, False, dot_func_dense, lhs_dns, rhs_dns, trans_lhs) + else: + lhs_dns = lhs_nd.asnumpy() + rhs_dns = rhs_nd.asnumpy() + lhs_nd = sp.csr_matrix(lhs_nd.asnumpy()) + rhs_nd = rhs_nd.asnumpy() + # One warm up run, verify correctness + lhs_nd_copy = sp.spmatrix.transpose(lhs_nd) if trans_lhs else lhs_nd + out = dot_func_sparse(lhs_nd_copy, rhs_dns) + sparse_cost = measure_cost(num_repeat, trans_lhs, False, dot_func_sparse, lhs_nd, rhs_nd) + dense_cost = measure_cost(num_repeat, trans_lhs, True, dot_func_dense, lhs_dns, rhs_dns) + + speedup = dense_cost / sparse_cost + # Print results + m = lhs_shape[0] + k = lhs_shape[1] + n = rhs_shape[1] + result_pattern = '{:15.1f} {:15.1f} {:>10} {:8d} {:8d} {:8d} {:13.2f} {:13.2f} {:8.2f}' + results = result_pattern.format(lhs_den*100, + rhs_den*100, + str(ctx), + m, + k, + n, + sparse_cost*1000, + dense_cost*1000, + speedup) + print(results) + + def print_benchmark_info(lhs, rhs, lhs_trans, fw): + trans_str = "^T" if lhs_trans else "" + print("========================================================") + print(" %s sparse dot benchmark: dot(%s, %s) = %s ") % (fw, lhs, rhs, rhs) + print(" (matrix multiplication: (m x k)%s * (k x n) = m x n) ") % (trans_str) + print("========================================================") + headline_pattern = '{:>15} {:>15} {:>10} {:>8} {:>8} {:>8} {:>13} {:>13} {:>8}' + headline = headline_pattern.format('lhs_density(%)', + 'rhs_density(%)', + 'context', + 'm', 'k', 'n', + 't_sparse(ms)', + 't_dense(ms)', + 'speedup') + print(headline) + + + def run_benchmark(ctx=None, lhs="csr", lhs_trans=False, rhs="dns", fw="mxnet", rhs_density=1, + distribution="uniform"): + if lhs != "csr": + raise ValueError("Value other than csr for lhs not supported") + if rhs_density > 1 or rhs_density < 0: + raise ValueError("rhs_density has to be between 0 and 1") + + print_benchmark_info(lhs, rhs, lhs_trans, fw) + + + lhs_stype = "csr" + rhs_stype = "row_sparse" if rhs == "rsp" else "default" + + feature_dim_list = data_dict['feature_dim'] + output_dim_list = data_dict['m'] + batch_size_list = data_dict['batch_size'] + density_list = data_dict['density'] + + default_output_index = data_dict['default_index']['output_dim'] + default_batch_size_index = data_dict['default_index']['batch_size'] + default_feature_index = data_dict['default_index']['feature_dim'] + default_density_index = data_dict['default_index']['density'] + num_repeat = data_dict['num_repeat'] + + for output_dim in output_dim_list: + if lhs_trans: + output_row_dim = batch_size_list[default_batch_size_index] + else: + output_row_dim = feature_dim_list[default_feature_index] + bench_dot((batch_size_list[default_batch_size_index], + feature_dim_list[default_feature_index]), + (output_row_dim, output_dim), + lhs_stype, rhs_stype, + density_list[default_density_index], rhs_density, + lhs_trans, ctx, num_repeat=num_repeat, + fw=fw, distribution=distribution) + + for feature_dim in feature_dim_list: + if lhs_trans: + output_row_dim = batch_size_list[default_batch_size_index] + else: + output_row_dim = feature_dim + bench_dot((batch_size_list[default_batch_size_index], feature_dim), + (output_row_dim, output_dim_list[default_output_index]), + lhs_stype, rhs_stype, density_list[default_density_index], rhs_density, + lhs_trans, ctx, num_repeat=num_repeat, fw=fw, distribution=distribution) + + for batch_size in batch_size_list: + if lhs_trans: + output_row_dim = batch_size + else: + output_row_dim = feature_dim_list[default_feature_index] + bench_dot((batch_size, feature_dim_list[default_feature_index]), + (output_row_dim, + output_dim_list[default_output_index]), + lhs_stype, rhs_stype, density_list[default_density_index], + rhs_density, lhs_trans, ctx, num_repeat=num_repeat, + fw=fw, distribution=distribution) + + for density in density_list: + if lhs_trans: + output_row_dim = batch_size_list[default_batch_size_index] + else: + output_row_dim = feature_dim_list[default_feature_index] + bench_dot((batch_size_list[default_batch_size_index], + feature_dim_list[default_feature_index]), + (output_row_dim, + output_dim_list[default_output_index]), + lhs_stype, rhs_stype, density, rhs_density, lhs_trans, ctx, + num_repeat=num_repeat, fw=fw, distribution=distribution) + + check_call(_LIB.MXSetNumOMPThreads(ctypes.c_int(ARGS.num_omp_threads))) + context = mx.gpu() if ARGS.gpu else mx.cpu() + # TODO(anirudh): make the data dicts to config which can be passed at runtime + distributions = ["uniform", "powerlaw"] + for distribution in distributions: + run_benchmark(context, lhs="csr", + rhs="default", lhs_trans=False, + fw="mxnet", rhs_density=1, + distribution=distribution) + run_benchmark(context, lhs="csr", + rhs="default", lhs_trans=True, + fw="mxnet", rhs_density=1, + distribution=distribution) + run_benchmark(context, lhs="csr", + rhs="rsp", lhs_trans=False, + fw="mxnet", rhs_density=0.05, + distribution=distribution) + if not ARGS.gpu: + run_benchmark(context, lhs="csr", + rhs="default", lhs_trans=False, + fw="scipy", rhs_density=1, + distribution=distribution) + run_benchmark(context, lhs="csr", + rhs="default", lhs_trans=True, + fw="scipy", rhs_density=1, + distribution=distribution) + + +if __name__ == "__main__": + begin_time = time.time() + test_dot_real(KDDA) + test_dot_real(AVAZU) + test_dot_real(CRITEO) + test_dot_synthetic(SYNTHETIC1) + test_dot_synthetic(SYNTHETIC2) + total_time = time.time() - begin_time + print("total time is %f") % total_time diff --git a/benchmark/python/sparse/sparse_end2end.py b/benchmark/python/sparse/sparse_end2end.py new file mode 100644 index 000000000000..e9d8bf884713 --- /dev/null +++ b/benchmark/python/sparse/sparse_end2end.py @@ -0,0 +1,249 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from mxnet.test_utils import * +import time +import argparse +import os + +parser = argparse.ArgumentParser(description="Run sparse linear regression " \ + "with distributed kvstore", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument('--profiler', type=int, default=0, + help='whether to use profiler') +parser.add_argument('--num-epoch', type=int, default=1, + help='number of epochs to train') +parser.add_argument('--batch-size', type=int, default=512, + help='number of examples per batch') +parser.add_argument('--num-batch', type=int, default=99999999, + help='number of batches per epoch') +parser.add_argument('--dummy-iter', type=int, default=0, + help='whether to use dummy iterator to exclude io cost') +parser.add_argument('--kvstore', type=str, default='local', + help='what kvstore to use [local, dist_sync, etc]') +parser.add_argument('--log-level', type=str, default='debug', + help='logging level [debug, info, error]') +parser.add_argument('--dataset', type=str, default='avazu', + help='what test dataset to use') +parser.add_argument('--num-gpu', type=int, default=0, + help='number of gpus to use. 0 means using cpu(0);' + 'otherwise, use gpu(0),...,gpu(num_gpu-1)') +parser.add_argument('--output-dim', type=int, default=4, + help='number of columns of the forward output') +parser.add_argument('--dummy-metric', type=int, default=0, + help='whether to call update_metric') + + +def get_libsvm_data(data_dir, data_name, url, data_origin_name): + if not os.path.isdir(data_dir): + os.system("mkdir " + data_dir) + os.chdir(data_dir) + if (not os.path.exists(data_name)): + import urllib + zippath = os.path.join(data_dir, data_origin_name) + urllib.urlretrieve(url, zippath) + os.system("bzip2 -d %r" % data_origin_name) + os.chdir("..") + + +class DummyIter(mx.io.DataIter): + "A dummy iterator that always return the same batch, used for speed testing" + def __init__(self, real_iter): + super(DummyIter, self).__init__() + self.real_iter = real_iter + self.provide_data = real_iter.provide_data + self.provide_label = real_iter.provide_label + self.batch_size = real_iter.batch_size + + for batch in real_iter: + self.the_batch = batch + break + + def __iter__(self): + return self + + def next(self): + return self.the_batch + +# testing dataset sources +avazu = { + 'data_name': 'avazu-app.t', + 'data_origin_name': 'avazu-app.t.bz2', + 'url': "https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary/avazu-app.t.bz2", + 'feature_dim': 1000000, +} + +kdda = { + 'data_name': 'kdda.t', + 'data_origin_name': 'kdda.t.bz2', + 'url': "https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary/kdda.t.bz2", + 'feature_dim': 20216830, +} + +datasets = { 'kdda' : kdda, 'avazu' : avazu } + + +def get_sym(feature_dim): + x = mx.symbol.Variable("data", stype='csr') + norm_init = mx.initializer.Normal(sigma=0.01) + w = mx.symbol.Variable("w", shape=(feature_dim, args.output_dim), init=norm_init, stype='row_sparse') + embed = mx.symbol.dot(x, w) + y = mx.symbol.Variable("softmax_label") + model = mx.symbol.SoftmaxOutput(data=embed, label=y, name="out") + return model + + +def row_sparse_pull(kv, key, data, slices, weight_array, priority): + # if have kvstore, need to pull corresponding rows of + # the weights to each context + # column indices (NDArray type) of the csr data + # used as the row_idx of the weight row-sparse matrix + row_indices = data.indices + if len(slices) == 1: + kv.row_sparse_pull(key, weight_array, priority=priority, row_ids=row_indices) + else: # more than one slices, multi-GPU training. Need to retain weight rows according to data slices + # TODO(junwu): + # the following line blocks, may need to pre-compute + # and cache it outside the for loop + indptr = data.indptr.asnumpy() + row_idx_array = [] + for s in slices: + row_idx_array.append(row_indices[indptr[s.start]:indptr[s.stop]]) + kv.row_sparse_pull(key, weight_array, priority=priority, row_ids=row_idx_array) + + +if __name__ == '__main__': + + # arg parser + args = parser.parse_args() + num_epoch = args.num_epoch + num_batch = args.num_batch + kvstore = args.kvstore + profiler = args.profiler > 0 + batch_size = args.batch_size if args.num_gpu == 0 else args.num_gpu * args.batch_size + dummy_iter = args.dummy_iter + dataset = args.dataset + log_level = args.log_level + contexts = mx.context.cpu(0) if args.num_gpu < 1\ + else [mx.context.gpu(i) for i in range(args.num_gpu)] + + # create kvstore when there are gpus + kv = mx.kvstore.create(kvstore) if args.num_gpu >= 1 else None + rank = kv.rank if kv is not None else 0 + num_worker = kv.num_workers if kv is not None else 1 + + # only print log for rank 0 worker + import logging + if rank != 0: + log_level = logging.ERROR + elif log_level == 'DEBUG': + log_level = logging.DEBUG + else: + log_level = logging.INFO + head = '%(asctime)-15s %(message)s' + logging.basicConfig(level=log_level, format=head) + + # dataset + assert(dataset in datasets), "unknown dataset " + dataset + metadata = datasets[dataset] + feature_dim = metadata['feature_dim'] + if logging: + logging.debug('preparing data ... ') + data_dir = os.path.join(os.getcwd(), 'data') + path = os.path.join(data_dir, metadata['data_name']) + if not os.path.exists(path): + get_libsvm_data(data_dir, metadata['data_name'], metadata['url'], + metadata['data_origin_name']) + assert os.path.exists(path) + + # data iterator + train_data = mx.io.LibSVMIter(data_libsvm=path, data_shape=(feature_dim,), + batch_size=batch_size, num_parts=num_worker, + part_index=rank) + if dummy_iter: + train_data = DummyIter(train_data) + + # model + model = get_sym(feature_dim) + + # module + mod = mx.mod.Module(symbol=model, data_names=['data'], + label_names=['softmax_label'], context=contexts) + mod.bind(data_shapes=train_data.provide_data, label_shapes=train_data.provide_label) + mod.init_params(initializer=mx.init.Uniform(scale=.1)) + sgd = mx.optimizer.SGD(momentum=0.0, clip_gradient=5.0, + learning_rate=0.1, rescale_grad=1.0/batch_size/num_worker) + mod.init_optimizer(optimizer=sgd, kvstore=kv) + # use accuracy as the metric + metric = mx.metric.create('acc') + + index = mod._exec_group.param_names.index('w') + # weight_array bound to executors of the contexts + weight_array = mod._exec_group.param_arrays[index] + + mx.nd.waitall() # sync point for initialization + # start profiler + if profiler: + device = 'cpu' + if args.num_gpu > 0: + device = 'gpu' + str(args.num_gpu) + name = 'profile_' + args.dataset + '_' + device + '_nworker' + str(num_worker)\ + + '_batchsize' + str(args.batch_size) + '_outdim' + str(args.output_dim) + '.json' + mx.profiler.profiler_set_config(mode='all', filename=name) + mx.profiler.profiler_set_state('run') + + logging.debug('start training ...') + start = time.time() + data_iter = iter(train_data) + for epoch in range(num_epoch): + nbatch = 0 + end_of_batch = False + data_iter.reset() + metric.reset() + next_batch = next(data_iter) + if kv is not None: + row_sparse_pull(kv, 'w', next_batch.data[0], mod._exec_group.slices, weight_array, -index) + while not end_of_batch: + nbatch += 1 + batch = next_batch + + mod.forward_backward(batch) + # update parameters + mod.update() + + try: + # pre fetch next batch + next_batch = next(data_iter) + if nbatch == num_batch: + raise StopIteration + if kv is not None: + row_sparse_pull(kv, 'w', next_batch.data[0], mod._exec_group.slices, weight_array, -index) + except StopIteration: + end_of_batch = True + # accumulate prediction accuracy + if args.dummy_metric == 0: + mod.update_metric(metric, batch.label) + else: # call waitall to replace update_metric as sync point + mx.nd.waitall() # sync point for the current minibatch + logging.info('epoch %d, %s' % (epoch, metric.get())) + if epoch == 0: + print "num_batches = ", nbatch + if profiler: + mx.profiler.profiler_set_state('stop') + end = time.time() + time_cost = end - start + logging.info('num_worker = ' + str(num_worker) + ', time cost = ' + str(time_cost)) diff --git a/benchmark/python/sparse/sparse_op.py b/benchmark/python/sparse/sparse_op.py new file mode 100644 index 000000000000..0683aa84eacb --- /dev/null +++ b/benchmark/python/sparse/sparse_op.py @@ -0,0 +1,245 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import ctypes + +from mxnet.test_utils import * +import scipy.sparse as sp +import os +import time +import argparse + +from mxnet.base import check_call, _LIB +from util import get_data, estimate_density + +parser = argparse.ArgumentParser(description="Benchmark sparse operators", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument('--num-omp-threads', type=int, default=1, help='number of omp threads to set in MXNet') +args = parser.parse_args() + +# some data information +kdda = { + 'data_mini': 'kdda.t.mini', + 'data_name': 'kdda.t', + 'data_origin_name': 'kdda.t.bz2', + 'url': "https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary/kdda.t.bz2", + 'feature_dim': 20216830, + 'm': 200, + 'batch_size': [64] +} + +avazu = { + 'data_mini': 'avazu-app.t.mini', + 'data_name': 'avazu-app.t', + 'data_origin_name': 'avazu-app.t.bz2', + 'url': "https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary/avazu-app.t.bz2", + 'feature_dim': 1000000, + 'm': 500, + 'batch_size': [64, 128] +} + + +def measure_cost(repeat, f, *args, **kwargs): + # start bench + start = time.time() + results = [] + for i in range(repeat): + results.append(f(*args, **kwargs)) + for result in results: + result.wait_to_read() + end = time.time() + diff = end - start + return diff / repeat + + +def test_dot_real(data_dict): + def get_iter(path, data_shape, batch_size): + data_train = mx.io.LibSVMIter(data_libsvm=path, + data_shape=data_shape, + batch_size=batch_size) + data_iter = iter(data_train) + return data_iter + + data_dir = os.path.join(os.getcwd(), 'data') + + path = os.path.join(data_dir, data_dict['data_name']) + if not os.path.exists(path): + get_data( + data_dir, + data_dict['data_name'], + data_dict['url'], + data_dict['data_origin_name'] + ) + assert os.path.exists(path) + + k = data_dict['feature_dim'] + m = data_dict['m'] + density = estimate_density(path, data_dict['feature_dim']) + + mini_path = os.path.join(data_dir, data_dict['data_mini']) + if not os.path.exists(mini_path): + os.system("head -n 2000 %r > %r" % (path, mini_path)) + assert os.path.exists(mini_path) + + print "Running Benchmarking on %r data" % data_dict['data_mini'] + for batch_size in data_dict['batch_size']: # iterator through different batch size of choice + print "batch_size is %d" % batch_size + # model + data_shape = (k, ) + train_iter = get_iter(mini_path, data_shape, batch_size) + weight = mx.nd.random_uniform(low=0, high=1, shape=(k, m)) + + csr_data = [] + dns_data = [] + num_batch = 0 + for batch in train_iter: + data = train_iter.getdata() + csr_data.append(data) + dns_data.append(data.tostype('default')) + num_batch += 1 + bag_of_data = [csr_data, dns_data] + num_repeat = 5 + costs = [] + for d in bag_of_data: + weight.wait_to_read() + cost = 0. + count = 0 + for d_batch in d: + d_batch.wait_to_read() + cost += measure_cost(num_repeat, mx.nd.dot, d_batch, weight) + count += 1 + costs.append(cost/count) + t_sparse = costs[0] + t_dense = costs[1] + ratio = t_dense / t_sparse + print('density(%)\tn\tm\tk\tt_dense/t_sparse\tt_dense\tt_sparse') + fmt = "%0.4f\t\t%d\t%d\t%d\t%0.2f\t\t\t%0.4f\t%0.6f" + print(fmt % (density * 100, batch_size, m, k, ratio, t_dense, t_sparse)) + + +def test_dot_synthetic(): + """benchmark mx.nd.dot(sparse_ndarray, dense_ndarray) with given density. + `t_sparse` is the time cost of dot(csr, dns), while `t_dense` is the time cost + of dot(dns, dns), with the same matrix except that it is in default storage type. + """ + def measure_cost_forward_baseline(repeat, dot, lhs, rhs): + start = time.time() + for i in range(repeat): + dot(lhs, rhs) + end = time.time() + diff = end - start + return diff / repeat + + def measure_cost_backward_baseline(repeat, dot, transpose, lhs, rhs): + start = time.time() + for i in range(repeat): + dot(transpose(lhs), rhs) + end = time.time() + diff = end - start + return diff / repeat + + def bench_dot_forward(m, k, n, density, ctx, repeat): + set_default_context(ctx) + dns = mx.nd.random_uniform(shape=(k, n)).copyto(ctx) + data_shape = (m, k) + csr_data = rand_ndarray(data_shape, 'csr', density) + dns_data = csr_data.tostype('default') + rhs_dns_np = dns.asnumpy() + lhs_csr_sp = sp.csr_matrix(dns_data.asnumpy()) # csr in scipy + lhs_dns_np = lhs_csr_sp.tostype('default') + + data = [dns_data, csr_data] + costs = [] + for d in data: + dns.wait_to_read() + d.wait_to_read() + cost = measure_cost(repeat, mx.nd.dot, d, dns) + costs.append(cost) + ratio = costs[0] / costs[1] + + costs_baseline = [] + cost = measure_cost_forward_baseline(repeat, np.dot, lhs_dns_np, rhs_dns_np) + costs_baseline.append(cost) + cost = measure_cost_forward_baseline(repeat, sp.spmatrix.dot, lhs_csr_sp, rhs_dns_np) + costs_baseline.append(cost) + ratio_baseline = costs_baseline[0] / costs_baseline[1] + fmt = "%0.1f\t\t%s\t%d\t%d\t%d\t%0.2f\t\t\t%0.2f\t%0.5f\t\t%0.2f\t\t\t\t%0.6f\t%0.5f" + print(fmt % (density * 100, str(ctx), n, m, k, ratio, costs[0], costs[1], + ratio_baseline, costs_baseline[0], costs_baseline[1])) + + def bench_dot_backward(m, k, n, density, ctx, repeat): + set_default_context(ctx) + dns = mx.nd.random_uniform(shape=(m, n)).copyto(ctx) + data_shape = (m, k) + csr_data = rand_ndarray(data_shape, 'csr', density) + dns_data = csr_data.tostype('default') + rhs_dns_np = dns.asnumpy() + lhs_csr_sp = sp.csr_matrix(dns_data.asnumpy()) + lhs_dns_np = lhs_csr_sp.tostype('default') + + data = [dns_data, csr_data] + costs = [] + for d in data: + dns.wait_to_read() + d.wait_to_read() + cost = measure_cost(repeat, mx.nd.dot, d, dns, transpose_a=True) + costs.append(cost) + ratio = costs[0] / costs[1] + + costs_baseline = [] + cost = measure_cost_backward_baseline(repeat, np.dot, np.transpose, lhs_dns_np, rhs_dns_np) + costs_baseline.append(cost) + cost = measure_cost_backward_baseline(repeat, sp.spmatrix.dot, sp.spmatrix.transpose, lhs_csr_sp, rhs_dns_np) + costs_baseline.append(cost) + ratio_baseline = costs_baseline[0] / costs_baseline[1] + fmt = "%0.1f\t\t%s\t%d\t%d\t%d\t%0.2f\t\t\t%0.2f\t%0.5f\t\t%0.2f\t\t\t\t%0.6f\t%0.5f" + print(fmt % (density * 100, str(ctx), n, m, k, ratio, costs[0], costs[1], + ratio_baseline, costs_baseline[0], costs_baseline[1])) + + print("A = sparse NDArray of shape(m, k)") + print("B = dense NDArray of shape(k, n)") + print("dot_forward\tdot(csr, dns)") + print('density(%)\tcontext\tn\tm\tk\tt_dense/t_sparse\tt_dense\tt_sparse' + '\tt_scipy_dense/t_scipy_sparse\tt_scipy_dense\tt_scipy_sparse') + + check_call(_LIB.MXSetNumOMPThreads(ctypes.c_int(args.num_omp_threads))) + # TODO(haibin) make these runtime options + m = 512 + k = [50000, 100000] + n = [64, 128] + density = [1.00, 0.90, 0.70, 0.50, 0.30, 0.20, 0.10, 0.07, 0.05, 0.02, 0.01, 0.005, 0.001] + num_repeat = 10 + # contexts = [mx.cpu(), mx.gpu(0)] + contexts = [mx.cpu()] + for i in range(2): + for ctx in contexts: + for den in density: + bench_dot_forward(m, k[i], n[i], den, ctx, num_repeat) + + print("dot_backward\tdot(csr.T, dns)") + print('density(%)\tcontext\tn\tm\tk\tt_dense/t_sparse\tt_dense\tt_sparse' + '\tt_scipy_dense/t_scipy_sparse\tt_scipy_dense\tt_scipy_sparse') + for i in range(2): + for ctx in contexts: + for den in density: + bench_dot_backward(m, k[i], n[i], den, ctx, num_repeat) + + +if __name__ == "__main__": + test_dot_real(avazu) + test_dot_real(kdda) + test_dot_synthetic() diff --git a/benchmark/python/sparse/util.py b/benchmark/python/sparse/util.py new file mode 100644 index 000000000000..947ff4a65037 --- /dev/null +++ b/benchmark/python/sparse/util.py @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os +import random + + +def get_data(data_dir, data_name, url, data_origin_name): + if not os.path.isdir(data_dir): + os.system("mkdir " + data_dir) + os.chdir(data_dir) + if (not os.path.exists(data_name)): + import urllib + zippath = os.path.join(data_dir, data_origin_name) + urllib.urlretrieve(url, zippath) + os.system("bzip2 -d %r" % data_origin_name) + os.chdir("..") + + +def estimate_density(DATA_PATH, feature_size): + """sample 10 times of a size of 1000 for estimating the density of the sparse dataset""" + if not os.path.exists(DATA_PATH): + raise Exception("Data is not there!") + density = [] + P = 0.01 + for _ in xrange(10): + num_non_zero = 0 + num_sample = 0 + with open(DATA_PATH) as f: + for line in f: + if (random.random() < P): + num_non_zero += len(line.split(" ")) - 1 + num_sample += 1 + density.append(num_non_zero * 1.0 / (feature_size * num_sample)) + return sum(density) / len(density) + diff --git a/docs/api/python/ndarray.md b/docs/api/python/ndarray.md index 5e9f7e1a1184..3f2cef24a73a 100644 --- a/docs/api/python/ndarray.md +++ b/docs/api/python/ndarray.md @@ -64,9 +64,21 @@ A detailed tutorial is available at ``` In the rest of this document, we first overview the methods provided by the -`ndarray.NDArray` class, and then list other routines provided by the -`ndarray` package. +`ndarray.NDArray` class and its subclasses, and then list other routines +provided by the `ndarray` package. +The `ndarray` package provides several classes: + +```eval_rst +.. autosummary:: + :nosignatures: + + NDArray + sparse.CSRNDArray + sparse.RowSparseNDArray +``` + +We summarize the interface for each class in the following sections. ## The `NDArray` class @@ -80,6 +92,7 @@ In the rest of this document, we first overview the methods provided by the NDArray.size NDArray.context NDArray.dtype + NDArray.stype ``` ### Array conversion @@ -94,6 +107,7 @@ In the rest of this document, we first overview the methods provided by the NDArray.asnumpy NDArray.asscalar NDArray.astype + NDArray.tostype ``` ### Array change shape @@ -171,6 +185,35 @@ In the rest of this document, we first overview the methods provided by the NDArray.wait_to_read ``` +## The `sparse.RowSparseNDArray` Class + +```eval_rst +.. autosummary:: + :nosignatures: + + sparse.RowSparseNDArray.copyto + sparse.RowSparseNDArray.tostype + sparse.RowSparseNDArray.__setitem__ + sparse.RowSparseNDArray.__getitem__ + sparse.RowSparseNDArray.data + sparse.RowSparseNDArray.indices +``` + +## The `sparse.CSRNDArray` Class + +```eval_rst +.. autosummary:: + :nosignatures: + + sparse.CSRNDArray.copyto + sparse.CSRNDArray.tostype + sparse.CSRNDArray.__setitem__ + sparse.CSRNDArray.__getitem__ + sparse.CSRNDArray.data + sparse.CSRNDArray.indices + sparse.CSRNDArray.indptr +``` + ## Array creation routines ```eval_rst @@ -499,8 +542,24 @@ The `contrib.ndarray` module contains many useful experimental APIs for new feat ```eval_rst + +.. autoclass:: mxnet.ndarray.NDArray + :members: + :special-members: + +.. autoclass:: mxnet.ndarray.sparse.CSRNDArray + :members: + :special-members: + +.. autoclass:: mxnet.ndarray.sparse.RowSparseNDArray + :members: + :special-members: + .. automodule:: mxnet.ndarray :members: + :imported-members: + :special-members: + :exclude-members: CachedOp, BaseSparseNDArray, NDArray, CSRNDArray, RowSparseNDArray .. automodule:: mxnet.random :members: diff --git a/example/sparse/get_data.py b/example/sparse/get_data.py new file mode 100644 index 000000000000..578cf2ce5226 --- /dev/null +++ b/example/sparse/get_data.py @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# pylint: skip-file +import os, gzip +import pickle as pickle +import sys + +def get_libsvm_data(data_dir, data_name, url, data_origin_name): + if not os.path.isdir(data_dir): + os.mkdir(data_dir) + os.chdir(data_dir) + if (not os.path.exists(data_name)): + import urllib + zippath = os.path.join(data_dir, data_origin_name) + urllib.urlretrieve(url, zippath) + os.system("bzip2 -d %r" % data_origin_name) + os.chdir("..") diff --git a/example/sparse/linear_classification.py b/example/sparse/linear_classification.py new file mode 100644 index 000000000000..567568c6eb80 --- /dev/null +++ b/example/sparse/linear_classification.py @@ -0,0 +1,185 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import mxnet as mx +from mxnet.test_utils import * +from get_data import get_libsvm_data +import time +import argparse +import os + +parser = argparse.ArgumentParser(description="Run sparse linear classification " \ + "with distributed kvstore", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument('--profiler', type=int, default=0, + help='whether to use profiler') +parser.add_argument('--num-epoch', type=int, default=1, + help='number of epochs to train') +parser.add_argument('--batch-size', type=int, default=8192, + help='number of examples per batch') +parser.add_argument('--num-batch', type=int, default=99999999, + help='number of batches per epoch') +parser.add_argument('--dummy-iter', type=int, default=0, + help='whether to use dummy iterator to exclude io cost') +parser.add_argument('--kvstore', type=str, default='dist_sync', + help='what kvstore to use [local, dist_sync, etc]') +parser.add_argument('--log-level', type=str, default='DEBUG', + help='logging level [debug, info, error]') +parser.add_argument('--dataset', type=str, default='avazu', + help='what test dataset to use') + +class DummyIter(mx.io.DataIter): + "A dummy iterator that always return the same batch, used for speed testing" + def __init__(self, real_iter): + super(DummyIter, self).__init__() + self.real_iter = real_iter + self.provide_data = real_iter.provide_data + self.provide_label = real_iter.provide_label + self.batch_size = real_iter.batch_size + + for batch in real_iter: + self.the_batch = batch + break + + def __iter__(self): + return self + + def next(self): + return self.the_batch + +# testing dataset sources +avazu = { + 'data_name': 'avazu-app.t', + 'data_origin_name': 'avazu-app.t.bz2', + 'url': "https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary/avazu-app.t.bz2", + 'feature_dim': 1000000, +} + +kdda = { + 'data_name': 'kdda.t', + 'data_origin_name': 'kdda.t.bz2', + 'url': "https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary/kdda.t.bz2", + 'feature_dim': 20216830, +} + +datasets = { 'kdda' : kdda, 'avazu' : avazu } + +def linear_model(feature_dim): + x = mx.symbol.Variable("data", stype='csr') + norm_init = mx.initializer.Normal(sigma=0.01) + weight = mx.symbol.Variable("weight", shape=(feature_dim, 1), init=norm_init, stype='row_sparse') + bias = mx.symbol.Variable("bias", shape=(1,), init=norm_init) + dot = mx.symbol.dot(x, weight) + pred = mx.symbol.broadcast_add(dot, bias) + y = mx.symbol.Variable("softmax_label") + model = mx.symbol.SoftmaxOutput(data=pred, label=y, name="out") + return model + +if __name__ == '__main__': + # arg parser + args = parser.parse_args() + num_epoch = args.num_epoch + num_batch = args.num_batch + kvstore = args.kvstore + profiler = args.profiler > 0 + batch_size = args.batch_size + dummy_iter = args.dummy_iter + dataset = args.dataset + log_level = args.log_level + + # create kvstore + kv = mx.kvstore.create(kvstore) + rank = kv.rank + num_worker = kv.num_workers + + # only print log for rank 0 worker + import logging + if rank != 0: + log_level = logging.ERROR + elif log_level == 'DEBUG': + log_level = logging.DEBUG + else: + log_level = logging.INFO + head = '%(asctime)-15s %(message)s' + logging.basicConfig(level=log_level, format=head) + + # dataset + assert(dataset in datasets), "unknown dataset " + dataset + metadata = datasets[dataset] + feature_dim = metadata['feature_dim'] + if logging: + logging.debug('preparing data ... ') + data_dir = os.path.join(os.getcwd(), 'data') + path = os.path.join(data_dir, metadata['data_name']) + if not os.path.exists(path): + get_libsvm_data(data_dir, metadata['data_name'], metadata['url'], + metadata['data_origin_name']) + assert os.path.exists(path) + + # data iterator + train_data = mx.io.LibSVMIter(data_libsvm=path, data_shape=(feature_dim,), + batch_size=batch_size, num_parts=num_worker, + part_index=rank) + if dummy_iter: + train_data = DummyIter(train_data) + + # model + model = linear_model(feature_dim) + + # module + mod = mx.mod.Module(symbol=model, data_names=['data'], label_names=['softmax_label']) + mod.bind(data_shapes=train_data.provide_data, label_shapes=train_data.provide_label) + mod.init_params(initializer=mx.init.Uniform(scale=.1)) + sgd = mx.optimizer.SGD(momentum=0.0, clip_gradient=5.0, + learning_rate=0.1, rescale_grad=1.0/batch_size/num_worker) + mod.init_optimizer(optimizer=sgd, kvstore=kv) + # use accuracy as the metric + metric = mx.metric.create('Accuracy') + + # start profiler + if profiler: + name = 'profile_output_' + str(num_worker) + '.json' + mx.profiler.profiler_set_config(mode='all', filename=name) + mx.profiler.profiler_set_state('run') + + logging.debug('start training ...') + start = time.time() + data_iter = iter(train_data) + for epoch in range(num_epoch): + nbatch = 0 + data_iter.reset() + metric.reset() + for batch in data_iter: + nbatch += 1 + row_ids = batch.data[0].indices + # pull sparse weight + index = mod._exec_group.param_names.index('weight') + kv.row_sparse_pull('weight', mod._exec_group.param_arrays[index], + priority=-index, row_ids=[row_ids]) + mod.forward_backward(batch) + # update parameters + mod.update() + # accumulate prediction accuracy + mod.update_metric(metric, batch.label) + if nbatch == num_batch: + break + logging.info('epoch %d, %s' % (epoch, metric.get())) + if profiler: + mx.profiler.profiler_set_state('stop') + end = time.time() + time_cost = end - start + logging.info('num_worker = ' + str(num_worker) + ', time cost = ' + str(time_cost)) diff --git a/include/mxnet/c_api.h b/include/mxnet/c_api.h index 2289354e8a5e..a43f73fe45ab 100644 --- a/include/mxnet/c_api.h +++ b/include/mxnet/c_api.h @@ -276,6 +276,38 @@ MXNET_DLL int MXNDArrayCreateEx(const mx_uint *shape, int delay_alloc, int dtype, NDArrayHandle *out); + + +/*! + * \brief create an empty sparse NDArray with specified shape and data type + * \param storage_type the storage type of the ndarray + * \param shape the pointer to the shape + * \param ndim the dimension of the shape + * \param dev_type device type, specify device we want to take + * \param dev_id the device id of the specific device + * \param delay_alloc whether to delay allocation until + * the narray is first mutated + * \param dtype data type of created array + * \param num_aux the number of aux data to support this ndarray + * \param aux_type data type of the aux data for the created array + * \param aux_ndims the dimension of the shapes of aux data + * \param aux_shape the shapes of aux data + * \param out the returning handle + * \return 0 when success, -1 when failure happens + */ +MXNET_DLL int MXNDArrayCreateSparseEx(int storage_type, + const mx_uint *shape, + mx_uint ndim, + int dev_type, + int dev_id, + int delay_alloc, + int dtype, + mx_uint num_aux, + int *aux_type, + mx_uint *aux_ndims, + const mx_uint *aux_shape, + NDArrayHandle *out); + /*! * \brief create a NDArray handle that is loaded from raw bytes. * \param buf the head of the raw bytes @@ -350,6 +382,17 @@ MXNET_DLL int MXNDArraySyncCopyFromCPU(NDArrayHandle handle, MXNET_DLL int MXNDArraySyncCopyToCPU(NDArrayHandle handle, void *data, size_t size); +/*! + * \brief Copy src.data() to dst.data() if i = -1, else dst.aux_data(i) if i >= 0 + * This function blocks. Do not use it in performance critical code. + * \param handle_dst handle of a dst ndarray whose data/aux_data has been allocated + * \param handle_src handle of a src ndarray which has default storage type + * \param i dst data blob indicator + */ +MXNET_DLL int MXNDArraySyncCopyFromNDArray(NDArrayHandle handle_dst, + const NDArrayHandle handle_src, + const int i); + /*! * \brief Wait until all the pending writes with respect NDArray are finished. * Always call this before read data out synchronizely. @@ -388,6 +431,7 @@ MXNET_DLL int MXNDArraySlice(NDArrayHandle handle, mx_uint slice_begin, mx_uint slice_end, NDArrayHandle *out); + /*! * \brief Index the NDArray along axis 0. * \param handle the handle to the NDArray @@ -398,6 +442,13 @@ MXNET_DLL int MXNDArraySlice(NDArrayHandle handle, MXNET_DLL int MXNDArrayAt(NDArrayHandle handle, mx_uint idx, NDArrayHandle *out); + +/*! + * \brief get the storage type of the array + */ +MXNET_DLL int MXNDArrayGetStorageType(NDArrayHandle handle, + int *out_storage_type); + /*! * \brief Reshape the NDArray. * \param handle the handle to the narray @@ -436,6 +487,34 @@ MXNET_DLL int MXNDArrayGetData(NDArrayHandle handle, */ MXNET_DLL int MXNDArrayGetDType(NDArrayHandle handle, int *out_dtype); + +/*! + * \brief get the type of the ith aux data in NDArray + * \param handle the handle to the narray + * \param i the index of the aux data + * \param out_type pointer holder to get type of aux data + * \return 0 when success, -1 when failure happens + */ +MXNET_DLL int MXNDArrayGetAuxType(NDArrayHandle handle, + mx_uint i, + int *out_type); + +/*! + * \brief Get a deep copy of the ith aux data blob + * in the form of an NDArray of default storage type. + * This function blocks. Do not use it in performance critical code. + */ +MXNET_DLL int MXNDArrayGetAuxNDArray(NDArrayHandle handle, + mx_uint i, + NDArrayHandle *out); + +/*! + * \brief Get a deep copy of the data blob + * in the form of an NDArray of default storage type. + * This function blocks. Do not use it in performance critical code. + */ +MXNET_DLL int MXNDArrayGetDataNDArray(NDArrayHandle handle, + NDArrayHandle *out); /*! * \brief get the context of the NDArray * \param handle the handle to the narray @@ -581,6 +660,28 @@ MXNET_DLL int MXImperativeInvoke(AtomicSymbolCreator creator, int num_params, const char **param_keys, const char **param_vals); +/*! + * \brief invoke a nnvm op and imperative function + * \param creator the op + * \param num_inputs number of input NDArrays + * \param inputs input NDArrays + * \param num_outputs number of output NDArrays + * \param outputs output NDArrays + * \param num_params number of keyword parameters + * \param param_keys keys for keyword parameters + * \param param_vals values for keyword parameters + * \param out_stypes output ndarrays' stypes + * \return 0 when success, -1 when failure happens + */ +MXNET_DLL int MXImperativeInvokeEx(AtomicSymbolCreator creator, + int num_inputs, + NDArrayHandle *inputs, + int *num_outputs, + NDArrayHandle **outputs, + int num_params, + const char **param_keys, + const char **param_vals, + const int **out_stypes); /*! * \brief set whether to record operator for autograd * \param is_recording 1 when recording, 0 when not recording. @@ -666,6 +767,30 @@ MXNET_DLL int MXCreateCachedOp(SymbolHandle handle, * \brief free cached operator */ MXNET_DLL int MXFreeCachedOp(CachedOpHandle handle); +/*! + * \brief invoke cached operator + */ +MXNET_DLL int MXInvokeCachedOp(CachedOpHandle handle, + int num_inputs, + NDArrayHandle *inputs, + int *num_outputs, + NDArrayHandle **outputs); +/*! + * \brief invoke a cached op + * \param handle the handle to the cached op + * \param num_inputs number of input NDArrays + * \param inputs input NDArrays + * \param num_outputs number of output NDArrays + * \param outputs output NDArrays + * \param out_stypes output ndarrays' stypes + * \return 0 when success, -1 when failure happens + */ +MXNET_DLL int MXInvokeCachedOpEx(CachedOpHandle handle, + int num_inputs, + NDArrayHandle *inputs, + int *num_outputs, + NDArrayHandle **outputs, + const int** out_stypes); /*! * \brief invoke cached operator */ @@ -1017,20 +1142,20 @@ MXNET_DLL int MXSymbolInferShape(SymbolHandle sym, * \return 0 when success, -1 when failure happens */ MXNET_DLL int MXSymbolInferShapePartial(SymbolHandle sym, - mx_uint num_args, - const char** keys, - const mx_uint *arg_ind_ptr, - const mx_uint *arg_shape_data, - mx_uint *in_shape_size, - const mx_uint **in_shape_ndim, - const mx_uint ***in_shape_data, - mx_uint *out_shape_size, - const mx_uint **out_shape_ndim, - const mx_uint ***out_shape_data, - mx_uint *aux_shape_size, - const mx_uint **aux_shape_ndim, - const mx_uint ***aux_shape_data, - int *complete); + mx_uint num_args, + const char** keys, + const mx_uint *arg_ind_ptr, + const mx_uint *arg_shape_data, + mx_uint *in_shape_size, + const mx_uint **in_shape_ndim, + const mx_uint ***in_shape_data, + mx_uint *out_shape_size, + const mx_uint **out_shape_ndim, + const mx_uint ***out_shape_data, + mx_uint *aux_shape_size, + const mx_uint **aux_shape_ndim, + const mx_uint ***aux_shape_data, + int *complete); /*! * \brief infer type of unknown input types given the known one. @@ -1061,6 +1186,10 @@ MXNET_DLL int MXSymbolInferType(SymbolHandle sym, mx_uint *aux_type_size, const int **aux_type_data, int *complete); + + + + //-------------------------------------------- // Part 4: Executor interface //-------------------------------------------- @@ -1222,36 +1351,39 @@ MXNET_DLL int MXExecutorBindEX(SymbolHandle symbol_handle, ExecutorHandle *out); MXNET_DLL int MXExecutorSimpleBind(SymbolHandle symbol_handle, - int dev_type, - int dev_id, - const mx_uint num_g2c_keys, - const char** g2c_keys, - const int* g2c_dev_types, - const int* g2c_dev_ids, - const mx_uint provided_grad_req_list_len, - const char** provided_grad_req_names, - const char** provided_grad_req_types, - const mx_uint num_provided_arg_shapes, - const char** provided_arg_shape_names, - const mx_uint* provided_arg_shape_data, - const mx_uint* provided_arg_shape_idx, - const mx_uint num_provided_arg_dtypes, - const char** provided_arg_dtype_names, - const int* provided_arg_dtypes, - const mx_uint num_shared_arg_names, - const char** shared_arg_name_list, - int* shared_buffer_len, - const char** shared_buffer_name_list, - NDArrayHandle* shared_buffer_handle_list, - const char*** updated_shared_buffer_name_list, - NDArrayHandle** updated_shared_buffer_handle_list, - mx_uint* num_in_args, - NDArrayHandle** in_args, - NDArrayHandle** arg_grads, - mx_uint* num_aux_states, - NDArrayHandle** aux_states, - ExecutorHandle shared_exec_handle, - ExecutorHandle* out); + int dev_type, + int dev_id, + const mx_uint num_g2c_keys, + const char** g2c_keys, + const int* g2c_dev_types, + const int* g2c_dev_ids, + const mx_uint provided_grad_req_list_len, + const char** provided_grad_req_names, + const char** provided_grad_req_types, + const mx_uint num_provided_arg_shapes, + const char** provided_arg_shape_names, + const mx_uint* provided_arg_shape_data, + const mx_uint* provided_arg_shape_idx, + const mx_uint num_provided_arg_dtypes, + const char** provided_arg_dtype_names, + const int* provided_arg_dtypes, + const mx_uint num_provided_arg_stypes, + const char** provided_arg_stype_names, + const int* provided_arg_stypes, + const mx_uint num_shared_arg_names, + const char** shared_arg_name_list, + int* shared_buffer_len, + const char** shared_buffer_name_list, + NDArrayHandle* shared_buffer_handle_list, + const char*** updated_shared_buffer_name_list, + NDArrayHandle** updated_shared_buffer_handle_list, + mx_uint* num_in_args, + NDArrayHandle** in_args, + NDArrayHandle** arg_grads, + mx_uint* num_aux_states, + NDArrayHandle** aux_states, + ExecutorHandle shared_exec_handle, + ExecutorHandle* out); /*! * \brief set a call back to notify the completion of operation */ @@ -1468,6 +1600,26 @@ MXNET_DLL int MXKVStorePullEx(KVStoreHandle handle, const char** keys, NDArrayHandle* vals, int priority); + +/*! + * \brief pull a list of (key, value) pairs from the kvstore, where each key is a string. + * The NDArray pulled back will be in row_sparse storage with only the specified + * row_ids present based row_ids (others rows are zeros). + * \param handle handle to the kvstore + * \param num the number of key-value pairs + * \param keys the list of keys + * \param vals the list of values + * \param row_ids the list of row_id NDArrays + * \param priority the priority of the action + * \return 0 when success, -1 when failure happens + */ +MXNET_DLL int MXKVStorePullRowSparse(KVStoreHandle handle, + mx_uint num, + const char** keys, + NDArrayHandle* vals, + const NDArrayHandle* row_ids, + int priority); + /*! * \brief user-defined updater for the kvstore * It's this updater's responsibility to delete \a recv and \a local diff --git a/include/mxnet/executor.h b/include/mxnet/executor.h index a74d3b07b5be..85d34778dd8c 100644 --- a/include/mxnet/executor.h +++ b/include/mxnet/executor.h @@ -133,6 +133,7 @@ class Executor { const std::vector& aux_state_ctxes, const std::unordered_map& arg_shape_map, const std::unordered_map& arg_dtype_map, + const std::unordered_map& arg_stype_map, const std::vector& grad_req_types, const std::unordered_set& param_names, std::vector* in_args, diff --git a/include/mxnet/graph_attr_types.h b/include/mxnet/graph_attr_types.h new file mode 100644 index 000000000000..3aba0119d8ca --- /dev/null +++ b/include/mxnet/graph_attr_types.h @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file graph_attr_types.h + * \brief Data structures that can appear in graph attributes. + */ +#ifndef MXNET_GRAPH_ATTR_TYPES_H_ +#define MXNET_GRAPH_ATTR_TYPES_H_ + +#include + +namespace mxnet { + +/*! + * \brief The result holder of storage type of each NodeEntry in the graph. + * \note Stored under graph.attrs["storage_type"], provided by Pass "InferStorageType" + * + * \code + * Graph g = ApplyPass(src_graph, "InferStorageType"); + * const StorageVector& stypes = g.GetAttr("storage_type"); + * // get shape by entry id + * int entry_type = stypes[g.indexed_graph().entry_id(my_entry)]; + * \endcode + * + * \sa FInferStorageType + */ +using StorageTypeVector = std::vector; + +} // namespace mxnet + +#endif // MXNET_GRAPH_ATTR_TYPES_H_ diff --git a/include/mxnet/kvstore.h b/include/mxnet/kvstore.h index d2924ecea1b5..9ea63b4cec79 100644 --- a/include/mxnet/kvstore.h +++ b/include/mxnet/kvstore.h @@ -25,6 +25,7 @@ #define MXNET_KVSTORE_H_ #include #include +#include #include #include #include @@ -173,6 +174,29 @@ class KVStore { const std::vector& values, int priority = 0) = 0; + /*! + * \brief pull a list of key-value pairs from the store. + * The NDArray pulled back will be in row_sparse storage with only the + * specified row_ids present (others rows are zeros). + * \param keys the list of keys + * \param values the list of buffers - row_id pairs + * \param priority the priority of the action. + */ + virtual void PullRowSparse(const std::vector& str_keys, + const std::vector>& val_rowids, + const int priority = 0) = 0; + + /*! + * \brief pull a list of key-value pairs from the store, where each key is a string. + * The NDArray pulled back will be in row_sparse storage with only the + * specified row_ids present (others rows are zeros). + * \param keys the list of keys in string format + * \param values the list of buffers - row_id pairs + * \param priority the priority of the action. + */ + virtual void PullRowSparse(const std::vector& str_keys, + const std::vector>& val_rowids, + const int priority = 0) = 0; /** * \brief the prototype of user-defined updater diff --git a/include/mxnet/ndarray.h b/include/mxnet/ndarray.h index d7dff4098b27..754bc28e7bed 100644 --- a/include/mxnet/ndarray.h +++ b/include/mxnet/ndarray.h @@ -47,7 +47,6 @@ namespace mxnet { -// forward declaration namespace autograd { class AGNode; @@ -71,6 +70,23 @@ class AGNodeEntry { class AutogradRuntime; } // namespace autograd +// enum for storage types +namespace csr { +enum CSRAuxType {kIndPtr, kIdx}; +} + +namespace rowsparse { +enum RowSparseAuxType {kIdx}; +} + +enum NDArrayStorageType { + kUndefinedStorage = -1, // undefined storage + kDefaultStorage, // dense + kRowSparseStorage, // row sparse + kCSRStorage, // csr +}; + + /*! * \brief ndarray interface */ @@ -91,10 +107,55 @@ class NDArray { */ NDArray(const TShape &shape, Context ctx, bool delay_alloc = false, int dtype = mshadow::default_type_flag) - : ptr_(std::make_shared(shape.Size(), ctx, delay_alloc, dtype)), + : ptr_(std::make_shared(shape, ctx, delay_alloc, dtype)), shape_(shape), dtype_(dtype), entry_({nullptr, 0, 0}) { #if MKL_EXPERIMENTAL == 1 Mkl_mem_ = std::make_shared(); +#endif + } + /*! \brief constructor for NDArray with storage type + */ + NDArray(const NDArrayStorageType stype, const TShape &shape, Context ctx, + bool delay_alloc = true, int dtype = mshadow::default_type_flag, + std::vector aux_types = {}, std::vector aux_shapes = {}, + TShape storage_shape = TShape(mshadow::Shape1(0))) + : shape_(shape), dtype_(dtype), entry_({nullptr, 0, 0}) { + // Assign default aux types if not given + if (aux_types.size() == 0) { + if (stype == kRowSparseStorage) { + aux_types = {mshadow::kInt64}; + } else if (stype == kCSRStorage) { + aux_types = {mshadow::kInt64, mshadow::kInt64}; + } else { + LOG(FATAL) << "Unknown storage type " << stype; + } + } + // Assign default shapes if not given + // unknown shapes are intialized as {0} such that Size() would return 0 + if (aux_shapes.size() == 0) { + if (stype == kRowSparseStorage) { + aux_shapes = {TShape(mshadow::Shape1(0))}; + } else if (stype == kCSRStorage) { + // aux shapes for indptr and indices + aux_shapes = {TShape(mshadow::Shape1(0)), TShape(mshadow::Shape1(0))}; + } else { + LOG(FATAL) << "Unknown storage type " << stype; + } + } + if (storage_shape.Size() == 0) { + if (stype == kRowSparseStorage) { + storage_shape = shape; + storage_shape[0] = aux_shapes[rowsparse::kIdx][0]; + } else if (stype == kCSRStorage) { + storage_shape = aux_shapes[csr::kIdx]; + } else { + LOG(FATAL) << "Unknown storage type " << stype; + } + } + ptr_ = std::make_shared(stype, storage_shape, ctx, delay_alloc, + dtype, aux_types, aux_shapes); +#if MKL_EXPERIMENTAL == 1 + Mkl_mem_ = std::make_shared(); #endif } /*! @@ -111,17 +172,86 @@ class NDArray { Mkl_mem_ = std::make_shared(); #endif } + /*! - * \return the shape of current NDArray + * \brief constructing a static NDArray of non-default storage that shares data with TBlob + * Use with caution: allocate ONLY ONE NDArray for each TBlob, + * make sure the memory region is available through out the life of NDArray + * \param stype the storage type of NDArray + * \param shape the shape of NDArray + * \param data the memory content of static data + * \param aux_data the memory content of static aux data + * \param dev_id the device id this tensor sits at + */ + NDArray(const NDArrayStorageType stype, const TShape &shape, + const TBlob &data, const std::vector &aux_data, int dev_id) + : ptr_(std::make_shared(stype, data, aux_data, dev_id)), shape_(shape), + dtype_(data.type_flag_), entry_({nullptr, 0, 0}) { +#if MKL_EXPERIMENTAL == 1 + Mkl_mem_ = std::make_shared(); +#endif + } + + + /*! + * \return the shape of current NDArray. */ inline const TShape& shape() const { return shape_; } + /*! + * \return the shape of underlying chunk which stores the NDArray data/value. + * It is only intended for non-default storage. For row-sparse storage, it is the shape of + * the tensor which stores the non-zero values. + */ + inline const TShape &storage_shape() const { + CHECK(ptr_ != nullptr); + CHECK_NE(storage_type(), kDefaultStorage) + << "storage_shape() is not intended for kDefaultStorage."; + return ptr_->storage_shape; + } + + /*! + * \brief get the shape of aux_data(index) + * \param index the index of the aux data + * \return the shape of aux data at given index + */ + inline const TShape& aux_shape(size_t index) const { + CHECK_NE(storage_type(), kDefaultStorage) + << "aux_shape() is not intended for kDefaultStorage."; + return ptr_->aux_shapes[index]; + } + + /* \return the shapes of all aux data */ + const std::vector& aux_shapes() const { + CHECK_NE(storage_type(), kDefaultStorage) + << "aux_shapes() is not intended for kDefaultStorage."; + return ptr_->aux_shapes; + } + + /*! returns the dtypes of all aux data */ + const std::vector& aux_types() const { + CHECK_NE(storage_type(), kDefaultStorage) + << "aux_types() is not intended for kDefaultStorage."; + return ptr_->aux_types; + } + + /*! + * \brief For a sparse operation on a csr matrix for example, + * the size of the column index array + * is an estimated value in the beginning for allocating enough capacity + * for the final result. After the operation is done, the exact size of + * the shape is known and need to be reset using this function. + */ + inline void set_aux_shape(size_t index, const TShape& shape) const { + ptr_->set_aux_shape(index, shape); + } + /*! * \return the data TBlob */ inline const TBlob& data() const { - CheckAndAlloc(); + if (storage_type() == kDefaultStorage) CheckAndAlloc(); SetTBlob(); return tblob_; } @@ -129,6 +259,26 @@ class NDArray { * \return the gradient ndarray. */ NDArray grad() const; + + /*! + * \return the aux TBlob + */ + inline TBlob aux_data(size_t i) const { + auto stype = storage_type(); + TBlob res; + auto shape = aux_shape(i); + auto type = aux_type(i); + MSHADOW_TYPE_SWITCH(type, DType, { + auto dptr = static_cast(ptr_->aux_handles[i].dptr); + CHECK(stype == kRowSparseStorage || stype == kCSRStorage) + << "Unexpected storage type: " << stype; + res = TBlob(dptr, shape, ptr_->aux_handles[i].ctx.dev_mask(), type); + }); +#if MKL_EXPERIMENTAL == 1 + res.Mkl_mem_ = Mkl_mem_; +#endif + return res; + } /*! * \return the context of NDArray, this function is only valid when the NDArray is not empty */ @@ -141,6 +291,15 @@ class NDArray { inline int dtype() const { return dtype_; } + inline int aux_type(size_t i) const { + CHECK(!is_none()); + return ptr_->aux_types[i]; + } + + inline NDArrayStorageType storage_type() const { + if (is_none()) return kUndefinedStorage; + return ptr_->storage_type; + } /*! \return whether this ndarray is not initialized */ inline bool is_none() const { return ptr_.get() == nullptr; @@ -149,6 +308,27 @@ class NDArray { bool fresh_out_grad() const; /*! \return updated grad state in entry_ */ void set_fresh_out_grad(bool state) const; + // returns true if a sparse ndarray's aux_data and storage are initialized + inline bool storage_initialized() const { + if (is_none()) return false; + auto stype = storage_type(); + CHECK_NE(stype, kDefaultStorage) + << "storage_initialized() is not intended for kDefaultStorage."; + if (stype == kRowSparseStorage) { + CHECK_EQ(aux_shape(rowsparse::kIdx)[0], storage_shape()[0]) + << "inconsistent storage shape " << storage_shape() + << " vs. aux shape " << aux_shape(rowsparse::kIdx); + return aux_shape(0).Size() != 0; + } else if (stype == kCSRStorage) { + CHECK_EQ(aux_shape(csr::kIdx)[0], storage_shape()[0]) + << "inconsistent storage shape " << storage_shape() + << " vs. aux shape " << aux_shape(csr::kIdx); + return aux_shape(0).Size() != 0; + } else { + LOG(FATAL) << "Unknown storage type"; + } + return true; + } /*! * \brief Block until all the pending write operations with respect * to current NDArray are finished, and read can be performed. @@ -179,6 +359,12 @@ class NDArray { * \param strm the output stream */ void Save(dmlc::Stream *strm) const; + /*! + * \brief load ndarrays before supporting sparse ndarrays + * \param strm the output stream + * \param magic the magic number used for version control + */ + bool LegacyLoad(dmlc::Stream *strm, const uint32_t magic); /*! * \brief load the content from binary stream * \param strm the output stream @@ -269,6 +455,12 @@ class NDArray { * \param size the size of the source array, in sizeof(DType) not raw btyes. */ void SyncCopyFromCPU(const void *data, size_t size) const; + + /*! + * \brief Copy from src.data()/aux_data(i) to this->data()/aux_data(j) + */ + void SyncCopyFromNDArray(const NDArray &src, int i = -1, int j = -1); + /*! * \brief Do a synchronize copy to a continugous CPU memory region. * @@ -282,17 +474,31 @@ class NDArray { void SyncCopyToCPU(void *data, size_t size) const; /*! * \brief Slice a NDArray - * \param begin begin index in first dim - * \param end end index in first dim + * \param begin begin index in first dim (inclusive) + * \param end end index in first dim (exclusive) * \return sliced NDArray */ NDArray Slice(index_t begin, index_t end) const; + /*! * \brief Index a NDArray * \param idx the index * \return idx-th sub array NDArray */ NDArray At(index_t idx) const; + + /*! + * \brief Generate a deep copy of aux_data(i) returned as + * a default storage type NDArray + */ + NDArray aux_ndarray(size_t i) const; + + /*! + * \brief Generate a deep copy of data() returned as a + * default storage type NDArray + */ + NDArray data_ndarray() const; + /*! * \brief Create a NDArray that shares memory with current one * The new array must have smaller memory size than the current array. @@ -301,6 +507,8 @@ class NDArray { * \return NDArray in new shape and type. */ inline NDArray AsArray(const TShape &shape, int dtype) const { + CHECK_EQ(storage_type(), kDefaultStorage) + << "AsArray is intended only for kDefaultStorage."; CHECK_GE(shape_.Size() * mshadow::mshadow_sizeof(dtype_), shape.Size() * mshadow::mshadow_sizeof(dtype)) << "NDArray.AsArray: target memory size is bigger"; @@ -342,8 +550,45 @@ class NDArray { * This is an internal function used by system that normal user should not use */ inline void CheckAndAlloc() const { + CHECK_EQ(storage_type(), kDefaultStorage); ptr_->CheckAndAlloc(); } + + /*! + * \brief Allocate the space if the allocation has been delayed + * or the requested size is bigger than the available one. + * This function can only be called by ndarray of default + * storage type and effectively changes the ndarray's shape_. + * Note: This function is named as this to avoid overload conflict + * with CheckAndAlloc(const std::vector &aux_shapes), since + * TShape tmp = some_shape is equivalent to TShape tmp = {some_shape}. + */ + void ReshapeAndAlloc(const TShape& shape) { + CHECK_EQ(storage_type(), kDefaultStorage); + CHECK(!is_none()); + shape_ = shape; + ptr_->CheckAndAlloc(shape.Size() * mshadow::mshadow_sizeof(dtype_)); + } + + /* ! + * \brief Alloc memory for non-default storage + * aux_shape is only known at run time + */ + inline void CheckAndAlloc(const std::vector &aux_shapes) const { + CHECK_NE(storage_type(), kDefaultStorage) + << "CheckAndAlloc(aux_shapes) is not intended for kDefaultStorage"; + ptr_->CheckAndAlloc(shape_, aux_shapes, dtype_); + } + inline void CheckAndAllocData(const TShape &storage_shape) const { + CHECK_NE(storage_type(), kDefaultStorage) + << "CheckAndAllocData is not intended for kDefaultStorage"; + ptr_->CheckAndAllocData(storage_shape, dtype_); + } + inline void CheckAndAllocAuxData(size_t i, const TShape &aux_shape) const { + CHECK_NE(storage_type(), kDefaultStorage) + << "CheckAndAllocAuxData is not intended for kDefaultStorage"; + ptr_->CheckAndAllocAuxData(i, aux_shape); + } /*! * \brief Save list of ndarray into the Stream.x * \param fo The stream of output. @@ -366,44 +611,138 @@ class NDArray { private: friend class autograd::AutogradRuntime; /*! \brief the real data chunk that backs NDArray */ + // shandle is used to store the actual values in the NDArray + // aux_handles store the aux data(such as indices) if it's needed by non-default storage. struct Chunk { - /*! \brief storage handlefrom storage engine */ + /*! \brief storage handle from storage engine. + for non-default storage, shandle stores the data(value) array. + */ Storage::Handle shandle; + /*! \brief storage handles for aux data (e.g index) + for row_sparse, aux_handles[0] = indices + for csr, aux_handles[0] = indptr, aux_handles[1] = indices + */ + std::vector aux_handles; /*! \brief variable from engine */ Engine::VarHandle var; /*! * \brief if this is true, this means the data do not come * from Storage, and do not need to be freed */ + /*! \brief construct from static data */ bool static_data; - /*! \brief whether allocation is delayed */ + /*! \brief whether data allocation is delayed. This doesn't indicate whether aux data + allocation is delayed. */ bool delay_alloc; + // the type of the storage. The storage_type is never kUndefinedStorage once the chunk + // is constructed. + NDArrayStorageType storage_type = kDefaultStorage; + /*! \brief type of aux */ + std::vector aux_types; + // context of data + Context ctx; + // The shape of the chunk data. + // This might not be the same shape as the NDArray, since the storage may be sparse. + // The default value for storage_shape is {0} when an empty non-default NDArray is created. + TShape storage_shape; + // The shape of aux data. The default value for the shape depends on the type of storage. + // If aux_shapes[i].Size() is zero, aux data i is empty. + std::vector aux_shapes; + /*! \brief default cosntructor */ - Chunk() : static_data(true), delay_alloc(false) { - var = Engine::Get()->NewVariable(); + Chunk() : static_data(true), delay_alloc(false) {} + + /*! \brief construct a new chunk */ + Chunk(TShape shape, Context ctx_, bool delay_alloc_, int dtype) + : static_data(false), delay_alloc(true), ctx(ctx_) { + auto size = shape.Size(); + storage_shape = shape; + var = Engine::Get()->NewVariable(); + shandle.size = size * mshadow::mshadow_sizeof(dtype); + shandle.ctx = ctx_; + if (!delay_alloc_) this->CheckAndAlloc(); } - /*! \brief construct from static data */ + Chunk(const TBlob &data, int dev_id) - : static_data(true), - delay_alloc(false) { + : static_data(true), delay_alloc(false) { + CHECK(storage_type == kDefaultStorage); var = Engine::Get()->NewVariable(); if (data.dev_mask() == cpu::kDevMask) { - shandle.ctx = Context::CPU(); + ctx = Context::CPU(); } else { CHECK_EQ(data.dev_mask(), gpu::kDevMask); - shandle.ctx = Context::GPU(dev_id); + ctx = Context::GPU(dev_id); } + // init shandle + shandle.ctx = ctx; shandle.dptr = data.dptr_; shandle.size = data.shape_.Size() * mshadow::mshadow_sizeof(data.type_flag_); + storage_shape = data.shape_; } - /*! \brief construct a new chunk */ - Chunk(uint64_t size, Context ctx, bool delay_alloc_, int dtype) - : static_data(false), delay_alloc(true) { + // Constructor for a non-default storage chunk + Chunk(NDArrayStorageType storage_type_, const TShape &storage_shape_, Context ctx_, + bool delay_alloc_, int dtype, const std::vector &aux_types_, + const std::vector &aux_shapes_) + : static_data(false), delay_alloc(delay_alloc_), storage_type(storage_type_), + aux_types(aux_types_), ctx(ctx_), storage_shape(storage_shape_), + aux_shapes(aux_shapes_) { + shandle.ctx = ctx; var = Engine::Get()->NewVariable(); - shandle.size = size * mshadow::mshadow_sizeof(dtype); + // aux_handles always reflect the correct number of aux data + for (size_t i = 0; i < aux_shapes.size(); i++) { + CheckAndAllocAuxData(i, aux_shapes[i]); + // this line is needed in case when aux_shapes[i].Size() = 0 + // aux_handles[i] will not be updated and take only default value. + aux_handles[i].ctx = ctx; + } + if (!delay_alloc) { + CheckAndAllocData(storage_shape, dtype); + } + } + + Chunk(const NDArrayStorageType storage_type_, const TBlob &data, + const std::vector &aux_data, int dev_id) + : static_data(true), delay_alloc(false), storage_type(storage_type_) { + using namespace mshadow; + CHECK_NE(storage_type, kDefaultStorage); + // init var + var = Engine::Get()->NewVariable(); + // init ctx + if (data.dev_mask() == cpu::kDevMask) { + ctx = Context::CPU(); + } else { + CHECK_EQ(data.dev_mask(), gpu::kDevMask); + ctx = Context::GPU(dev_id); + } + // init shandle shandle.ctx = ctx; - if (!delay_alloc_) this->CheckAndAlloc(); + shandle.dptr = data.dptr_; + shandle.size = data.shape_.Size() * mshadow_sizeof(data.type_flag_); + storage_shape = data.shape_; + // init aux handles + for (const auto &aux : aux_data) { + Storage::Handle aux_handle; + aux_handle.ctx = ctx; + aux_handle.dptr = aux.dptr_; + aux_handle.size = aux.shape_.Size() * mshadow_sizeof(aux.type_flag_); + aux_handles.push_back(aux_handle); + aux_types.emplace_back(aux.type_flag_); + aux_shapes.emplace_back(aux.shape_); + } + } + + /*! \brief set the shape for ith aux data, and update storage shape if necessary */ + inline void set_aux_shape(const size_t i, const TShape& shape) { + aux_shapes[i] = shape; + if (storage_shape.ndim() > 0) { + if (storage_type == kRowSparseStorage && i == rowsparse::kIdx) { + storage_shape[0] = shape[0]; + } else if (storage_type == kCSRStorage && i == csr::kIdx) { + storage_shape[0] = shape[0]; + } + } } + /*! \brief check if delay alloc is on, do alloc if not yet done */ inline void CheckAndAlloc(void) { if (delay_alloc) { @@ -411,22 +750,113 @@ class NDArray { delay_alloc = false; } } - /*! \brief destructor */ - ~Chunk() { - if (static_data || delay_alloc) { - Engine::Get()->DeleteVariable([](RunContext s) {}, shandle.ctx, var); + + /*! \brief Check and alloc memory for a dense ndarray */ + // size is the number of bytes + void CheckAndAlloc(uint64_t dbytes) { + CHECK_EQ(kDefaultStorage, storage_type) + << "CheckAndAlloc(dbytes) is not intended for kDefaultStorage"; + if (delay_alloc) { + shandle = Storage::Get()->Alloc(dbytes, shandle.ctx); + delay_alloc = false; + } else if (shandle.size < dbytes) { + // free storage if necessary and alloc again + if (shandle.size > 0) Storage::Get()->Free(shandle); + // init storage + shandle = Storage::Get()->Alloc(dbytes, shandle.ctx); + } + } + + inline void CheckAndAlloc(const TShape &shape, const std::vector &aux_shapes, + int dtype) { + // calculate size, perform allocation + if (kRowSparseStorage == storage_type) { + // For row sparse, aux_shape indicates the number of rows to allocate + auto aux_shape = aux_shapes[rowsparse::kIdx]; + CheckAndAllocAuxData(rowsparse::kIdx, aux_shape); + TShape storage_shape(shape); + storage_shape[0] = aux_shape[0]; + CheckAndAllocData(storage_shape, dtype); + } else if (kCSRStorage == storage_type) { + CheckAndAllocAuxData(csr::kIndPtr, aux_shapes[csr::kIndPtr]); + CheckAndAllocAuxData(csr::kIdx, aux_shapes[csr::kIdx]); + CheckAndAllocData(aux_shapes[csr::kIdx], dtype); } else { - Storage::Handle h = this->shandle; - Engine::Get()->DeleteVariable([h](RunContext s) { - Storage::Get()->Free(h); - }, shandle.ctx, var); + LOG(FATAL) << "Storage type " << storage_type << " not implemented for CheckAndAlloc"; + } + } + // create storage handle for data based on shape and dtype, assuming ctx is set + // storage shape is also updated + // if data is already allocated, try reuse the storage. Otherwise, free the current one + // and allocate new storage + inline void CheckAndAllocData(const TShape &shape, int dtype) { + CHECK_NE(aux_shapes.size(), 0) << "data is expected to be allocated after aux_data"; + auto dbytes = shape.Size() * mshadow::mshadow_sizeof(dtype); + if (shandle.size < dbytes) { + // free storage if necessary and alloc again + if (shandle.size > 0) Storage::Get()->Free(shandle); + // init storage + shandle = Storage::Get()->Alloc(dbytes, ctx); } + // init shape + storage_shape = shape; + // delay_alloc is only set when data storage handle is present + delay_alloc = false; + } + // create storage handle for aux data based on shape + // this function assumes ctx, aux shapes and aux types are set + // aux shape is also updated + // if aux data is already allocated, try reuse the storage. Otherwise, free the current one + // and allocate new storage + inline void CheckAndAllocAuxData(size_t i, const TShape &shape) { + CHECK_EQ(shape.ndim(), 1) << "shape must be 1D in CheckAndAllocAuxData"; + CHECK_NE(storage_type, kUndefinedStorage) + << "storage type cannot be kUndefinedStorage in CheckAndAllocAuxData"; + CHECK_NE(storage_type, kDefaultStorage) + << "storage type cannot be kDefaultStorage in CheckAndAllocAuxData"; + if (aux_handles.size() <= i) { + aux_handles.resize(i + 1); + } + size_t aux_bytes = shape.Size() * mshadow::mshadow_sizeof(aux_types[i]); + if (aux_handles[i].size < aux_bytes) { + // free storage if necessary and alloc again + if (aux_handles[i].size > 0) Storage::Get()->Free(aux_handles[i]); + // init aux storage + aux_handles[i] = Storage::Get()->Alloc(aux_bytes, ctx); + } + // init shape + set_aux_shape(i, shape); + } + /*! \brief destructor */ + ~Chunk() { + bool skip_free = static_data || delay_alloc; + Storage::Handle h = this->shandle; + std::vector aux_h = this->aux_handles; + Engine::Get()->DeleteVariable([h, aux_h, skip_free](RunContext s) { + if (skip_free == false) { + Storage::Get()->Free(h); + for (size_t i = 0; i < aux_h.size(); i++) { + if (aux_h[i].size > 0) Storage::Get()->Free(aux_h[i]); + } + } + }, shandle.ctx, var); } - }; + }; // struct Chunk void SetTBlob() const { - tblob_.dptr_ = static_cast(ptr_->shandle.dptr) + byte_offset_; - tblob_.shape_ = shape_; + CHECK(ptr_ != nullptr); + TShape shape = shape_; + char *dptr = static_cast(ptr_->shandle.dptr); + auto stype = storage_type(); + if (stype == kDefaultStorage) { + dptr += byte_offset_; + } else if (stype == kCSRStorage || stype == kRowSparseStorage) { + shape = storage_shape(); + } else { + LOG(FATAL) << "unknown storage type " << stype; + } + tblob_.dptr_ = dptr; + tblob_.shape_ = shape; tblob_.type_flag_ = dtype_; tblob_.SetDLTensor(ptr_->shandle.ctx.dev_mask(), ptr_->shandle.ctx.dev_id); #if MKL_EXPERIMENTAL == 1 @@ -438,7 +868,7 @@ class NDArray { std::shared_ptr Mkl_mem_; #endif /*! \brief internal data of NDArray */ - std::shared_ptr ptr_; + std::shared_ptr ptr_{nullptr}; /*! \brief shape of current NDArray */ TShape shape_; /*! \brief byte offset in chunk */ @@ -455,7 +885,12 @@ class NDArray { * this situation. */ mutable TBlob tblob_; -}; +}; // class NDArray + +/*! + * \return the number of aux data used for given storage type + */ +size_t num_aux_data(NDArrayStorageType stype); /*! * \brief issue an copy operation from one NDArray to another @@ -470,7 +905,6 @@ class NDArray { */ void CopyFromTo(const NDArray &from, NDArray *to, int priority = 0); - /*! * \brief Perform elementwise sum over each data from source, store result into out. * \param source the ndarray we want to sum diff --git a/include/mxnet/op_attr_types.h b/include/mxnet/op_attr_types.h index 1bcae0d29348..f559a921c522 100644 --- a/include/mxnet/op_attr_types.h +++ b/include/mxnet/op_attr_types.h @@ -25,7 +25,6 @@ #ifndef MXNET_OP_ATTR_TYPES_H_ #define MXNET_OP_ATTR_TYPES_H_ - #include #include @@ -226,6 +225,23 @@ using FCompute = std::function& inputs, const std::vector& req, const std::vector& outputs)>; +/*! + * \brief Resiger an NDArray compute function for simple stateless forward only operator + * + * \note Register under "FComputeEx" and "FComputeEx" + * Dispatched only when operators process non-default storage inputs or outputs + */ +using FComputeEx = std::function& inputs, + const std::vector& req, + const std::vector& outputs)>; + +using FInferStorageType = std::function* in_attrs, + std::vector* out_attrs)>; + } // namespace mxnet #endif // MXNET_OP_ATTR_TYPES_H_ diff --git a/include/mxnet/storage.h b/include/mxnet/storage.h index bfb42de8771a..7e3af8eeca81 100644 --- a/include/mxnet/storage.h +++ b/include/mxnet/storage.h @@ -41,11 +41,11 @@ class Storage { /*! * \brief Pointer to the data. */ - void* dptr; + void* dptr{nullptr}; /*! * \brief Size of the storage. */ - size_t size; + size_t size{0}; /*! * \brief Context information about device and ID. */ diff --git a/perl-package/AI-MXNetCAPI/mxnet.i b/perl-package/AI-MXNetCAPI/mxnet.i index fd1a471bcf16..b4c1336de624 100644 --- a/perl-package/AI-MXNetCAPI/mxnet.i +++ b/perl-package/AI-MXNetCAPI/mxnet.i @@ -1203,6 +1203,12 @@ int MXExecutorSimpleBind(SymbolHandle symbol_handle, const mx_uint num_provided_arg_dtypes, const char** in, // provided_arg_dtype_names, const int* in, // provided_arg_dtypes, + +//--------------- sparse related variables, ignored for now + const mx_uint num_provided_arg_stypes, + const char** provided_arg_stype_names, + const int* provided_arg_stypes, +//--------------- const mx_uint num_shared_arg_names, const char** in, // shared_arg_name_list, //------------ diff --git a/perl-package/AI-MXNetCAPI/mxnet_typemaps.i b/perl-package/AI-MXNetCAPI/mxnet_typemaps.i index 640215fd7792..5d2fbd6880a1 100644 --- a/perl-package/AI-MXNetCAPI/mxnet_typemaps.i +++ b/perl-package/AI-MXNetCAPI/mxnet_typemaps.i @@ -820,6 +820,17 @@ } } +%typemap(in,numinputs=0) (const mx_uint num_provided_arg_stypes, const char** provided_arg_stype_names, + const int* provided_arg_stypes) + (mx_uint temp1, char* temp2, int temp3) +{ + $2 = &temp2; + $3 = &temp3; + $1 = 0; + *$2 = NULL; + *$3 = 0; +} + %typemap(in,numinputs=0) (mx_uint* num_aux_states, NDArrayHandle** aux_states) (mx_uint temp1, diff --git a/python/mxnet/__init__.py b/python/mxnet/__init__.py index 3c3ce76a9284..72dc2b2fec8d 100644 --- a/python/mxnet/__init__.py +++ b/python/mxnet/__init__.py @@ -26,6 +26,7 @@ from . import base from . import contrib from . import ndarray +from . import ndarray as nd from . import name # use mx.sym as short for symbol from . import symbol as sym @@ -34,8 +35,6 @@ from . import io from . import recordio from . import operator -# use mx.nd as short for mx.ndarray -from . import ndarray as nd # use mx.rnd as short for mx.random from . import random as rnd from . import random diff --git a/python/mxnet/_ctypes/ndarray.py b/python/mxnet/_ctypes/ndarray.py index 5a50f80498ec..c2e6fce40de8 100644 --- a/python/mxnet/_ctypes/ndarray.py +++ b/python/mxnet/_ctypes/ndarray.py @@ -32,10 +32,19 @@ from ..ndarray_doc import _build_doc +_STORAGE_TYPE_ID_TO_STR = { + -1 : 'undefined', + 0 : 'default', + 1 : 'row_sparse', + 2 : 'csr', +} + + class NDArrayBase(object): """Base data structure for ndarray""" __slots__ = ["handle", "writable"] # pylint: disable= no-member + def __init__(self, handle, writable=True): """initialize a new NDArray @@ -78,7 +87,11 @@ def _imperative_invoke(handle, ndargs, keys, vals, out): output_vars = ctypes.POINTER(NDArrayHandle)() num_output = ctypes.c_int(0) - check_call(_LIB.MXImperativeInvoke( + # return output stypes to avoid the c_api call for checking + # a handle's stype in _ndarray_cls + out_stypes = ctypes.POINTER(ctypes.c_int)() + + check_call(_LIB.MXImperativeInvokeEx( ctypes.c_void_p(handle), ctypes.c_int(len(ndargs)), c_array(NDArrayHandle, [arr.handle for arr in ndargs]), @@ -86,14 +99,17 @@ def _imperative_invoke(handle, ndargs, keys, vals, out): ctypes.byref(output_vars), ctypes.c_int(len(keys)), c_array(ctypes.c_char_p, [c_str(key) for key in keys]), - c_array(ctypes.c_char_p, [c_str(str(val)) for val in vals]))) + c_array(ctypes.c_char_p, [c_str(str(val)) for val in vals]), + ctypes.byref(out_stypes))) if original_output is not None: return original_output if num_output.value == 1: - return _ndarray_cls(ctypes.cast(output_vars[0], NDArrayHandle)) + return _ndarray_cls(ctypes.cast(output_vars[0], NDArrayHandle), + stype=_STORAGE_TYPE_ID_TO_STR[out_stypes[0]]) else: - return [_ndarray_cls(ctypes.cast(output_vars[i], NDArrayHandle)) + return [_ndarray_cls(ctypes.cast(output_vars[i], NDArrayHandle), + stype=_STORAGE_TYPE_ID_TO_STR[out_stypes[i]]) for i in range(num_output.value)] @@ -128,17 +144,24 @@ def __call__(self, *args, **kwargs): "CachedOp.__call__ got unexpected keyword argument(s): " + \ ', '.join(kwargs.keys())) - check_call(_LIB.MXInvokeCachedOp( + # return output stypes to avoid the c_api call for checking + # a handle's stype in _ndarray_cls + out_stypes = ctypes.POINTER(ctypes.c_int)() + + check_call(_LIB.MXInvokeCachedOpEx( self.handle, ctypes.c_int(len(args)), c_array(NDArrayHandle, [arr.handle for arr in args]), ctypes.byref(num_output), - ctypes.byref(output_vars))) + ctypes.byref(output_vars), + ctypes.byref(out_stypes))) if original_output is not None: return original_output if num_output.value == 1: - return _ndarray_cls(ctypes.cast(output_vars[0], NDArrayHandle)) + return _ndarray_cls(ctypes.cast(output_vars[0], NDArrayHandle), + stype=_STORAGE_TYPE_ID_TO_STR[out_stypes[0]]) else: - return [_ndarray_cls(ctypes.cast(output_vars[i], NDArrayHandle)) + return [_ndarray_cls(ctypes.cast(output_vars[i], NDArrayHandle), + stype=_STORAGE_TYPE_ID_TO_STR[out_stypes[i]]) for i in range(num_output.value)] diff --git a/python/mxnet/base.py b/python/mxnet/base.py index aad0580e7d07..d446355da0b5 100644 --- a/python/mxnet/base.py +++ b/python/mxnet/base.py @@ -72,6 +72,20 @@ def __str__(self): msg += ' is not implemented for Symbol and only available in NDArray.' return msg +class NotSupportedForSparseNDArray(MXNetError): + def __init__(self, function, alias, *args): + super(NotSupportedForSparseNDArray, self).__init__() + self.function = function.__name__ + self.alias = alias + self.args = [str(type(a)) for a in args] + def __str__(self): + msg = 'Function {}'.format(self.function) + if self.alias: + msg += ' (namely operator "{}")'.format(self.alias) + if self.args: + msg += ' with arguments ({})'.format(', '.join(self.args)) + msg += ' is not supported for SparseNDArray and only available in NDArray.' + return msg class MXCallbackList(ctypes.Structure): """Structure that holds Callback information. Passed to CustomOpProp.""" diff --git a/python/mxnet/contrib/autograd.py b/python/mxnet/contrib/autograd.py index c7fb6e17803a..2d2500e7a217 100644 --- a/python/mxnet/contrib/autograd.py +++ b/python/mxnet/contrib/autograd.py @@ -24,6 +24,7 @@ import functools from ..base import _LIB, check_call, string_types from ..base import mx_uint, NDArrayHandle, c_array +# pylint: disable= unused-import from ..ndarray import NDArray, zeros_like from ..symbol import _GRAD_REQ_MAP diff --git a/python/mxnet/executor.py b/python/mxnet/executor.py index baff834bb33a..5cc94a5e80ac 100644 --- a/python/mxnet/executor.py +++ b/python/mxnet/executor.py @@ -27,6 +27,7 @@ from .base import mx_uint, NDArrayHandle, ExecutorHandle from .base import check_call, c_array, py_str from .ndarray import NDArray +from .ndarray import _ndarray_cls from . import ndarray as nd # those functions are not used here, we just import them to keep backward compatibility @@ -105,7 +106,9 @@ def _get_outputs(self): handles = ctypes.POINTER(NDArrayHandle)() check_call(_LIB.MXExecutorOutputs(self.handle, ctypes.byref(out_size), ctypes.byref(handles))) - return [NDArray(NDArrayHandle(handles[i])) for i in range(out_size.value)] + num_output = out_size.value + outputs = [_ndarray_cls(NDArrayHandle(handles[i])) for i in range(num_output)] + return outputs def forward(self, is_train=False, **kwargs): """Calculate the outputs specified by the bound symbol. diff --git a/python/mxnet/image/detection.py b/python/mxnet/image/detection.py index 8ac1aebe72dd..f67b05de5de3 100644 --- a/python/mxnet/image/detection.py +++ b/python/mxnet/image/detection.py @@ -27,7 +27,7 @@ from ..base import numeric_types from .. import ndarray as nd -from .._ndarray_internal import _cvcopyMakeBorder as copyMakeBorder +from ..ndarray._internal import _cvcopyMakeBorder as copyMakeBorder from .. import io from .image import RandomOrderAug, ColorJitterAug, LightingAug, ColorNormalizeAug from .image import ResizeAug, ForceResizeAug, CastAug, HueJitterAug, RandomGrayAug diff --git a/python/mxnet/image/image.py b/python/mxnet/image/image.py index 2e40019971ac..d99db214222c 100644 --- a/python/mxnet/image/image.py +++ b/python/mxnet/image/image.py @@ -34,9 +34,9 @@ from ..base import numeric_types from .. import ndarray as nd -from .. import _ndarray_internal as _internal -from .._ndarray_internal import _cvimresize as imresize -from .._ndarray_internal import _cvcopyMakeBorder as copyMakeBorder +from ..ndarray import _internal +from ..ndarray._internal import _cvimresize as imresize +from ..ndarray._internal import _cvcopyMakeBorder as copyMakeBorder from .. import io from .. import recordio diff --git a/python/mxnet/io.py b/python/mxnet/io.py index 0404e34ea36c..4e69a8a801cb 100644 --- a/python/mxnet/io.py +++ b/python/mxnet/io.py @@ -34,6 +34,7 @@ from .base import mx_real_t from .base import check_call, build_param_doc as _build_param_doc from .ndarray import NDArray +from .ndarray import _ndarray_cls from .ndarray import array from .ndarray import concatenate @@ -801,12 +802,12 @@ def iter_next(self): def getdata(self): hdl = NDArrayHandle() check_call(_LIB.MXDataIterGetData(self.handle, ctypes.byref(hdl))) - return NDArray(hdl, False) + return _ndarray_cls(hdl, False) def getlabel(self): hdl = NDArrayHandle() check_call(_LIB.MXDataIterGetLabel(self.handle, ctypes.byref(hdl))) - return NDArray(hdl, False) + return _ndarray_cls(hdl, False) def getindex(self): index_size = ctypes.c_uint64(0) diff --git a/python/mxnet/kvstore.py b/python/mxnet/kvstore.py index fd0091182aea..2af70e36e60a 100644 --- a/python/mxnet/kvstore.py +++ b/python/mxnet/kvstore.py @@ -22,6 +22,7 @@ import ctypes import pickle from .ndarray import NDArray +from .ndarray import _ndarray_cls from .base import _LIB from .base import check_call, c_array, c_str, string_types, mx_uint, py_str from .base import NDArrayHandle, KVStoreHandle @@ -53,8 +54,8 @@ def _updater_wrapper(updater): """A wrapper for the user-defined handle.""" def updater_handle(key, lhs_handle, rhs_handle, _): """ ctypes function """ - lhs = NDArray(NDArrayHandle(lhs_handle)) - rhs = NDArray(NDArrayHandle(rhs_handle)) + lhs = _ndarray_cls(NDArrayHandle(lhs_handle)) + rhs = _ndarray_cls(NDArrayHandle(rhs_handle)) updater(key, lhs, rhs) return updater_handle @@ -186,6 +187,8 @@ def pull(self, key, out=None, priority=0): The returned values are gauranteed to be the latest values in the store. + For row_sparse values, please use `row_sparse_pull` instead. + Parameters ---------- key : int or list of int @@ -236,6 +239,66 @@ def pull(self, key, out=None, priority=0): self.handle, mx_uint(len(ckeys)), ckeys, cvals, ctypes.c_int(priority))) + def row_sparse_pull(self, key, out=None, priority=0, row_ids=None): + """ Pulls a single row_sparse value or a sequence of row_sparse values from the store + with specified row_ids. + + `row_sparse_pull` is executed asynchronously after all previous + `push`/`pull`/`row_sparse_pull` calls for the same input key(s) are finished. + + The returned values are guaranteed to be the latest values in the store. + + Parameters + ---------- + key : str or list of str + Keys. + + out: NDArray or list of NDArray or list of list of NDArray + Values corresponding to the keys. The stype is expected to be row_sparse + + priority : int, optional + The priority of the pull operation. + Higher priority pull operations are likely to be executed before + other pull actions. + + row_ids : NDArray or list of NDArray + The row_ids for which to pull for each value. Each row_id is an 1D-NDArray \ + whose values don't have to be unique nor sorted. + + Examples + -------- + >>> shape = (3, 3) + >>> kv.init('3', mx.nd.ones(shape).tostype('row_sparse')) + >>> a = mx.nd.zeros(shape, stype='row_sparse') + >>> row_ids = mx.nd.array([0, 2], dtype='int64') + >>> kv.row_sparse_pull('3', out=a, row_ids=row_ids) + >>> print a.asnumpy() + [[ 1. 1. 1.] + [ 0. 0. 0.] + [ 1. 1. 1.]] + >>> duplicate_row_ids = mx.nd.array([2, 2], dtype='int64') + >>> kv.row_sparse_pull('3', out=a, row_ids=duplicate_row_ids) + >>> print a.asnumpy() + [[ 0. 0. 0.] + [ 0. 0. 0.] + [ 1. 1. 1.]] + >>> unsorted_row_ids = mx.nd.array([1, 0], dtype='int64') + >>> kv.row_sparse_pull('3', out=a, row_ids=unsorted_row_ids) + >>> print a.asnumpy() + [[ 1. 1. 1.] + [ 1. 1. 1.] + [ 0. 0. 0.]] + """ + assert(out is not None) + assert(row_ids is not None) + ckeys, cvals = _ctype_key_value(key, out) + _, crow_ids = _ctype_key_value(key, row_ids) + assert(len(crow_ids) == len(cvals)), "number of row_ids doesn't match number of values" + + check_call(_LIB.MXKVStorePullRowSparse( + self.handle, mx_uint(len(ckeys)), ckeys, cvals, crow_ids, ctypes.c_int(priority))) + + def set_optimizer(self, optimizer): """ Registers an optimizer with the kvstore. diff --git a/python/mxnet/model.py b/python/mxnet/model.py index 01b3fa50e18f..2444ca0dc59e 100644 --- a/python/mxnet/model.py +++ b/python/mxnet/model.py @@ -93,8 +93,7 @@ def _create_kvstore(kvstore, num_device, arg_params): return (kv, update_on_kvstore) -def _initialize_kvstore(kvstore, param_arrays, arg_params, param_names, - update_on_kvstore): +def _initialize_kvstore(kvstore, param_arrays, arg_params, param_names, update_on_kvstore): """Initialize kvstore""" for idx, param_on_devs in enumerate(param_arrays): name = param_names[idx] @@ -118,10 +117,11 @@ def _update_params_on_kvstore(param_arrays, grad_arrays, kvstore, param_names): def _update_params(param_arrays, grad_arrays, updater, num_device, kvstore=None, param_names=None): """Perform update of param_arrays from grad_arrays not on kvstore.""" - for index, pair in enumerate(zip(param_arrays, grad_arrays)): + for i, pair in enumerate(zip(param_arrays, grad_arrays)): arg_list, grad_list = pair if grad_list[0] is None: continue + index = i if kvstore: name = param_names[index] # push gradient, priority is negative index @@ -131,7 +131,7 @@ def _update_params(param_arrays, grad_arrays, updater, num_device, for k, p in enumerate(zip(arg_list, grad_list)): # faked an index here, to make optimizer create diff # state for the same index but on diff devs, TODO(mli) - # use a better solution latter + # use a better solution later w, g = p updater(index*num_device+k, g, w) diff --git a/python/mxnet/module/base_module.py b/python/mxnet/module/base_module.py index 3123462f9c7c..bae166e3ffd8 100644 --- a/python/mxnet/module/base_module.py +++ b/python/mxnet/module/base_module.py @@ -957,7 +957,8 @@ def bind(self, data_shapes, label_shapes=None, for_training=True, def init_optimizer(self, kvstore='local', optimizer='sgd', optimizer_params=(('learning_rate', 0.01),), force_init=False): - """Installs and initializes optimizers. + """Installs and initializes optimizers, as well as initialize kvstore for + distributed training Parameters ---------- diff --git a/python/mxnet/module/module.py b/python/mxnet/module/module.py index 058edd57eb3d..d55b2117ebd3 100644 --- a/python/mxnet/module/module.py +++ b/python/mxnet/module/module.py @@ -25,7 +25,6 @@ import warnings from .. import context as ctx -from .. import ndarray as nd from .. import optimizer as opt from .executor_group import DataParallelExecutorGroup @@ -33,6 +32,7 @@ from ..model import load_checkpoint from ..initializer import Uniform, InitDesc from ..io import DataDesc +from ..ndarray import zeros from .base_module import BaseModule, _check_input_names, _parse_data_desc @@ -427,13 +427,13 @@ def bind(self, data_shapes, label_shapes=None, for_training=True, else: assert self._arg_params is None and self._aux_params is None param_arrays = [ - nd.zeros(x[0].shape, dtype=x[0].dtype) + zeros(shape=x[0].shape, dtype=x[0].dtype, stype=x[0].stype) for x in self._exec_group.param_arrays ] self._arg_params = {name:arr for name, arr in zip(self._param_names, param_arrays)} aux_arrays = [ - nd.zeros(x[0].shape, dtype=x[0].dtype) + zeros(x[0].shape, dtype=x[0].dtype) for x in self._exec_group.aux_arrays ] self._aux_params = {name:arr for name, arr in zip(self._aux_names, aux_arrays)} @@ -441,7 +441,6 @@ def bind(self, data_shapes, label_shapes=None, for_training=True, if shared_module is not None and shared_module.optimizer_initialized: self.borrow_optimizer(shared_module) - def reshape(self, data_shapes, label_shapes=None): """Reshapes the module for new input shapes. @@ -483,6 +482,7 @@ def init_optimizer(self, kvstore='local', optimizer='sgd', if self._params_dirty: self._sync_params_from_devices() + (kvstore, update_on_kvstore) = \ _create_kvstore(kvstore, len(self._context), self._arg_params) diff --git a/python/mxnet/ndarray/__init__.py b/python/mxnet/ndarray/__init__.py new file mode 100644 index 000000000000..63220787a43c --- /dev/null +++ b/python/mxnet/ndarray/__init__.py @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""NDArray API of MXNet.""" + +from . import _internal, sparse, op +from .op import CachedOp +# pylint: disable=wildcard-import, redefined-builtin +from .ndarray import * +from .utils import load, save, zeros, empty, array +from .sparse import _ndarray_cls diff --git a/python/mxnet/_ndarray_internal.py b/python/mxnet/ndarray/_internal.py similarity index 100% rename from python/mxnet/_ndarray_internal.py rename to python/mxnet/ndarray/_internal.py diff --git a/python/mxnet/ndarray.py b/python/mxnet/ndarray/ndarray.py similarity index 87% rename from python/mxnet/ndarray.py rename to python/mxnet/ndarray/ndarray.py index 42f0ff5e87cf..20ca2262f0cd 100644 --- a/python/mxnet/ndarray.py +++ b/python/mxnet/ndarray/ndarray.py @@ -21,6 +21,7 @@ """NDArray API of MXNet.""" from __future__ import absolute_import from __future__ import division + try: from __builtin__ import slice as py_slice except ImportError: @@ -28,40 +29,25 @@ import ctypes import warnings - -import os as _os -import sys as _sys - import operator import numpy as np -from .base import _LIB, string_types, numeric_types, integer_types -from .base import c_array, py_str, c_str, mx_real_t, _Null # pylint: disable=unused-import -from .base import mx_uint, NDArrayHandle, check_call, OpHandle -from .base import ctypes2buffer -from .context import Context -from . import _ndarray_internal as _internal -from .ndarray_doc import _build_doc - - -# Use different version of SymbolBase -# When possible, use cython to speedup part of computation. -# pylint: disable=unused-import -try: - if int(_os.environ.get("MXNET_ENABLE_CYTHON", True)) == 0: - from ._ctypes.ndarray import NDArrayBase, _set_ndarray_class - from ._ctypes.ndarray import CachedOp, _imperative_invoke - elif _sys.version_info >= (3, 0): - from ._cy3.ndarray import NDArrayBase, _set_ndarray_class, _imperative_invoke - from ._cy3.ndarray import CachedOp, _imperative_invoke - else: - from ._cy2.ndarray import NDArrayBase, _set_ndarray_class, _imperative_invoke - from ._cy2.ndarray import CachedOp, _imperative_invoke -except ImportError: - if int(_os.environ.get("MXNET_ENFORCE_CYTHON", False)) != 0: - raise ImportError("Cython Module cannot be loaded but MXNET_ENFORCE_CYTHON=1") - from ._ctypes.ndarray import NDArrayBase, _set_ndarray_class, _imperative_invoke - from ._ctypes.ndarray import CachedOp, _imperative_invoke -# pylint: enable=unused-import +from ..base import _LIB, numeric_types, integer_types +from ..base import c_array, mx_real_t +from ..base import mx_uint, NDArrayHandle, check_call +from ..base import ctypes2buffer +from ..context import Context +from . import _internal +from .op import NDArrayBase, _STORAGE_TYPE_ID_TO_STR +from . import broadcast_add, broadcast_mul, transpose, broadcast_not_equal, broadcast_power +from . import broadcast_sub, broadcast_div, broadcast_to, broadcast_equal, cast_storage +from . import broadcast_greater, broadcast_greater_equal, broadcast_lesser, broadcast_lesser_equal +from . import zeros_like, slice + +__all__ = ["NDArray", "concatenate", "_DTYPE_NP_TO_MX", "_DTYPE_MX_TO_NP", "_GRAD_REQ_MAP", + "ones", "add", "arange", "divide", "equal", "full", "greater", "greater_equal", + "imdecode", "lesser", "lesser_equal", "maximum", "minimum", "moveaxis", + "multiply", "negative", "not_equal", "onehot_encode", "power", "subtract", + "true_divide", "waitall", "_new_empty_handle"] # pylint: disable= no-member _DTYPE_NP_TO_MX = { @@ -74,7 +60,6 @@ np.int8 : 5, np.int64 : 6, } - _DTYPE_MX_TO_NP = { -1 : None, 0 : np.float32, @@ -85,7 +70,12 @@ 5 : np.int8, 6 : np.int64, } - +_STORAGE_TYPE_STR_TO_ID = { + 'undefined' : -1, + 'default' : 0, + 'row_sparse' : 1, + 'csr' : 2, +} _GRAD_REQ_MAP = { 'null': 0, 'write': 1, @@ -93,6 +83,7 @@ } # pylint: enable= no-member + def _new_empty_handle(): """Returns a new empty handle. @@ -107,6 +98,7 @@ def _new_empty_handle(): check_call(_LIB.MXNDArrayCreateNone(ctypes.byref(hdl))) return hdl + def _new_alloc_handle(shape, ctx, delay_alloc, dtype=mx_real_t): """Return a new handle with specified shape and context. @@ -128,6 +120,7 @@ def _new_alloc_handle(shape, ctx, delay_alloc, dtype=mx_real_t): ctypes.byref(hdl))) return hdl + def waitall(): """Wait for all async operations to finish in MXNet. @@ -135,6 +128,13 @@ def waitall(): """ check_call(_LIB.MXNDArrayWaitAll()) + +def _storage_type(handle): + storage_type = ctypes.c_int(0) + check_call(_LIB.MXNDArrayGetStorageType(handle, ctypes.byref(storage_type))) + return _STORAGE_TYPE_ID_TO_STR[storage_type.value] + + class NDArray(NDArrayBase): """An array object representing a multidimensional, homogeneous array of fixed-size items. @@ -144,6 +144,7 @@ class NDArray(NDArrayBase): # make numpy functions return NDArray instead of numpy object array __array_priority__ = 1000.0 # pylint: disable= no-member, undefined-variable + def __repr__(self): """Returns a string representation of the array.""" shape_info = 'x'.join(['%d' % x for x in self.shape]) @@ -151,6 +152,9 @@ def __repr__(self): self.__class__.__name__, shape_info, self.context) + def __reduce__(self): + return NDArray, (None,), self.__getstate__() + def __add__(self, other): """x.__add__(y) <=> x+y <=> mx.nd.add(x, y) """ return add(self, other) @@ -742,7 +746,6 @@ def wait_to_read(self): """ check_call(_LIB.MXNDArrayWaitToRead(self.handle)) - @property def ndim(self): """Returns the number of dimensions of this array @@ -777,6 +780,7 @@ def shape(self): self.handle, ctypes.byref(ndim), ctypes.byref(pdata))) return tuple(pdata[:ndim.value]) + @property def size(self): """Number of elements in the array. @@ -841,6 +845,12 @@ def dtype(self): self.handle, ctypes.byref(mx_dtype))) return _DTYPE_MX_TO_NP[mx_dtype.value] + @property + def stype(self): + """Storage-type of the array. + """ + return _storage_type(self.handle) + @property # pylint: disable= invalid-name, undefined-variable def T(self): @@ -964,7 +974,7 @@ def copyto(self, other): Returns ------- - NDArray + NDArray, CSRNDArray, RowSparseNDArray The copied array. If ``other`` is an ``NDArray``, then the return value and ``other`` will point to the same ``NDArray``. @@ -1101,6 +1111,20 @@ def backward(self, out_grad=None, retain_graph=False, train_mode=True): ctypes.c_int(retain_graph), ctypes.c_int(train_mode))) + def tostype(self, stype): + """Return a copy of the array with chosen storage type. + + See Also + ---------- + :meth:`mxnet.ndarray.cast_storage`. + + Returns + ------- + NDArray, CSRNDArray or RowSparseNDArray + A copy of the array with the chosen storage stype + """ + return cast_storage(self, stype=stype) + def onehot_encode(indices, out): """One-hot encoding indices into matrix out. @@ -1113,74 +1137,7 @@ def onehot_encode(indices, out): # pylint: enable= no-member, protected-access -def empty(shape, ctx=None, dtype=mx_real_t): - """Returns a new array of given shape and type, without initializing entries. - - Parameters - ---------- - shape : int or tuple of int - The shape of the empty array. - ctx : Context, optional - An optional device context (default is the current default context). - dtype : str or numpy.dtype, optional - An optional value type (default is `float32`). - - Returns - ------- - NDArray - A created array. - - Examples - -------- - >>> mx.nd.empty(1) - - >>> mx.nd.empty((1,2), mx.gpu(0)) - - >>> mx.nd.empty((1,2), mx.gpu(0), 'float16') - - """ - if isinstance(shape, integer_types): - shape = (shape, ) - if ctx is None: - ctx = Context.default_ctx - return NDArray(handle=_new_alloc_handle(shape, ctx, False, dtype)) - -def zeros(shape, ctx=None, dtype=mx_real_t, **kwargs): - """Returns a new array filled with all zeros, with the given shape and type. - - Parameters - ---------- - shape : int or tuple of int - The shape of the empty array. - ctx : Context, optional - An optional device context (default is the current default context). - dtype : str or numpy.dtype, optional - An optional value type (default is `float32`). - out : NDArray, optional - The output NDArray (default is `None`). - - Returns - ------- - NDArray - A created array - - Examples - -------- - >>> mx.nd.zeros(1).asnumpy() - array([ 0.], dtype=float32) - >>> mx.nd.zeros((1,2), mx.gpu(0)) - - >>> mx.nd.zeros((1,2), mx.gpu(0), 'float16').asnumpy() - array([[ 0., 0.]], dtype=float16) - """ - # pylint: disable= unused-argument - if ctx is None: - ctx = Context.default_ctx - # pylint: disable= no-member, protected-access - return _internal._zeros(shape=shape, ctx=ctx, dtype=dtype, **kwargs) - # pylint: enable= no-member, protected-access - -def ones(shape, ctx=None, dtype=mx_real_t, **kwargs): +def ones(shape, ctx=None, dtype=None, **kwargs): """Returns a new array filled with all ones, with the given shape and type. Parameters @@ -1212,10 +1169,12 @@ def ones(shape, ctx=None, dtype=mx_real_t, **kwargs): # pylint: disable= unused-argument if ctx is None: ctx = Context.default_ctx + dtype = mx_real_t if dtype is None else dtype # pylint: disable= no-member, protected-access return _internal._ones(shape=shape, ctx=ctx, dtype=dtype, **kwargs) # pylint: enable= no-member, protected-access + def full(shape, val, ctx=None, dtype=mx_real_t, out=None): """Returns a new array of given shape and type, filled with the given value `val`. @@ -1269,18 +1228,6 @@ def array(source_array, ctx=None, dtype=None): ------- NDArray An `NDArray` with the same contents as the `source_array`. - - Examples - -------- - >>> import numpy as np - >>> mx.nd.array([1, 2, 3]) - - >>> mx.nd.array([[1, 2], [3, 4]]) - - >>> mx.nd.array(np.zeros((3, 2))) - - >>> mx.nd.array(np.zeros((3, 2)), mx.gpu(0)) - """ if isinstance(source_array, NDArray): dtype = source_array.dtype if dtype is None else dtype @@ -1382,6 +1329,7 @@ def arange(start, stop=None, step=1.0, repeat=1, ctx=None, dtype=mx_real_t): dtype=dtype, ctx=str(ctx)) # pylint: enable= no-member, protected-access, too-many-arguments + #pylint: disable= too-many-arguments, no-member, protected-access def _ufunc_helper(lhs, rhs, fn_array, fn_scalar, lfn_scalar, rfn_scalar=None): """ Helper function for element-wise operation. @@ -1430,6 +1378,7 @@ def _ufunc_helper(lhs, rhs, fn_array, fn_scalar, lfn_scalar, rfn_scalar=None): raise TypeError('type %s not supported' % str(type(rhs))) #pylint: enable= too-many-arguments, no-member, protected-access + def add(lhs, rhs): """Returns element-wise sum of the input arrays with broadcasting. @@ -1491,6 +1440,7 @@ def add(lhs, rhs): None) # pylint: enable= no-member, protected-access + def subtract(lhs, rhs): """Returns element-wise difference of the input arrays with broadcasting. @@ -1552,6 +1502,7 @@ def subtract(lhs, rhs): _internal._rminus_scalar) # pylint: enable= no-member, protected-access + def multiply(lhs, rhs): """Returns element-wise product of the input arrays with broadcasting. @@ -1612,6 +1563,7 @@ def multiply(lhs, rhs): None) # pylint: enable= no-member, protected-access + def divide(lhs, rhs): """Returns element-wise division of the input arrays with broadcasting. @@ -1668,6 +1620,7 @@ def divide(lhs, rhs): _internal._rdiv_scalar) # pylint: enable= no-member, protected-access + def modulo(lhs, rhs): """Returns element-wise modulo of the input arrays with broadcasting. @@ -1724,6 +1677,7 @@ def modulo(lhs, rhs): _internal._rmod_scalar) # pylint: enable= no-member, protected-access + def power(base, exp): """Returns result of first array elements raised to powers from second array, element-wise with broadcasting. @@ -1785,6 +1739,7 @@ def power(base, exp): _internal._rpower_scalar) # pylint: enable= no-member, protected-access + def maximum(lhs, rhs): """Returns element-wise maximum of the input arrays with broadcasting. @@ -1841,6 +1796,7 @@ def maximum(lhs, rhs): None) # pylint: enable= no-member, protected-access + def minimum(lhs, rhs): """Returns element-wise minimum of the input arrays with broadcasting. @@ -1897,6 +1853,7 @@ def minimum(lhs, rhs): None) # pylint: enable= no-member, protected-access + def equal(lhs, rhs): """Returns the result of element-wise **equal to** (==) comparison operation with broadcasting. @@ -1960,6 +1917,7 @@ def equal(lhs, rhs): None) # pylint: enable= no-member, protected-access + def not_equal(lhs, rhs): """Returns the result of element-wise **not equal to** (!=) comparison operation with broadcasting. @@ -2026,6 +1984,7 @@ def not_equal(lhs, rhs): None) # pylint: enable= no-member, protected-access + def greater(lhs, rhs): """Returns the result of element-wise **greater than** (>) comparison operation with broadcasting. @@ -2089,6 +2048,7 @@ def greater(lhs, rhs): _internal._lesser_scalar) # pylint: enable= no-member, protected-access + def greater_equal(lhs, rhs): """Returns the result of element-wise **greater than or equal to** (>=) comparison operation with broadcasting. @@ -2152,6 +2112,7 @@ def greater_equal(lhs, rhs): _internal._lesser_equal_scalar) # pylint: enable= no-member, protected-access + def lesser(lhs, rhs): """Returns the result of element-wise **lesser than** (<) comparison operation with broadcasting. @@ -2279,12 +2240,14 @@ def lesser_equal(lhs, rhs): _internal._greater_equal_scalar) # pylint: enable= no-member, protected-access + def true_divide(lhs, rhs): """This function is similar to :meth:`divide`. """ return divide(lhs, rhs) + def negative(arr): """Numerical negative, element-wise. @@ -2310,95 +2273,6 @@ def negative(arr): return multiply(arr, -1.0) -def load(fname): - """Loads an array from file. - - See more details in ``save``. - - Parameters - ---------- - fname : str - The filename. - - Returns - ------- - list of NDArray or dict of str to NDArray - Loaded data. - """ - if not isinstance(fname, string_types): - raise TypeError('fname required to be a string') - out_size = mx_uint() - out_name_size = mx_uint() - handles = ctypes.POINTER(NDArrayHandle)() - names = ctypes.POINTER(ctypes.c_char_p)() - check_call(_LIB.MXNDArrayLoad(c_str(fname), - ctypes.byref(out_size), - ctypes.byref(handles), - ctypes.byref(out_name_size), - ctypes.byref(names))) - if out_name_size.value == 0: - return [NDArray(NDArrayHandle(handles[i])) for i in range(out_size.value)] - else: - assert out_name_size.value == out_size.value - return dict( - (py_str(names[i]), NDArray(NDArrayHandle(handles[i]))) for i in range(out_size.value)) - - -def save(fname, data): - """Saves a list of arrays or a dict of str->array to file. - - Examples of filenames: - - - ``/path/to/file`` - - ``s3://my-bucket/path/to/file`` (if compiled with AWS S3 supports) - - ``hdfs://path/to/file`` (if compiled with HDFS supports) - - Parameters - ---------- - fname : str - The filename. - data : ``NDArray``, list of ``NDArray` or dict of str to ``NDArray`` - The data to save. - - Examples - -------- - >>> x = mx.nd.zeros((2,3)) - >>> y = mx.nd.ones((1,4)) - >>> mx.nd.save('my_list', [x,y]) - >>> mx.nd.save('my_dict', {'x':x, 'y':y}) - >>> mx.nd.load('my_list') - [, ] - >>> mx.nd.load('my_dict') - {'y': , 'x': } - """ - if isinstance(data, NDArray): - data = [data] - handles = [] - if isinstance(data, dict): - keys = [] - for key, val in data.items(): - if not isinstance(key, string_types): - raise TypeError('save only accept dict str->NDArray or list of NDArray') - if not isinstance(val, NDArray): - raise TypeError('save only accept dict str->NDArray or list of NDArray') - keys.append(c_str(key)) - handles.append(val.handle) - keys = c_array(ctypes.c_char_p, keys) - elif isinstance(data, list): - for val in data: - if not isinstance(val, NDArray): - raise TypeError('save only accept dict str->NDArray or list of NDArray') - handles.append(val.handle) - keys = None - else: - raise ValueError("data needs to either be a NDArray, dict of str, NDArray pairs " - "or a list of NDarrays.") - check_call(_LIB.MXNDArraySave(c_str(fname), - mx_uint(len(handles)), - c_array(NDArrayHandle, handles), - keys)) - - def concatenate(arrays, axis=0, always_copy=True): """DEPRECATED, use ``concat`` instead @@ -2455,6 +2329,7 @@ def concatenate(arrays, axis=0, always_copy=True): return ret + def imdecode(str_img, clip_rect=(0, 0, 0, 0), out=None, index=0, channels=3, mean=None): """DEPRECATED, use mx.img instead @@ -2497,159 +2372,65 @@ def imdecode(str_img, clip_rect=(0, 0, 0, 0), out=None, index=0, channels=3, mea out=out) -# pylint: disable=too-many-locals, invalid-name -def _make_ndarray_function(handle, name): - """Create a NDArray function from the FunctionHandle.""" - real_name = ctypes.c_char_p() - desc = ctypes.c_char_p() - num_args = mx_uint() - arg_names = ctypes.POINTER(ctypes.c_char_p)() - arg_types = ctypes.POINTER(ctypes.c_char_p)() - arg_descs = ctypes.POINTER(ctypes.c_char_p)() - key_var_num_args = ctypes.c_char_p() - ret_type = ctypes.c_char_p() - - check_call(_LIB.MXSymbolGetAtomicSymbolInfo( - handle, ctypes.byref(real_name), ctypes.byref(desc), - ctypes.byref(num_args), - ctypes.byref(arg_names), - ctypes.byref(arg_types), - ctypes.byref(arg_descs), - ctypes.byref(key_var_num_args), - ctypes.byref(ret_type))) - narg = int(num_args.value) - arg_names = [py_str(arg_names[i]) for i in range(narg)] - arg_types = [py_str(arg_types[i]) for i in range(narg)] - func_name = name - key_var_num_args = py_str(key_var_num_args.value) - ret_type = py_str(ret_type.value) if ret_type.value is not None else '' - doc_str = _build_doc(func_name, - py_str(desc.value), - arg_names, - arg_types, - [py_str(arg_descs[i]) for i in range(narg)], - key_var_num_args, - ret_type) - - dtype_name = None - arr_name = None - ndsignature = [] - signature = [] - ndarg_names = [] - kwarg_names = [] - for i in range(narg): - name, atype = arg_names[i], arg_types[i] - if name == 'dtype': - dtype_name = name - signature.append('%s=_Null'%name) - elif atype.startswith('NDArray') or atype.startswith('Symbol'): - assert not arr_name, \ - "Op can only have one argument with variable " \ - "size and it must be the last argument." - if atype.endswith('[]'): - ndsignature.append('*%s'%name) - arr_name = name - else: - ndsignature.append('%s=None'%name) - ndarg_names.append(name) - else: - signature.append('%s=_Null'%name) - kwarg_names.append(name) - signature.append('out=None') - signature.append('name=None') - signature.append('**kwargs') - signature = ndsignature + signature - - code = [] - if arr_name: - code.append(""" -def %s(*%s, **kwargs):"""%(func_name, arr_name)) - code.append(""" - ndargs = [] - for i in {}: - assert isinstance(i, NDArrayBase), \\ - "Positional arguments must have NDArray type, " \\ - "but got %s"%str(i) - ndargs.append(i)""".format(arr_name)) - if dtype_name is not None: - code.append(""" - if '%s' in kwargs: - kwargs['%s'] = np.dtype(kwargs['%s']).name"""%( - dtype_name, dtype_name, dtype_name)) - code.append(""" - _ = kwargs.pop('name', None) - out = kwargs.pop('out', None) - keys = list(kwargs.keys()) - vals = list(kwargs.values())""") - else: - code.append(""" -def %s(%s): - ndargs = [] - keys = list(kwargs.keys()) - vals = list(kwargs.values())"""%(func_name, ', '.join(signature))) - # NDArray args - for name in ndarg_names: # pylint: disable=redefined-argument-from-local - code.append(""" - if {name} is not None: - assert isinstance({name}, NDArrayBase), \\ - "Argument {name} must have NDArray type, but got %s"%str({name}) - ndargs.append({name})""".format(name=name)) - # kwargs - for name in kwarg_names: # pylint: disable=redefined-argument-from-local - code.append(""" - if %s is not _Null: - keys.append('%s') - vals.append(%s)"""%(name, name, name)) - # dtype - if dtype_name is not None: - code.append(""" - if %s is not _Null: - keys.append('%s') - vals.append(np.dtype(%s).name)"""%(dtype_name, dtype_name, dtype_name)) - - code.append(""" - return _imperative_invoke(%d, ndargs, keys, vals, out)"""%( - handle.value)) - - local = {} - exec(''.join(code), None, local) # pylint: disable=exec-used - ndarray_function = local[func_name] - ndarray_function.__name__ = func_name - ndarray_function.__doc__ = doc_str - ndarray_function.__module__ = 'mxnet.ndarray' - return ndarray_function - - -# pylint: enable=too-many-locals, invalid-name -def _init_ndarray_module(ndarray_class, root_namespace): - """List and add all the ndarray functions to current module.""" - _set_ndarray_class(ndarray_class) - plist = ctypes.POINTER(ctypes.c_char_p)() - size = ctypes.c_uint() - - check_call(_LIB.MXListAllOpNames(ctypes.byref(size), - ctypes.byref(plist))) - op_names = [] - for i in range(size.value): - op_names.append(py_str(plist[i])) - - module_obj = _sys.modules["%s.ndarray" % root_namespace] - module_internal = _sys.modules["%s._ndarray_internal" % root_namespace] - module_contrib = _sys.modules["%s.contrib.ndarray" % root_namespace] - for name in op_names: - hdl = OpHandle() - check_call(_LIB.NNGetOpHandle(c_str(name), ctypes.byref(hdl))) - function = _make_ndarray_function(hdl, name) - if function.__name__.startswith('_contrib_'): - function.__name__ = function.__name__[9:] - function.__module__ = 'mxnet.contrib.ndarray' - setattr(module_contrib, function.__name__, function) - elif function.__name__.startswith('_'): - setattr(module_internal, function.__name__, function) - else: - setattr(module_obj, function.__name__, function) +def zeros(shape, ctx=None, dtype=None, **kwargs): + """Returns a new array filled with all zeros, with the given shape and type. + + Parameters + ---------- + shape : int or tuple of int + The shape of the empty array. + ctx : Context, optional + An optional device context (default is the current default context). + dtype : str or numpy.dtype, optional + An optional value type (default is `float32`). + out : NDArray, optional + The output NDArray (default is `None`). + + Returns + ------- + NDArray + A created array + + Examples + -------- + >>> mx.nd.zeros(1).asnumpy() + array([ 0.], dtype=float32) + >>> mx.nd.zeros((1,2), mx.gpu(0)) + + >>> mx.nd.zeros((1,2), mx.gpu(0), 'float16').asnumpy() + array([[ 0., 0.]], dtype=float16) + """ + # pylint: disable= unused-argument + if ctx is None: + ctx = Context.default_ctx + dtype = mx_real_t if dtype is None else dtype + # pylint: disable= no-member, protected-access + return _internal._zeros(shape=shape, ctx=ctx, dtype=dtype, **kwargs) + # pylint: enable= no-member, protected-access + + +def empty(shape, ctx=None, dtype=None): + """Returns a new array of given shape and type, without initializing entries. + + Parameters + ---------- + shape : int or tuple of int + The shape of the empty array. + ctx : Context, optional + An optional device context (default is the current default context). + dtype : str or numpy.dtype, optional + An optional value type (default is `float32`). -_init_ndarray_module(NDArray, "mxnet") + Returns + ------- + NDArray + A created array. -# from .base import add_fileline_to_docstring -# add_fileline_to_docstring(__name__) + """ + if isinstance(shape, int): + shape = (shape, ) + if ctx is None: + ctx = Context.default_ctx + if dtype is None: + dtype = mx_real_t + return NDArray(handle=_new_alloc_handle(shape, ctx, False, dtype)) diff --git a/python/mxnet/ndarray/op.py b/python/mxnet/ndarray/op.py new file mode 100644 index 000000000000..e4a1ab0df48b --- /dev/null +++ b/python/mxnet/ndarray/op.py @@ -0,0 +1,209 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Register backend ops in mxnet.ndarray namespace""" + +import sys as _sys +import os as _os +import ctypes +import numpy as np # pylint: disable=unused-import + +from ..ndarray_doc import _build_doc + +# Use different version of SymbolBase +# When possible, use cython to speedup part of computation. +# pylint: disable=unused-import +try: + if int(_os.environ.get("MXNET_ENABLE_CYTHON", True)) == 0: + from .._ctypes.ndarray import NDArrayBase, _STORAGE_TYPE_ID_TO_STR + from .._ctypes.ndarray import CachedOp, _imperative_invoke + elif _sys.version_info >= (3, 0): + from .._cy3.ndarray import NDArrayBase, _imperative_invoke, _STORAGE_TYPE_ID_TO_STR + from .._cy3.ndarray import CachedOp, _imperative_invoke + else: + from .._cy2.ndarray import NDArrayBase, _imperative_invoke, _STORAGE_TYPE_ID_TO_STR + from .._cy2.ndarray import CachedOp, _imperative_invoke +except ImportError: + if int(_os.environ.get("MXNET_ENFORCE_CYTHON", False)) != 0: + raise ImportError("Cython Module cannot be loaded but MXNET_ENFORCE_CYTHON=1") + from .._ctypes.ndarray import NDArrayBase, _imperative_invoke, _STORAGE_TYPE_ID_TO_STR + from .._ctypes.ndarray import CachedOp, _imperative_invoke + +from ..base import mx_uint, check_call, _LIB, py_str, OpHandle, c_str, _Null +# pylint: enable=unused-import + + +# pylint: disable=too-many-locals, invalid-name +def _make_ndarray_function(handle, name): + """Create a NDArray function from the FunctionHandle.""" + real_name = ctypes.c_char_p() + desc = ctypes.c_char_p() + num_args = mx_uint() + arg_names = ctypes.POINTER(ctypes.c_char_p)() + arg_types = ctypes.POINTER(ctypes.c_char_p)() + arg_descs = ctypes.POINTER(ctypes.c_char_p)() + key_var_num_args = ctypes.c_char_p() + ret_type = ctypes.c_char_p() + + check_call(_LIB.MXSymbolGetAtomicSymbolInfo( + handle, ctypes.byref(real_name), ctypes.byref(desc), + ctypes.byref(num_args), + ctypes.byref(arg_names), + ctypes.byref(arg_types), + ctypes.byref(arg_descs), + ctypes.byref(key_var_num_args), + ctypes.byref(ret_type))) + narg = int(num_args.value) + arg_names = [py_str(arg_names[i]) for i in range(narg)] + arg_types = [py_str(arg_types[i]) for i in range(narg)] + func_name = name + key_var_num_args = py_str(key_var_num_args.value) + ret_type = py_str(ret_type.value) if ret_type.value is not None else '' + doc_str = _build_doc(func_name, + py_str(desc.value), + arg_names, + arg_types, + [py_str(arg_descs[i]) for i in range(narg)], + key_var_num_args, + ret_type) + + dtype_name = None + arr_name = None + ndsignature = [] + signature = [] + ndarg_names = [] + kwarg_names = [] + for i in range(narg): + name, atype = arg_names[i], arg_types[i] + if name == 'dtype': + dtype_name = name + signature.append('%s=_Null'%name) + elif atype.startswith('NDArray') or atype.startswith('Symbol'): + assert not arr_name, \ + "Op can only have one argument with variable " \ + "size and it must be the last argument." + if atype.endswith('[]'): + ndsignature.append('*%s'%name) + arr_name = name + else: + ndsignature.append('%s=None'%name) + ndarg_names.append(name) + else: + signature.append('%s=_Null'%name) + kwarg_names.append(name) + signature.append('out=None') + signature.append('name=None') + signature.append('**kwargs') + signature = ndsignature + signature + + code = [] + if arr_name: + code.append(""" +def %s(*%s, **kwargs):"""%(func_name, arr_name)) + code.append(""" + ndargs = [] + for i in {}: + assert isinstance(i, NDArrayBase), \\ + "Positional arguments must have NDArray type, " \\ + "but got %s"%str(i) + ndargs.append(i)""".format(arr_name)) + if dtype_name is not None: + code.append(""" + if '%s' in kwargs: + kwargs['%s'] = np.dtype(kwargs['%s']).name"""%( + dtype_name, dtype_name, dtype_name)) + code.append(""" + _ = kwargs.pop('name', None) + out = kwargs.pop('out', None) + keys = list(kwargs.keys()) + vals = list(kwargs.values())""") + else: + code.append(""" +def %s(%s): + ndargs = [] + keys = list(kwargs.keys()) + vals = list(kwargs.values())"""%(func_name, ', '.join(signature))) + # NDArray args + for name in ndarg_names: # pylint: disable=redefined-argument-from-local + code.append(""" + if {name} is not None: + assert isinstance({name}, NDArrayBase), \\ + "Argument {name} must have NDArray type, but got %s"%str({name}) + ndargs.append({name})""".format(name=name)) + # kwargs + for name in kwarg_names: # pylint: disable=redefined-argument-from-local + code.append(""" + if %s is not _Null: + keys.append('%s') + vals.append(%s)"""%(name, name, name)) + # dtype + if dtype_name is not None: + code.append(""" + if %s is not _Null: + keys.append('%s') + vals.append(np.dtype(%s).name)"""%(dtype_name, dtype_name, dtype_name)) + + code.append(""" + return _imperative_invoke(%d, ndargs, keys, vals, out)"""%( + handle.value)) + + local = {} + exec(''.join(code), None, local) # pylint: disable=exec-used + ndarray_function = local[func_name] + ndarray_function.__name__ = func_name + ndarray_function.__doc__ = doc_str + ndarray_function.__module__ = 'mxnet.ndarray' + return ndarray_function + + +# pylint: enable=too-many-locals, invalid-name +def _init_ndarray_module(root_namespace): + """List and add all the ndarray functions to current module.""" + plist = ctypes.POINTER(ctypes.c_char_p)() + size = ctypes.c_uint() + + check_call(_LIB.MXListAllOpNames(ctypes.byref(size), + ctypes.byref(plist))) + op_names = [] + for i in range(size.value): + op_names.append(py_str(plist[i])) + + module_obj = _sys.modules["%s.ndarray" % root_namespace] + module_sparse = _sys.modules["%s.ndarray.sparse" % root_namespace] + module_internal = _sys.modules["%s.ndarray._internal" % root_namespace] + module_contrib = _sys.modules["%s.contrib.ndarray" % root_namespace] + for name in op_names: + hdl = OpHandle() + check_call(_LIB.NNGetOpHandle(c_str(name), ctypes.byref(hdl))) + function = _make_ndarray_function(hdl, name) + if function.__name__.startswith('_contrib_'): + function.__name__ = function.__name__[9:] + function.__module__ = 'mxnet.contrib.ndarray' + setattr(module_contrib, function.__name__, function) + elif function.__name__.startswith('_'): + setattr(module_internal, function.__name__, function) + else: + setattr(module_obj, function.__name__, function) + + # register sparse ops under mxnet.ndarray.sparse + if function.__name__.startswith('_sparse_'): + function.__name__ = function.__name__[8:] + function.__module__ = 'mxnet.ndarray.sparse' + setattr(module_sparse, function.__name__, function) + +# register backend operators in mx.nd +_init_ndarray_module("mxnet") diff --git a/python/mxnet/ndarray/sparse.py b/python/mxnet/ndarray/sparse.py new file mode 100644 index 000000000000..97e43f5ebe79 --- /dev/null +++ b/python/mxnet/ndarray/sparse.py @@ -0,0 +1,923 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# coding: utf-8 +"""Sparse NDArray API of MXNet.""" + +from __future__ import absolute_import +from __future__ import division +try: + from __builtin__ import slice as py_slice +except ImportError: + from builtins import slice as py_slice + +import ctypes +import warnings + +import os as _os +import sys as _sys + +# import operator +import numpy as np +from ..base import NotSupportedForSparseNDArray +from ..base import _LIB, numeric_types +from ..base import c_array, mx_real_t +from ..base import mx_uint, NDArrayHandle, check_call +from ..context import Context +from . import _internal +from .ndarray import _DTYPE_NP_TO_MX, _DTYPE_MX_TO_NP +from .ndarray import _STORAGE_TYPE_STR_TO_ID +from .ndarray import NDArray, _storage_type +from .ndarray import zeros as _zeros_ndarray +from .ndarray import array as _array +from . import cast_storage +from . import slice as nd_slice + +# Use different verison of SymbolBase +# When possible, use cython to speedup part of computation. +# pylint: disable=unused-import +try: + if int(_os.environ.get("MXNET_ENABLE_CYTHON", True)) == 0: + from .._ctypes.ndarray import _set_ndarray_class + elif _sys.version_info >= (3, 0): + from .._cy3.ndarray import _set_ndarray_class + else: + from .._cy2.ndarray import _set_ndarray_class +except ImportError: + if int(_os.environ.get("MXNET_ENFORCE_CYTHON", False)) != 0: + raise ImportError("Cython Module cannot be loaded but MXNET_ENFORCE_CYTHON=1") + from .._ctypes.ndarray import _set_ndarray_class +# pylint: enable=unused-import + + +__all__ = ["_ndarray_cls", "csr_matrix", "row_sparse_array", + "BaseSparseNDArray", "CSRNDArray", "RowSparseNDArray"] + + +_STORAGE_AUX_TYPES = { + 'row_sparse': [np.int64], + 'csr': [np.int64, np.int64] +} + + +def _new_alloc_handle(stype, shape, ctx, delay_alloc, dtype, aux_types, aux_shapes=None): + """Return a new handle with specified storage type, shape, dtype and context. + + Empty handle is only used to hold results + + Returns + ------- + handle + A new empty ndarray handle + """ + hdl = NDArrayHandle() + aux_type_ids = [int(_DTYPE_NP_TO_MX[np.dtype(aux_t).type]) for aux_t in aux_types] + aux_shapes = [(0,) for aux_t in aux_types] if aux_shapes is None else aux_shapes + aux_shape_lens = [len(aux_shape) for aux_shape in aux_shapes] + aux_shapes = sum(aux_shapes, ()) + num_aux = mx_uint(len(aux_types)) + check_call(_LIB.MXNDArrayCreateSparseEx( + ctypes.c_int(int(_STORAGE_TYPE_STR_TO_ID[stype])), + c_array(mx_uint, shape), + mx_uint(len(shape)), + ctypes.c_int(ctx.device_typeid), + ctypes.c_int(ctx.device_id), + ctypes.c_int(int(delay_alloc)), + ctypes.c_int(int(_DTYPE_NP_TO_MX[np.dtype(dtype).type])), + num_aux, + c_array(ctypes.c_int, aux_type_ids), + c_array(mx_uint, aux_shape_lens), + c_array(mx_uint, aux_shapes), + ctypes.byref(hdl))) + return hdl + + +class BaseSparseNDArray(NDArray): + """The base class of an NDArray stored in a sparse storage format. + + See CSRNDArray and RowSparseNDArray for more details. + """ + + def __iadd__(self, other): + raise NotImplementedError() + + def __isub__(self, other): + raise NotImplementedError() + + def __imul__(self, other): + raise NotImplementedError() + + def __idiv__(self, other): + raise NotImplementedError() + + def __itruediv__(self, other): + raise NotImplementedError() + + def _sync_copyfrom(self, source_array): + raise NotImplementedError() + + def _at(self, idx): + raise NotSupportedForSparseNDArray(self._at, '[idx]', idx) + + def _slice(self, start, stop): + raise NotSupportedForSparseNDArray(self._slice, None, start, stop) + + def reshape(self, shape): + raise NotSupportedForSparseNDArray(self.reshape, None, shape) + + def _aux_type(self, i): + """Data-type of the array's ith aux data. + + Returns + ------- + numpy.dtype + This BaseSparseNDArray's aux data type. + """ + aux_type = ctypes.c_int() + check_call(_LIB.MXNDArrayGetAuxType(self.handle, i, ctypes.byref(aux_type))) + return _DTYPE_MX_TO_NP[aux_type.value] + + @property + def _num_aux(self): + """The number of aux data used to help store the sparse ndarray. + """ + return len(_STORAGE_AUX_TYPES[self.stype]) + + @property + def _aux_types(self): + """The data types of the aux data for the BaseSparseNDArray. + """ + aux_types = [] + num_aux = self._num_aux + for i in range(num_aux): + aux_types.append(self._aux_type(i)) + return aux_types + + def asnumpy(self): + """Return a dense ``numpy.ndarray`` object with value copied from this array + """ + return self.tostype('default').asnumpy() + + def astype(self, dtype): + """Returns a copy of the array after casting to a specified type. + Parameters + ---------- + dtype : numpy.dtype or str + The type of the returned array. + Examples + -------- + >>> x = mx.nd.zeros('row_sparse', (2,3), dtype='float32') + >>> y = x.astype('int32') + >>> y.dtype + + """ + res = zeros(shape=self.shape, ctx=self.context, + dtype=dtype, stype=self.stype) + self.copyto(res) + return res + + def copyto(self, other): + """Copies the value of this array to another array. + + Parameters + ---------- + other : NDArray or CSRNDArray or RowSparseNDArray or Context + The destination array or context. + + Returns + ------- + NDArray or CSRNDArray or RowSparseNDArray + The copied array. + """ + if isinstance(other, NDArray): + if other.handle is self.handle: + warnings.warn('You are attempting to copy an array to itself', RuntimeWarning) + return + return _internal._copyto(self, out=other) + elif isinstance(other, Context): + hret = _ndarray_cls(_new_alloc_handle(self.stype, self.shape, other, + True, self.dtype, self._aux_types)) + return _internal._copyto(self, out=hret) + else: + raise TypeError('copyto does not support type ' + str(type(other))) + + def _data(self): + """A deep copy NDArray of the data array associated with the BaseSparseNDArray. + + This function blocks. Do not use it in performance critical code. + """ + self.wait_to_read() + hdl = NDArrayHandle() + check_call(_LIB.MXNDArrayGetDataNDArray(self.handle, ctypes.byref(hdl))) + return NDArray(hdl) + + + def _aux_data(self, i): + """ Get a deep copy NDArray of the i-th aux data array associated with the + BaseSparseNDArray. + + This function blocks. Do not use it in performance critical code. + """ + self.wait_to_read() + hdl = NDArrayHandle() + check_call(_LIB.MXNDArrayGetAuxNDArray(self.handle, i, ctypes.byref(hdl))) + return NDArray(hdl) + + +# pylint: disable=abstract-method +class CSRNDArray(BaseSparseNDArray): + """A sparse representation of 2D NDArray in the standard CSR format. + + A CSRNDArray represents an NDArray as three separate arrays: `data`, + `indptr` and `indices`. It uses the standard CSR representation where the column indices for + row i are stored in indices[indptr[i]:indptr[i+1]] and their corresponding values are stored + in values[indptr[i]:indptr[i+1]]. + + The column indices for a given row are expected to be sorted in ascending order. + Duplicate column entries for the same row are not allowed. + + Example + ------- + >>> a = mx.nd.array([[0, 1, 0], [2, 0, 0], [0, 0, 0], [0, 0, 3]]) + >>> a = a.tostype('csr') + >>> a.indices.asnumpy() + array([1, 0, 2]) + >>> a.indptr.asnumpy() + array([0, 1, 2, 2, 3]) + >>> a.data.asnumpy() + array([ 1., 2., 3.], dtype=float32) + """ + + def __reduce__(self): + return CSRNDArray, (None,), super(CSRNDArray, self).__getstate__() + + def __iadd__(self, other): + (self + other).copyto(self) + return self + + def __isub__(self, other): + (self - other).copyto(self) + return self + + def __imul__(self, other): + (self * other).copyto(self) + return self + + def __idiv__(self, other): + (self / other).copyto(self) + return self + + def __itruediv__(self, other): + (self / other).copyto(self) + return self + + def __getitem__(self, key): + """x.__getitem__(i) <=> x[i] + + Returns a sliced view of this array. + + Parameters + ---------- + key : slice + Indexing key. + + Examples + -------- + >>> indptr = np.array([0, 2, 3, 6]) + >>> indices = np.array([0, 2, 2, 0, 1, 2]) + >>> data = np.array([1, 2, 3, 4, 5, 6]) + >>> a = mx.nd.csr_matrix(data, indptr, indices, (3, 3)) + >>> a.asnumpy() + array([[1, 0, 2], + [0, 0, 3], + [4, 5, 6]]) + >>> a[1:2].asnumpy() + array([[0, 0, 3]], dtype=float32) + """ + if isinstance(key, int): + raise ValueError("__getitem__ with int key is not implemented for CSRNDArray") + if isinstance(key, py_slice): + if key.step is not None: + raise ValueError('CSRNDArray only supports continuous slicing on axis 0') + if key.start is not None or key.stop is not None: + begin = key.start if key.start else 0 + end = key.stop if key.stop else self.shape[0] + return nd_slice(self, begin=begin, end=end) + else: + return self + if isinstance(key, tuple): + raise ValueError('Multi-dimension indexing is not supported') + + def __setitem__(self, key, value): + """x.__setitem__(i, y) <=> x[i]=y + + Set self[key] to value. Only slice key [:] is supported. + + Parameters + ---------- + key : slice + The indexing key. + value : NDArray or CSRNDArray or numpy.ndarray + The value to set. + + Examples + -------- + >>> src = mx.nd.zeros((3,3), stype='csr') + >>> src.asnumpy() + array([[ 0., 0., 0.], + [ 0., 0., 0.], + [ 0., 0., 0.]], dtype=float32) + >>> # assign CSRNDArray with same storage type + >>> x = mx.nd.ones('row_sparse', (3,3)).tostype('csr') + >>> x[:] = src + >>> x.asnumpy() + array([[ 1., 1., 1.], + [ 1., 1., 1.], + [ 1., 1., 1.]], dtype=float32) + >>> # assign NDArray to CSRNDArray + >>> x[:] = mx.nd.ones((3,3)) * 2 + >>> x.asnumpy() + array([[ 2., 2., 2.], + [ 2., 2., 2.], + [ 2., 2., 2.]], dtype=float32) + """ + if not self.writable: + raise ValueError('Failed to assign to a readonly CSRNDArray') + if isinstance(key, py_slice): + if key.step is not None or key.start is not None or key.stop is not None: + raise ValueError('Assignment with slice for CSRNDArray is not ' \ + 'implmented yet.') + if isinstance(value, NDArray): + # avoid copying to itself + if value.handle is not self.handle: + value.copyto(self) + elif isinstance(value, numeric_types): + raise ValueError("Assigning numeric types to CSRNDArray is " \ + "not implemented yet.") + elif isinstance(value, (np.ndarray, np.generic)): + # TODO(haibin/anisub) check scipy.sparse and use _sync_copy_from to + # avoid the temporary copy + warnings.warn('Assigning non-NDArray object to CSRNDArray is not efficient', + RuntimeWarning) + tmp = _array(value) + tmp.copyto(self) + else: + raise TypeError('type %s not supported' % str(type(value))) + else: + assert(isinstance(key, (int, tuple))) + raise Exception('CSRNDArray only supports [:] for assignment') + + @property + def indices(self): + """A deep copy NDArray of the indices array of the CSRNDArray. + This generates a deep copy of the column indices of the current `csr` matrix. + + Returns + ------- + NDArray + This CSRNDArray's indices array. + """ + return self._aux_data(1) + + @property + def indptr(self): + """A deep copy NDArray of the indptr array of the CSRNDArray. + This generates a deep copy of the `indptr` of the current `csr` matrix. + + Returns + ------- + NDArray + This CSRNDArray's indptr array. + """ + return self._aux_data(0) + + @property + def data(self): + """A deep copy NDArray of the data array of the CSRNDArray. + This generates a deep copy of the `data` of the current `csr` matrix. + + Returns + ------- + NDArray + This CSRNDArray's data array. + """ + return self._data() + + def tostype(self, stype): + """Return a copy of the array with chosen storage type. + + Returns + ------- + NDArray or CSRNDArray + A copy of the array with the chosen storage stype + """ + if stype == 'row_sparse': + raise ValueError("cast_storage from csr to row_sparse is not supported") + return cast_storage(self, stype=stype) + + def copyto(self, other): + """Copies the value of this array to another array. + + If ``other`` is a ``NDArray`` or ``CSRNDArray`` object, then ``other.shape`` and + ``self.shape`` should be the same. This function copies the value from + ``self`` to ``other``. + + If ``other`` is a context, a new ``CSRNDArray`` will be first created on + the target context, and the value of ``self`` is copied. + + Parameters + ---------- + other : NDArray or CSRNDArray or Context + The destination array or context. + + Returns + ------- + NDArray or CSRNDArray + The copied array. If ``other`` is an ``NDArray`` or ``CSRNDArray``, then the return + value and ``other`` will point to the same ``NDArray`` or ``CSRNDArray``. + """ + if isinstance(other, Context): + return super(CSRNDArray, self).copyto(other) + elif isinstance(other, NDArray): + stype = other.stype + if stype == 'default' or stype == 'csr': + return super(CSRNDArray, self).copyto(other) + else: + raise TypeError('copyto does not support destination NDArray stype ' + str(stype)) + else: + raise TypeError('copyto does not support type ' + str(type(other))) + + +# pylint: disable=abstract-method +class RowSparseNDArray(BaseSparseNDArray): + """A sparse representation of a set of NDArray row slices at given indices. + + A RowSparseNDArray represents a multidimensional NDArray using two separate arrays: `data` and + `indices`. + + - data: an NDArray of any dtype with shape [D0, D1, ..., Dn]. + - indices: a 1-D int64 NDArray with shape [D0]. + + The `indices` stores the indices of the row slices with non-zeros, + while the values are stored in `data`. The corresponding NDArray ``dense`` + represented by RowSparseNDArray ``rsp`` has + + ``dense[rsp.indices[i], :, :, :, ...] = rsp.data[i, :, :, :, ...]`` + + >>> dense.asnumpy() + array([[ 1., 2., 3.], + [ 0., 0., 0.], + [ 4., 0., 5.], + [ 0., 0., 0.], + [ 0., 0., 0.]], dtype=float32) + >>> rsp = dense.tostype('row_sparse') + >>> rsp.indices.asnumpy() + array([0, 2], dtype=int64) + >>> rsp.data.asnumpy() + array([[ 1., 2., 3.], + [ 4., 0., 5.]], dtype=float32) + + A RowSparseNDArray is typically used to represent non-zero row-slices of a large NDArray + of shape [LARGE0, D1, .. , Dn] where LARGE0 >> D0 and most row slices are zeros. + + The indices are expected to be sorted in ascending order. + + RowSparseNDArray is used principally in the definition of gradients for operations + that have sparse gradients (e.g. sparse dot and sparse embedding). + """ + def __reduce__(self): + return RowSparseNDArray, (None,), super(RowSparseNDArray, self).__getstate__() + + def __iadd__(self, other): + (self + other).copyto(self) + return self + + def __isub__(self, other): + (self - other).copyto(self) + return self + + def __imul__(self, other): + (self * other).copyto(self) + return self + + def __idiv__(self, other): + (self / other).copyto(self) + return self + + def __itruediv__(self, other): + (self / other).copyto(self) + return self + + def __getitem__(self, key): + """x.__getitem__(i) <=> x[i] + + Returns a sliced view of this array. + + Parameters + ---------- + key : slice + Indexing key. + + Examples + -------- + >>> x = mx.nd.zeros((2, 3), stype='row_sparse') + >>> x[:].asnumpy() + array([[ 0., 0., 0.], + [ 0., 0., 0.]], dtype=float32) + """ + if isinstance(key, int): + raise Exception("__getitem__ with int key is not implemented for RowSparseNDArray yet") + if isinstance(key, py_slice): + if key.step is not None or key.start is not None or key.stop is not None: + raise Exception('RowSparseNDArray only supports [:] for __getitem__') + else: + return self + if isinstance(key, tuple): + raise ValueError('Multi-dimension indexing is not supported') + + def __setitem__(self, key, value): + """x.__setitem__(i, y) <=> x[i]=y + + Set self[key] to value. Only slice key [:] is supported. + + Parameters + ---------- + key : slice + The indexing key. + value : NDArray or numpy.ndarray + The value to set. + + Examples + -------- + >>> src = mx.nd.row_sparse([[1, 0, 2], [4, 5, 6]], [0, 2], (3,3)) + >>> src.asnumpy() + array([[ 1., 0., 2.], + [ 0., 0., 0.], + [ 4., 5., 6.]], dtype=float32) + >>> # assign RowSparseNDArray with same storage type + >>> x = mx.nd.zeros('row_sparse', (3,3)) + >>> x[:] = src + >>> x.asnumpy() + array([[ 1., 0., 2.], + [ 0., 0., 0.], + [ 4., 5., 6.]], dtype=float32) + >>> # assign NDArray to RowSparseNDArray + >>> x[:] = mx.nd.ones((3,3)) + >>> x.asnumpy() + array([[ 1., 1., 1.], + [ 1., 1., 1.], + [ 1., 1., 1.]], dtype=float32) + """ + if not self.writable: + raise ValueError('Failed to assign to a readonly RowSparseNDArray') + if isinstance(key, py_slice): + if key.step is not None or key.start is not None or key.stop is not None: + raise ValueError('Assignment with slice for RowSparseNDArray ' \ + 'is not implmented yet.') + if isinstance(value, NDArray): + # avoid copying to itself + if value.handle is not self.handle: + value.copyto(self) + elif isinstance(value, numeric_types): + raise ValueError("Assigning numeric types to RowSparseNDArray " \ + "is not implemented yet.") + elif isinstance(value, (np.ndarray, np.generic)): + warnings.warn('Assigning non-NDArray object to RowSparseNDArray is not efficient', + RuntimeWarning) + tmp = _array(value) + tmp.copyto(self) + else: + raise TypeError('type %s not supported' % str(type(value))) + else: + assert(isinstance(key, (int, tuple))) + raise TypeError('RowSparseNDArray only supports [:] for assignment') + + @property + def indices(self): + """A deep copy NDArray of the indices array of the RowSparseNDArray. + This generates a deep copy of the row indices of the current `row_sparse` matrix. + + Returns + ------- + NDArray + This RowSparseNDArray's indices array. + """ + return self._aux_data(0) + + @property + def data(self): + """A deep copy NDArray of the data array of the RowSparseNDArray. + This generates a deep copy of the `data` of the current `row_sparse` matrix. + + Returns + ------- + NDArray + This RowSparseNDArray's data array. + """ + return self._data() + + def tostype(self, stype): + """Return a copy of the array with chosen storage type. + + Returns + ------- + NDArray or RowSparseNDArray + A copy of the array with the chosen storage stype + """ + if stype == 'csr': + raise ValueError("cast_storage from row_sparse to csr is not supported") + return cast_storage(self, stype=stype) + + def copyto(self, other): + """Copies the value of this array to another array. + + If ``other`` is a ``NDArray`` or ``RowSparseNDArray`` object, then ``other.shape`` + and ``self.shape`` should be the same. This function copies the value from + ``self`` to ``other``. + + If ``other`` is a context, a new ``RowSparseNDArray`` will be first created on + the target context, and the value of ``self`` is copied. + + Parameters + ---------- + other : NDArray or RowSparseNDArray or Context + The destination array or context. + + Returns + ------- + NDArray or RowSparseNDArray + The copied array. If ``other`` is an ``NDArray`` or ``RowSparseNDArray``, then the + return value and ``other`` will point to the same ``NDArray`` or ``RowSparseNDArray``. + """ + if isinstance(other, Context): + return super(RowSparseNDArray, self).copyto(other) + elif isinstance(other, NDArray): + stype = other.stype + if stype == 'default' or stype == 'row_sparse': + return super(RowSparseNDArray, self).copyto(other) + else: + raise TypeError('copyto does not support destination NDArray stype ' + str(stype)) + else: + raise TypeError('copyto does not support type ' + str(type(other))) + + +def _prepare_src_array(src, dtype, default_dtype): + """Prepare `src` and its dtype so that they can be used to construct NDArray. + `src` is converted to a `np.ndarray` if it's neither an `NDArray` nor an `np.ndarray`. + """ + if isinstance(src, NDArray): + dtype = src.dtype if dtype is None else dtype + else: + dtype = default_dtype if dtype is None else dtype + if not isinstance(src, np.ndarray): + try: + src = np.array(src, dtype=dtype) + except: + raise TypeError('values must be array like object') + return src, dtype + + +def csr_matrix(data, indptr, indices, shape, ctx=None, dtype=None, indptr_type=None, + indices_type=None): + """Creates a 2D array with compressed sparse row(CSR) format. + + Parameters + ---------- + data: array_like + An object exposing the array interface, with shape [nnz], where D0 is the number of + non-zero entries. + indptr: array_like + An object exposing the array interface, with shape [D0 + 1]. The first element in indptr + should always be zero. + indices: array_like + An object exposing the array interface, with shape [nnz]. + ctx: Context, optional + Device context (default is the current default context). + dtype: str or numpy.dtype, optional + The data type of the output array. The default dtype is ``values.dtype`` + if `values` is an `NDArray`, `float32` otherwise. + indptr_type: str or numpy.dtype, optional + The data type of the indices array. The default dtype is ``indptr.dtype`` + if `indptr` is an `NDArray`, `int64` otherwise. + indices_type: str or numpy.dtype, optional + The data type of the indices array. The default dtype is ``indices.dtype`` + if `indicies` is an `NDArray`, `int64` otherwise. + + Returns + ------- + CSRNDArray + A `CSRNDArray` with the `csr` storage representation. + + Example + ------- + >>> import mxnet as mx + >>> a = mx.nd.csr_matrix([1, 2, 3], [0, 1, 2, 2, 3], [1, 0, 2], (4, 3)) + >>> a.asnumpy() + array([[ 0., 1., 0.], + [ 2., 0., 0.], + [ 0., 0., 0.], + [ 0., 0., 3.]], dtype=float32) + """ + storage_type = 'csr' + # context + if ctx is None: + ctx = Context.default_ctx + # prepare src array and types + data, dtype = _prepare_src_array(data, dtype, mx_real_t) + indptr, indptr_type = _prepare_src_array(indptr, indptr_type, + _STORAGE_AUX_TYPES[storage_type][0]) + indices, indices_type = _prepare_src_array(indices, indices_type, + _STORAGE_AUX_TYPES[storage_type][1]) + # verify types + assert('int64' in str(indptr_type)), "expected int64 for indptr" + assert('int64' in str(indices_type)), "expected int64 for indices" + # verify shapes + aux_shapes = [indptr.shape, indices.shape] + assert(data.ndim == 1) + assert(indptr.ndim == 1) + assert(indices.ndim == 1) + assert(len(shape) == 2) + result = CSRNDArray(_new_alloc_handle(storage_type, shape, ctx, False, dtype, + [indptr_type, indices_type], aux_shapes)) + # TODO(junwu): Convert data, indptr, and indices to mxnet NDArrays + # if they are not for now. In the future, we should provide a c-api + # to accept np.ndarray types to copy from to result.data and aux_data + if not isinstance(data, NDArray): + data = _array(data, ctx, dtype) + if not isinstance(indptr, NDArray): + indptr = _array(indptr, ctx, indptr_type) + if not isinstance(indices, NDArray): + indices = _array(indices, ctx, indices_type) + check_call(_LIB.MXNDArraySyncCopyFromNDArray(result.handle, data.handle, ctypes.c_int(-1))) + check_call(_LIB.MXNDArraySyncCopyFromNDArray(result.handle, indptr.handle, ctypes.c_int(0))) + check_call(_LIB.MXNDArraySyncCopyFromNDArray(result.handle, indices.handle, ctypes.c_int(1))) + return result + + +def row_sparse_array(data, indices, shape, ctx=None, dtype=None, indices_type=None): + """Creates a multidimensional row sparse array with a set of tensor slices at given indices. + + Parameters + ---------- + data: array_like + An object exposing the array interface, with shape [D0, D1, .. DK], where D0 is + the number of rows with non-zeros entries. + indices: array_like + An object exposing the array interface, with shape [D0]. + ctx : Context, optional + Device context (default is the current default context). + dtype : str or numpy.dtype, optional + The data type of the output array. The default dtype is ``data.dtype`` + if `data` is an `NDArray`, `float32` otherwise. + indices_type: str or numpy.dtype, optional + The data type of the indices array. The default dtype is ``indices.dtype`` + if `indicies` is an `NDArray`, `int64` otherwise. + + Returns + ------- + RowSparseNDArray + An `RowSparseNDArray` with the `row_sparse` storage representation. + + Example + ------- + >>> a = mx.nd.row_sparse_array([[1, 2], [3, 4]], [1, 4], (6, 2)) + >>> a.asnumpy() + array([[ 0., 0.], + [ 1., 2.], + [ 0., 0.], + [ 0., 0.], + [ 3., 4.], + [ 0., 0.]], dtype=float32) + """ + storage_type = 'row_sparse' + # context + if ctx is None: + ctx = Context.default_ctx + # prepare src array and types + data, dtype = _prepare_src_array(data, dtype, mx_real_t) + indices, indices_type = _prepare_src_array(indices, indices_type, + _STORAGE_AUX_TYPES[storage_type][0]) + # verify types + assert('int64' in str(indices_type)), "expected int64 for indices" + # verify shapes + assert(data.ndim == len(shape)) + assert(indices.ndim == 1) + result = RowSparseNDArray(_new_alloc_handle(storage_type, shape, ctx, False, dtype, + [indices_type], [indices.shape])) + + # TODO(junwu): Convert data, indptr, and indices to mxnet NDArrays + # if they are not for now. In the future, we should provide a c-api + # to accept np.ndarray types to copy from to result.data and aux_data + if not isinstance(data, NDArray): + data = _array(data, ctx, dtype) + if not isinstance(indices, NDArray): + indices = _array(indices, ctx, indices_type) + check_call(_LIB.MXNDArraySyncCopyFromNDArray(result.handle, data.handle, ctypes.c_int(-1))) + check_call(_LIB.MXNDArraySyncCopyFromNDArray(result.handle, indices.handle, ctypes.c_int(0))) + return result + + +def _ndarray_cls(handle, writable=True, stype=None): + if stype is None: + stype = _storage_type(handle) + if stype == 'default': + return NDArray(handle, writable=writable) + elif stype == 'csr': + return CSRNDArray(handle, writable=writable) + elif stype == 'row_sparse': + return RowSparseNDArray(handle, writable=writable) + else: + raise Exception("unknown storage type") + + +_set_ndarray_class(_ndarray_cls) + + +def zeros(stype, shape, ctx=None, dtype=None, aux_types=None, **kwargs): + """Return a new array of given shape and type, filled with zeros. + + Parameters + ---------- + stype: string + The storage type of the empty array, such as 'row_sparse', 'csr', etc + shape : int or tuple of int + The shape of the empty array + ctx : Context, optional + An optional device context (default is the current default context) + dtype : str or numpy.dtype, optional + An optional value type (default is `float32`) + aux_types: list of numpy.dtype, optional + An optional list of types of the aux data for RowSparseNDArray or CSRNDArray + (default values depends on the storage type) + + Returns + ------- + RowSparseNDArray or CSRNDArray + A created array + Examples + -------- + >>> mx.nd.zeros((1,2), mx.cpu(), stype='csr') + + >>> mx.nd.zeros((1,2), mx.cpu(), 'float16', stype='row_sparse').asnumpy() + array([[ 0., 0.]], dtype=float16) + """ + if stype == 'default': + return _zeros_ndarray(shape, ctx=ctx, dtype=dtype, **kwargs) + if ctx is None: + ctx = Context.default_ctx + dtype = mx_real_t if dtype is None else dtype + if aux_types is None: + if stype == 'row_sparse' or stype == 'csr': + aux_types = _STORAGE_AUX_TYPES[stype] + else: + raise Exception("unknown storage type") + assert(len(aux_types) == len(_STORAGE_AUX_TYPES[stype])) + out = _ndarray_cls(_new_alloc_handle(stype, shape, ctx, True, dtype, aux_types)) + return _internal._zeros(shape=shape, ctx=ctx, dtype=dtype, out=out, **kwargs) + + +def empty(stype, shape, ctx=None, dtype=None, aux_types=None): + """Returns a new array of given shape and type, without initializing entries. + """ + if isinstance(shape, int): + shape = (shape, ) + if ctx is None: + ctx = Context.default_ctx + if dtype is None: + dtype = mx_real_t + assert(stype is not None) + if stype == 'csr' or stype == 'row_sparse': + return zeros(stype, shape, ctx=ctx, dtype=dtype, aux_types=aux_types) + else: + raise Exception("unknown stype : " + str(stype)) + + +def array(source_array, ctx=None, dtype=None, aux_types=None): + """Creates a sparse array from any object exposing the array interface. + """ + if isinstance(source_array, NDArray): + assert(source_array.stype != 'default'), \ + "Please use `cast_storage` to create BaseSparseNDArray from an NDArray" + dtype = source_array.dtype if dtype is None else dtype + aux_types = source_array._aux_types if aux_types is None else aux_types + else: + # TODO(haibin/anisub) support creation from scipy object when `_sync_copy_from` is ready + raise NotImplementedError('creating BaseSparseNDArray from ' \ + ' a non-NDArray object is not implemented.') + arr = empty(source_array.stype, source_array.shape, ctx, dtype, aux_types) + arr[:] = source_array + return arr diff --git a/python/mxnet/ndarray/utils.py b/python/mxnet/ndarray/utils.py new file mode 100644 index 000000000000..a0dd83692b87 --- /dev/null +++ b/python/mxnet/ndarray/utils.py @@ -0,0 +1,240 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# coding: utf-8 +"""Utility functions for NDArray and BaseSparseNDArray.""" +import ctypes + +from ..base import _LIB, check_call, py_str, c_str, string_types, mx_uint, NDArrayHandle, c_array +from .ndarray import NDArray +from .ndarray import array as _array +from .ndarray import empty as _empty_ndarray +from .ndarray import zeros as _zeros_ndarray +from .sparse import zeros as _zeros_sparse_ndarray +from .sparse import empty as _empty_sparse_ndarray +from .sparse import array as _sparse_array +from .sparse import _ndarray_cls + + +def zeros(shape, ctx=None, dtype=None, stype=None, aux_types=None, **kwargs): + """Return a new array of given shape and type, filled with zeros. + + Parameters + ---------- + shape : int or tuple of int + The shape of the empty array + ctx : Context, optional + An optional device context (default is the current default context) + dtype : str or numpy.dtype, optional + An optional value type (default is `float32`) + stype: string, optional + The storage type of the empty array, such as 'row_sparse', 'csr', etc. + aux_types: list of numpy.dtype, optional + An optional list of types of the aux data for RowSparseNDArray or CSRNDArray + (default values depend on the storage type) + + Returns + ------- + NDArray, CSRNDArray or RowSparseNDArray + A created array + Examples + -------- + >>> mx.nd.zeros((1,2), mx.cpu(), stype='csr') + + >>> mx.nd.zeros((1,2), mx.cpu(), 'float16', stype='row_sparse').asnumpy() + array([[ 0., 0.]], dtype=float16) + """ + + if stype is None or stype == 'default': + return _zeros_ndarray(shape, ctx, dtype, **kwargs) + else: + return _zeros_sparse_ndarray(stype, shape, ctx, dtype, aux_types, **kwargs) + + +def empty(shape, ctx=None, dtype=None, stype=None, aux_types=None): + """Returns a new array of given shape and type, without initializing entries. + + Parameters + ---------- + shape : int or tuple of int + The shape of the empty array. + ctx : Context, optional + An optional device context (default is the current default context). + dtype : str or numpy.dtype, optional + An optional value type (default is `float32`). + stype : str, optional + An optional storage type (default is `default`). + aux_types: list of numpy.dtype, optional + An optional list of types of the aux data for RowSparseNDArray or CSRNDArray + (default values depend on the storage type) + + Returns + ------- + NDArray, CSRNDArray or RowSparseNDArray + A created array. + + Examples + -------- + >>> mx.nd.empty(1) + + >>> mx.nd.empty((1,2), mx.gpu(0)) + + >>> mx.nd.empty((1,2), mx.gpu(0), 'float16') + + >>> mx.nd.empty((1,2), stype='csr') + + """ + if stype is None or stype == 'default': + return _empty_ndarray(shape, ctx, dtype) + else: + return _empty_sparse_ndarray(stype, shape, ctx, dtype, aux_types) + + +def array(source_array, ctx=None, dtype=None, aux_types=None): + """Creates an array from any object exposing the array interface. + + Parameters + ---------- + source_array : array_like + An object exposing the array interface, an object whose `__array__` + method returns an array, or any (nested) sequence. + ctx : Context, optional + Device context (default is the current default context). + dtype : str or numpy.dtype, optional + The data type of the output array. The default dtype is ``source_array.dtype`` + if `source_array` is an `NDArray`, `float32` otherwise. + aux_types: list of numpy.dtype, optional + An optional list of types of the aux data for RowSparseNDArray or CSRNDArray + (default values depend on the storage type) + + Returns + ------- + NDArray, RowSparseNDArray or CSRNDArray + An array with the same contents as the `source_array`. + + Examples + -------- + >>> import numpy as np + >>> mx.nd.array([1, 2, 3]) + + >>> mx.nd.array([[1, 2], [3, 4]]) + + >>> mx.nd.array(np.zeros((3, 2))) + + >>> mx.nd.array(np.zeros((3, 2)), mx.gpu(0)) + + >>> mx.nd.array(mx.nd.zeros((3, 2), stype='row_sparse')) + + """ + # TODO(haibin/anisub) Check if input is scipy.sparse object with `scipy.sparse.issparse` + if isinstance(source_array, NDArray) and source_array.stype != 'default': + return _sparse_array(source_array, ctx=ctx, dtype=dtype, aux_types=aux_types) + else: + return _array(source_array, ctx=ctx, dtype=dtype) + + +def load(fname): + """Loads an array from file. + + See more details in ``save``. + + Parameters + ---------- + fname : str + The filename. + + Returns + ------- + list of NDArray, RowSparseNDArray or CSRNDArray, or \ + dict of str to NDArray, RowSparseNDArray or CSRNDArray + Loaded data. + """ + if not isinstance(fname, string_types): + raise TypeError('fname required to be a string') + out_size = mx_uint() + out_name_size = mx_uint() + handles = ctypes.POINTER(NDArrayHandle)() + names = ctypes.POINTER(ctypes.c_char_p)() + check_call(_LIB.MXNDArrayLoad(c_str(fname), + ctypes.byref(out_size), + ctypes.byref(handles), + ctypes.byref(out_name_size), + ctypes.byref(names))) + if out_name_size.value == 0: + return [_ndarray_cls(NDArrayHandle(handles[i])) for i in range(out_size.value)] + else: + assert out_name_size.value == out_size.value + return dict( + (py_str(names[i]), _ndarray_cls(NDArrayHandle(handles[i]))) + for i in range(out_size.value)) + + +def save(fname, data): + """Saves a list of arrays or a dict of str->array to file. + + Examples of filenames: + + - ``/path/to/file`` + - ``s3://my-bucket/path/to/file`` (if compiled with AWS S3 supports) + - ``hdfs://path/to/file`` (if compiled with HDFS supports) + + Parameters + ---------- + fname : str + The filename. + data : NDArray, RowSparseNDArray or CSRNDArray, \ + or list of NDArray, RowSparseNDArray or CSRNDArray, \ + or dict of str to NDArray, RowSparseNDArray or CSRNDArray + The data to save. + + Examples + -------- + >>> x = mx.nd.zeros((2,3)) + >>> y = mx.nd.ones((1,4)) + >>> mx.nd.save('my_list', [x,y]) + >>> mx.nd.save('my_dict', {'x':x, 'y':y}) + >>> mx.nd.load('my_list') + [, ] + >>> mx.nd.load('my_dict') + {'y': , 'x': } + """ + if isinstance(data, NDArray): + data = [data] + handles = [] + if isinstance(data, dict): + keys = [] + for key, val in data.items(): + if not isinstance(key, string_types): + raise TypeError('save only accept dict str->NDArray or list of NDArray') + if not isinstance(val, NDArray): + raise TypeError('save only accept dict str->NDArray or list of NDArray') + keys.append(c_str(key)) + handles.append(val.handle) + keys = c_array(ctypes.c_char_p, keys) + elif isinstance(data, list): + for val in data: + if not isinstance(val, NDArray): + raise TypeError('save only accept dict str->NDArray or list of NDArray') + handles.append(val.handle) + keys = None + else: + raise ValueError("data needs to either be a NDArray, dict of str, NDArray pairs " + "or a list of NDarrays.") + check_call(_LIB.MXNDArraySave(c_str(fname), + mx_uint(len(handles)), + c_array(NDArrayHandle, handles), + keys)) diff --git a/python/mxnet/optimizer.py b/python/mxnet/optimizer.py index 1ef9cc845036..e7e283f88e43 100644 --- a/python/mxnet/optimizer.py +++ b/python/mxnet/optimizer.py @@ -339,8 +339,8 @@ class SGD(Optimizer): state = momentum * state + lr * rescale_grad * clip(grad, clip_gradient) + wd * weight weight = weight - state - For details of the update algorithm see :class:`~mxnet.ndarray.sgd_update` and - :class:`~mxnet.ndarray.sgd_mom_update`. + Sparse updating is supported. For details of the update algorithm see + :class:`~mxnet.ndarray.sgd_update` and :class:`~mxnet.ndarray.sgd_mom_update`. This optimizer accepts the following parameters in addition to those accepted by :class:`.Optimizer`. @@ -367,7 +367,8 @@ def create_state(self, index, weight): if self.multi_precision and weight.dtype == numpy.float16: weight_master_copy = array(weight, ctx=weight.context, dtype=numpy.float32) if self.momentum != 0.0: - momentum = zeros(weight.shape, weight.context, dtype=numpy.float32) + momentum = zeros(weight.shape, weight.context, dtype=numpy.float32, + stype=weight.stype) return (momentum, weight_master_copy) if weight.dtype == numpy.float16 and not self.multi_precision: warnings.warn("Accumulating with float16 in optimizer can lead to " @@ -375,7 +376,7 @@ def create_state(self, index, weight): "Consider using multi_precision=True option of the " "SGD optimizer") if self.momentum != 0.0: - momentum = zeros(weight.shape, weight.context, dtype=weight.dtype) + momentum = zeros(weight.shape, weight.context, dtype=weight.dtype, stype=weight.stype) return momentum def update(self, index, weight, grad, state): @@ -563,8 +564,10 @@ def __init__(self, learning_rate=0.001, beta1=0.9, beta2=0.999, epsilon=1e-8, self.epsilon = epsilon def create_state(self, index, weight): - return (zeros(weight.shape, weight.context, dtype=weight.dtype), # mean - zeros(weight.shape, weight.context, dtype=weight.dtype)) # variance + return (zeros(weight.shape, weight.context, dtype=weight.dtype, + stype=weight.stype), # mean + zeros(weight.shape, weight.context, dtype=weight.dtype, + stype=weight.stype)) # variance def update(self, index, weight, grad, state): assert(isinstance(weight, NDArray)) @@ -669,11 +672,11 @@ def __init__(self, learning_rate=0.001, gamma1=0.9, gamma2=0.9, def create_state(self, index, weight): if self.centered: return ( - zeros(weight.shape, weight.context), # n - zeros(weight.shape, weight.context), # g - zeros(weight.shape, weight.context)) # delta + zeros(weight.shape, weight.context, stype=weight.stype), # n + zeros(weight.shape, weight.context, stype=weight.stype), # g + zeros(weight.shape, weight.context, stype=weight.stype)) # delta else: - return (zeros(weight.shape, weight.context), ) # n + return (zeros(weight.shape, weight.context, stype=weight.stype),) # n def update(self, index, weight, grad, state): assert(isinstance(weight, NDArray)) diff --git a/python/mxnet/random.py b/python/mxnet/random.py index 29b250d980ce..14bfc2731bd6 100644 --- a/python/mxnet/random.py +++ b/python/mxnet/random.py @@ -22,13 +22,13 @@ import ctypes from .base import _LIB, check_call -from ._ndarray_internal import _sample_uniform as uniform -from ._ndarray_internal import _sample_normal as normal -from ._ndarray_internal import _sample_gamma as gamma -from ._ndarray_internal import _sample_exponential as exponential -from ._ndarray_internal import _sample_poisson as poisson -from ._ndarray_internal import _sample_negbinomial as negative_binomial -from ._ndarray_internal import _sample_gennegbinomial as generalized_negative_binomial +from .ndarray._internal import _sample_uniform as uniform +from .ndarray._internal import _sample_normal as normal +from .ndarray._internal import _sample_gamma as gamma +from .ndarray._internal import _sample_exponential as exponential +from .ndarray._internal import _sample_poisson as poisson +from .ndarray._internal import _sample_negbinomial as negative_binomial +from .ndarray._internal import _sample_gennegbinomial as generalized_negative_binomial def seed(seed_state): """Seeds the random number generators in MXNet. diff --git a/python/mxnet/symbol/__init__.py b/python/mxnet/symbol/__init__.py new file mode 100644 index 000000000000..d93a230f490d --- /dev/null +++ b/python/mxnet/symbol/__init__.py @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Symbol API of MXNet.""" + +from . import _internal, sparse, op +# pylint: disable=wildcard-import, redefined-builtin +from .symbol import * +from ..ndarray import _GRAD_REQ_MAP diff --git a/python/mxnet/_symbol_internal.py b/python/mxnet/symbol/_internal.py similarity index 100% rename from python/mxnet/_symbol_internal.py rename to python/mxnet/symbol/_internal.py diff --git a/python/mxnet/symbol/op.py b/python/mxnet/symbol/op.py new file mode 100644 index 000000000000..82884a5cc6a2 --- /dev/null +++ b/python/mxnet/symbol/op.py @@ -0,0 +1,242 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Register backend ops in mxnet.symbol namespace.""" + +import sys as _sys +import os as _os +import ctypes +import numpy as _numpy # pylint: disable=unused-import + +from mxnet.base import mx_uint, check_call, _LIB, py_str, OpHandle, c_str +from mxnet.symbol_doc import _build_doc + +# Use different version of SymbolBase +# When possible, use cython to speedup part of computation. +# pylint: disable=unused-import +try: + if int(_os.environ.get("MXNET_ENABLE_CYTHON", True)) == 0: + from .._ctypes.symbol import SymbolBase, _set_symbol_class + from .._ctypes.symbol import _symbol_creator + elif _sys.version_info >= (3, 0): + from .._cy3.symbol import SymbolBase, _set_symbol_class + from .._cy3.symbol import _symbol_creator + else: + from .._cy2.symbol import SymbolBase, _set_symbol_class + from .._cy2.symbol import _symbol_creator +except ImportError: + if int(_os.environ.get("MXNET_ENFORCE_CYTHON", False)) != 0: + raise ImportError("Cython Module cannot be loaded but MXNET_ENFORCE_CYTHON=1") + from .._ctypes.symbol import SymbolBase, _set_symbol_class + from .._ctypes.symbol import _symbol_creator + +from ..base import _Null +from ..name import NameManager +from ..attribute import AttrScope +# pylint: enable=unused-import + + +def _make_atomic_symbol_function(handle, name): + """Create an atomic symbol function by handle and function name.""" + real_name = ctypes.c_char_p() + desc = ctypes.c_char_p() + num_args = mx_uint() + arg_names = ctypes.POINTER(ctypes.c_char_p)() + arg_types = ctypes.POINTER(ctypes.c_char_p)() + arg_descs = ctypes.POINTER(ctypes.c_char_p)() + key_var_num_args = ctypes.c_char_p() + ret_type = ctypes.c_char_p() + + check_call(_LIB.MXSymbolGetAtomicSymbolInfo( + handle, ctypes.byref(real_name), ctypes.byref(desc), + ctypes.byref(num_args), + ctypes.byref(arg_names), + ctypes.byref(arg_types), + ctypes.byref(arg_descs), + ctypes.byref(key_var_num_args), + ctypes.byref(ret_type))) + narg = int(num_args.value) + arg_names = [py_str(arg_names[i]) for i in range(narg)] + arg_types = [py_str(arg_types[i]) for i in range(narg)] + func_name = name + key_var_num_args = py_str(key_var_num_args.value) + ret_type = py_str(ret_type.value) if ret_type.value is not None else '' + doc_str = _build_doc(func_name, + py_str(desc.value), + arg_names, + arg_types, + [py_str(arg_descs[i]) for i in range(narg)], + key_var_num_args, + ret_type) + + dtype_name = None + arr_name = None + ndsignature = [] + signature = [] + ndarg_names = [] + kwarg_names = [] + for i in range(narg): + name, atype = arg_names[i], arg_types[i] + if name == 'dtype': + dtype_name = name + signature.append('%s=_Null'%name) + elif atype.startswith('NDArray') or atype.startswith('Symbol'): + assert not arr_name, \ + "Op can only have one argument with variable " \ + "size and it must be the last argument." + if atype.endswith('[]'): + ndsignature.append('*%s'%name) + arr_name = name + else: + ndsignature.append('%s=None'%name) + ndarg_names.append(name) + else: + signature.append('%s=_Null'%name) + kwarg_names.append(name) + #signature.append('is_train=False') + signature.append('name=None') + signature.append('attr=None') + signature.append('out=None') + signature.append('**kwargs') + signature = ndsignature + signature + + code = [] + if arr_name: + code.append(""" +def %s(*%s, **kwargs):"""%(func_name, arr_name)) + code.append(""" + sym_args = [] + for i in {}: + assert isinstance(i, SymbolBase), \\ + "Positional arguments must be Symbol instances, " \\ + "but got %s"%str(i) + sym_args.append(i)""".format(arr_name)) + if dtype_name is not None: + code.append(""" + if '%s' in kwargs: + kwargs['%s'] = _numpy.dtype(kwargs['%s']).name"""%( + dtype_name, dtype_name, dtype_name)) + code.append(""" + attr = kwargs.pop('attr', None) + kwargs.update(AttrScope.current.get(attr)) + name = kwargs.pop('name', None) + name = NameManager.current.get(name, '%s') + _ = kwargs.pop('out', None) + keys = [] + vals = [] + sym_kwargs = dict() + for k, v in kwargs.items(): + if isinstance(v, SymbolBase): + sym_kwargs[k] = v + else: + keys.append(k) + vals.append(v)"""%(func_name.lower())) + if key_var_num_args: + code.append(""" + if '%s' not in kwargs: + keys.append('%s') + vals.append(len(sym_args) + len(sym_kwargs))"""%( + key_var_num_args, key_var_num_args)) + + code.append(""" + return _symbol_creator(%d, sym_args, sym_kwargs, keys, vals, name)"""%( + handle.value)) + else: + code.append(""" +def %s(%s): + kwargs.update(AttrScope.current.get(attr)) + sym_kwargs = dict() + keys = [] + vals = []"""%(func_name, ', '.join(signature))) + code.append(""" + for k, v in kwargs.items(): + if isinstance(v, SymbolBase): + sym_kwargs[k] = v + else: + keys.append(k) + vals.append(v)""") + # NDArray args + for name in ndarg_names: # pylint: disable=redefined-argument-from-local + code.append(""" + if {name} is not None: + assert isinstance({name}, SymbolBase), \\ + "Argument {name} must be Symbol instances, but got %s"%str({name}) + sym_kwargs['{name}'] = {name}""".format(name=name)) + # kwargs + for name in kwarg_names: # pylint: disable=redefined-argument-from-local + code.append(""" + if %s is not _Null: + keys.append('%s') + vals.append(%s)"""%(name, name, name)) + # dtype + if dtype_name is not None: + code.append(""" + if %s is not _Null: + keys.append('%s') + vals.append(_numpy.dtype(%s).name)"""%(dtype_name, dtype_name, dtype_name)) + + code.append(""" + name = NameManager.current.get(name, '%s') + return _symbol_creator(%d, None, sym_kwargs, keys, vals, name)"""%( + func_name.lower(), handle.value)) + + local = {} + exec(''.join(code), None, local) # pylint: disable=exec-used + symbol_function = local[func_name] + symbol_function.__name__ = func_name + symbol_function.__doc__ = doc_str + symbol_function.__module__ = 'mxnet.symbol' + return symbol_function + + +def _init_symbol_module(root_namespace): + """List and add all the atomic symbol functions to current module.""" + plist = ctypes.POINTER(ctypes.c_char_p)() + size = ctypes.c_uint() + + check_call(_LIB.MXListAllOpNames(ctypes.byref(size), + ctypes.byref(plist))) + op_names = [] + for i in range(size.value): + op_names.append(py_str(plist[i])) + + module_obj = _sys.modules["%s.symbol" % root_namespace] + module_sparse = _sys.modules["%s.symbol.sparse" % root_namespace] + module_internal = _sys.modules["%s.symbol._internal" % root_namespace] + module_contrib = _sys.modules["%s.contrib.symbol" % root_namespace] + for name in op_names: + hdl = OpHandle() + check_call(_LIB.NNGetOpHandle(c_str(name), ctypes.byref(hdl))) + function = _make_atomic_symbol_function(hdl, name) + if function.__name__.startswith('_contrib_'): + function.__name__ = function.__name__[9:] + function.__module__ = 'mxnet.contrib.symbol' + setattr(module_contrib, function.__name__, function) + elif function.__name__.startswith('_'): + setattr(module_internal, function.__name__, function) + else: + setattr(module_obj, function.__name__, function) + + # register sparse ops under mxnet.symbol.sparse + if function.__name__.startswith('_sparse_'): + function.__name__ = function.__name__[8:] + function.__module__ = 'mxnet.symbol.sparse' + setattr(module_sparse, function.__name__, function) + + +# Initialize the atomic symbol in startups +_init_symbol_module("mxnet") diff --git a/python/mxnet/symbol/sparse.py b/python/mxnet/symbol/sparse.py new file mode 100644 index 000000000000..1d94f2b85bc7 --- /dev/null +++ b/python/mxnet/symbol/sparse.py @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Sparse Symbol API of MXNet.""" diff --git a/python/mxnet/symbol.py b/python/mxnet/symbol/symbol.py similarity index 90% rename from python/mxnet/symbol.py rename to python/mxnet/symbol/symbol.py index 14cb3811deeb..aa8ca0b8dd53 100644 --- a/python/mxnet/symbol.py +++ b/python/mxnet/symbol/symbol.py @@ -29,39 +29,19 @@ import warnings from numbers import Number -import os as _os -import sys as _sys import numpy as _numpy -from .base import _LIB, numeric_types -from .base import c_array, c_str, mx_uint, py_str, string_types -from .base import NDArrayHandle, ExecutorHandle, SymbolHandle, OpHandle -from .base import check_call, MXNetError, NotImplementedForSymbol, _Null # pylint: disable=unused-import -from .context import Context -from .ndarray import NDArray, _DTYPE_NP_TO_MX, _DTYPE_MX_TO_NP, _GRAD_REQ_MAP -from .name import NameManager # pylint: disable=unused-import -from .executor import Executor -from . import _symbol_internal as _internal -from .attribute import AttrScope -from .symbol_doc import _build_doc - -# Use different version of SymbolBase -# When possible, use cython to speedup part of computation. -try: - if int(_os.environ.get("MXNET_ENABLE_CYTHON", True)) == 0: - from ._ctypes.symbol import SymbolBase, _set_symbol_class - from ._ctypes.symbol import _symbol_creator # pylint: disable=unused-import - elif _sys.version_info >= (3, 0): - from ._cy3.symbol import SymbolBase, _set_symbol_class - from ._cy3.symbol import _symbol_creator # pylint: disable=unused-import - else: - from ._cy2.symbol import SymbolBase, _set_symbol_class - from ._cy2.symbol import _symbol_creator # pylint: disable=unused-import -except ImportError: - if int(_os.environ.get("MXNET_ENFORCE_CYTHON", False)) != 0: - raise ImportError("Cython Module cannot be loaded but MXNET_ENFORCE_CYTHON=1") - from ._ctypes.symbol import SymbolBase, _set_symbol_class - from ._ctypes.symbol import _symbol_creator # pylint: disable=unused-import +from ..base import _LIB, numeric_types +from ..base import c_array, c_str, mx_uint, py_str, string_types +from ..base import NDArrayHandle, ExecutorHandle, SymbolHandle +from ..base import check_call, MXNetError, NotImplementedForSymbol +from ..context import Context +from ..ndarray import NDArray, _DTYPE_NP_TO_MX, _DTYPE_MX_TO_NP, _GRAD_REQ_MAP +from ..ndarray.ndarray import _STORAGE_TYPE_STR_TO_ID +from ..ndarray import _ndarray_cls +from ..executor import Executor +from . import _internal, reshape +from .op import SymbolBase, _set_symbol_class, AttrScope, _Null # pylint: disable=unused-import class Symbol(SymbolBase): @@ -1263,8 +1243,9 @@ def _get_ndarray_inputs(arg_key, args, arg_names, allow_missing): raise TypeError('Only accept list of NDArrays or dict of str to NDArray') return c_array(NDArrayHandle, arg_handles), arg_arrays - def simple_bind(self, ctx, grad_req='write', type_dict=None, group2ctx=None, - shared_arg_names=None, shared_exec=None, shared_buffer=None, **kwargs): + def simple_bind(self, ctx, grad_req='write', type_dict=None, stype_dict=None, + group2ctx=None, shared_arg_names=None, shared_exec=None, + shared_buffer=None, **kwargs): """Bind current symbol to get an executor, allocate all the arguments needed. Allows specifying data types. @@ -1306,6 +1287,9 @@ def simple_bind(self, ctx, grad_req='write', type_dict=None, group2ctx=None, type_dict : Dict of str->numpy.dtype Input type dictionary, name->dtype + stype_dict : Dict of str->str + Input storage type dictionary, name->storage_type + group2ctx : Dict of string to mx.Context The dict mapping the `ctx_group` attribute to the context assignment. @@ -1320,7 +1304,8 @@ def simple_bind(self, ctx, grad_req='write', type_dict=None, group2ctx=None, shared_buffer : Dict of string to `NDArray` The dict mapping argument names to the `NDArray` that can be reused for initializing the current executor. This buffer will be checked for reuse if one argument name - of the current executor is not found in `shared_arg_names`. + of the current executor is not found in `shared_arg_names`. The `NDArray`s are + expected have default storage type. kwargs : Dict of str->shape Input shape dictionary, name->shape @@ -1330,6 +1315,7 @@ def simple_bind(self, ctx, grad_req='write', type_dict=None, group2ctx=None, executor : mxnet.Executor The generated executor """ + # data types num_provided_arg_types = 0 provided_arg_type_names = ctypes.POINTER(ctypes.c_char_p)() # provided type argument names provided_arg_type_data = ctypes.POINTER(mx_uint)() # provided types @@ -1345,6 +1331,22 @@ def simple_bind(self, ctx, grad_req='write', type_dict=None, group2ctx=None, provided_arg_type_names = c_array(ctypes.c_char_p, provided_arg_type_names) provided_arg_type_data = c_array(ctypes.c_int, provided_arg_type_data) + # storage types + num_provided_arg_stypes = 0 + # provided storage type argument names + provided_arg_stype_names = ctypes.POINTER(ctypes.c_char_p)() + provided_arg_stype_data = ctypes.POINTER(mx_uint)() # provided storage types + if stype_dict is not None: + provided_arg_stype_names = [] + provided_arg_stype_data = [] + for k, v in stype_dict.items(): + if v in _STORAGE_TYPE_STR_TO_ID: + provided_arg_stype_names.append(c_str(k)) + provided_arg_stype_data.append(ctypes.c_int(_STORAGE_TYPE_STR_TO_ID[v])) + num_provided_arg_stypes = mx_uint(len(provided_arg_stype_names)) + provided_arg_stype_names = c_array(ctypes.c_char_p, provided_arg_stype_names) + provided_arg_stype_data = c_array(ctypes.c_int, provided_arg_stype_data) + provided_arg_shape_data = [] # shape data # argument shape index in sdata, # e.g. [sdata[indptr[0]], sdata[indptr[1]]) is the shape of the first arg @@ -1418,6 +1420,8 @@ def simple_bind(self, ctx, grad_req='write', type_dict=None, group2ctx=None, shared_buffer_names = [] shared_buffer_handles = [] for k, v in shared_buffer.items(): + assert(v.stype == 'default'), \ + "shared_buffer is expected to only contain NDArrays with default storage" shared_buffer_names.append(c_str(k)) shared_buffer_handles.append(v.handle) shared_buffer_names = c_array(ctypes.c_char_p, shared_buffer_names) @@ -1457,6 +1461,9 @@ def simple_bind(self, ctx, grad_req='write', type_dict=None, group2ctx=None, num_provided_arg_types, provided_arg_type_names, provided_arg_type_data, + num_provided_arg_stypes, + provided_arg_stype_names, + provided_arg_stype_data, mx_uint(len(shared_arg_name_list)), c_array(ctypes.c_char_p, shared_arg_name_list), ctypes.byref(shared_buffer_len), @@ -1486,11 +1493,12 @@ def simple_bind(self, ctx, grad_req='write', type_dict=None, group2ctx=None, shared_buffer[k] = v # create in_args, arg_grads, and aux_states for the current executor - arg_arrays = [NDArray(NDArrayHandle(in_arg_handles[i])) for i in range(num_in_args.value)] - grad_arrays = [NDArray(NDArrayHandle(arg_grad_handles[i])) + arg_arrays = [_ndarray_cls(NDArrayHandle(in_arg_handles[i])) \ + for i in range(num_in_args.value)] + grad_arrays = [_ndarray_cls(NDArrayHandle(arg_grad_handles[i])) if arg_grad_handles[i] is not None else None for i in range(num_in_args.value)] - aux_arrays = [NDArray(NDArrayHandle(aux_state_handles[i])) + aux_arrays = [_ndarray_cls(NDArrayHandle(aux_state_handles[i])) for i in range(num_aux_states.value)] executor = Executor(exe_handle, self, ctx, grad_req, group2ctx) @@ -1767,7 +1775,8 @@ def detach(self): def backward(self): raise NotImplementedForSymbol(self.backward, None) -def var(name, attr=None, shape=None, lr_mult=None, wd_mult=None, dtype=None, init=None, **kwargs): +def var(name, attr=None, shape=None, lr_mult=None, wd_mult=None, dtype=None, + init=None, stype=None, **kwargs): """Creates a symbolic variable with specified name. Example usage: @@ -1794,6 +1803,8 @@ def var(name, attr=None, shape=None, lr_mult=None, wd_mult=None, dtype=None, ini The dtype for input variable. If not specified, this value will be inferred. init : initializer (mxnet.init.*) Initializer for this variable to (optionally) override the default initializer. + stype : str + The storage type of the variable. kwargs : Additional attribute variables Additional attributes must start and end with double underscores. @@ -1821,6 +1832,8 @@ def var(name, attr=None, shape=None, lr_mult=None, wd_mult=None, dtype=None, ini if not isinstance(init, string_types): init = init.dumps() attr['__init__'] = init + if stype is not None: + attr['__storage_type__'] = str(_STORAGE_TYPE_STR_TO_ID[stype]) for k, v in kwargs.items(): if k.startswith('__') and k.endswith('__'): attr[k] = str(v) @@ -2195,188 +2208,4 @@ def arange(start, stop=None, step=1.0, repeat=1, name=None, dtype=None): return _internal._arange(start=start, stop=stop, step=step, repeat=repeat, name=name, dtype=dtype) - -def _make_atomic_symbol_function(handle, name): - """Create an atomic symbol function by handle and function name.""" - real_name = ctypes.c_char_p() - desc = ctypes.c_char_p() - num_args = mx_uint() - arg_names = ctypes.POINTER(ctypes.c_char_p)() - arg_types = ctypes.POINTER(ctypes.c_char_p)() - arg_descs = ctypes.POINTER(ctypes.c_char_p)() - key_var_num_args = ctypes.c_char_p() - ret_type = ctypes.c_char_p() - - check_call(_LIB.MXSymbolGetAtomicSymbolInfo( - handle, ctypes.byref(real_name), ctypes.byref(desc), - ctypes.byref(num_args), - ctypes.byref(arg_names), - ctypes.byref(arg_types), - ctypes.byref(arg_descs), - ctypes.byref(key_var_num_args), - ctypes.byref(ret_type))) - narg = int(num_args.value) - arg_names = [py_str(arg_names[i]) for i in range(narg)] - arg_types = [py_str(arg_types[i]) for i in range(narg)] - func_name = name - key_var_num_args = py_str(key_var_num_args.value) - ret_type = py_str(ret_type.value) if ret_type.value is not None else '' - doc_str = _build_doc(func_name, - py_str(desc.value), - arg_names, - arg_types, - [py_str(arg_descs[i]) for i in range(narg)], - key_var_num_args, - ret_type) - - dtype_name = None - arr_name = None - ndsignature = [] - signature = [] - ndarg_names = [] - kwarg_names = [] - for i in range(narg): - name, atype = arg_names[i], arg_types[i] - if name == 'dtype': - dtype_name = name - signature.append('%s=_Null'%name) - elif atype.startswith('NDArray') or atype.startswith('Symbol'): - assert not arr_name, \ - "Op can only have one argument with variable " \ - "size and it must be the last argument." - if atype.endswith('[]'): - ndsignature.append('*%s'%name) - arr_name = name - else: - ndsignature.append('%s=None'%name) - ndarg_names.append(name) - else: - signature.append('%s=_Null'%name) - kwarg_names.append(name) - #signature.append('is_train=False') - signature.append('name=None') - signature.append('attr=None') - signature.append('out=None') - signature.append('**kwargs') - signature = ndsignature + signature - - code = [] - if arr_name: - code.append(""" -def %s(*%s, **kwargs):"""%(func_name, arr_name)) - code.append(""" - sym_args = [] - for i in {}: - assert isinstance(i, SymbolBase), \\ - "Positional arguments must be Symbol instances, " \\ - "but got %s"%str(i) - sym_args.append(i)""".format(arr_name)) - if dtype_name is not None: - code.append(""" - if '%s' in kwargs: - kwargs['%s'] = _numpy.dtype(kwargs['%s']).name"""%( - dtype_name, dtype_name, dtype_name)) - code.append(""" - attr = kwargs.pop('attr', None) - kwargs.update(AttrScope.current.get(attr)) - name = kwargs.pop('name', None) - name = NameManager.current.get(name, '%s') - _ = kwargs.pop('out', None) - keys = [] - vals = [] - sym_kwargs = dict() - for k, v in kwargs.items(): - if isinstance(v, SymbolBase): - sym_kwargs[k] = v - else: - keys.append(k) - vals.append(v)"""%(func_name.lower())) - if key_var_num_args: - code.append(""" - if '%s' not in kwargs: - keys.append('%s') - vals.append(len(sym_args) + len(sym_kwargs))"""%( - key_var_num_args, key_var_num_args)) - - code.append(""" - return _symbol_creator(%d, sym_args, sym_kwargs, keys, vals, name)"""%( - handle.value)) - else: - code.append(""" -def %s(%s): - kwargs.update(AttrScope.current.get(attr)) - sym_kwargs = dict() - keys = [] - vals = []"""%(func_name, ', '.join(signature))) - code.append(""" - for k, v in kwargs.items(): - if isinstance(v, SymbolBase): - sym_kwargs[k] = v - else: - keys.append(k) - vals.append(v)""") - # NDArray args - for name in ndarg_names: # pylint: disable=redefined-argument-from-local - code.append(""" - if {name} is not None: - assert isinstance({name}, SymbolBase), \\ - "Argument {name} must be Symbol instances, but got %s"%str({name}) - sym_kwargs['{name}'] = {name}""".format(name=name)) - # kwargs - for name in kwarg_names: # pylint: disable=redefined-argument-from-local - code.append(""" - if %s is not _Null: - keys.append('%s') - vals.append(%s)"""%(name, name, name)) - # dtype - if dtype_name is not None: - code.append(""" - if %s is not _Null: - keys.append('%s') - vals.append(_numpy.dtype(%s).name)"""%(dtype_name, dtype_name, dtype_name)) - - code.append(""" - name = NameManager.current.get(name, '%s') - return _symbol_creator(%d, None, sym_kwargs, keys, vals, name)"""%( - func_name.lower(), handle.value)) - - local = {} - exec(''.join(code), None, local) # pylint: disable=exec-used - symbol_function = local[func_name] - symbol_function.__name__ = func_name - symbol_function.__doc__ = doc_str - symbol_function.__module__ = 'mxnet.symbol' - return symbol_function - - -def _init_symbol_module(symbol_class, root_namespace): - """List and add all the atomic symbol functions to current module.""" - _set_symbol_class(symbol_class) - plist = ctypes.POINTER(ctypes.c_char_p)() - size = ctypes.c_uint() - - check_call(_LIB.MXListAllOpNames(ctypes.byref(size), - ctypes.byref(plist))) - op_names = [] - for i in range(size.value): - op_names.append(py_str(plist[i])) - - module_obj = _sys.modules["%s.symbol" % root_namespace] - module_internal = _sys.modules["%s._symbol_internal" % root_namespace] - module_contrib = _sys.modules["%s.contrib.symbol" % root_namespace] - for name in op_names: - hdl = OpHandle() - check_call(_LIB.NNGetOpHandle(c_str(name), ctypes.byref(hdl))) - function = _make_atomic_symbol_function(hdl, name) - if function.__name__.startswith('_contrib_'): - function.__name__ = function.__name__[9:] - function.__module__ = 'mxnet.contrib.symbol' - setattr(module_contrib, function.__name__, function) - elif function.__name__.startswith('_'): - setattr(module_internal, function.__name__, function) - else: - setattr(module_obj, function.__name__, function) - - -# Initialize the atomic symbol in startups -_init_symbol_module(Symbol, "mxnet") +_set_symbol_class(Symbol) diff --git a/python/mxnet/test_utils.py b/python/mxnet/test_utils.py index c5587f8d80a8..e1210fbd3e6e 100644 --- a/python/mxnet/test_utils.py +++ b/python/mxnet/test_utils.py @@ -31,15 +31,17 @@ from contextlib import contextmanager import numpy as np import numpy.testing as npt -import mxnet as mx -from .context import Context -from .ndarray import array -from .symbol import Symbol +import numpy.random as rnd try: import requests except ImportError: # in rare cases requests may be not installed pass +import mxnet as mx +from .context import Context +from .ndarray.ndarray import _STORAGE_TYPE_STR_TO_ID +from .ndarray import array +from .symbol import Symbol _rng = np.random.RandomState(1234) @@ -85,6 +87,184 @@ def random_arrays(*shapes): return arrays +def random_sample(population, k): + """Return a k length list of the elements chosen from the population sequence.""" + assert 0 <= k <= len(population) + population_copy = population[:] + np.random.shuffle(population_copy) + return population_copy[0:k] + + +def _validate_csr_generation_inputs(num_rows, num_cols, density, + distribution="uniform"): + """Validates inputs for csr generation helper functions + """ + total_nnz = int(num_rows * num_cols * density) + if density < 0 or density > 1: + raise ValueError("density has to be between 0 and 1") + + if num_rows <= 0 or num_cols <= 0: + raise ValueError("num_rows or num_cols should be greater than 0") + + if distribution == "powerlaw": + if total_nnz < 2 * num_rows: + raise ValueError("not supported for this density: %s" + " for this shape (%s, %s)" + " Please keep :" + " num_rows * num_cols * density >= 2 * num_rows" + % (density, num_rows, num_cols)) + + +def _get_uniform_dataset_csr(num_rows, num_cols, density=0.1, dtype=None): + """Returns CSRNDArray with uniform distribution + This generates a csr matrix with totalnnz unique randomly chosen numbers + from num_rows*num_cols and arranges them in the 2d array in the + following way: row_index = (random_number_generated / num_rows) + col_index = random_number_generated - row_index * num_cols + """ + _validate_csr_generation_inputs(num_rows, num_cols, density, + distribution="uniform") + from scipy import sparse as sp + csr = sp.rand(num_rows, num_cols, density, dtype=dtype, format="csr") + result = mx.nd.sparse.csr_matrix(csr.data, csr.indptr, csr.indices, + (num_rows, num_cols), dtype=dtype) + return result + + +def _get_powerlaw_dataset_csr(num_rows, num_cols, density=0.1, dtype=None): + """Returns CSRNDArray with powerlaw distribution + with exponentially increasing number of non zeros in each row. + Not supported for cases where total_nnz < 2*num_rows. This is because + the algorithm first tries to ensure that there are rows with no zeros by + putting non zeros at beginning of each row. + """ + + _validate_csr_generation_inputs(num_rows, num_cols, density, + distribution="powerlaw") + + total_nnz = int(num_rows * num_cols * density) + + unused_nnz = total_nnz + output_arr = np.zeros((num_rows, num_cols), dtype=dtype) + # Start with ones on each row so that no row is empty + for row in range(num_rows): + output_arr[row][0] = 1 + rnd.uniform(0.001, 2) + unused_nnz = unused_nnz - 1 + if unused_nnz <= 0: + return mx.nd.array(output_arr).tostype("csr") + + # Populate rest of matrix with 2^i items in ith row. + # if we have used all total nnz return the sparse matrix + # else if we reached max column size then fill up full columns until we use all nnz + col_max = 2 + for row in range(num_rows): + col_limit = min(num_cols, col_max) + # In case col_limit reached assign same value to all elements, which is much faster + if col_limit == num_cols and unused_nnz > col_limit: + output_arr[row] = 1 + rnd.uniform(0.001, 2) + unused_nnz = unused_nnz - col_limit + 1 + if unused_nnz <= 0: + return mx.nd.array(output_arr).tostype("csr") + else: + continue + for col_index in range(1, col_limit): + output_arr[row][col_index] = 1 + rnd.uniform(0.001, 2) + unused_nnz = unused_nnz - 1 + if unused_nnz <= 0: + return mx.nd.array(output_arr).tostype("csr") + col_max = col_max * 2 + + if unused_nnz > 0: + raise ValueError("not supported for this density: %s" + " for this shape (%s,%s)" % (density, num_rows, num_cols)) + else: + return mx.nd.array(output_arr).tostype("csr") + + +def rand_sparse_ndarray(shape, stype, density=None, distribution=None, dtype=None): + """Generate a random sparse ndarray. Returns the ndarray, value(np) and indices(np) + Parameters + ---------- + shape: list or tuple + stype: str, valid values: "csr" or "row_sparse" + density, optional: float, should be between 0 and 1 + distribution, optional: str, valid values: "uniform" or "powerlaw" + dtype, optional: numpy.dtype, default value is None + Returns + ------- + Result of type CSRNDArray or RowSparseNDArray + Examples + -------- + Below is an example of the powerlaw distribution with csr as the stype. + It calculates the nnz using the shape and density. + It fills up the ndarray with exponentially increasing number of elements. + If there are enough unused_nnzs, n+1th row will have twice more nnzs compared to nth row. + else, remaining unused_nnzs will be used in n+1th row + If number of cols is too small and we have already reached column size it will fill up + all following columns in all followings rows until we reach the required density. + + >>> csr_arr, _ = rand_sparse_ndarray(shape=(5, 16), stype="csr", + density=0.50, distribution="powerlaw") + >>> indptr = csr_arr.indptr.asnumpy() + >>> indices = csr_arr.indices.asnumpy() + >>> data = csr_arr.data.asnumpy() + >>> row2nnz = len(data[indptr[1]:indptr[2]]) + >>> row3nnz = len(data[indptr[2]:indptr[3]]) + >>> assert(row3nnz == 2*row2nnz) + >>> row4nnz = len(data[indptr[3]:indptr[4]]) + >>> assert(row4nnz == 2*row3nnz) + """ + density = rnd.rand() if density is None else density + dtype = default_dtype() if dtype is None else dtype + distribution = "uniform" if distribution is None else distribution + if stype == 'row_sparse': + assert (distribution == "uniform"), \ + "Distribution %s not supported for row_sparse" % (distribution) + # sample index + idx_sample = rnd.rand(shape[0]) + indices = np.argwhere(idx_sample < density).flatten() + if indices.shape[0] == 0: + result = mx.nd.zeros(shape, stype='row_sparse', dtype=dtype) + return result, (np.array([], dtype=dtype), np.array([], dtype='int64')) + # generate random values + val = rnd.rand(indices.shape[0], *shape[1:]).astype(dtype) + arr = mx.nd.sparse.row_sparse_array(val, indices, shape, indices_type=np.int64, dtype=dtype) + return arr, (val, indices) + elif stype == 'csr': + assert len(shape) == 2 + if distribution == "uniform": + csr = _get_uniform_dataset_csr(shape[0], shape[1], density, dtype=dtype) + return csr, (csr.indptr, csr.indices, csr.data) + elif distribution == "powerlaw": + csr = _get_powerlaw_dataset_csr(shape[0], shape[1], density, dtype=dtype) + return csr, (csr.indptr, csr.indices, csr.data) + else: + assert(False), "Distribution not supported: %s" % (distribution) + else: + assert(False), "unknown storage type" + + +def rand_ndarray(shape, stype, density=None, dtype=None, distribution=None): + if stype == 'default': + arr = mx.nd.array(random_arrays(shape), dtype=dtype) + else: + arr, _ = rand_sparse_ndarray(shape, stype, density=density, dtype=dtype, + distribution=distribution) + return arr + + +def rand_shape_2d(dim0=10, dim1=10): + return rnd.randint(1, dim0 + 1), rnd.randint(1, dim1 + 1) + + +def rand_shape_3d(dim0=10, dim1=10, dim2=10): + return rnd.randint(1, dim0 + 1), rnd.randint(1, dim1 + 1), rnd.randint(1, dim2 + 1) + + +def rand_shape_nd(n, dim=10): + return rnd.randint(1, dim+1, size=n) + + def np_reduce(dat, axis, keepdims, numpy_reduce_func): """Compatible reduce for old version of NumPy. @@ -316,7 +496,8 @@ def _parse_location(sym, location, ctx): % (str(set(sym.list_arguments())), str(set(location.keys())))) else: location = {k: v for k, v in zip(sym.list_arguments(), location)} - location = {k: mx.nd.array(v, ctx=ctx) for k, v in location.items()} + location = {k: mx.nd.array(v, ctx=ctx) if isinstance(v, np.ndarray) \ + else v for k, v in location.items()} return location @@ -437,7 +618,8 @@ def numeric_grad(executor, location, aux_states=None, eps=1e-4, use_forward_trai def check_numeric_gradient(sym, location, aux_states=None, numeric_eps=1e-3, rtol=1e-2, - atol=None, grad_nodes=None, use_forward_train=True, ctx=None): + atol=None, grad_nodes=None, use_forward_train=True, ctx=None, + grad_stype_dict=None): """Verify an operation by checking backward pass via finite difference method. Based on Theano's `theano.gradient.verify_grad` [1] @@ -454,7 +636,7 @@ def check_numeric_gradient(sym, location, aux_states=None, numeric_eps=1e-3, rto - if type is dict of str -> numpy.ndarray maps the name of arguments to the corresponding numpy.ndarray. *In either case, value of all the arguments must be provided.* - aux_states : ist or tuple or dict, optional + aux_states : list or tuple or dict, optional The auxiliary states required when generating the executor for the symbol. numeric_eps : float, optional Delta for the finite difference method that approximates the gradient. @@ -466,6 +648,8 @@ def check_numeric_gradient(sym, location, aux_states=None, numeric_eps=1e-3, rto Whether to use is_train=True when computing the finite-difference. ctx : Context, optional Check the gradient computation on the specified device. + grad_stype_dict : dict of str->str, optional + Storage type dictionary for gradient ndarrays. References --------- ..[1] https://github.com/Theano/Theano/blob/master/theano/gradient.py @@ -489,7 +673,7 @@ def random_projection(shape): location_npy = {k:v.asnumpy() for k, v in location.items()} aux_states = _parse_aux_states(sym=sym, aux_states=aux_states, ctx=ctx) if aux_states is not None: - aux_states_npy = {k:v.asnumpy() for k, v in aux_states.items()} + aux_states_npy = {k: v.asnumpy() for k, v in aux_states.items()} else: aux_states_npy = None if grad_nodes is None: @@ -516,6 +700,14 @@ def random_projection(shape): + [("__random_proj", _rng.normal(0, 0.01, size=out_shape[0]))]) args_grad = {k: mx.nd.array(v, ctx=ctx) for k, v in args_grad_npy.items()} + if grad_stype_dict is not None: + assert isinstance(grad_stype_dict, dict), "grad_stype_dict must be a dict" + for k, v in grad_stype_dict.items(): + if k in args_grad and v in _STORAGE_TYPE_STR_TO_ID and v != 'default': + # create an uninitialized sparse ndarray for executor + # if the symbolic grad is expected to be zero, it should not be initialized at all + args_grad[k] = mx.nd.zeros(args_grad[k].shape, args_grad[k].context, + args_grad[k].dtype, v) executor = out.bind(ctx, grad_req=grad_req, args=location, args_grad=args_grad, aux_states=aux_states) @@ -607,15 +799,15 @@ def check_symbolic_forward(sym, location, expected, rtol=1E-4, atol=None, g[:] = 0 executor.forward(is_train=False) - outputs = [x.asnumpy() for x in executor.outputs] + outputs = [x.asnumpy() for x in executor.outputs] for output_name, expect, output in zip(sym.list_outputs(), expected, outputs): assert_almost_equal(expect, output, rtol, atol, ("EXPECTED_%s"%output_name, "FORWARD_%s"%output_name)) def check_symbolic_backward(sym, location, out_grads, expected, rtol=1e-5, atol=None, - aux_states=None, grad_req='write', ctx=None): + aux_states=None, grad_req='write', ctx=None, grad_stypes=None): """Compares a symbol's backward results with the expected ones. Prints error messages if the backward results are not the same as the expected results. @@ -651,6 +843,8 @@ def check_symbolic_backward(sym, location, out_grads, expected, rtol=1e-5, atol= Gradient requirements. 'write', 'add' or 'null'. ctx : Context, optional Running context. + grad_stypes: dict of str->str + dictionary of mapping argument name to stype for the gradient Example ------- @@ -676,14 +870,23 @@ def check_symbolic_backward(sym, location, out_grads, expected, rtol=1e-5, atol= if isinstance(expected, (list, tuple)): expected = {k:v for k, v in zip(sym.list_arguments(), expected)} args_grad_npy = {k:_rng.normal(size=v.shape) for k, v in expected.items()} - args_grad_data = {k: mx.nd.array(v, ctx=ctx) for k, v in args_grad_npy.items()} + args_grad_data = {} + for k, v in args_grad_npy.items(): + nd = mx.nd.array(v, ctx=ctx) + if grad_stypes is not None and k in grad_stypes: + args_grad_data[k] = nd.tostype(grad_stypes[k]) + else: + args_grad_data[k] = nd + if isinstance(grad_req, str): grad_req = {k:grad_req for k in sym.list_arguments()} elif isinstance(grad_req, (list, tuple)): grad_req = {k:v for k, v in zip(sym.list_arguments(), grad_req)} - executor = sym.bind(ctx=ctx, args=location, args_grad=args_grad_data, aux_states=aux_states) + executor = sym.bind(ctx=ctx, args=location, args_grad=args_grad_data, + aux_states=aux_states, grad_req=grad_req) executor.forward(is_train=True) + if isinstance(out_grads, (tuple, list)): out_grads = [mx.nd.array(v, ctx=ctx) for v in out_grads] elif isinstance(out_grads, (dict)): diff --git a/src/c_api/c_api.cc b/src/c_api/c_api.cc index 93458d21ac5a..0fe3fe3e302e 100644 --- a/src/c_api/c_api.cc +++ b/src/c_api/c_api.cc @@ -172,6 +172,39 @@ int MXNDArrayCreateEx(const mx_uint *shape, API_END(); } +int MXNDArrayCreateSparseEx(int storage_type, + const mx_uint *shape, + mx_uint ndim, + int dev_type, + int dev_id, + int delay_alloc, + int dtype, + mx_uint num_aux, + int *aux_type, + mx_uint *aux_ndims, + const mx_uint *aux_shape, + NDArrayHandle *out) { + API_BEGIN(); + std::vector aux_types; + std::vector aux_shapes; + auto shape_start = aux_shape; + for (size_t i = 0; i < num_aux; i++) { + // types + aux_types.push_back(aux_type[i]); + // shapes + aux_shapes.emplace_back(shape_start, shape_start + aux_ndims[i]); + shape_start += aux_ndims[i]; + } + *out = new NDArray( + NDArrayStorageType(storage_type), + TShape(shape, shape + ndim), + Context::Create(static_cast(dev_type), dev_id), + delay_alloc != 0, + dtype, aux_types, aux_shapes); + API_END(); +} + + int MXNDArrayLoadFromRawBytes(const void *buf, size_t size, NDArrayHandle *out) { @@ -215,6 +248,23 @@ int MXNDArraySyncCopyToCPU(NDArrayHandle handle, API_END(); } +/*! + * \brief Copy src.data() to dst.data() if i = -1, else dst.aux_data(i) if i >= 0 + * This function blocks. Do not use it in performance critical code. + * \param handle_dst handle of a dst ndarray whose data/aux_data has been allocated + * \param handle_src handle of a src ndarray which has default storage type + * \param i dst data blob indicator + */ +int MXNDArraySyncCopyFromNDArray(NDArrayHandle handle_dst, + const NDArrayHandle handle_src, + const int i) { + API_BEGIN(); + NDArray* dst = static_cast(handle_dst); + NDArray* src = static_cast(handle_src); + dst->SyncCopyFromNDArray(*src, -1, i); + API_END(); +} + int MXNDArrayWaitToRead(NDArrayHandle handle) { API_BEGIN(); static_cast(handle)->WaitToRead(); @@ -351,6 +401,18 @@ MXNET_DLL int MXNDArrayReshape(NDArrayHandle handle, API_END_HANDLE_ERROR(delete ptr); } +int MXNDArrayGetStorageType(NDArrayHandle handle, + int *out_storage_type) { + API_BEGIN(); + NDArray *arr = static_cast(handle); + if (!arr->is_none()) { + *out_storage_type = arr->storage_type(); + } else { + *out_storage_type = kUndefinedStorage; + } + API_END(); +} + int MXNDArrayGetShape(NDArrayHandle handle, mx_uint *out_dim, const mx_uint **out_pdata) { @@ -400,6 +462,42 @@ int MXNDArrayGetDType(NDArrayHandle handle, API_END(); } +int MXNDArrayGetAuxType(NDArrayHandle handle, + mx_uint i, + int *out_type) { + API_BEGIN(); + NDArray *arr = static_cast(handle); + *out_type = arr->aux_type(i); + API_END(); +} + +/*! + * \brief Get a deep copy of the ith aux data blob + * in the form of an NDArray of default storage type. + * This function blocks. Do not use it in performance critical code. + */ +int MXNDArrayGetAuxNDArray(NDArrayHandle handle, + mx_uint i, + NDArrayHandle *out) { + API_BEGIN(); + NDArray *arr = static_cast(handle); + *out = new NDArray(arr->aux_ndarray(i)); + API_END(); +} + +/*! + * \brief Get a deep copy of the data blob + * in the form of an NDArray of default storage type. + * This function blocks. Do not use it in performance critical code. + */ +int MXNDArrayGetDataNDArray(NDArrayHandle handle, + NDArrayHandle *out) { + API_BEGIN(); + NDArray *arr = static_cast(handle); + *out = new NDArray(arr->data_ndarray()); + API_END(); +} + int MXNDArrayGetContext(NDArrayHandle handle, int *out_dev_type, int *out_dev_id) { @@ -735,6 +833,24 @@ int MXKVStorePullEx(KVStoreHandle handle, API_END(); } +int MXKVStorePullRowSparse(KVStoreHandle handle, + mx_uint num, + const char** keys, + NDArrayHandle* vals, + const NDArrayHandle* row_ids, + int priority) { + API_BEGIN(); + std::vector v_keys(num); + std::vector> v_val_rowids(num); + for (mx_uint i = 0; i < num; ++i) { + v_keys[i] = keys[i]; + v_val_rowids[i] = std::make_pair(static_cast(vals[i]), + *static_cast(row_ids[i])); + } + static_cast(handle)->PullRowSparse(v_keys, v_val_rowids, priority); + API_END(); +} + int MXKVStoreSetUpdater(KVStoreHandle handle, MXKVStoreUpdater updater, void* updater_handle) { diff --git a/src/c_api/c_api_common.h b/src/c_api/c_api_common.h index 846b53973b07..fee3f03f6db0 100644 --- a/src/c_api/c_api_common.h +++ b/src/c_api/c_api_common.h @@ -76,6 +76,8 @@ struct MXAPIThreadLocalEntry { std::vector arg_shapes, out_shapes, aux_shapes; /*! \brief result holder for returning type flags */ std::vector arg_types, out_types, aux_types; + /*! \brief result holder for returning storage types */ + std::vector arg_storage_types, out_storage_types, aux_storage_types; /*! \brief result holder for returning shape dimensions */ std::vector arg_shape_ndim, out_shape_ndim, aux_shape_ndim; /*! \brief result holder for returning shape pointer */ diff --git a/src/c_api/c_api_executor.cc b/src/c_api/c_api_executor.cc index a4c48e426879..631c1a7d93eb 100644 --- a/src/c_api/c_api_executor.cc +++ b/src/c_api/c_api_executor.cc @@ -198,6 +198,9 @@ int MXExecutorBindEX(SymbolHandle symbol_handle, * \param num_provided_arg_dtypes number of user provided in_arg and axu_state dtypes * \param provided_arg_dtype_names argument name list of provided dtypes * \param provided_arg_dtypes data of provided dtypes + * \param num_provided_arg_stypes number of user provided in_arg and axu_state storage types + * \param provided_arg_stype_names argument name list of provided storage types + * \param provided_arg_stypes data of provided storage types * \param num_shared_arg_names number of parameter names passed from _bind_ith_exec * \param shared_arg_name_list parameter name list passed from _bind_ith_exec * \param shared_buffer_len number of shared data arrays passed from _bind_ith_exec @@ -230,6 +233,9 @@ int MXExecutorSimpleBind(SymbolHandle symbol_handle, const mx_uint num_provided_arg_dtypes, const char** provided_arg_dtype_names, const int* provided_arg_dtypes, + const mx_uint num_provided_arg_stypes, + const char** provided_arg_stype_names, + const int* provided_arg_stypes, const mx_uint num_shared_arg_names, const char** shared_arg_name_list, int* shared_buffer_len, @@ -254,7 +260,7 @@ int MXExecutorSimpleBind(SymbolHandle symbol_handle, // attr_dict for setting up type_dict and arg/aux ctx std::unordered_map> attr_dict; - if (nullptr == provided_arg_dtypes || nullptr != g2c_keys) { + if (nullptr == provided_arg_dtypes || nullptr != g2c_keys || nullptr == provided_arg_stypes) { std::vector> attrs = sym->ListAttrsRecursive(); attr_dict.reserve(attrs.size()); @@ -280,6 +286,23 @@ int MXExecutorSimpleBind(SymbolHandle symbol_handle, } } + // setup arg_stype_map + std::unordered_map arg_stype_map; + if (nullptr == provided_arg_stypes) { // use attr_dict + for (const auto& arg_name : in_arg_names) { + const auto it = attr_dict.find(arg_name); + if (it == attr_dict.end() || !it->second.count("__storage_type__")) { + arg_stype_map[arg_name] = kDefaultStorage; + } + } + } else { // use user input type_dict + // create stype map for in_args and aux_states + arg_stype_map.reserve(num_provided_arg_stypes); + for (mx_uint i = 0; i < num_provided_arg_stypes; ++i) { + arg_stype_map[provided_arg_stype_names[i]] = provided_arg_stypes[i]; + } + } + // create default ctx Context ctx = Context::Create(static_cast(dev_type), dev_id); // create ctx map @@ -420,9 +443,10 @@ int MXExecutorSimpleBind(SymbolHandle symbol_handle, std::vector aux_state_vec; *out = Executor::SimpleBind(*sym, ctx, ctx_map, in_arg_ctx_vec, arg_grad_ctx_vec, - aux_state_ctx_vec, arg_shape_map, arg_dtype_map, grad_req_type_vec, - shared_arg_name_set, &in_arg_vec, &arg_grad_vec, &aux_state_vec, - use_shared_buffer? &shared_buffer_map : nullptr, + aux_state_ctx_vec, arg_shape_map, arg_dtype_map, arg_stype_map, + grad_req_type_vec, shared_arg_name_set, &in_arg_vec, + &arg_grad_vec, &aux_state_vec, + use_shared_buffer ? &shared_buffer_map : nullptr, reinterpret_cast(shared_exec_handle)); // copy ndarray ptrs to ret->handles so that front end diff --git a/src/c_api/c_api_ndarray.cc b/src/c_api/c_api_ndarray.cc index 3202f55abea7..d392baf45d3e 100644 --- a/src/c_api/c_api_ndarray.cc +++ b/src/c_api/c_api_ndarray.cc @@ -18,7 +18,8 @@ */ /*! - * \file c_api_symbolic.cc + * Copyright (c) 2016 by Contributors + * \file c_api_ndarray.cc * \brief C API of mxnet */ @@ -150,14 +151,17 @@ void SetContext(Context* p_ctx, #endif // MXNET_USE_CUDA } +// Set the shape, dtype and storage type void SetShapeType(const nnvm::Op* op, const nnvm::NodeAttrs& attrs, const Context& ctx, const std::vector& ndinputs, - std::vector* p_ndoutputs) { + std::vector* p_ndoutputs, + int* dispatch_stype) { std::vector& ndoutputs = *p_ndoutputs; static auto& infershape = nnvm::Op::GetAttr("FInferShape"); static auto& infertype = nnvm::Op::GetAttr("FInferType"); + static auto& inferstorage = nnvm::Op::GetAttr("FInferStorageType"); MXAPIThreadLocalEntry *ret = MXAPIThreadLocalStore::Get(); // infer shape std::vector& in_shapes = ret->arg_shapes; @@ -193,9 +197,35 @@ void SetShapeType(const nnvm::Op* op, CHECK(infertype[op](attrs, &in_types, &out_types)); CHECK_EQ(out_types.size(), ndoutputs.size()); + // infer storage type + auto& in_storage_types = ret->arg_storage_types; + auto& out_storage_types = ret->out_storage_types; + in_storage_types.clear(); + out_storage_types.clear(); + for (auto& i : ndinputs) { + in_storage_types.push_back(i.storage_type()); + } + for (auto& i : ndoutputs) { + out_storage_types.push_back(i.storage_type()); + } + if (inferstorage.count(op)) { + CHECK(inferstorage[op](attrs, ctx, &in_storage_types, &out_storage_types)); + CHECK_EQ(out_storage_types.size(), ndoutputs.size()); + } + + bool contains_non_default = common::ContainsNonDefaultStorage(in_storage_types); + contains_non_default |= common::ContainsNonDefaultStorage(out_storage_types); + int kNonDefaultStorage = -2; + *dispatch_stype = contains_non_default ? kNonDefaultStorage : kDefaultStorage; for (size_t i = 0; i < ndoutputs.size(); ++i) { + NDArrayStorageType storage_type = static_cast(out_storage_types[i]); if (ndoutputs[i].is_none()) { - ndoutputs[i] = NDArray(out_shapes[i], ctx, true, out_types[i]); + // if failed to infer the storage type, assume the output storage is dense + if (storage_type == kDefaultStorage || out_storage_types[i] == kUndefinedStorage) { + ndoutputs[i] = NDArray(out_shapes[i], ctx, true, out_types[i]); + } else { + ndoutputs[i] = NDArray(storage_type, out_shapes[i], ctx, true, out_types[i]); + } } else { CHECK_EQ(ndoutputs[i].shape(), out_shapes[i]) << i << "th output has invalid shape. " @@ -212,7 +242,7 @@ void SetShapeType(const nnvm::Op* op, void SetDependency(std::vector *p_read_vars, std::vector *p_write_vars, std::vector *p_requested, - std::vector *p_auxidx, + std::vector *p_mutate_idx, const nnvm::Op* op, const nnvm::NodeAttrs& attrs, const Context& ctx, @@ -224,7 +254,7 @@ void SetDependency(std::vector *p_read_vars, std::vector& read_vars = *p_read_vars; std::vector& write_vars = *p_write_vars; std::vector& requested = *p_requested; - std::vector& auxidx = *p_auxidx; + std::vector& mutate_idx = *p_mutate_idx; if (tmp_resource.count(op)) { int ntmp = 0; @@ -250,15 +280,30 @@ void SetDependency(std::vector *p_read_vars, write_vars.push_back(i.var()); } if (mutate.count(op)) { - auxidx = mutate[op](attrs); - std::sort(auxidx.begin(), auxidx.end()); - for (auto & i : auxidx) { + mutate_idx = mutate[op](attrs); + std::sort(mutate_idx.begin(), mutate_idx.end()); + for (auto & i : mutate_idx) { write_vars.push_back(ndinputs[i].var()); } } Engine::Get()->DeduplicateVarHandle(&read_vars, &write_vars); } +inline void SetWriteInplaceReq(const std::vector &ndinputs, + const std::vector &ndoutputs, + std::vector *req) { + std::unordered_set in_vars; + for (auto &nd : ndinputs) { + in_vars.insert(nd.var()); + } + for (size_t i = 0; i < ndoutputs.size(); i++) { + // output NDArray shares the memory with the input NDArray + if (in_vars.find(ndoutputs[i].var()) != in_vars.end()) { + req->at(i) = kWriteInplace; + } + } +} + void PushFCompute(const FCompute& fn, const nnvm::Op* op, const nnvm::NodeAttrs& attrs, @@ -267,24 +312,75 @@ void PushFCompute(const FCompute& fn, const std::vector& write_vars, const std::vector& requested, const std::vector& ndinputs, - const std::vector& ndoutputs) { + const std::vector& ndoutputs, + const std::vector& mutate_idx) { + using namespace common; bool is_train = AutogradRuntime::Get()->IsTraining(); Engine::Get()->PushAsync( - [ctx, attrs, fn, ndinputs, ndoutputs, requested, is_train]( + [ctx, attrs, fn, ndinputs, ndoutputs, requested, is_train, mutate_idx]( RunContext rctx, engine::CallbackOnComplete on_complete) { std::vector input_blobs, output_blobs; - for (auto& i : ndinputs) { - input_blobs.push_back(i.data()); - } - for (auto& i : ndoutputs) { - output_blobs.push_back(i.data()); + // pre-fcompute and post-fcompute storage fallback src NDArrays and dst NDArrays + std::vector pre_temp_src, pre_temp_dst, post_temp_dst, post_temp_src; + // mapping from index in input_blobs to index in pre_temp_dst + std::unordered_map in_temp_idx_map; + // populate input blobs and output blobs + SetupDefaultBlobs(ndinputs, &input_blobs, &pre_temp_src, &pre_temp_dst, &in_temp_idx_map); + SetupDefaultBlobs(ndoutputs, &output_blobs, &post_temp_dst, &post_temp_src); + // add mutable inputs to post temp list + for (const auto idx : mutate_idx) { + auto map_iter = in_temp_idx_map.find(idx); + if (map_iter != in_temp_idx_map.end()) { + post_temp_src.push_back(pre_temp_dst[map_iter->second]); + post_temp_dst.push_back(ndinputs[idx]); + } } OpContext opctx{is_train, rctx, engine::CallbackOnComplete(), requested}; std::vector req(output_blobs.size(), kWriteTo); - fn(attrs, opctx, input_blobs, req, output_blobs); + if (ctx.dev_mask() == gpu::kDevMask) { +#if MXNET_USE_CUDA + CastNonDefaultStorage(pre_temp_src, pre_temp_dst, opctx); + fn(attrs, opctx, input_blobs, req, output_blobs); + // cast to original storage type, if necessary + CastNonDefaultStorage(post_temp_src, post_temp_dst, opctx); + rctx.get_stream()->Wait(); +#else + LOG(FATAL) << MXNET_GPU_NOT_ENABLED_ERROR; +#endif + } else { + CastNonDefaultStorage(pre_temp_src, pre_temp_dst, opctx); + fn(attrs, opctx, input_blobs, req, output_blobs); + // cast to original storage type, if necessary + CastNonDefaultStorage(post_temp_src, post_temp_dst, opctx); + } + on_complete(); + }, ctx, read_vars, write_vars, FnProperty::kNormal, + 0, PROFILER_MESSAGE(op->name.c_str())); +} + +void PushFComputeEx(const FComputeEx& fn, + const nnvm::Op* op, + const nnvm::NodeAttrs& attrs, + const Context& ctx, + const std::vector& read_vars, + const std::vector& write_vars, + const std::vector& requested, + const std::vector& ndinputs, + const std::vector& ndoutputs) { + Engine::Get()->PushAsync( + [ctx, attrs, fn, ndinputs, ndoutputs, requested]( + RunContext rctx, + engine::CallbackOnComplete on_complete) { + std::vector input_blobs, output_blobs; + OpContext opctx{false, rctx, + engine::CallbackOnComplete(), + requested}; + std::vector req(ndoutputs.size(), kWriteTo); + SetWriteInplaceReq(ndinputs, ndoutputs, &req); + fn(attrs, opctx, ndinputs, req, ndoutputs); if (ctx.dev_mask() == gpu::kDevMask) { rctx.get_stream()->Wait(); } @@ -301,7 +397,9 @@ void PushOperator(const OpStatePtr& state, const std::vector& write_vars, const std::vector& requested, const std::vector& ndinputs, - const std::vector& ndoutputs) { + const std::vector& ndoutputs, + const std::vector& mutate_idx) { + using namespace common; static auto& fexec_type = nnvm::Op::GetAttr("FExecType"); bool is_train = AutogradRuntime::Get()->IsTraining(); @@ -314,15 +412,40 @@ void PushOperator(const OpStatePtr& state, if (fcompute != nullptr) { CHECK(exec_type == ExecType::kSync || exec_type == ExecType::kAsync); Engine::Get()->PushAsync( - [state, fcompute, ndinputs, ndoutputs, requested, is_train, exec_type]( + [state, fcompute, ndinputs, ndoutputs, requested, is_train, exec_type, mutate_idx]( RunContext rctx, engine::CallbackOnComplete on_complete) { OpContext opctx{is_train, rctx, on_complete, requested}; + std::vector input_blobs, output_blobs; - for (const auto& i : ndinputs) input_blobs.push_back(i.data()); - for (const auto& i : ndoutputs) output_blobs.push_back(i.data()); + // pre-fcompute and post-fcompute storage fallback src NDArrays and dst NDArrays + std::vector pre_temp_src, pre_temp_dst, post_temp_dst, post_temp_src; + // mapping from index in input_blobs to index in pre_temp_dst + std::unordered_map in_temp_idx_map; + // populate input blobs and output blobs + SetupDefaultBlobs(ndinputs, &input_blobs, &pre_temp_src, &pre_temp_dst, &in_temp_idx_map); + SetupDefaultBlobs(ndoutputs, &output_blobs, &post_temp_dst, &post_temp_src); + // add mutable inputs to post temp list + for (const auto idx : mutate_idx) { + if (in_temp_idx_map.find(idx) != in_temp_idx_map.end()) { + post_temp_src.push_back(pre_temp_dst[in_temp_idx_map[idx]]); + post_temp_dst.push_back(ndinputs[idx]); + } + } std::vector req(output_blobs.size(), kWriteTo); - fcompute(state, opctx, input_blobs, req, output_blobs); + if (rctx.get_ctx().dev_mask() == gpu::kDevMask) { +#if MXNET_USE_CUDA + CastNonDefaultStorage(pre_temp_src, pre_temp_dst, opctx); + fcompute(state, opctx, input_blobs, req, output_blobs); + CastNonDefaultStorage(post_temp_src, post_temp_dst, opctx); +#else + LOG(FATAL) << MXNET_GPU_NOT_ENABLED_ERROR; +#endif + } else { + CastNonDefaultStorage(pre_temp_src, pre_temp_dst, opctx); + fcompute(state, opctx, input_blobs, req, output_blobs); + CastNonDefaultStorage(post_temp_src, post_temp_dst, opctx); + } if (exec_type == ExecType::kSync) { if (rctx.get_ctx().dev_mask() == gpu::kDevMask) { rctx.get_stream()->Wait(); @@ -342,6 +465,7 @@ void PushOperator(const OpStatePtr& state, engine::CallbackOnComplete on_complete) { OpContext opctx{is_train, rctx, on_complete, requested}; std::vector req(ndoutputs.size(), kWriteTo); + SetWriteInplaceReq(ndinputs, ndoutputs, &req); fcompute_ex(state, opctx, ndinputs, req, ndoutputs); if (exec_type == ExecType::kSync) { if (rctx.get_ctx().dev_mask() == gpu::kDevMask) { @@ -363,8 +487,6 @@ void ImperativeInvokeImpl(const Context& default_ctx, const nnvm::NodeAttrs& attrs, std::vector* p_ndinputs, std::vector* p_ndoutputs) { - static auto& fcpu = nnvm::Op::GetAttr("FCompute"); - static auto& fgpu = nnvm::Op::GetAttr("FCompute"); static auto& ndfunc = nnvm::Op::GetAttr("FNDArrayFunction"); static auto& createop = nnvm::Op::GetAttr("FCreateOpState"); MXAPIThreadLocalEntry *ret = MXAPIThreadLocalStore::Get(); @@ -379,29 +501,32 @@ void ImperativeInvokeImpl(const Context& default_ctx, } else { // TODO(piiswrong): infer ctx Context ctx; + int stype; SetContext(&ctx, attrs, ndinputs, ndoutputs, default_ctx); - SetShapeType(op, attrs, ctx, ndinputs, &ndoutputs); + SetShapeType(op, attrs, ctx, ndinputs, &ndoutputs, &stype); std::vector read_vars, write_vars; std::vector requested; - std::vector auxidx; - SetDependency(&read_vars, &write_vars, &requested, &auxidx, + std::vector mutate_idx; + SetDependency(&read_vars, &write_vars, &requested, &mutate_idx, op, attrs, ctx, ndinputs, ndoutputs); - FCompute fn; - if (ctx.dev_mask() == cpu::kDevMask && fcpu.count(op)) { - fn = fcpu[op]; - } else if (ctx.dev_mask() == gpu::kDevMask && fgpu.count(op)) { - fn = fgpu[op]; - } - - if (fn) { + FCompute fn = common::GetFCompute(op, "FCompute", ctx); + FComputeEx fn_ex = common::GetFCompute(op, "FComputeEx", ctx); + if (fn_ex && stype != kDefaultStorage) { if (AutogradRuntime::Get()->IsRecording()) { AutogradRuntime::Get()->RecordImperativeFCompute(op, attrs, &ndinputs, &ndoutputs); } - PushFCompute(fn, op, attrs, ctx, read_vars, write_vars, + PushFComputeEx(fn_ex, op, attrs, ctx, read_vars, write_vars, requested, ndinputs, ndoutputs); + } else if (fn) { + if (AutogradRuntime::Get()->IsRecording()) { + AutogradRuntime::Get()->RecordImperativeFCompute(op, + attrs, &ndinputs, &ndoutputs); + } + PushFCompute(fn, op, attrs, ctx, read_vars, write_vars, + requested, ndinputs, ndoutputs, mutate_idx); } else if (createop.count(op)) { auto state = createop[op](attrs, ctx, ret->arg_shapes, ret->arg_types); @@ -411,7 +536,7 @@ void ImperativeInvokeImpl(const Context& default_ctx, } write_vars.push_back(state.get_var()); PushOperator(state, op, attrs, ctx, read_vars, write_vars, - requested, ndinputs, ndoutputs); + requested, ndinputs, ndoutputs, mutate_idx); } else { LOG(FATAL) << "Operator " << op->name << " is not implemented for " @@ -461,6 +586,28 @@ int MXImperativeInvoke(AtomicSymbolCreator creator, API_END(); } +int MXImperativeInvokeEx(AtomicSymbolCreator creator, + int num_inputs, + NDArrayHandle *inputs, + int *num_outputs, + NDArrayHandle **outputs, + int num_params, + const char **param_keys, + const char **param_vals, + const int **out_stypes) { // outputs storage types + API_BEGIN(); + MXImperativeInvoke(creator, num_inputs, inputs, num_outputs, outputs, + num_params, param_keys, param_vals); + MXAPIThreadLocalEntry *ret = MXAPIThreadLocalStore::Get(); + NDArray** output_nds = reinterpret_cast(*outputs); + ret->out_types.resize(*num_outputs); + for (int i = 0; i < *num_outputs; ++i) { + ret->out_types[i] = output_nds[i]->storage_type(); + } + *out_stypes = dmlc::BeginPtr(ret->out_types); + API_END(); +} + int MXCreateCachedOp(SymbolHandle handle, CachedOpHandle *out) { nnvm::Symbol* sym = static_cast(handle); @@ -540,6 +687,24 @@ int MXInvokeCachedOp(CachedOpHandle handle, API_END(); } +int MXInvokeCachedOpEx(CachedOpHandle handle, + int num_inputs, + NDArrayHandle *inputs, + int *num_outputs, + NDArrayHandle **outputs, + const int **out_stypes) { // outputs storage types + API_BEGIN(); + MXInvokeCachedOp(handle, num_inputs, inputs, num_outputs, outputs); + MXAPIThreadLocalEntry *ret = MXAPIThreadLocalStore::Get(); + NDArray** output_nds = reinterpret_cast(*outputs); + ret->out_types.resize(*num_outputs); + for (int i = 0; i < *num_outputs; ++i) { + ret->out_types[i] = output_nds[i]->storage_type(); + } + *out_stypes = dmlc::BeginPtr(ret->out_types); + API_END(); +} + int MXAutogradIsTraining(bool* curr) { API_BEGIN(); *curr = AutogradRuntime::Get()->IsTraining(); diff --git a/src/c_api/c_api_symbolic.cc b/src/c_api/c_api_symbolic.cc index e2c29b888ada..d526aea0d35f 100644 --- a/src/c_api/c_api_symbolic.cc +++ b/src/c_api/c_api_symbolic.cc @@ -29,6 +29,7 @@ #include #include "./c_api_common.h" #include "../operator/operator_common.h" +#include "../executor/exec_pass.h" namespace mxnet { namespace op { @@ -459,7 +460,7 @@ int MXSymbolInferShape(SymbolHandle sym, } try { - g = nnvm::pass::InferShape(std::move(g), arg_shapes, "__shape__"); + g = mxnet::exec::InferShape(std::move(g), arg_shapes, "__shape__"); } catch (const mxnet::op::InferShapeError &err) { throw dmlc::Error(err.msg); } @@ -544,7 +545,7 @@ int MXSymbolInferType(SymbolHandle sym, mxnet::MatchArguments(g.indexed_graph(), kwargs, &arg_types, "InferType"); } - g = nnvm::pass::InferType(std::move(g), arg_types, "__dtype__"); + g = mxnet::exec::InferType(std::move(g), arg_types, "__dtype__"); // copy back CopyAttr(g.indexed_graph(), g.GetAttr("dtype"), &(ret->arg_types), &(ret->out_types), &(ret->aux_types)); diff --git a/src/c_api/c_predict_api.cc b/src/c_api/c_predict_api.cc index 5ca01492800e..dda4fda1ed8f 100644 --- a/src/c_api/c_predict_api.cc +++ b/src/c_api/c_predict_api.cc @@ -32,6 +32,7 @@ #include #include "./c_api_common.h" #include "../operator/operator_common.h" +#include "../executor/exec_pass.h" using namespace mxnet; @@ -194,7 +195,7 @@ int MXPredCreatePartialOut(const char* symbol_json_str, } } nnvm::Graph g; g.outputs = sym.outputs; - g = nnvm::pass::InferShape(std::move(g), in_shapes, "__shape__"); + g = mxnet::exec::InferShape(std::move(g), in_shapes, "__shape__"); bool infer_complete = (g.GetAttr("shape_num_unknown_nodes") == 0); CHECK(infer_complete) << "The shape information of is not enough to get the shapes"; diff --git a/src/common/utils.cc b/src/common/utils.cc new file mode 100644 index 000000000000..125e4e5dc7d7 --- /dev/null +++ b/src/common/utils.cc @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file utils.cc + * \brief cpu implementation of util functions + */ + +#include "./utils.h" +#include "../operator/tensor/cast_storage-inl.h" + +namespace mxnet { +namespace common { + +template<> +void CastStorageDispatch(const OpContext& ctx, + const NDArray& input, + const NDArray& output) { + mxnet::op::CastStorageComputeImpl(ctx, input, output); +} + +} // namespace common +} // namespace mxnet diff --git a/src/common/utils.cu b/src/common/utils.cu new file mode 100644 index 000000000000..093480a98907 --- /dev/null +++ b/src/common/utils.cu @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file utils.cu + * \brief gpu implementation of util functions + */ + +#include "./utils.h" +#include "../operator/tensor/cast_storage-inl.h" + +namespace mxnet { +namespace common { + +template<> +void CastStorageDispatch(const OpContext& ctx, + const NDArray& input, + const NDArray& output) { + mxnet::op::CastStorageComputeImpl(ctx, input, output); +} + +} // namespace common +} // namespace mxnet diff --git a/src/common/utils.h b/src/common/utils.h index 85e30970f1a0..92631a9b5c34 100644 --- a/src/common/utils.h +++ b/src/common/utils.h @@ -24,7 +24,14 @@ #ifndef MXNET_COMMON_UTILS_H_ #define MXNET_COMMON_UTILS_H_ -#if DMLC_USE_CXX11 +#include +#include +#include +#include +#include +#include +#include + #include #include #include @@ -33,15 +40,100 @@ #include #include #include -#endif // DMLC_USE_CXX11 - -#include -#include +#include namespace mxnet { namespace common { -#if DMLC_USE_CXX11 +template +void CastStorageDispatch(const OpContext& ctx, const NDArray& input, const NDArray& output); + +/* + * \brief setup default-storage tblobs from source NDArrays. If any source NDArray has non-default + * storage, it creates a temp NDArray with default storage and uses the temp tblob. The + * function also records the indices of non-default source NDArrays and the indices of + * their corresponding temporary NDArrays in the temp array. + * \param src list of source NDArray + * \param blobs list of tblobs to return + * \param temp_src list of source NDArrays which requires temporary default storage representation + * \param temp_dst list of temporary destination NDArrays for default storage representation + * \param idx_map mapping from indices in source NDArrays to indices in temp_dst. When not set, + indices are not recorded + * \return true if any source NDArray need to cast storage + */ +inline bool SetupDefaultBlobs(const std::vector& src, + std::vector *blobs, + std::vector *temp_src, + std::vector *temp_dst, + std::unordered_map *idx_map = nullptr) { + bool require_cast = false; + for (size_t i = 0; i < src.size(); i++) { + auto& nd = src[i]; + if (nd.storage_type() != kDefaultStorage) { + if (idx_map != nullptr) { + (*idx_map)[i] = temp_dst->size(); + } + NDArray temp(nd.shape(), nd.ctx(), false, nd.dtype()); + temp_src->emplace_back(nd); + temp_dst->emplace_back(temp); + blobs->emplace_back(temp.data()); + require_cast = true; + } else { + blobs->push_back(nd.data()); + } + } + return require_cast; +} + +/* + * \brief cast the NDArrays in `src` and store the result in NDArrays in `dst`. + * This is only used for storage fallback in executor. + * When storage_fallback is false, and `MXNET_EXEC_STORAGE_FALLBACK` == 0, + * storage fallback is disallowed. + * \param src list of source NDArray to cast + * \param dst list of destionation NDArray which hold the result of cast_storage operation + * \param ctx operator context for cast_storage operation + * \param storage_fallback whether storage_fallback is allowed. When set to false, + * its value depends on `MXNET_EXEC_STORAGE_FALLBACK`. + */ +template +inline void CastNonDefaultStorage(const std::vector& src, + const std::vector& dst, + const OpContext& ctx, + bool storage_fallback = false) { + CHECK_GE(dst.size(), src.size()); + if (src.size() == 0) return; + if (storage_fallback == false) { + storage_fallback = dmlc::GetEnv("MXNET_EXEC_STORAGE_FALLBACK", true); + } + if (storage_fallback == false) { + LOG(FATAL) << "Storage type conversion detected during execution. " + << "You are probably executing an operator which " + << "doesn't support NDArray inputs with non-default storage."; + } + for (size_t i = 0; i < src.size(); i++) { + CastStorageDispatch(ctx, src[i], dst[i]); + } +} + +// Check if any storage type is not default storage +inline bool ContainsNonDefaultStorage(const StorageTypeVector& vstorage) { + for (const auto& i : vstorage) { + if (i != kUndefinedStorage && i != kDefaultStorage) return true; + } + return false; +} + +// Check if any NDArray in the list has default storage +inline bool ContainsDefaultStorage(const std::vector& ndarrays) { + for (const auto &nd : ndarrays) { + if (nd.storage_type() == kDefaultStorage) { + return true; + } + } + return false; +} + // heuristic to dermine number of threads per GPU inline int GetNumThreadPerGPU() { // This is resource efficient option. @@ -56,6 +148,67 @@ inline int GetExecNumMatchColor() { return std::min(num_match_color, GetNumThreadPerGPU()); } +template +V ParallelAccumulate(const T* a, const int n, V start) { + V sum = start; +#pragma omp parallel for reduction(+:sum) + for (int i = 0; i < n; ++i) { + sum += a[i]; + } + return sum; +} + +/*! + * \brief + * Helper function for ParallelSort. + * DO NOT call this function directly. + * Use the interface ParallelSort instead. + * Ref: https://github.com/dmlc/difacto/blob/master/src/common/parallel_sort.h + */ +template +void ParallelSortHelper(RandomIt first, size_t len, + size_t grainsize, const Compare& comp) { + if (len < grainsize) { + std::sort(first, first+len, comp); + } else { + std::thread thr(ParallelSortHelper, first, len/2, grainsize, comp); + ParallelSortHelper(first+len/2, len - len/2, grainsize, comp); + thr.join(); + std::inplace_merge(first, first+len/2, first+len, comp); + } +} + +/*! + * \brief + * Sort the elements in the range [first, last) into the ascending order defined by + * the comparator comp. + * If the length of the range [first, last) is greater than a certain threshold, + * the range will be recursively divided into two and assign two threads + * to sort each half range. + * Ref: https://github.com/dmlc/difacto/blob/master/src/common/parallel_sort.h + */ +template +void ParallelSort(RandomIt first, RandomIt last, size_t num_threads, Compare comp) { + const auto num = std::distance(first, last); + size_t grainsize = std::max(num / num_threads + 5, static_cast(1024*16)); + ParallelSortHelper(first, num, grainsize, comp); +} + +/*! + * \brief + * Sort the elements in the range [first, last) into ascending order. + * The elements are compared using the default < operator. + * If the length of the range [first, last) is greater than a certain threshold, + * the range will be recursively divided into two and assign two threads + * to sort each half range. + * Ref: https://github.com/dmlc/difacto/blob/master/src/common/parallel_sort.h + */ +template +void ParallelSort(RandomIt first, RandomIt last, size_t num_threads) { + ParallelSort(first, last, num_threads, + std::less::value_type>()); +} + /*! * \brief Random Engine */ @@ -159,8 +312,6 @@ FCompType GetFCompute(const nnvm::Op* op, const std::string& name, } } -#endif // DMLC_USE_CXX11 - } // namespace common } // namespace mxnet #endif // MXNET_COMMON_UTILS_H_ diff --git a/src/executor/attach_op_execs_pass.cc b/src/executor/attach_op_execs_pass.cc index 47b74758d702..fe8cc653bbc3 100644 --- a/src/executor/attach_op_execs_pass.cc +++ b/src/executor/attach_op_execs_pass.cc @@ -24,6 +24,7 @@ #include #include #include +#include #include #include "../common/utils.h" #include "./exec_pass.h" @@ -40,33 +41,98 @@ const OperatorProperty* OpPropGetOpProperty(const NodeAttrs& attrs); namespace exec { -// forward executor -class StatefulComputeExecutor : public OpExecutor { +// abstract OpExecutor which provides storage fallback procedure on +// non-default inputs and outputs +// FComputeExecutor and FStatefulComputeExecutor inherit from this class +class StorageFallbackOpExecutor : public OpExecutor { public: - void Run(RunContext rctx) override { + explicit StorageFallbackOpExecutor(const std::vector &mutate_idx) + : mutate_idx_(mutate_idx) {} + + void Setup() override { + init_ = false; + } + + protected: + // initialize the data blobs + void InitBlobs() { + using namespace common; if (!init_) { - in_data_.clear(); - for (size_t i = 0; i < in_array.size(); ++i) { - in_data_.push_back(in_array[i].data()); - } - out_data_.clear(); - for (size_t i = 0; i < out_array.size(); ++i) { - out_data_.push_back(out_array[i].data()); + in_data_.clear(); out_data_.clear(); + pre_temp_src_.clear(); pre_temp_dst_.clear(); + post_temp_src_.clear(); post_temp_dst_.clear(); + in_temp_idx_map_.clear(); + SetupDefaultBlobs(in_array, &in_data_, &pre_temp_src_, &pre_temp_dst_, &in_temp_idx_map_); + SetupDefaultBlobs(out_array, &out_data_, &post_temp_dst_, &post_temp_src_); + for (const auto idx : mutate_idx_) { + auto map_iter = in_temp_idx_map_.find(idx); + if (map_iter != in_temp_idx_map_.end()) { + post_temp_src_.push_back(pre_temp_dst_[map_iter->second]); + post_temp_dst_.push_back(in_array[idx]); + } } init_ = true; } + } + + // storage fallback before fcompute is launched + void PreFCompute(bool is_gpu) { + using namespace common; + InitBlobs(); + if (is_gpu) { +#if MXNET_USE_CUDA + CastNonDefaultStorage(pre_temp_src_, pre_temp_dst_, op_ctx); +#else + LOG(FATAL) << MXNET_GPU_NOT_ENABLED_ERROR; +#endif + } else { + CastNonDefaultStorage(pre_temp_src_, pre_temp_dst_, op_ctx); + } + } + + // storage fallback after fcompute is completed + void PostFCompute(bool is_gpu) { + using namespace common; + if (is_gpu) { +#if MXNET_USE_CUDA + CastNonDefaultStorage(post_temp_src_, post_temp_dst_, op_ctx); +#else + LOG(FATAL) << MXNET_GPU_NOT_ENABLED_ERROR; +#endif + } else { + CastNonDefaultStorage(post_temp_src_, post_temp_dst_, op_ctx); + } + } + + // default storage tensor blobs for fcompute + std::vector in_data_, out_data_; + // source NDArray for cast storage + std::vector pre_temp_src_, post_temp_src_; + // destination NDArray for cast storage + std::vector pre_temp_dst_, post_temp_dst_; + // mapping from index in input_blobs to index in pre_temp_dst + std::unordered_map in_temp_idx_map_; + // indices of mutatable inputs + std::vector mutate_idx_; + // whether blobs are initialized + bool init_; +}; + + +// stateful compute executor +class StatefulComputeExecutor : public StorageFallbackOpExecutor { + public: + void Run(RunContext rctx, bool is_gpu) override { op_ctx.run_ctx = rctx; + PreFCompute(is_gpu); fcompute_(state_, op_ctx, in_data_, req, out_data_); + PostFCompute(is_gpu); #if MKL_EXPERIMENTAL == 1 mkl_tblobs_prv_to_cpu(in_data_); mkl_tblobs_prv_to_cpu(out_data_); #endif } - void Setup() override { - init_ = false; - } - ExecType exec_type() const override { return exec_type_; } @@ -77,23 +143,23 @@ class StatefulComputeExecutor : public OpExecutor { explicit StatefulComputeExecutor(const OpStatePtr& state, const FStatefulCompute& fcompute, - ExecType exec_type) - : state_(state), fcompute_(fcompute), exec_type_(exec_type) {} + ExecType exec_type, + const std::vector &mutate_idx) + : StorageFallbackOpExecutor(mutate_idx), + state_(state), fcompute_(fcompute), exec_type_(exec_type) {} private: friend Graph AttachOpExecs(Graph g); OpStatePtr state_; FStatefulCompute fcompute_; ExecType exec_type_; - bool init_; - std::vector in_data_, out_data_; }; -// forward executor +// stateful compute_ex executor class StatefulComputeExExecutor : public OpExecutor { public: - void Run(RunContext rctx) override { + void Run(RunContext rctx, bool is_gpu) override { op_ctx.run_ctx = rctx; fcompute_(state_, op_ctx, in_array, req, out_array); } @@ -121,47 +187,60 @@ class StatefulComputeExExecutor : public OpExecutor { }; -// fcompute executor executor -class FComputeExecutor : public OpExecutor { +// fcompute executor +class FComputeExecutor : public StorageFallbackOpExecutor { public: - void Run(RunContext rctx) override { - if (!init_) { - in_data_.resize(in_array.size()); - out_data_.resize(out_array.size()); - auto get_blob = [](const NDArray& nd) { - return nd.data(); - }; - std::transform(in_array.begin(), in_array.end(), in_data_.begin(), get_blob); - std::transform(out_array.begin(), out_array.end(), out_data_.begin(), get_blob); - init_ = true; - } + void Run(RunContext rctx, bool is_gpu) override { + using namespace common; op_ctx.run_ctx = rctx; + PreFCompute(is_gpu); fcompute_(attrs_, op_ctx, in_data_, req, out_data_); + PostFCompute(is_gpu); #if MKL_EXPERIMENTAL == 1 mkl_tblobs_prv_to_cpu(in_data_); mkl_tblobs_prv_to_cpu(out_data_); #endif } - void Setup() override { - init_ = false; + ExecType exec_type() const override { + return exec_type_; } + explicit FComputeExecutor(const NodeAttrs& attrs, FCompute fcompute, + ExecType exec_type, const std::vector &mutate_idx) + : StorageFallbackOpExecutor(mutate_idx), + attrs_(attrs), fcompute_(fcompute), exec_type_(exec_type) { + } + + private: + NodeAttrs attrs_; + FCompute fcompute_; + ExecType exec_type_; +}; + +// fcompute_ex executor +class FComputeExExecutor : public OpExecutor { + public: + void Run(RunContext rctx, bool is_gpu) override { + op_ctx.run_ctx = rctx; + fcompute_(attrs_, op_ctx, in_array, req, out_array); + } + + void Setup() override {} + ExecType exec_type() const override { return exec_type_; } - explicit FComputeExecutor(const NodeAttrs& attrs, FCompute fcompute, - ExecType exec_type) + explicit FComputeExExecutor(const NodeAttrs& attrs, FComputeEx fcompute, + ExecType exec_type) : attrs_(attrs), fcompute_(fcompute), exec_type_(exec_type) { } private: NodeAttrs attrs_; - FCompute fcompute_; + FComputeEx fcompute_; ExecType exec_type_; - bool init_; - std::vector in_data_, out_data_; }; // pass to attach operator executors @@ -180,6 +259,8 @@ Graph AttachOpExecs(Graph g) { const auto& vctx = g.GetAttr("context"); const auto& saved_states = g.GetAttr< std::unordered_map >("saved_states"); + const auto& dispatch_stypes = g.GetAttr("dispatch_stypes"); + // get the graph const auto& idx = g.indexed_graph(); @@ -217,7 +298,8 @@ Graph AttachOpExecs(Graph g) { FStatefulCompute fcompute = common::GetFCompute( op, "FStatefulCompute", vctx[i]); if (fcompute != nullptr) { - ret[i] = std::make_shared(state, fcompute, exec_type); + ret[i] = std::make_shared(state, fcompute, + exec_type, mutate_index); } else { FStatefulComputeEx fcompute_ex = common::GetFCompute( op, "FStatefulComputeEx", vctx[i]); @@ -236,7 +318,7 @@ Graph AttachOpExecs(Graph g) { if (fcompute != nullptr) { ret[i] = std::make_shared( dynamic_cast(ret[fwd_id].get())->state_, - fcompute, exec_type); + fcompute, exec_type, mutate_index); } else { FStatefulComputeEx fcompute_ex = common::GetFCompute( op, "FStatefulComputeEx", vctx[i]); @@ -249,11 +331,15 @@ Graph AttachOpExecs(Graph g) { } } else { FCompute fcompute = common::GetFCompute(op, "FCompute", vctx[i]); - if (fcompute != nullptr) { + FComputeEx fcomp_ex = common::GetFCompute(op, "FComputeEx", vctx[i]); + if (fcomp_ex != nullptr && dispatch_stypes[i] != kDefaultStorage) { + ret[i] = std::make_shared( + inode.source->attrs, fcomp_ex, exec_type); + } else if (fcompute != nullptr) { ret[i] = std::make_shared( - inode.source->attrs, fcompute, exec_type); + inode.source->attrs, fcompute, exec_type, mutate_index); } else { - LOG(FATAL) << "FCompute not registered " << op->name; + LOG(INFO) << "Neither FCompute nor FComputeEx registered " << op->name; } } } diff --git a/src/executor/exec_pass.h b/src/executor/exec_pass.h index 0eda71d98214..326262147b9f 100644 --- a/src/executor/exec_pass.h +++ b/src/executor/exec_pass.h @@ -27,9 +27,12 @@ #include #include #include +#include #include +#include #include #include +#include namespace mxnet { namespace exec { @@ -37,6 +40,12 @@ namespace exec { /*! \brief reuse graph definition */ using nnvm::Graph; +const int kBadStorageID = -1; +const int kExternalStorageID = -2; +const int kDynamicStorageID = -3; + +const int kNonDefaultStorage = -2; + /*! * \brief executor to execute an operator * This is a graph executor dependent interface @@ -44,7 +53,7 @@ using nnvm::Graph; */ class OpExecutor { public: - /*! \brief input arrays */ + /*! \brief input data arrays, which may be either input or aux */ std::vector in_array; /*! \brief output data arrays */ std::vector out_array; @@ -65,7 +74,7 @@ class OpExecutor { * This function call do not synchronize the stream. * \param rctx The runtime context passed in by environment. */ - virtual void Run(RunContext rctx) = 0; + virtual void Run(RunContext rctx, bool is_gpu) = 0; /*! \return the execution type */ virtual ExecType exec_type() const = 0; /*! \return return engine variable for operator states */ @@ -123,6 +132,45 @@ Graph AttachOpResources(Graph g); */ Graph DetectInplaceAddTo(Graph g); +/*! + * \brief Infer shapes in the graph given the information. + * \param graph The input graph. + * \param shape_inputs The shapes of input symbols to the graph. + * \param shape_attr_key The key to the node attribute that can indicate shape. This is + * the place where manual hint for shapes could be injected. + * \return A graph with new attribute "shape" containing inferred shape of each NodeEntry. + * The index of ShapeVector is given by graph.indexed_graph().entry_id. + */ +Graph InferShape(Graph graph, + nnvm::ShapeVector shape_inputs, + const std::string& shape_attr_key = ""); + +/*! + * \brief Infer types in the graph given the information. + * \param graph The input graph. + * \param dtype_inputs The types of input symbols to the graph. + * \param dtype_attr_key The key to the node attribute that can indicate types. This is + * the place where manual hint for types could be injected. + * \return A graph with new attribute "dtype" containing inferred type of each NodeEntry. + * The index of ShapeVector is given by graph.indexed_graph().entry_id. + */ +Graph InferType(Graph graph, + nnvm::DTypeVector dtype_inputs, + const std::string& dtype_attr_key = ""); + +/*! + * \brief Infer storage types in the graph given the information. + * \param graph The input graph. + * \param storage_type_inputs The storage types of input symbols to the graph. + * \param storage_type_attr_key The key to the node attribute that can indicate storage types. + This is the place where manual hint for types could be injected. + * \return A graph with new attribute "storage_type" containing inferred type of each NodeEntry. + * The index of StorageTypeVector is given by graph.indexed_graph().entry_id. + */ +Graph InferStorageType(Graph graph, + StorageTypeVector storage_type_inputs, + const std::string& storage_type_attr_key = ""); + } // namespace exec } // namespace mxnet diff --git a/src/executor/graph_executor.cc b/src/executor/graph_executor.cc index 6dc8cf39970e..9c4398343b1c 100644 --- a/src/executor/graph_executor.cc +++ b/src/executor/graph_executor.cc @@ -30,9 +30,15 @@ #include "./exec_pass.h" #include "./graph_executor.h" #include "../engine/profiler.h" +#include "../common/utils.h" namespace mxnet { namespace exec { + +GraphExecutor::GraphExecutor() { + log_verbose_ = dmlc::GetEnv("MXNET_EXEC_VERBOSE_LOGGING", false); +} + GraphExecutor::~GraphExecutor() { for (auto& n : op_nodes_) { if (n.cached_opr != nullptr) { @@ -47,6 +53,30 @@ GraphExecutor::~GraphExecutor() { } } +inline NDArray InitZeros(const NDArrayStorageType stype, const TShape &shape, + const Context &ctx, const int dtype) { + // NDArray with default storage + if (stype == kDefaultStorage) { + NDArray ret(shape, ctx, false, dtype); + ret = 0; + return ret; + } + // NDArray with non-default storage. Storage allocation is always delayed. + return NDArray(stype, shape, ctx, true, dtype); +} + +inline void EmplaceBackZeros(const NDArrayStorageType stype, const TShape &shape, + const Context &ctx, const int dtype, + std::vector *vec) { + // NDArray with default storage + if (stype == kDefaultStorage) { + vec->emplace_back(shape, ctx, false, dtype); + vec->back() = 0; + } else { + // NDArray with non-default storage. Storage allocation is always delayed. + vec->emplace_back(stype, shape, ctx, true, dtype); + } +} void GraphExecutor::Forward(bool is_train) { RunOps(is_train, 0, num_forward_nodes_); } @@ -438,6 +468,29 @@ void HandleInferTypeError(const size_t num_forward_inputs, << oss.str(); } +void HandleInferStorageTypeError(const size_t num_forward_inputs, + const nnvm::IndexedGraph& idx, + const StorageTypeVector& inferred_stypes) { + int cnt = 10; + std::ostringstream oss; + for (size_t i = 0; i < num_forward_inputs; ++i) { + const uint32_t nid = idx.input_nodes().at(i); + const uint32_t eid = idx.entry_id(nid, 0); + const int inferred_stype = inferred_stypes[eid]; + if (inferred_stype == -1) { + const std::string& arg_name = idx[nid].source->attrs.name; + oss << arg_name << ": " << inferred_stype << ", "; + if (--cnt == 0) { + oss << "..."; + break; + } + } + } + LOG(FATAL) << "InferStoragetType pass cannot decide storage type for the following arguments " + "(-1 means unknown stype). Please consider providing them as inputs:\n" + << oss.str(); +} + /*! * \brief GraphExecutor initializer for regular bind flow in which * input arguments and gradients are provided by users. This initializer @@ -475,21 +528,25 @@ void GraphExecutor::Init(nnvm::Symbol symbol, data_entry_.resize(idx.num_node_entries()); nnvm::ShapeVector arg_shapes; nnvm::DTypeVector arg_dtypes; + StorageTypeVector arg_stypes; for (size_t i = 0; i < num_forward_inputs_; ++i) { const uint32_t nid = idx.input_nodes().at(i); const std::string& arg_name = idx[nid].source->attrs.name; + size_t eid = idx.entry_id(nid, 0); if (mutable_nodes.count(nid)) { CHECK_LT(aux_top, aux_states.size()); - data_entry_[idx.entry_id(nid, 0)] = aux_states[aux_top]; + data_entry_[eid] = aux_states[aux_top]; arg_shapes.push_back(aux_states[aux_top].shape()); arg_dtypes.push_back(aux_states[aux_top].dtype()); + arg_stypes.push_back(aux_states[aux_top].storage_type()); aux_state_map_.emplace(arg_name, aux_states[aux_top]); ++aux_top; } else { CHECK_LT(arg_top, in_args.size()); - data_entry_[idx.entry_id(nid, 0)] = in_args[arg_top]; + data_entry_[eid] = in_args[arg_top]; arg_shapes.push_back(in_args[arg_top].shape()); arg_dtypes.push_back(in_args[arg_top].dtype()); + arg_stypes.push_back(in_args[arg_top].storage_type()); in_arg_map_.emplace(arg_name, in_args[arg_top]); if (kNullOp != grad_req_types[arg_top]) { grad_store_.emplace_back(grad_req_types[arg_top], arg_grad_store[arg_top]); @@ -497,23 +554,33 @@ void GraphExecutor::Init(nnvm::Symbol symbol, } ++arg_top; } + if (log_verbose_) { + LOG(INFO) << "\tassign data entry\t" << eid << " as stype " + << data_entry_[eid].storage_type() << " (input)"; + } } // expand arg_shapes and arg_dtypes to contain backward inputs arg_shapes.resize(idx.input_nodes().size(), TShape()); - g = nnvm::pass::InferShape(g, arg_shapes, "__shape__"); + g = InferShape(std::move(g), arg_shapes, "__shape__"); if (g.GetAttr("shape_num_unknown_nodes") != 0U) { HandleInferShapeError(num_forward_inputs_, g.indexed_graph(), g.GetAttr("shape")); } arg_dtypes.resize(idx.input_nodes().size(), -1); - g = nnvm::pass::InferType(g, arg_dtypes, "__dtype__"); + g = InferType(std::move(g), arg_dtypes, "__dtype__"); if (g.GetAttr("dtype_num_unknown_nodes") != 0U) { HandleInferTypeError(num_forward_inputs_, g.indexed_graph(), g.GetAttr("dtype")); } + g = InferStorageType(std::move(g), arg_stypes, "__storage_type__"); + if (g.GetAttr("storage_type_num_unknown_nodes") != 0U) { + HandleInferStorageTypeError(num_forward_inputs_, g.indexed_graph(), + g.GetAttr("storage_type")); + } + // Initialize the rest attributes of the graph. // This function can be called by regular bind // operation flow as well. @@ -529,6 +596,7 @@ void GraphExecutor::Init(nnvm::Symbol symbol, void GraphExecutor::InitArguments(const nnvm::IndexedGraph& idx, const nnvm::ShapeVector& inferred_shapes, const nnvm::DTypeVector& inferred_dtypes, + const StorageTypeVector& inferred_stypes, const std::vector& in_arg_ctxes, const std::vector& arg_grad_ctxes, const std::vector& aux_state_ctxes, @@ -546,22 +614,37 @@ void GraphExecutor::InitArguments(const nnvm::IndexedGraph& idx, const uint32_t eid = idx.entry_id(nid, 0); const TShape& inferred_shape = inferred_shapes[eid]; const int inferred_dtype = inferred_dtypes[eid]; + const NDArrayStorageType inferred_stype = (NDArrayStorageType) inferred_stypes[eid]; const std::string& arg_name = idx[nid].source->attrs.name; if (mutable_nodes.count(nid)) { // aux_states - aux_state_vec->emplace_back(inferred_shape, aux_state_ctxes[aux_top], false, inferred_dtype); - aux_state_vec->back() = 0; + EmplaceBackZeros(inferred_stype, inferred_shape, aux_state_ctxes[aux_top], + inferred_dtype, aux_state_vec); data_entry_[eid] = aux_state_vec->back(); aux_state_map_.emplace(arg_name, aux_state_vec->back()); ++aux_top; + if (log_verbose_) { + LOG(INFO) << "\tassign aux entry\t" << eid << "\t as stype " << inferred_stype; + } } else { // in_args - in_arg_vec->emplace_back(inferred_shape, in_arg_ctxes[arg_top], false, inferred_dtype); - in_arg_vec->back() = 0; + EmplaceBackZeros(inferred_stype, inferred_shape, in_arg_ctxes[arg_top], + inferred_dtype, in_arg_vec); data_entry_[eid] = in_arg_vec->back(); + if (log_verbose_) { + LOG(INFO) << "\tassign data entry\t" << eid << "\tas stype " << inferred_stype; + } + // Get the storage type for grad if (kNullOp == grad_req_types[arg_top]) { arg_grad_vec->emplace_back(); } else { - arg_grad_vec->emplace_back(inferred_shape, arg_grad_ctxes[arg_top], false, inferred_dtype); - arg_grad_vec->back() = 0; + // Init based on storage type + auto grad_oid = grad_store_.size() + num_forward_outputs_; + auto grad_eid = idx.entry_id(idx.outputs()[grad_oid]); + auto grad_stype = (NDArrayStorageType) inferred_stypes[grad_eid]; + EmplaceBackZeros(grad_stype, inferred_shape, arg_grad_ctxes[arg_top], + inferred_dtype, arg_grad_vec); + if (log_verbose_) { + LOG(INFO) << "\tassign grad entry\t" << grad_eid << "\tas stype " << grad_stype; + } grad_store_.emplace_back(grad_req_types[arg_top], arg_grad_vec->back()); arg_grad_map_.emplace(arg_name, arg_grad_vec->back()); } @@ -573,33 +656,40 @@ void GraphExecutor::InitArguments(const nnvm::IndexedGraph& idx, /*! * \brief If the requested ndarray's shape size is less than - * the corresponding shared_data_array's shape size, reuse - * the memory allocation; otherwise, create a zero ndarray. + * the corresponding shared_data_array's shape size and the + * storage type is default storage, reuse the memory allocation + * in shared_buffer; otherwise, create a zero ndarray. */ NDArray ReshapeOrCreate(const std::string& name, const TShape& dest_arg_shape, const int dest_arg_dtype, + const NDArrayStorageType dest_arg_stype, const Context& ctx, std::unordered_map* shared_buffer) { + if (dest_arg_dtype != kDefaultStorage) { + return InitZeros(dest_arg_stype, dest_arg_shape, ctx, dest_arg_dtype); + } auto it = shared_buffer->find(name); if (it != shared_buffer->end()) { if (it->second.shape().Size() >= dest_arg_shape.Size()) { // memory can be reused CHECK_EQ(it->second.dtype(), dest_arg_dtype) << "Requested arg array's dtype does not match the reusable ndarray"; + CHECK_EQ(it->second.storage_type(), kDefaultStorage) + << "shared_buffer should only contain NDArrays with default storage type."; return it->second.Reshape(dest_arg_shape); } else { LOG(WARNING) << "Bucketing: data " << name << " has a shape " << dest_arg_shape << ", which is larger than already allocated shape " << it->second.shape() << ". Need to re-allocate. Consider putting default bucket key to be " << "the bucket taking the largest input for better memory sharing."; - it->second = NDArray(dest_arg_shape, ctx, false, dest_arg_dtype); - it->second = 0; + // the NDArrays in shared_buffer are guaranteed to be of default storage + it->second = InitZeros(dest_arg_stype, dest_arg_shape, ctx, dest_arg_dtype); return it->second; } // arg_array.shape().Size() >= arg_shape.Size() } else { - auto p = shared_buffer->emplace(name, NDArray(dest_arg_shape, ctx, false, dest_arg_dtype)); - p.first->second = 0; - return p.first->second; + auto ret = InitZeros(dest_arg_stype, dest_arg_shape, ctx, dest_arg_dtype); + shared_buffer->emplace(name, ret); + return ret; } // if (it != shared_buffer->end()) } @@ -612,6 +702,7 @@ NDArray ReshapeOrCreate(const std::string& name, void GraphExecutor::InitArguments(const nnvm::IndexedGraph& idx, const nnvm::ShapeVector& inferred_shapes, const nnvm::DTypeVector& inferred_dtypes, + const StorageTypeVector& inferred_stypes, const std::vector& in_arg_ctxes, const std::vector& arg_grad_ctxes, const std::vector& aux_state_ctxes, @@ -631,9 +722,12 @@ void GraphExecutor::InitArguments(const nnvm::IndexedGraph& idx, const uint32_t eid = idx.entry_id(nid, 0); const TShape& inferred_shape = inferred_shapes[eid]; const int inferred_dtype = inferred_dtypes[eid]; + const NDArrayStorageType inferred_stype = (NDArrayStorageType) inferred_stypes[eid]; const std::string& arg_name = idx[nid].source->attrs.name; - if (mutable_nodes.count(nid)) { // aux_states - if (nullptr != shared_exec) { + // aux_states + if (mutable_nodes.count(nid)) { + if (nullptr != shared_exec && inferred_stype == kDefaultStorage && + shared_exec->aux_state_map().at(arg_name).storage_type() == kDefaultStorage) { const NDArray& aux_nd = shared_exec->aux_state_map().at(arg_name); CHECK_EQ(inferred_shape, aux_nd.shape()) << "Inferred shape does not match shared_exec.aux_array's shape." @@ -647,16 +741,18 @@ void GraphExecutor::InitArguments(const nnvm::IndexedGraph& idx, << arg_name << " for the current executor"; aux_state_vec->emplace_back(aux_nd); } else { - aux_state_vec->emplace_back(inferred_shape, aux_state_ctxes[aux_top], - false, inferred_dtype); - aux_state_vec->back() = 0; + EmplaceBackZeros(inferred_stype, inferred_shape, aux_state_ctxes[aux_top], + inferred_dtype, aux_state_vec); } // if (has_shared_exec) data_entry_[eid] = aux_state_vec->back(); aux_state_map_.emplace(arg_name, aux_state_vec->back()); ++aux_top; - } else { // in_args + } else { // in_args and grad for in_args if (shared_arg_names.count(arg_name)) { // model parameter - if (nullptr != shared_exec) { + // model parameter + if (nullptr != shared_exec && inferred_stype == kDefaultStorage && + shared_exec->in_arg_map().at(arg_name).storage_type() == kDefaultStorage) { + // try to reuse memory from shared_exec const NDArray& in_arg_nd = shared_exec->in_arg_map().at(arg_name); CHECK_EQ(inferred_shape, in_arg_nd.shape()) << "Inferred shape does not match shared_exec.arg_array's shape" @@ -669,33 +765,43 @@ void GraphExecutor::InitArguments(const nnvm::IndexedGraph& idx, " be resued for creating NDArray of the argument" << arg_name << " for the current executor"; in_arg_vec->emplace_back(in_arg_nd); - if (kNullOp == grad_req_types[arg_top]) { - arg_grad_vec->emplace_back(); - } else { + } else { + // doesn't have shared_exec, or non-default storage + EmplaceBackZeros(inferred_stype, inferred_shape, in_arg_ctxes[arg_top], + inferred_dtype, in_arg_vec); + } + // gradient for model parameter + if (kNullOp == grad_req_types[arg_top]) { + arg_grad_vec->emplace_back(); + } else { + auto grad_oid = grad_store_.size() + num_forward_outputs_; + auto grad_eid = idx.entry_id(idx.outputs()[grad_oid]); + auto grad_stype = (NDArrayStorageType) inferred_stypes[grad_eid]; + if (nullptr != shared_exec && grad_stype == kDefaultStorage && + shared_exec->arg_grad_map().at(arg_name).storage_type() == kDefaultStorage) { + // try to reuse memory from shared_exec arg_grad_vec->emplace_back(shared_exec->arg_grad_map().at(arg_name)); - grad_store_.emplace_back(grad_req_types[arg_top], arg_grad_vec->back()); - } // if (kNullOp == grad_req_types[arg_top]) - } else { // !has shared_exec - in_arg_vec->emplace_back(inferred_shape, in_arg_ctxes[arg_top], false, inferred_dtype); - in_arg_vec->back() = 0; - if (kNullOp == grad_req_types[arg_top]) { - arg_grad_vec->emplace_back(); } else { - arg_grad_vec->emplace_back(inferred_shape, arg_grad_ctxes[arg_top], - false, inferred_dtype); - arg_grad_vec->back() = 0; - grad_store_.emplace_back(grad_req_types[arg_top], arg_grad_vec->back()); - } // if (kNullOp == grad_req_types[arg_top]) - } // if (has_shared_exec) + EmplaceBackZeros(grad_stype, inferred_shape, arg_grad_ctxes[arg_top], + inferred_dtype, arg_grad_vec); + } + grad_store_.emplace_back(grad_req_types[arg_top], arg_grad_vec->back()); + } } else { // !shared_arg_names.count(arg_name) + // model parameter in_arg_vec->emplace_back(ReshapeOrCreate(arg_name, inferred_shape, inferred_dtype, - in_arg_ctxes[arg_top], shared_buffer)); + inferred_stype, in_arg_ctxes[arg_top], + shared_buffer)); + // gradient for model parameter if (kNullOp == grad_req_types[arg_top]) { arg_grad_vec->emplace_back(); } else { + auto grad_oid = grad_store_.size() + num_forward_outputs_; + auto grad_eid = idx.entry_id(idx.outputs()[grad_oid]); + auto grad_stype = (NDArrayStorageType) inferred_stypes[grad_eid]; arg_grad_vec->emplace_back(ReshapeOrCreate("grad of " + arg_name, inferred_shape, - inferred_dtype, arg_grad_ctxes[arg_top], - shared_buffer)); + inferred_dtype, grad_stype, + arg_grad_ctxes[arg_top], shared_buffer)); grad_store_.emplace_back(grad_req_types[arg_top], arg_grad_vec->back()); } // if (kNullOp == grad_req_types[arg_top]) } // if (shared_arg_names.count(arg_name)) @@ -718,14 +824,35 @@ void GraphExecutor::FinishInitGraph(nnvm::Symbol symbol, Executor* shared_exec, const nnvm::NodeEntryMap& feed_dict) { const auto& idx = g.indexed_graph(); + // dispatch based on stype per operator + const auto& vstorage_type = g.GetAttr("storage_type"); + StorageTypeVector dispatch_stypes(idx.num_nodes(), kUndefinedStorage); + for (size_t nid = 0; nid < idx.num_nodes(); nid++) { + const auto& inode = idx[nid]; + auto num_outputs = inode.source->num_outputs(); + auto num_inputs = inode.inputs.size(); + StorageTypeVector vs(num_inputs + num_outputs, kUndefinedStorage); + for (size_t i = 0; i < num_inputs; i++) { + auto e = inode.inputs[i]; + vs[i] = vstorage_type[idx.entry_id(e)]; + CHECK_NE(vs[i], kUndefinedStorage); + } + for (uint32_t i = 0; i < num_outputs; ++i) { + uint32_t eid = idx.entry_id(nid, i); + vs[i + num_inputs] = vstorage_type[eid]; + } + bool contains_non_default = common::ContainsNonDefaultStorage(vs); + dispatch_stypes[nid] = contains_non_default ? kNonDefaultStorage : kDefaultStorage; + } + g.attrs["dispatch_stypes"] = std::make_shared(std::move(dispatch_stypes)); + + // data entries for output gradients for (size_t j = num_forward_outputs_; j < idx.outputs().size(); ++j) { data_entry_[idx.entry_id(idx.outputs()[j])] = grad_store_[j - num_forward_outputs_].second; } { // memory allocator - const int kBadStorageID = -1; - const int kExternalStorageID = -2; nnvm::StorageVector arg_storage_id(idx.num_node_entries(), kBadStorageID); for (size_t j = num_forward_outputs_; j < idx.outputs().size(); ++j) { arg_storage_id[idx.entry_id(idx.outputs()[j])] = kExternalStorageID; @@ -735,6 +862,9 @@ void GraphExecutor::FinishInitGraph(nnvm::Symbol symbol, data_entry_[eid] = kv.second; arg_storage_id[eid] = kExternalStorageID; } + for (size_t i = 0; i < idx.num_node_entries(); i++) { + if (vstorage_type[i] != kDefaultStorage) arg_storage_id[i] = kDynamicStorageID; + } g.attrs["storage"] = std::make_shared(std::move(arg_storage_id)); g = nnvm::ApplyPass(g, "PlanMemory"); } @@ -792,6 +922,7 @@ void GraphExecutor::Init(nnvm::Symbol symbol, const std::vector& aux_state_ctxes, const std::unordered_map& arg_shape_map, const std::unordered_map& arg_dtype_map, + const std::unordered_map& arg_stype_map, const std::vector& grad_req_types, const std::unordered_set& shared_arg_names, std::vector* in_arg_vec, @@ -811,6 +942,7 @@ void GraphExecutor::Init(nnvm::Symbol symbol, const nnvm::IndexedGraph& idx = g.indexed_graph(); nnvm::ShapeVector arg_shapes(idx.input_nodes().size(), TShape()); nnvm::DTypeVector arg_dtypes(idx.input_nodes().size(), -1); + StorageTypeVector arg_stypes(idx.input_nodes().size(), kUndefinedStorage); for (size_t i = 0; i < num_forward_inputs_; ++i) { const uint32_t nid = idx.input_nodes().at(i); const std::string& name = idx[nid].source->attrs.name; @@ -822,29 +954,41 @@ void GraphExecutor::Init(nnvm::Symbol symbol, if (arg_dtype_map.end() != it2) { arg_dtypes[i] = it2->second; } + auto it3 = arg_stype_map.find(name); + if (arg_stype_map.end() != it3) { + arg_stypes[i] = it3->second; + } } - g = nnvm::pass::InferShape(g, arg_shapes, "__shape__"); + g = InferShape(std::move(g), arg_shapes, "__shape__"); if (g.GetAttr("shape_num_unknown_nodes") != 0U) { HandleInferShapeError(num_forward_inputs_, g.indexed_graph(), g.GetAttr("shape")); } - g = nnvm::pass::InferType(g, arg_dtypes, "__dtype__"); + g = InferType(std::move(g), arg_dtypes, "__dtype__"); if (g.GetAttr("dtype_num_unknown_nodes") != 0U) { HandleInferTypeError(num_forward_inputs_, g.indexed_graph(), g.GetAttr("dtype")); } + g = InferStorageType(std::move(g), arg_stypes, "__storage_type__"); + if (g.GetAttr("storage_type_num_unknown_nodes") != 0U) { + HandleInferStorageTypeError(num_forward_inputs_, g.indexed_graph(), + g.GetAttr("storage_type")); + } + // Create in_args, arg_grads, and aux_states using // the inferred shapes and dtypes. if (nullptr == shared_buffer) { // regular simple bind InitArguments(idx, g.GetAttr("shape"), g.GetAttr("dtype"), + g.GetAttr("storage_type"), in_arg_ctxes, arg_grad_ctxes, aux_state_ctxes, grad_req_types, in_arg_vec, arg_grad_vec, aux_state_vec); } else { // simple bind using shared data arrays and shared_exec InitArguments(idx, g.GetAttr("shape"), g.GetAttr("dtype"), + g.GetAttr("storage_type"), in_arg_ctxes, arg_grad_ctxes, aux_state_ctxes, grad_req_types, shared_arg_names, shared_exec, shared_buffer, in_arg_vec, arg_grad_vec, aux_state_vec); @@ -905,20 +1049,29 @@ void GraphExecutor::InitDataEntryMemory(std::vector* shared_pool) { const auto& vdtype = graph_.GetAttr("dtype"); const auto& vshape = graph_.GetAttr("shape"); const auto& vstorage = graph_.GetAttr("storage_id"); + const auto& vstorage_type = graph_.GetAttr("storage_type"); const auto& vctx = graph_.GetAttr("context"); CHECK_EQ(idx.num_node_entries(), vshape.size()); CHECK_EQ(idx.num_node_entries(), vdtype.size()); CHECK_EQ(idx.num_node_entries(), vstorage.size()); CHECK_EQ(data_entry_.size(), vshape.size()); std::vector data_context(idx.num_node_entries()); + std::vector data_storage_type(idx.num_node_entries(), kUndefinedStorage); for (uint32_t nid = 0; nid < idx.num_nodes(); ++nid) { for (uint32_t i = 0; i < idx[nid].source->num_outputs(); ++i) { - data_context[idx.entry_id(nid, i)] = vctx[nid]; + auto eid = idx.entry_id(nid, i); + data_context[eid] = vctx[nid]; + CHECK_NE(vstorage_type[nid], kUndefinedStorage); + data_storage_type[eid] = (NDArrayStorageType) vstorage_type[nid]; } } // information about the pool - using PoolEntry = std::pair; + struct PoolEntry { + Context ctx; + size_t bytes; + NDArrayStorageType stype; + }; std::vector pool_info; // assign array to head gradient @@ -926,26 +1079,36 @@ void GraphExecutor::InitDataEntryMemory(std::vector* shared_pool) { uint32_t nid = idx.input_nodes().at(i); uint32_t oid = head_grad_map_.at(idx[nid].source); uint32_t eid = idx.entry_id(idx.outputs()[oid]); + NDArrayStorageType stype = (NDArrayStorageType) vstorage_type[eid]; CHECK_NE(vshape[eid].ndim(), 0U); CHECK_NE(vdtype[eid], -1); - data_entry_[idx.entry_id(nid, 0)] = - NDArray(vshape[eid], data_context[eid], false, vdtype[eid]); + auto data_eid = idx.entry_id(nid, 0); + // initialize based on storage_type + if (stype != kDefaultStorage) { + data_entry_[data_eid] = NDArray(stype, vshape[eid], data_context[eid], true, vdtype[eid]); + } else { + data_entry_[data_eid] = NDArray(vshape[eid], data_context[eid], false, vdtype[eid]); + } + if (log_verbose_) { + LOG(INFO) << "\tinit head_g entry\t" << data_eid << "\tas stype " << stype; + } } // get maximum bytes in each pool for (size_t i = 0; i < vshape.size(); ++i) { if (!data_entry_[i].is_none()) continue; size_t bytes = vshape[i].Size() * mshadow::mshadow_sizeof(vdtype[i]); int storage_id = vstorage[i]; + // skip pool allocation for kBadStorageID, kExternalStorageID and kDynamicStorageID if (storage_id < 0) continue; size_t sid = static_cast(storage_id); if (sid >= pool_info.size()) { - pool_info.resize(sid + 1, PoolEntry{Context::CPU(), size_t(0)}); + pool_info.resize(sid + 1, PoolEntry{Context::CPU(), size_t(0), kUndefinedStorage}); } PoolEntry& info = pool_info[sid]; - if (info.second == 0) { - info = PoolEntry{data_context[i], bytes}; + if (info.bytes == 0) { + info = PoolEntry{data_context[i], bytes, data_storage_type[i]}; } else { - info.second = std::max(info.second, bytes); + info.bytes = std::max(info.bytes, bytes); } } // construct the re-use pool, if needed @@ -966,13 +1129,14 @@ void GraphExecutor::InitDataEntryMemory(std::vector* shared_pool) { sorted_pool_index.push_back(i); } auto pool_comparator = [&pool_info](int lhs, int rhs){ - return pool_info[lhs].second > pool_info[rhs].second; + return pool_info[lhs].bytes > pool_info[rhs].bytes; }; std::sort(sorted_pool_index.begin(), sorted_pool_index.end(), pool_comparator); for (size_t i : sorted_pool_index) { - const Context& ctx = pool_info[i].first; - size_t bytes = pool_info[i].second; + const Context& ctx = pool_info[i].ctx; + size_t bytes = pool_info[i].bytes; + NDArrayStorageType storage_type = pool_info[i].stype; bool allocated = false; for (auto it = free_pool.lower_bound(bytes); it != free_pool.end(); ++it) { if (it->second.ctx() == ctx && it->first >= bytes) { @@ -987,7 +1151,9 @@ void GraphExecutor::InitDataEntryMemory(std::vector* shared_pool) { CHECK_LE(nword, std::numeric_limits::max()); // allocate float arrays TShape shape{static_cast(nword)}; - NDArray nd(shape, ctx); + // TODO(junwu): adding delay_alloc=true to create nd + // is a temporary solution. + NDArray nd(shape, ctx, true); data_pool_[i] = nd; // put the new allocated arrays to shared pool if (shared_pool != nullptr) { @@ -997,15 +1163,22 @@ void GraphExecutor::InitDataEntryMemory(std::vector* shared_pool) { } CHECK_EQ(data_pool_.size(), pool_info.size()); // assign the data entries - for (size_t i = 0; i < data_entry_.size(); ++i) { // avoid pre-allocated arrays if (!data_entry_[i].is_none()) continue; // assign allocated array by storage id int storage_id = vstorage[i]; - CHECK_GE(storage_id, 0) << "Do not support runtime shape op yet"; - const NDArray& src = data_pool_.at(storage_id); - data_entry_[i] = src.AsArray(vshape[i], vdtype[i]); + auto storage_type = (NDArrayStorageType) vstorage_type[i]; + if (storage_type == kDefaultStorage) { + CHECK_GE(storage_id, 0) << "Do not support runtime shape op yet"; + const NDArray& src = data_pool_.at(storage_id); + data_entry_[i] = src.AsArray(vshape[i], vdtype[i]); + } else { + data_entry_[i] = NDArray(storage_type, vshape[i], data_context[i]); + } + if (log_verbose_) { + LOG(INFO) << "\tinit data entry\t" << i << "\tas stype " << storage_type; + } } } @@ -1020,11 +1193,28 @@ void GraphExecutor::InitCachedOps() { const auto& vctx = graph_.GetAttr("context"); const auto& addto_entry = graph_.GetAttr >("addto_entry"); const auto& skip_plus_node = graph_.GetAttr >("skip_plus_node"); + const auto& vstorage_type = graph_.GetAttr("storage_type"); op_nodes_.resize(idx.num_nodes()); // setup the array and requirements. for (uint32_t nid = 0; nid < idx.num_nodes(); ++nid) { const auto& inode = idx[nid]; + if (log_verbose_) { + if (inode.source->is_variable()) { + LOG(INFO) << "node " << nid << " var"; + } else { + LOG(INFO) << "node " << nid << " " << inode.source->attrs.op->name; + auto exec = op_execs[nid]; + for (const auto& e : inode.inputs) { + auto eid = idx.entry_id(e); + LOG(INFO) << "\t\tinput " << eid << " stype: " << vstorage_type[eid]; + } + for (uint32_t index = 0; index < inode.source->num_outputs(); ++index) { + uint32_t eid = idx.entry_id(nid, index); + LOG(INFO) << "\t\toutput " << eid << " stype: " << vstorage_type[eid]; + } + } + } if (inode.source->is_variable()) continue; #if MXNET_USE_PROFILER op_nodes_[nid].opr_name = inode.source->op()->name.c_str(); @@ -1104,7 +1294,7 @@ void GraphExecutor::InitCachedOps() { if (is_async) { exec->op_ctx.async_on_complete = on_complete; } - exec->Run(ctx); + exec->Run(ctx, is_gpu); // call on complete only if it is async op if (!is_async) { if (is_gpu) { @@ -1265,7 +1455,8 @@ void GraphExecutor::RunOps(bool is_train, size_t topo_start, size_t topo_end) { CHECK_EQ(opnode.exec->out_array.size(), 1U); CopyFromTo(opnode.exec->in_array[0], &(opnode.exec->out_array[0])); } else if (opnode.exec->exec_type() == ExecType::kLocal) { - opnode.exec->Run(RunContext{opnode.ctx, nullptr}); + bool is_gpu = opnode.ctx.dev_mask() == gpu::kDevMask; + opnode.exec->Run(RunContext{opnode.ctx, nullptr}, is_gpu); } else if (opnode.cached_opr != nullptr) { #if MXNET_USE_PROFILER bool profiling = engine::Profiler::Get()->GetState() == engine::Profiler::kRunning; @@ -1335,7 +1526,7 @@ GraphExecutor::CachedSegOpr GraphExecutor::CreateCachedSegOpr(size_t topo_start, RunContext ctx, Engine::CallbackOnComplete on_complete) { // Run all opr in the sub-graph for (auto &exec : exec_list) { - exec->Run(ctx); + exec->Run(ctx, is_gpu); } if (is_gpu) { #if MXNET_USE_CUDA @@ -1370,6 +1561,7 @@ Executor *Executor::SimpleBind(nnvm::Symbol symbol, const std::vector& aux_state_ctxes, const std::unordered_map& arg_shape_map, const std::unordered_map& arg_dtype_map, + const std::unordered_map& arg_stype_map, const std::vector& grad_req_types, const std::unordered_set& shared_arg_names, std::vector* in_args, @@ -1380,7 +1572,7 @@ Executor *Executor::SimpleBind(nnvm::Symbol symbol, auto exec = new exec::GraphExecutor(); exec->Init(symbol, default_ctx, group2ctx, in_arg_ctxes, arg_grad_ctxes, aux_state_ctxes, - arg_shape_map, arg_dtype_map, + arg_shape_map, arg_dtype_map, arg_stype_map, grad_req_types, shared_arg_names, in_args, arg_grads, aux_states, shared_buffer, shared_exec); diff --git a/src/executor/graph_executor.h b/src/executor/graph_executor.h index dc50bef002ab..48222f05fae2 100644 --- a/src/executor/graph_executor.h +++ b/src/executor/graph_executor.h @@ -59,6 +59,7 @@ class GraphExecutor : public Executor { friend class autograd::AutogradRuntime; using Executor::MonitorCallback; + GraphExecutor(); virtual ~GraphExecutor(); void Forward(bool is_train) override; void PartialForward(bool is_train, int step, int *step_left) override; @@ -96,6 +97,7 @@ class GraphExecutor : public Executor { const std::vector& aux_state_ctxes, const std::unordered_map& arg_shape_map, const std::unordered_map& arg_dtype_map, + const std::unordered_map& arg_stype_map, const std::vector& grad_req_types, const std::unordered_set& shared_arg_names, std::vector* in_arg_vec, @@ -141,6 +143,7 @@ class GraphExecutor : public Executor { void InitArguments(const nnvm::IndexedGraph& idx, const nnvm::ShapeVector& inferred_shapes, const nnvm::DTypeVector& inferred_dtypes, + const StorageTypeVector& inferred_stypes, const std::vector& in_arg_ctxes, const std::vector& arg_grad_ctxes, const std::vector& aux_state_ctxes, @@ -153,6 +156,7 @@ class GraphExecutor : public Executor { void InitArguments(const nnvm::IndexedGraph& idx, const nnvm::ShapeVector& inferred_shapes, const nnvm::DTypeVector& inferred_dtypes, + const StorageTypeVector& inferred_stypes, const std::vector& in_arg_ctxes, const std::vector& arg_grad_ctxes, const std::vector& aux_state_ctxes, @@ -201,7 +205,8 @@ class GraphExecutor : public Executor { std::vector op_nodes_; // internal data entry of each node std::vector data_entry_; - // internal data pool of allocated entries + // internal data pool of allocated entries. + // these allocated entries can be used for static memory sharing between executors. std::vector data_pool_; // output arrays std::vector output_arrays_; @@ -233,6 +238,8 @@ class GraphExecutor : public Executor { bool prefer_bulk_execution_; // cached segment operator std::vector cached_seg_opr_; + // verbose logging + bool log_verbose_ = false; }; } // namespace exec diff --git a/src/executor/infer_graph_attr_pass.cc b/src/executor/infer_graph_attr_pass.cc new file mode 100644 index 000000000000..144c3713e205 --- /dev/null +++ b/src/executor/infer_graph_attr_pass.cc @@ -0,0 +1,356 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file infer_graph_attr_pass.cc + * \brief infer graph shape, dtype, and storage type + */ + +#include +#include +#include "./exec_pass.h" + +namespace mxnet { +namespace exec { + +template +bool ApplyOpInferAttr(const nnvm::Graph& g, + const FInfer& finfer, + const NodeAttrs& attrs, + const uint32_t nid, + std::vector* in_attrs, + std::vector* out_attrs) { + return finfer(attrs, in_attrs, out_attrs); +} + +template<> +bool ApplyOpInferAttr(const nnvm::Graph& g, + const FInferStorageType& finfer, + const NodeAttrs& attrs, + const uint32_t nid, + std::vector* in_attrs, + std::vector* out_attrs) { + const ContextVector& ctxes = g.GetAttr("context"); + return finfer(attrs, ctxes[nid], in_attrs, out_attrs); +} + +/*!\brief + * This is a duplicate of the InferAttr function in nnvm with minor modification + * to support inferring storage type whose function signature is different from + * shape/type inference functions'. The nnvm InferAttr will be deprecated + * in the future. Please use interfaces InferShape, InferType, and InferStorageType + * to call this function. + */ +template +nnvm::Graph InferAttr(nnvm::Graph &&ret, + const AttrType empty_val, + const char* infer_name, + const char* input_name, + const char* attr_key_name, + const char* attr_name, + const char* unknown_name, + IsNone fis_none, + FDefault fdefault, + bool backward_identity_assign) { + using nnvm::IndexedGraph; + using nnvm::Op; + using AttrVector = std::vector; + using dmlc::any; + + const IndexedGraph& idx = ret.indexed_graph(); + static auto& finfer_shape = + Op::GetAttr(infer_name); + static auto& is_backward = + Op::GetAttr("TIsBackward"); + // gradient function, used to get node correspondence. + static auto& fgrad = + Op::GetAttr("FGradient"); + // reshape shape vector + AttrVector rshape; + if (ret.attrs.count(attr_name) != 0) { + rshape = ret.MoveCopyAttr(attr_name); + } else { + rshape.resize(idx.num_node_entries(), empty_val); + } + + if (ret.attrs.count(input_name) != 0) { + const AttrVector& shape_args = ret.GetAttr(input_name); + CHECK_LE(shape_args.size(), idx.input_nodes().size()) + << "More provided " << attr_name << "s than number of arguments."; + for (size_t i = 0; i < shape_args.size(); ++i) { + rshape[idx.entry_id(idx.input_nodes()[i], 0)] = shape_args[i]; + } + // erase the provided arguments + ret.attrs.erase(input_name); + } + + // get the shape hints + std::string shape_hints_key = std::string(attr_name) + "_hints"; + if (ret.attrs.count(shape_hints_key)) { + nnvm::NodeEntryMap shape_hints = + ret.GetAttr>(shape_hints_key); + for (const auto& kv : shape_hints) { + nnvm::NodeEntry e = kv.first; + if (idx.exist(e.node.get())) { + rshape[idx.entry_id(kv.first)] = kv.second; + } + } + } + + std::string shape_attr_key; + if (ret.attrs.count(attr_key_name) != 0) { + shape_attr_key = ret.GetAttr(attr_key_name); + // erase the provided arguments + ret.attrs.erase(attr_key_name); + } + // Temp space for shape inference. + std::vector ishape, oshape; + + // inference step function for nid + auto infer_step = [&](uint32_t nid, bool last_iter) { + const auto& inode = idx[nid]; + const uint32_t num_inputs = inode.inputs.size(); + const uint32_t num_outputs = inode.source->num_outputs(); + if (inode.source->is_variable()) { + // Variable node. No operator. Only one output entry. + CHECK(inode.source->op() == nullptr); + CHECK_EQ(num_outputs, 1U); + const uint32_t out_ent_id = idx.entry_id(nid, 0); + if (shape_attr_key.length() != 0 && fis_none(rshape[out_ent_id])) { + auto it = inode.source->attrs.dict.find(shape_attr_key); + if (it != inode.source->attrs.dict.end()) { + std::istringstream is(it->second); + CHECK(is >> rshape[out_ent_id]) << "Invalid attribute"; + } + } + } else if (is_backward.get(inode.source->op(), false) && + inode.control_deps.size() && backward_identity_assign) { + CHECK_GE(inode.control_deps.size(), 1U) + << "BackwardOp need to have control_deps to its forward op"; + const IndexedGraph::Node& fnode = idx[inode.control_deps[0]]; + nnvm::NodePtr fwd_ptr = inode.source->control_deps[0]; + CHECK(fwd_ptr->op() != nullptr) << "Forward op cannot be a variable"; + // use gradient function to find out the correspondence. + std::vector ograd(fwd_ptr->num_outputs()); + for (size_t i = 0; i < ograd.size(); ++i) { + ograd[i].index = static_cast(i); + } + // input gradient list + auto igrad = fgrad[fwd_ptr->op()](fwd_ptr, ograd); + const nnvm::Node* igrad_node = nullptr; + // Input gradient assignement + for (size_t i = 0; i < igrad.size(); ++i) { + if (igrad[i].node->op() == inode.source->op()) { + uint32_t eid = idx.entry_id(nid, igrad[i].index); + if (fis_none(rshape[eid])) { + rshape[eid] = rshape[idx.entry_id(fnode.inputs[i])]; + } else { + CHECK_EQ(rshape[eid], rshape[idx.entry_id(fnode.inputs[i])]) + << "Backward shape inconsistent with the forward shape"; + } + if (igrad_node == nullptr) { + igrad_node = igrad[i].node.get(); + } else { + CHECK(igrad_node == igrad[i].node.get()); + } + } + } + // out grad entries + CHECK(igrad_node != nullptr) + << "Cannot find matching backward op for " << inode.source->attrs.name; + for (size_t i = 0; i < igrad_node->inputs.size(); ++i) { + const nnvm::NodeEntry& e = igrad_node->inputs[i]; + if (e.node == nullptr) { + uint32_t eid = idx.entry_id(inode.inputs[i]); + if (fis_none(rshape[eid])) { + rshape[eid] = rshape[idx.entry_id(inode.control_deps[0], e.index)]; + } + } + } + } else { + bool forward_known = true; + // Forward operator inference. + ishape.resize(num_inputs, empty_val); + for (uint32_t i = 0; i < ishape.size(); ++i) { + ishape[i] = rshape[idx.entry_id(inode.inputs[i])]; + if (fis_none(ishape[i])) forward_known = false; + } + oshape.resize(num_outputs, empty_val); + for (uint32_t i = 0; i < oshape.size(); ++i) { + oshape[i] = rshape[idx.entry_id(nid, i)]; + if (fis_none(oshape[i])) forward_known = false; + } + auto finfer = finfer_shape.get(inode.source->op(), fdefault); + if (!forward_known) { + if (finfer != nullptr) { + // Call inference function of the operator. + try { + forward_known = ApplyOpInferAttr(ret, finfer, inode.source->attrs, + nid, &ishape, &oshape); + } catch (const std::exception& e) { + throw dmlc::Error("Error in operator " + inode.source->attrs.name + ": " + e.what()); + } + } else { + CHECK(!last_iter) + << "Attribute " << infer_name + << " is not registed by op " << inode.source->op()->name + << " we are not able to complete the inference because of this"; + } + } + // Save to the result map. + for (uint32_t i = 0; i < num_inputs; ++i) { + rshape[idx.entry_id(inode.inputs[i])] = ishape[i]; + } + for (uint32_t i = 0; i < num_outputs; ++i) { + rshape[idx.entry_id(nid, i)] = oshape[i]; + } + } + }; + + size_t last_num_unknown; + size_t num_unknown = rshape.size(); + int i = 0; + do { + if (i % 2 == 0) { + for (uint32_t nid = 0; nid < idx.num_nodes(); ++nid) { + infer_step(nid, false); + } + } else { + // backward inference + for (uint32_t i = idx.num_nodes(); i != 0; --i) { + infer_step(i - 1, false); + } + } + last_num_unknown = num_unknown; + num_unknown = 0; + for (size_t j = 0; j < idx.num_node_entries(); ++j) { + if (fis_none(rshape[j])) { + ++num_unknown; + } + } + ++i; + } while (num_unknown > 0 && last_num_unknown > num_unknown); + // set the shapes + ret.attrs[attr_name] = std::make_shared(std::move(rshape)); + // number of nodes who knows the shape. + ret.attrs[unknown_name] = std::make_shared(num_unknown); + return ret; +} + +// inference fucntion for same type +inline bool SameType(const nnvm::NodeAttrs& attrs, + std::vector *iattr, + std::vector *oattr) { + int def_v = -1; + for (int v : *oattr) { + if (v != -1) { + def_v = v; break; + } + } + if (def_v == -1) { + for (int v : *iattr) { + if (v != -1) { + def_v = v; break; + } + } + } + if (def_v == -1) return false; + for (int& v : *oattr) { + v = def_v; + } + for (int& v : *iattr) { + v = def_v; + } + return true; +} + +// assigning default type N to both input and output attrs with value -1 +template +inline bool DefaultType(const nnvm::NodeAttrs& attrs, + const Context& ctx, + std::vector *iattr, + std::vector *oattr) { + // TODO(junwu): check whether need to use ctx + for (int& v : *oattr) { + if (v == none) v = default_val; + } + for (int& v : *iattr) { + if (v == none) v = default_val; + } + return true; +} + +nnvm::Graph InferShape(nnvm::Graph graph, + nnvm::ShapeVector shape_inputs, + const std::string& shape_attr_key) { + using dmlc::any; + if (shape_inputs.size() != 0) { + graph.attrs["shape_inputs"] = std::make_shared(std::move(shape_inputs)); + } + if (shape_attr_key.length() != 0) { + graph.attrs["shape_attr_key"] = std::make_shared(std::move(shape_attr_key)); + } + return InferAttr( + std::move(graph), nnvm::TShape(), + "FInferShape", "shape_inputs", "shape_attr_key", + "shape", "shape_num_unknown_nodes", + [](const nnvm::TShape& s) { return s.ndim() == 0 || s.Size() == 0; }, + nullptr, true); +} + +nnvm::Graph InferType(nnvm::Graph graph, + nnvm::DTypeVector dtype_inputs, + const std::string& dtype_attr_key) { + using dmlc::any; + if (dtype_inputs.size() != 0) { + graph.attrs["dtype_inputs"] = std::make_shared(std::move(dtype_inputs)); + } + if (dtype_attr_key.length() != 0) { + graph.attrs["dtype_attr_key"] = std::make_shared(std::move(dtype_attr_key)); + } + return InferAttr( + std::move(graph), -1, + "FInferType", "dtype_inputs", "dtype_attr_key", + "dtype", "dtype_num_unknown_nodes", + [](const int t) { return t == -1; }, + SameType, true); +} + +nnvm::Graph InferStorageType(nnvm::Graph graph, + StorageTypeVector storage_type_inputs, + const std::string& storage_type_attr_key) { + using dmlc::any; + if (storage_type_inputs.size() != 0) { + graph.attrs["storage_type_inputs"] = std::make_shared(std::move(storage_type_inputs)); + } + if (storage_type_attr_key.length() != 0) { + graph.attrs["storage_type_attr_key"] = std::make_shared(std::move(storage_type_attr_key)); + } + // for storage type, the backward attr is not necessarily the same as it's correspondence + const int kDefaultStorage = 0; + return InferAttr( + std::move(graph), -1, + "FInferStorageType", "storage_type_inputs", "storage_type_attr_key", + "storage_type", "storage_type_num_unknown_nodes", + [](const int t) { return t == -1; }, + DefaultType, false); +} + +} // namespace exec +} // namespace mxnet diff --git a/src/executor/inplace_addto_detect_pass.cc b/src/executor/inplace_addto_detect_pass.cc index 26a91e3f1b5e..9359d8863594 100644 --- a/src/executor/inplace_addto_detect_pass.cc +++ b/src/executor/inplace_addto_detect_pass.cc @@ -62,6 +62,8 @@ Graph DetectInplaceAddTo(Graph g) { uint32_t eid_rhs = idx.entry_id(inode.inputs[1]); if (ref_count[eid_rhs] != 1) continue; if (inode.inputs[0].node_id >= inode.inputs[1].node_id) continue; + // TODO(haibin) support inplace addto for Dynamic Storage + if (storage_id[eid_rhs] == kDynamicStorageID) continue; CHECK_NE(storage_id[eid_rhs], sid); storage_id[eid_rhs] = sid; addto_entry[eid_rhs] = 1; diff --git a/src/io/iter_batchloader.h b/src/io/iter_batchloader.h index c5ec10618080..ade7c1a53bd2 100644 --- a/src/io/iter_batchloader.h +++ b/src/io/iter_batchloader.h @@ -41,7 +41,7 @@ namespace io { class BatchLoader : public IIterator { public: explicit BatchLoader(IIterator *base): - base_(base), head_(1), num_overflow_(0) { + head_(1), num_overflow_(0), base_(base) { } virtual ~BatchLoader(void) { @@ -52,7 +52,7 @@ class BatchLoader : public IIterator { std::vector > kwargs_left; // init batch param, it could have similar param with kwargs_left = param_.InitAllowUnknown(kwargs); - // Init space for out_ + // Init space for out out_.inst_index = new unsigned[param_.batch_size]; out_.batch_size = param_.batch_size; out_.data.clear(); @@ -69,6 +69,7 @@ class BatchLoader : public IIterator { } head_ = 1; } + virtual bool Next(void) { out_.num_batch_padd = 0; out_.batch_size = param_.batch_size; @@ -128,23 +129,25 @@ class BatchLoader : public IIterator { return out_; } - private: + protected: /*! \brief batch parameters */ BatchParam param_; /*! \brief output data */ TBlobBatch out_; - /*! \brief base iterator */ - IIterator *base_; /*! \brief on first */ int head_; /*! \brief number of overflow instances that readed in round_batch mode */ int num_overflow_; + /*! \brief tensor to hold data */ + std::vector data_; + + private: + /*! \brief base iterator */ + IIterator *base_; /*! \brief data shape */ std::vector shape_; /*! \brief unit size */ std::vector unit_size_; - /*! \brief tensor to hold data */ - std::vector data_; // initialize the data holder by using from the first batch. inline void InitData(const DataInst& first_batch) { shape_.resize(first_batch.data.size()); diff --git a/src/io/iter_libsvm.cc b/src/io/iter_libsvm.cc new file mode 100644 index 000000000000..803d19e74481 --- /dev/null +++ b/src/io/iter_libsvm.cc @@ -0,0 +1,288 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file iter_libsvm.cc + * \brief define a LibSVM Reader to read in arrays + */ +#include +#include +#include +#include +#include +#include "./iter_sparse_prefetcher.h" +#include "./iter_sparse_batchloader.h" + +namespace mxnet { +namespace io { +// LibSVM parameters +struct LibSVMIterParam : public dmlc::Parameter { + /*! \brief path to data libsvm file */ + std::string data_libsvm; + /*! \brief data shape */ + TShape data_shape; + /*! \brief path to label libsvm file */ + std::string label_libsvm; + /*! \brief label shape */ + TShape label_shape; + /*! \brief partition the data into multiple parts */ + int num_parts; + /*! \brief the index of the part will read*/ + int part_index; + // declare parameters + DMLC_DECLARE_PARAMETER(LibSVMIterParam) { + DMLC_DECLARE_FIELD(data_libsvm) + .describe("The input LibSVM file or a directory path."); + DMLC_DECLARE_FIELD(data_shape) + .describe("The shape of one example."); + DMLC_DECLARE_FIELD(label_libsvm).set_default("NULL") + .describe("The input LibSVM file or a directory path. " + "If NULL, all labels will be read from ``data_libsvm``."); + index_t shape1[] = {1}; + DMLC_DECLARE_FIELD(label_shape).set_default(TShape(shape1, shape1 + 1)) + .describe("The shape of one label."); + DMLC_DECLARE_FIELD(num_parts).set_default(1) + .describe("partition the data into multiple parts"); + DMLC_DECLARE_FIELD(part_index).set_default(0) + .describe("the index of the part will read"); + } +}; + +class LibSVMIter: public SparseIIterator { + public: + LibSVMIter() {} + virtual ~LibSVMIter() {} + + // intialize iterator loads data in + virtual void Init(const std::vector >& kwargs) { + param_.InitAllowUnknown(kwargs); + CHECK_EQ(param_.data_shape.ndim(), 1) << "dimension of data_shape is expected to be 1"; + CHECK_GT(param_.num_parts, 0) << "number of parts should be positive"; + CHECK_GE(param_.part_index, 0) << "part index should be non-negative"; + data_parser_.reset(dmlc::Parser::Create(param_.data_libsvm.c_str(), + param_.part_index, + param_.num_parts, "libsvm")); + if (param_.label_libsvm != "NULL") { + label_parser_.reset(dmlc::Parser::Create(param_.label_libsvm.c_str(), + param_.part_index, + param_.num_parts, "libsvm")); + CHECK_GT(param_.label_shape.Size(), 1) + << "label_shape is not expected to be (1,) when param_.label_libsvm is set."; + } else { + CHECK_EQ(param_.label_shape.Size(), 1) + << "label_shape is expected to be (1,) when param_.label_libsvm is NULL"; + } + // both data and label are of CSRStorage in libsvm format + if (param_.label_shape.Size() > 1) { + out_.data.resize(6); + } else { + // only data is of CSRStorage in libsvm format. + out_.data.resize(4); + } + } + + virtual void BeforeFirst() { + data_parser_->BeforeFirst(); + if (label_parser_.get() != nullptr) { + label_parser_->BeforeFirst(); + } + data_ptr_ = label_ptr_ = 0; + data_size_ = label_size_ = 0; + inst_counter_ = 0; + end_ = false; + } + + virtual bool Next() { + if (end_) return false; + while (data_ptr_ >= data_size_) { + if (!data_parser_->Next()) { + end_ = true; return false; + } + data_ptr_ = 0; + data_size_ = data_parser_->Value().size; + } + out_.index = inst_counter_++; + CHECK_LT(data_ptr_, data_size_); + const auto data_row = data_parser_->Value()[data_ptr_++]; + // data, indices and indptr + out_.data[0] = AsDataBlob(data_row); + out_.data[1] = AsIdxBlob(data_row); + out_.data[2] = AsIndPtrPlaceholder(data_row); + + if (label_parser_.get() != nullptr) { + while (label_ptr_ >= label_size_) { + CHECK(label_parser_->Next()) + << "Data LibSVM's row is smaller than the number of rows in label_libsvm"; + label_ptr_ = 0; + label_size_ = label_parser_->Value().size; + } + CHECK_LT(label_ptr_, label_size_); + const auto label_row = label_parser_->Value()[label_ptr_++]; + // data, indices and indptr + out_.data[3] = AsDataBlob(label_row); + out_.data[4] = AsIdxBlob(label_row); + out_.data[5] = AsIndPtrPlaceholder(label_row); + } else { + out_.data[3] = AsScalarLabelBlob(data_row); + } + return true; + } + + virtual const DataInst &Value(void) const { + return out_; + } + + virtual const NDArrayStorageType GetStorageType(bool is_data) const { + if (is_data) return kCSRStorage; + return param_.label_shape.Size() > 1 ? kCSRStorage : kDefaultStorage; + } + + virtual const TShape GetShape(bool is_data) const { + if (is_data) return param_.data_shape; + return param_.label_shape; + } + + private: + inline TBlob AsDataBlob(const dmlc::Row& row) { + const real_t* ptr = row.value; + TShape shape(mshadow::Shape1(row.length)); + return TBlob((real_t*) ptr, shape, cpu::kDevMask); // NOLINT(*) + } + + inline TBlob AsIdxBlob(const dmlc::Row& row) { + const uint64_t* ptr = row.index; + TShape shape(mshadow::Shape1(row.length)); + return TBlob((int64_t*) ptr, shape, cpu::kDevMask, mshadow::kInt64); // NOLINT(*) + } + + inline TBlob AsIndPtrPlaceholder(const dmlc::Row& row) { + return TBlob(nullptr, mshadow::Shape1(0), cpu::kDevMask, mshadow::kInt64); + } + + inline TBlob AsScalarLabelBlob(const dmlc::Row& row) { + const real_t* ptr = row.label; + return TBlob((real_t*) ptr, mshadow::Shape1(1), cpu::kDevMask); // NOLINT(*) + } + + LibSVMIterParam param_; + // output instance + DataInst out_; + // internal instance counter + unsigned inst_counter_{0}; + // at end + bool end_{false}; + // label parser + size_t label_ptr_{0}, label_size_{0}; + size_t data_ptr_{0}, data_size_{0}; + std::unique_ptr > label_parser_; + std::unique_ptr > data_parser_; +}; + + +DMLC_REGISTER_PARAMETER(LibSVMIterParam); + +MXNET_REGISTER_IO_ITER(LibSVMIter) +.describe(R"code(Returns the LibSVM file iterator. This iterator is experimental and +should be used with care. + +The input data is similar to libsvm file format, except that the indices are expected to be +zero-based instead of one-based. Details of the libsvm format are available at +`https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/` + +In this function, the `data_shape` parameter is used to set the shape of each line of the data. +The dimension of both `data_shape` and `label_shape` are expected to be 1. + +When `label_libsvm` is set to ``NULL``, both data and label are read from the same file specified +by `data_libsvm`. Otherwise, data is read from `data_libsvm` and label from `label_libsvm`, +in this case, if `data_libsvm` contains label, it will ignored. + +The `LibSVMIter` only support `round_batch` parameter set to ``True`` for now. So, if `batch_size` +is 3 and there are 4 total rows in libsvm file, 2 more examples +are consumed at the first round. If `reset` function is called after first round, +the call is ignored and remaining examples are returned in the second round. + +If ``data_libsvm = 'data/'`` is set, then all the files in this directory will be read. + +Examples:: + + // Contents of libsvm file ``data.t``. + 1.0 0:0.5 2:1.2 + -2.0 + -3.0 0:0.6 1:2.4 2:1.2 + 4 2:-1.2 + + // Creates a `LibSVMIter` with `batch_size`=3. + LibSVMIter = mx.io.LibSVMIter(data_libsvm = 'data.t', data_shape = (3,), + batch_size = 3) + + // The first batch (data and label) + [[ 0.5 0. 1.2 ] + [ 0. 0. 0. ] + [ 0.6 2.4 1.2 ]] + + [ 1. -2. -3.] + + // The second batch (data and label) + [[ 0. 0. -1.2 ] + [ 0.5 0. 1.2 ] + [ 0. 0. 0. ]] + + [ 4. 1. -2.] + + // Contents of libsvm file ``label.t`` + 1.0 + -2.0 0:0.125 + -3.0 2:1.2 + 4 1:1.0 2:-1.2 + + // Creates a `LibSVMIter` with specified label file + LibSVMIter = mx.io.LibSVMIter(data_libsvm = 'data.t', data_shape = (3,), + label_libsvm = 'label.t', label_shape = (3,), batch_size = 3) + + // Two batches of data read from the above iterator are as follows(data and label): + // The first batch + [[ 0.5 0. 1.2 ] + [ 0. 0. 0. ] + [ 0.6 2.4 1.2 ]] + + [[ 0. 0. 0. ] + [ 0.125 0. 0. ] + [ 0. 0. 1.2 ]] + + // The second batch + [[ 0. 0. -1.2 ] + [ 0.5 0. 1.2 ] + [ 0. 0. 0. ]] + + [[ 0. 1. -1.2 ] + [ 0. 0. 0. ] + [ 0.125 0. 0. ]] + +)code" ADD_FILELINE) +.add_arguments(LibSVMIterParam::__FIELDS__()) +.add_arguments(BatchParam::__FIELDS__()) +.add_arguments(PrefetcherParam::__FIELDS__()) +.set_body([]() { + return new SparsePrefetcherIter( + new SparseBatchLoader( + new LibSVMIter())); + }); + +} // namespace io +} // namespace mxnet diff --git a/src/io/iter_prefetcher.h b/src/io/iter_prefetcher.h index 89960c71a12f..a743b5132821 100644 --- a/src/io/iter_prefetcher.h +++ b/src/io/iter_prefetcher.h @@ -46,8 +46,7 @@ namespace io { class PrefetcherIter : public IIterator { public: explicit PrefetcherIter(IIterator* base) - : loader_(base), out_(nullptr) { - } + : loader_(base), out_(nullptr) {} ~PrefetcherIter() { while (recycle_queue_.size() != 0) { @@ -56,21 +55,24 @@ class PrefetcherIter : public IIterator { delete batch; } delete out_; - iter_.Destroy(); + iter.Destroy(); } - virtual void Init(const std::vector >& kwargs) { + void InitParams(const std::vector >& kwargs) { std::vector > kwargs_left; // init image rec param kwargs_left = param_.InitAllowUnknown(kwargs); - // use the kwarg to init batch loader - loader_->Init(kwargs); // maximum prefetch threaded iter internal size const int kMaxPrefetchBuffer = 16; // init thread iter - iter_.set_max_capacity(kMaxPrefetchBuffer); + iter.set_max_capacity(kMaxPrefetchBuffer); + } - iter_.Init([this](DataBatch **dptr) { + virtual void Init(const std::vector >& kwargs) { + InitParams(kwargs); + // use the kwarg to init batch loader + loader_->Init(kwargs); + iter.Init([this](DataBatch **dptr) { if (!loader_->Next()) return false; const TBlobBatch& batch = loader_->Value(); if (*dptr == nullptr) { @@ -109,7 +111,7 @@ class PrefetcherIter : public IIterator { } virtual void BeforeFirst(void) { - iter_.BeforeFirst(); + iter.BeforeFirst(); } virtual bool Next(void) { @@ -124,9 +126,9 @@ class PrefetcherIter : public IIterator { arr.WaitToWrite(); } recycle_queue_.pop(); - iter_.Recycle(&old_batch); + iter.Recycle(&old_batch); } - return iter_.Next(&out_); + return iter.Next(&out_); } virtual const DataBatch &Value(void) const { return *out_; @@ -135,16 +137,16 @@ class PrefetcherIter : public IIterator { protected: /*! \brief prefetcher parameters */ PrefetcherParam param_; - /*! \brief internal batch loader */ - std::unique_ptr > loader_; + /*! \brief backend thread */ + dmlc::ThreadedIter iter; private: + /*! \brief internal batch loader */ + std::unique_ptr > loader_; /*! \brief output data */ DataBatch *out_; /*! \brief queue to be recycled */ std::queue recycle_queue_; - /*! \brief backend thread */ - dmlc::ThreadedIter iter_; }; } // namespace io } // namespace mxnet diff --git a/src/io/iter_sparse.h b/src/io/iter_sparse.h new file mode 100644 index 000000000000..beaf5c682998 --- /dev/null +++ b/src/io/iter_sparse.h @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file iter_sparse.h + * \brief mxnet sparse data iterator + */ +#ifndef MXNET_IO_ITER_SPARSE_H_ +#define MXNET_IO_ITER_SPARSE_H_ + +#include +#include + +namespace mxnet { +/*! + * \brief iterator type + * \param DType data type + */ +template +class SparseIIterator : public IIterator { + public: + /*! \brief storage type of the data or label */ + virtual const NDArrayStorageType GetStorageType(bool is_data) const = 0; + /*! \brief shape of the data or label */ + virtual const TShape GetShape(bool is_data) const = 0; +}; // class SparseIIterator + +} // namespace mxnet +#endif // MXNET_IO_ITER_SPARSE_H_ diff --git a/src/io/iter_sparse_batchloader.h b/src/io/iter_sparse_batchloader.h new file mode 100644 index 000000000000..d5c9bd2f4578 --- /dev/null +++ b/src/io/iter_sparse_batchloader.h @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file iter_sparse_batchloader.h + * \brief define a batch adapter to create sparse tblob batch + */ +#ifndef MXNET_IO_ITER_SPARSE_BATCHLOADER_H_ +#define MXNET_IO_ITER_SPARSE_BATCHLOADER_H_ + +#include +#include +#include +#include +#include +#include +#include +#include "./inst_vector.h" +#include "./image_iter_common.h" +#include "./iter_batchloader.h" +#include "./iter_sparse.h" + +namespace mxnet { +namespace io { + +/*! \brief create a batch iterator from single instance iterator */ +class SparseBatchLoader : public BatchLoader, public SparseIIterator { + public: + explicit SparseBatchLoader(SparseIIterator *base): + BatchLoader(base), sparse_base_(base) { + } + + virtual ~SparseBatchLoader(void) {} + + inline void Init(const std::vector >& kwargs) { + BatchLoader::Init(kwargs); + data_stype_ = sparse_base_->GetStorageType(true); + label_stype_ = sparse_base_->GetStorageType(false); + if (param_.round_batch == 0) { + LOG(FATAL) << "sparse batch loader doesn't support round_batch == false yet"; + } + } + + virtual void BeforeFirst(void) { + BatchLoader::BeforeFirst(); + } + + virtual bool Next(void) { + out_.num_batch_padd = 0; + out_.batch_size = param_.batch_size; + this->head_ = 0; + // if overflown from previous round, directly return false, until before first is called + if (num_overflow_ != 0) return false; + index_t top = 0; + inst_cache_.clear(); + while (sparse_base_->Next()) { + inst_cache_.emplace_back(sparse_base_->Value()); + if (inst_cache_.size() >= param_.batch_size) break; + } + // no more data instance + if (inst_cache_.size() == 0) { + return false; + } + if (inst_cache_.size() < param_.batch_size) { + CHECK_GT(param_.round_batch, 0); + num_overflow_ = 0; + sparse_base_->BeforeFirst(); + for (; inst_cache_.size() < param_.batch_size; ++num_overflow_) { + CHECK(sparse_base_->Next()) << "number of input must be bigger than batch size"; + inst_cache_.emplace_back(sparse_base_->Value()); + } + } + out_.num_batch_padd = num_overflow_; + CHECK_EQ(inst_cache_.size(), param_.batch_size); + this->InitDataFromBatch(); + for (size_t j = 0; j < inst_cache_.size(); j++) { + const auto& d = inst_cache_[j]; + out_.inst_index[top] = d.index; + // TODO(haibin) double check the type? + int64_t unit_size = 0; + for (size_t i = 0; i < d.data.size(); ++i) { + // indptr tensor + if (IsIndPtr(i)) { + auto indptr = data_[i].get(); + if (j == 0) indptr[0] = 0; + indptr[j + 1] = indptr[j] + unit_size; + offsets_[i] = j; + } else { + // indices and values tensor + unit_size = d.data[i].shape_.Size(); + MSHADOW_TYPE_SWITCH(data_[i].type_flag_, DType, { + const auto begin = offsets_[i]; + const auto end = offsets_[i] + unit_size; + mshadow::Copy(data_[i].get().Slice(begin, end), + d.data[i].get_with_shape(mshadow::Shape1(unit_size))); + }); + offsets_[i] += unit_size; + } + } + } + return true; + } + + virtual const TBlobBatch &Value(void) const { + return BatchLoader::Value(); + } + + virtual const NDArrayStorageType GetStorageType(bool is_data) const { + return sparse_base_->GetStorageType(is_data); + } + + virtual const TShape GetShape(bool is_data) const { + TShape inst_shape = sparse_base_->GetShape(is_data); + std::vector shape_vec; + shape_vec.push_back(param_.batch_size); + for (index_t dim = 0; dim < inst_shape.ndim(); ++dim) { + shape_vec.push_back(inst_shape[dim]); + } + return TShape(shape_vec.begin(), shape_vec.end()); + } + + private: + /*! \brief base sparse iterator */ + SparseIIterator *sparse_base_; + /*! \brief data instances */ + std::vector inst_cache_; + /*! \brief data storage type */ + NDArrayStorageType data_stype_; + /*! \brief data label type */ + NDArrayStorageType label_stype_; + /*! \brief tensor offset for slicing */ + std::vector offsets_; + + // check whether ith position is the indptr tensor for a CSR tensor + inline bool IsIndPtr(size_t i) { + auto data_num_aux = num_aux_data(data_stype_); + auto label_num_aux = num_aux_data(label_stype_); + auto label_indptr_offset = data_num_aux + 1 + label_num_aux; + // data indptr + if (i == data_num_aux && data_stype_ == kCSRStorage) { + return true; + } + // label indptr + if (i == label_indptr_offset && label_stype_ == kCSRStorage && data_stype_ == kCSRStorage) { + return true; + } + return false; + } + + // initialize the data holder by using from the batch + inline void InitDataFromBatch() { + CHECK(data_stype_ == kCSRStorage || label_stype_ == kCSRStorage); + CHECK_GT(inst_cache_.size(), 0); + out_.data.clear(); + data_.clear(); + offsets_.clear(); + + size_t total_size = inst_cache_[0].data.size(); + data_.resize(total_size); + offsets_.resize(total_size, 0); + std::vector vec_sizes(total_size, 0); + // accumulate the memory required for a batch + for (size_t i = 0; i < total_size; ++i) { + size_t size = 0; + // vec_size for indptr + if (IsIndPtr(i)) { + size = param_.batch_size + 1; + } else { + for (const auto &d : inst_cache_) size += d.data[i].shape_.Size(); + } + vec_sizes[i] = size; + } + + CHECK_EQ(vec_sizes[0], vec_sizes[1]); + for (size_t i = 0; i < total_size; ++i) { + int src_type_flag = inst_cache_[0].data[i].type_flag_; + // init object attributes + TShape dst_shape(mshadow::Shape1(vec_sizes[i])); + data_[i].resize(mshadow::Shape1(vec_sizes[i]), src_type_flag); + CHECK(data_[i].dptr_ != nullptr); + out_.data.push_back(TBlob(data_[i].dptr_, dst_shape, cpu::kDevMask, src_type_flag)); + } + } +}; // class BatchLoader +} // namespace io +} // namespace mxnet +#endif // MXNET_IO_ITER_SPARSE_BATCHLOADER_H_ diff --git a/src/io/iter_sparse_prefetcher.h b/src/io/iter_sparse_prefetcher.h new file mode 100644 index 000000000000..3908f9bd3826 --- /dev/null +++ b/src/io/iter_sparse_prefetcher.h @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file iter_sparse_prefetcher.h + * \brief define a prefetcher using threaditer to keep k batch fetched + */ +#ifndef MXNET_IO_ITER_SPARSE_PREFETCHER_H_ +#define MXNET_IO_ITER_SPARSE_PREFETCHER_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "./inst_vector.h" +#include "./image_iter_common.h" +#include "./iter_prefetcher.h" +#include "./iter_sparse.h" + +namespace mxnet { +namespace io { +// iterator on sparse data +class SparsePrefetcherIter : public PrefetcherIter { + public: + explicit SparsePrefetcherIter(SparseIIterator* base) + : PrefetcherIter(base), sparse_loader_(base) {} + + ~SparsePrefetcherIter() {} + + virtual void Init(const std::vector >& kwargs) { + PrefetcherIter::InitParams(kwargs); + // use the kwarg to init batch loader + sparse_loader_->Init(kwargs); + iter.Init([this](DataBatch **dptr) { + if (!sparse_loader_->Next()) return false; + const TBlobBatch& batch = sparse_loader_->Value(); + if (*dptr == nullptr) { + // allocate databatch + *dptr = new DataBatch(); + (*dptr)->num_batch_padd = batch.num_batch_padd; + // (*dptr)->data.at(0) => data + // (*dptr)->data.at(1) => label + (*dptr)->data.resize(2); + (*dptr)->index.resize(batch.batch_size); + size_t data_iter = 0; + for (size_t i = 0; i < (*dptr)->data.size(); ++i) { + bool is_data = i == 0; + auto stype = this->GetStorageType(is_data); + auto dtype = param_.dtype ? param_.dtype.value() : batch.data[data_iter].type_flag_; + if (stype == kDefaultStorage) { + (*dptr)->data.at(i) = NDArray(batch.data[data_iter].shape_, + Context::CPU(), false, dtype); + } else { + (*dptr)->data.at(i) = NDArray(stype, this->GetShape(is_data), + Context::CPU(), false, dtype); + } + data_iter += num_aux_data(stype) + 1; + } + } + // copy data over + size_t data_iter = 0; + for (size_t i = 0; i < (*dptr)->data.size(); ++i) { + auto& nd = ((*dptr)->data)[i]; + auto stype = nd.storage_type(); + auto& data_i = ((*dptr)->data)[i]; + if (stype == kDefaultStorage) { + CopyFromTo(data_i.data(), batch.data[data_iter]); + } else if (stype == kCSRStorage) { + auto& values = batch.data[data_iter]; + auto& indices = batch.data[data_iter + 1]; + auto& indptr = batch.data[data_iter + 2]; + // allocate memory + CHECK_EQ(indices.shape_.Size(), values.shape_.Size()); + nd.CheckAndAllocAuxData(csr::kIdx, indices.shape_); + nd.CheckAndAllocData(values.shape_); + nd.CheckAndAllocAuxData(csr::kIndPtr, indptr.shape_); + // copy values, indices and indptr + CopyFromTo(data_i.data(), values); + CopyFromTo(data_i.aux_data(csr::kIdx), indices); + CopyFromTo(data_i.aux_data(csr::kIndPtr), indptr); + } else { + LOG(FATAL) << "Storage type not implemented: " << stype; + } + data_iter += num_aux_data(stype) + 1; + (*dptr)->num_batch_padd = batch.num_batch_padd; + } + if (batch.inst_index) { + std::copy(batch.inst_index, + batch.inst_index + batch.batch_size, + (*dptr)->index.begin()); + } + return true; + }, + [this]() { sparse_loader_->BeforeFirst(); }); + } + + virtual void BeforeFirst(void) { + PrefetcherIter::BeforeFirst(); + } + + virtual bool Next(void) { + return PrefetcherIter::Next(); + } + virtual const DataBatch &Value(void) const { + return PrefetcherIter::Value(); + } + + virtual const NDArrayStorageType GetStorageType(bool is_data) const { + return sparse_loader_->GetStorageType(is_data); + } + + virtual const TShape GetShape(bool is_data) const { + return sparse_loader_->GetShape(is_data); + } + + private: + /*! \brief internal sparse batch loader */ + SparseIIterator* sparse_loader_; + + inline void CopyFromTo(TBlob dst, const TBlob src) { + MSHADOW_TYPE_SWITCH(src.type_flag_, DType, { + mshadow::Copy(dst.FlatTo1D(), src.FlatTo1D()); + }); + } +}; +} // namespace io +} // namespace mxnet +#endif // MXNET_IO_ITER_SPARSE_PREFETCHER_H_ diff --git a/src/kvstore/comm.h b/src/kvstore/comm.h index ade9c95feda7..cd0d3ab02825 100644 --- a/src/kvstore/comm.h +++ b/src/kvstore/comm.h @@ -21,13 +21,17 @@ */ #ifndef MXNET_KVSTORE_COMM_H_ #define MXNET_KVSTORE_COMM_H_ +#include #include #include #include #include #include #include +#include #include "mxnet/ndarray.h" +#include "../ndarray/ndarray_function.h" +#include "../operator/tensor/sparse_retain-inl.h" namespace mxnet { namespace kvstore { /** @@ -40,9 +44,10 @@ class Comm { } virtual ~Comm() { } /** - * \brief init key with the data shape + * \brief init key with the data shape and storage shape */ - virtual void Init(int key, const TShape& shape, int dtype = mshadow::kFloat32) = 0; + virtual void Init(int key, const NDArrayStorageType stype, + const TShape& shape, int dtype = mshadow::kFloat32) = 0; /** * \brief returns src[0] + .. + src[src.size()-1] */ @@ -55,6 +60,18 @@ class Comm { int key, const NDArray& src, const std::vector dst, int priority) = 0; + /** + * \brief broadcast src to dst[i] with target row_ids for every i + * \param dst a list of destination row_sparse NDArray and its target row_ids to broadcast, + where the row_ids are expected to be unique and sorted + * \param use_copy if set to true, directly copy src to dst[i] without looking up the + provided row_ids + */ + virtual void BroadcastRowSparse(int key, const NDArray& src, + const std::vector>& dst, + const bool use_copy, + const int priority) = 0; + /** * \brief return a pinned contex */ @@ -75,43 +92,85 @@ class CommCPU : public Comm { CommCPU() { nthread_reduction_ = dmlc::GetEnv("MXNET_KVSTORE_REDUCTION_NTHREADS", 4); bigarray_bound_ = dmlc::GetEnv("MXNET_KVSTORE_BIGARRAY_BOUND", 1000 * 1000); + // TODO(junwu) delete the following data member, now for benchmark only + is_serial_push_ = dmlc::GetEnv("MXNET_KVSTORE_SERIAL_PUSH", 0); } virtual ~CommCPU() { } - void Init(int key, const TShape& shape, int type = mshadow::kFloat32) override { - merge_buf_[key].merged = NDArray(shape, pinned_ctx_, false, type); + void Init(int key, const NDArrayStorageType stype, const TShape& shape, + int type = mshadow::kFloat32) override { + if (stype == kDefaultStorage) { + merge_buf_[key].merged = NDArray(shape, pinned_ctx_, false, type); + } else { + merge_buf_[key].merged = NDArray(stype, shape, pinned_ctx_, true, type); + } } const NDArray& Reduce(int key, const std::vector& src, int priority) override { + auto& buf = merge_buf_[key]; // avoid extra copy for single device, but it may bring problems for // abnormal usage of kvstore if (src.size() == 1) { - return src[0]; + if (src[0].storage_type() == kDefaultStorage) { + return src[0]; + } else { // if sparse and only one GPU, always update weight on CPU + CopyFromTo(src[0], &buf.merged, priority); + return buf.merged; + } } - std::vector const_vars(src.size() - 1); - std::vector reduce(src.size()); - auto& buf = merge_buf_[key]; - CopyFromTo(src[0], &buf.merged, priority); - reduce[0] = buf.merged; - if (buf.copy_buf.empty()) { - buf.copy_buf.resize(src.size()-1); - for (size_t j = 0; j < src.size() - 1; ++j) { - buf.copy_buf[j] = NDArray( - src[0].shape(), pinned_ctx_, false, src[0].dtype()); + if (buf.merged.storage_type() == kDefaultStorage) { + std::vector const_vars(src.size() - 1); + std::vector reduce(src.size()); + CopyFromTo(src[0], &buf.merged, priority); + reduce[0] = buf.merged; + + if (buf.copy_buf.empty()) { + buf.copy_buf.resize(src.size()-1); + for (size_t j = 0; j < src.size() - 1; ++j) { + // allocate NDArray basd on storage type + buf.copy_buf[j] = NDArray( + src[0].shape(), pinned_ctx_, false, src[0].dtype()); + } } - } - for (size_t i = 1; i < src.size(); ++i) { - CopyFromTo(src[i], &(buf.copy_buf[i-1]), priority); - reduce[i] = buf.copy_buf[i-1]; - const_vars[i-1] = reduce[i].var(); - } + for (size_t i = 1; i < src.size(); ++i) { + CopyFromTo(src[i], &(buf.copy_buf[i-1]), priority); + reduce[i] = buf.copy_buf[i-1]; + const_vars[i-1] = reduce[i].var(); + } + + Engine::Get()->PushSync([reduce, this](RunContext rctx) { + ReduceSumCPU(reduce); + }, Context::CPU(), const_vars, {reduce[0].var()}, + FnProperty::kCPUPrioritized, priority, PROFILER_MESSAGE("KVStoreReduce")); - Engine::Get()->PushSync([reduce, this](RunContext rctx) { - ReduceSumCPU(reduce); - }, Context::CPU(), const_vars, {reduce[0].var()}, - FnProperty::kCPUPrioritized, priority, PROFILER_MESSAGE("KVStoreReduce")); + } else { + // buf.merged is a sparse ndarray. + std::vector const_vars(src.size()); + std::vector reduce(src.size()); + + if (buf.copy_buf.empty()) { + buf.copy_buf.resize(src.size()); + for (size_t j = 0; j < src.size(); ++j) { + buf.copy_buf[j] = NDArray( + src[0].storage_type(), src[0].shape(), pinned_ctx_, true, src[0].dtype()); + } + } + for (size_t i = 0; i < src.size(); ++i) { + CopyFromTo(src[i], &(buf.copy_buf[i]), priority); + reduce[i] = buf.copy_buf[i]; + const_vars[i] = reduce[i].var(); + } + auto result = buf.merged; + Engine::Get()->PushSync([reduce, result, this](RunContext rctx) { + NDArray out = result; + is_serial_push_? + ReduceSumCPUExSerial(reduce, &out) + : mxnet::ndarray::ElementwiseSum(rctx.get_stream(), reduce, &out); + }, Context::CPU(), const_vars, {result.var()}, + FnProperty::kCPUPrioritized, priority, PROFILER_MESSAGE("KVStoreReduce")); + } return buf.merged; } @@ -129,7 +188,113 @@ class CommCPU : public Comm { } } + void BroadcastRowSparse(int key, const NDArray& src, + const std::vector>& dst, + const bool use_copy, + const int priority) override { + using namespace mshadow; + CHECK_EQ(src.storage_type(), kRowSparseStorage) + << "BroadcastRowSparse expects row-sparse src NDArray"; + CHECK_EQ(src.ctx().dev_mask(), Context::kCPU) + << "BroadcastRowSparse with src on gpu context not supported"; + for (size_t i = 0; i < dst.size(); ++i) { + NDArray* out = dst[i].first; + NDArray row_id = dst[i].second; + if (use_copy) { + CopyFromTo(src, out, priority); + } else { + CHECK_EQ(out->storage_type(), kRowSparseStorage) + << "BroadcastRowSparse expects row_sparse dst NDArray"; + CHECK_EQ(row_id.ctx().dev_mask(), Context::kCPU) + << "BroadcastRowSparse with row_indices on gpu context not supported"; + // retain according to unique indices + const bool use_sparse_retain = (src.shape()[0] != src.storage_shape()[0]) + || (row_id.dtype() != out->aux_type(rowsparse::kIdx)) + || (out->ctx().dev_mask() != Context::kGPU); + if (use_sparse_retain) { // use sparse_retain op + const bool is_to_gpu = out->ctx().dev_mask() == Context::kGPU; + NDArray out_cpu = is_to_gpu? NDArray(kRowSparseStorage, src.shape(), + src.ctx(), true, src.dtype(), src.aux_types()) : *out; + Engine::Get()->PushSync([=](RunContext rctx) { + const TBlob& indices = row_id.data(); + NDArray temp = out_cpu; // get rid of const qualifier + op::SparseRetainOpForwardRspImpl(rctx.get_stream(), + src, indices, kWriteTo, + &temp); + }, Context::CPU(), {src.var(), row_id.var()}, {out_cpu.var()}, + FnProperty::kNormal, priority, PROFILER_MESSAGE("KVStoreSparseRetain")); + if (is_to_gpu) { + CopyFromTo(out_cpu, out, priority); + } + } else { // direct copy rows + Engine::Get()->PushSync([=](RunContext rctx) { + CopyRetainedRowsToGPU(rctx.get_stream(), rctx.get_stream(), + src, row_id, out); + }, out->ctx(), {src.var(), row_id.var()}, {out->var()}, + FnProperty::kCopyToGPU, priority, PROFILER_MESSAGE("KVStoreCopyRetainedRowsToGPU")); + } + } + } + } + private: + /*! + * \brief When src is a rsp with full rows, + * simply copy retained rows directly from cpu to gpu + * without invoking sparse_retain op. + */ + void CopyRetainedRowsToGPU(mshadow::Stream* cpu_stream, + mshadow::Stream* gpu_stream, + const NDArray& src, + const NDArray& indices, + NDArray* dst) { +#if MXNET_USE_CUDA == 1 + CHECK_EQ(src.storage_type(), kRowSparseStorage) + << "CopyRetainedRowsToGPU expects row-sparse src NDArray"; + CHECK_EQ(src.ctx().dev_mask(), Context::kCPU) + << "CopyRetainedRowsToGPU with src on gpu context not supported"; + CHECK_EQ(src.storage_shape()[0], src.shape()[0]) + << "CopyRetainedRowsToGPU only supports src rsp with full rows"; + CHECK_EQ(indices.storage_type(), kDefaultStorage); + CHECK_EQ(indices.ctx().dev_mask(), Context::kCPU); + CHECK_EQ(dst->storage_type(), kRowSparseStorage); + CHECK_EQ(dst->ctx().dev_mask(), Context::kGPU); + CHECK_EQ(indices.dtype(), dst->aux_type(rowsparse::kIdx)) + << "CopyRetainedRowsToGPU only supports same data type for idx array and dst aux_data(0)"; + if (!src.storage_initialized() || indices.data().Size() == 0U) { + op::FillZerosRspImpl(gpu_stream, dst); + return; + } + using namespace mshadow; + + const TBlob& src_data = src.data(); + const TBlob& idx_data = indices.data(); + const size_t row_length = src.shape().ProdShape(1, src.shape().ndim()); + const size_t num_rows_retained = idx_data.Size(); + dst->CheckAndAlloc({Shape1(num_rows_retained)}); + TBlob dst_data = dst->data(); + TBlob dst_idx_data = dst->aux_data(rowsparse::kIdx); + MSHADOW_TYPE_SWITCH(src.dtype(), DType, { + MSHADOW_IDX_TYPE_SWITCH(indices.dtype(), IType, { + // copy idx array + Tensor dst_idx_tensor = dst_idx_data.FlatTo1D(gpu_stream); + const Tensor idx_tensor = idx_data.FlatTo1D(cpu_stream); + Copy(dst_idx_tensor, idx_tensor, gpu_stream); + // copy src data + const Tensor src_data_tensor = src_data.get_with_shape( + Shape2(src_data.shape_[0], row_length), cpu_stream); + Tensor dst_data_tensor = dst_data.get_with_shape( + Shape2(dst_data.shape_[0], row_length), gpu_stream); + for (size_t i = 0; i < num_rows_retained; ++i) { + Copy(dst_data_tensor[i], src_data_tensor[idx_tensor[i]], gpu_stream); + } + }) + }) +#else + LOG(FATAL) << "GPU not enabled"; +#endif + } + // reduce sum into val[0] inline void ReduceSumCPU(const std::vector &in_data) { MSHADOW_TYPE_SWITCH(in_data[0].dtype(), DType, { @@ -144,6 +309,78 @@ class CommCPU : public Comm { }); } + // serial implementation of reduce sum for row sparse NDArray. + inline void ReduceSumCPUExSerial(const std::vector &in, NDArray *out) { + using namespace rowsparse; + using namespace mshadow; + auto stype = out->storage_type(); + CHECK_EQ(stype, kRowSparseStorage) << "Unexpected storage type " << stype; + size_t total_num_rows = 0; + size_t num_in = in.size(); + // skip the ones with empty indices and values + std::vector skip(num_in, false); + // the values tensor of the inputs + MSHADOW_TYPE_SWITCH(out->dtype(), DType, { + MSHADOW_IDX_TYPE_SWITCH(out->aux_type(kIdx), IType, { + std::vector> in_vals(num_in); + std::vector> in_indices(num_in); + // offset to the values tensor of all inputs + std::vector offsets(num_in, 0); + std::vector num_rows(num_in, 0); + for (size_t i = 0; i < num_in; i++) { + if (!in[i].storage_initialized()) { + skip[i] = true; + continue; + } + auto size = in[i].aux_shape(kIdx).Size(); + num_rows[i] = size; + total_num_rows += size; + in_vals[i] = in[i].data().FlatTo2D(); + in_indices[i] = in[i].aux_data(kIdx).FlatTo1D(); + } + std::vector indices; + indices.reserve(total_num_rows); + // gather indices from all inputs + for (size_t i = 0; i < num_in; i++) { + for (size_t j = 0; j < num_rows[i]; j++) { + indices.emplace_back(in_indices[i][j]); + } + } + CHECK_EQ(indices.size(), total_num_rows); + // dedup indices + std::sort(indices.begin(), indices.end()); + indices.resize(std::unique(indices.begin(), indices.end()) - indices.begin()); + // the one left are unique non-zero rows + size_t nnr = indices.size(); + // allocate memory for output + out->CheckAndAlloc({Shape1(nnr)}); + auto idx_data = out->aux_data(kIdx).FlatTo1D(); + auto val_data = out->data().FlatTo2D(); + + for (size_t i = 0; i < nnr; i++) { + // copy indices back + idx_data[i] = indices[i]; + bool zeros = true; + for (size_t j = 0; j < num_in; j++) { + if (skip[j]) continue; + size_t offset = offsets[j]; + if (offset < num_rows[j]) { + if (indices[i] == in_indices[j][offset]) { + if (zeros) { + Copy(val_data[i], in_vals[j][offset], nullptr); + zeros = false; + } else { + val_data[i] += in_vals[j][offset]; + } + offsets[j] += 1; + } + } + } + } + }); + }); + } + template inline static void ReduceSumCPU( const std::vector &dptr, size_t offset, index_t size) { @@ -209,6 +446,7 @@ class CommCPU : public Comm { std::unordered_map merge_buf_; size_t bigarray_bound_; int nthread_reduction_; + bool is_serial_push_; }; /** @@ -227,8 +465,13 @@ class CommDevice : public Comm { virtual ~CommDevice() { } - void Init(int key, const TShape& shape, int dtype = mshadow::kFloat32) override { - sorted_key_attrs_.push_back(std::make_tuple(key, shape, dtype)); + void Init(int key, const NDArrayStorageType stype, const TShape& shape, + int dtype = mshadow::kFloat32) override { + if (stype == kDefaultStorage) { + sorted_key_attrs_.push_back(std::make_tuple(key, shape, dtype)); + } else { + LOG(FATAL) << "storage type " << stype << " not implemented for device yet"; + } } const NDArray& Reduce(int key, const std::vector& src, @@ -296,6 +539,13 @@ class CommDevice : public Comm { } } + void BroadcastRowSparse(int key, const NDArray& src, + const std::vector>& dst, + const bool use_copy, + const int priority) override { + LOG(FATAL) << "Not implemented yet"; + } + private: void EnableP2P(const std::vector& devs) { #if MXNET_USE_CUDA diff --git a/src/kvstore/kvstore_dist.h b/src/kvstore/kvstore_dist.h index b64d7c6369bc..399754f5406d 100644 --- a/src/kvstore/kvstore_dist.h +++ b/src/kvstore/kvstore_dist.h @@ -25,6 +25,8 @@ #define MXNET_KVSTORE_KVSTORE_DIST_H_ #include #include +#include +#include #include "./kvstore_local.h" #include "mxnet/engine.h" #include "ps/ps.h" @@ -60,6 +62,7 @@ class KVStoreDist : public KVStoreLocal { } } bigarray_bound_ = dmlc::GetEnv("MXNET_KVSTORE_BIGARRAY_BOUND", 1000 * 1000); + log_verbose_ = dmlc::GetEnv("MXNET_KVSTORE_DIST_ROW_SPARSE_VERBOSE", false); } virtual ~KVStoreDist() { @@ -81,7 +84,7 @@ class KVStoreDist : public KVStoreLocal { const std::vector& values) override { CheckUnique(keys); for (size_t i = 0; i < keys.size(); ++i) { - comm_->Init(keys[i], values[i].shape(), values[i].dtype()); + comm_->Init(keys[i], values[i].storage_type(), values[i].shape(), values[i].dtype()); } if (get_rank() == 0) { Push_(keys, values, 0, false); @@ -108,17 +111,20 @@ class KVStoreDist : public KVStoreLocal { int priority) override { std::vector uniq_keys; std::vector > grouped_vals; - GroupKVPairs(keys, values, &uniq_keys, &grouped_vals); + GroupKVPairsPull(keys, values, &uniq_keys, &grouped_vals); for (size_t i = 0; i < uniq_keys.size(); ++i) { int key = uniq_keys[i]; // use the same array for merging to guarantee that pull always happens // after the previous push on this key auto& recv_buf = comm_buf_[key]; + const auto storage_type = grouped_vals[i][0]->storage_type(); + CHECK_EQ(storage_type, kDefaultStorage) + << "Expected stype of value to be kDefaultStorage"; if (recv_buf.is_none()) { // it may happen for the first time a no-rank-0 worker pull the weight. - recv_buf = NDArray( - grouped_vals[i][0]->shape(), pinned_ctx_, true, grouped_vals[i][0]->dtype()); + recv_buf = NDArray(grouped_vals[i][0]->shape(), pinned_ctx_, + true, grouped_vals[i][0]->dtype()); } auto pull_from_servers = [this, key, recv_buf]( RunContext rctx, Engine::CallbackOnComplete cb) { @@ -133,7 +139,7 @@ class KVStoreDist : public KVStoreLocal { auto vals = new ps::SArray(data, size, false); // issue pull CHECK_NOTNULL(ps_worker_)->ZPull( - pskv.keys, vals, &pskv.lens, 0, [vals, cb](){ delete vals; cb(); }); + pskv.keys, vals, &pskv.lens, kDefaultPushPull, [vals, cb](){ delete vals; cb(); }); }; CHECK_NOTNULL(Engine::Get())->PushAsync( @@ -143,12 +149,55 @@ class KVStoreDist : public KVStoreLocal { {recv_buf.var()}, FnProperty::kNormal, priority, - PROFILER_MESSAGE("KVStoreDistPull")); + PROFILER_MESSAGE("KVStoreDistDefaultPull")); comm_->Broadcast(key, recv_buf, grouped_vals[i], priority); } } + void PullRowSparse(const std::vector& keys, + const std::vector>& val_rowids, + const int priority = 0) { + std::vector uniq_keys; + std::vector>> grouped_val_rowids; + GroupKVPairsPullRsp(keys, val_rowids, &uniq_keys, &grouped_val_rowids); + + for (size_t i = 0; i < uniq_keys.size(); ++i) { + int key = uniq_keys[i]; + // use the same array for merging to guarantee that pull always happens + // after the previous push on this key + auto& recv_buf = comm_buf_[key]; + auto& grouped_val_rowid = grouped_val_rowids[i]; + const auto storage_type = grouped_val_rowid[0].first->storage_type(); + CHECK_EQ(storage_type, kRowSparseStorage) + << "expected kRowSparseStorage, but got " << storage_type; + if (recv_buf.is_none()) { + // it may happen for the first time a no-rank-0 worker pull the weight. + recv_buf = NDArray(storage_type, grouped_val_rowid[0].first->shape(), + pinned_ctx_, true, grouped_val_rowid[0].first->dtype()); + } + auto &target_val_rowids = grouped_val_rowids[i]; + const size_t num_vals = target_val_rowids.size(); + size_t num_rows = 0; + // TODO(haibin) refactor this for loop + for (size_t i = 0; i < num_vals; i++) { + auto &row_id = target_val_rowids[i].second; + NDArray indices = row_id.Copy(pinned_ctx_); + Unique(&indices, priority); + target_val_rowids[i].second = indices; + num_rows += indices.shape().Size(); + } + if (num_vals > 1) { + // TODO(haibin) aggregate over all unique indices + LOG(FATAL) << "RowSparsePull with multiple values is not implemented yet"; + } else { + auto& indices = target_val_rowids[0].second; + PullRowSparse_(key, &recv_buf, indices, priority); + comm_->BroadcastRowSparse(key, recv_buf, grouped_val_rowid, num_vals == 1, priority); + } + } + } + void set_updater(const Updater& updater) override { CHECK(updater) << "invalid updater"; if (IsServerNode()) { @@ -212,7 +261,7 @@ class KVStoreDist : public KVStoreLocal { // first aggregate the values over keys std::vector uniq_keys; std::vector > grouped_vals; - GroupKVPairs(keys, values, &uniq_keys, &grouped_vals); + GroupKVPairsPush(keys, values, &uniq_keys, &grouped_vals); for (size_t i = 0; i < uniq_keys.size(); ++i) { // merge over devcies @@ -221,42 +270,132 @@ class KVStoreDist : public KVStoreLocal { NDArray merged = do_merge ? comm_->Reduce(key, vals, priority) : vals[0]; auto& send_buf = comm_buf_[key]; + const auto storage_type = merged.storage_type(); if (merged.ctx().dev_mask() == cpu::kDevMask) { + // make sure the previous push/pull is completed + send_buf.WaitToWrite(); send_buf = merged; // avoid memory copy } else { if (send_buf.is_none()) { - send_buf = NDArray(merged.shape(), pinned_ctx_, true, merged.dtype()); + if (storage_type == kDefaultStorage) { + send_buf = NDArray(merged.shape(), pinned_ctx_, true, merged.dtype()); + } else { + send_buf = NDArray(storage_type, merged.shape(), pinned_ctx_, true, merged.dtype()); + } } CopyFromTo(merged, &send_buf); } // push to servers + if (storage_type == kDefaultStorage) { auto push_to_servers = [this, key, send_buf](RunContext rctx, Engine::CallbackOnComplete cb) { - // convert to ps keys - size_t size = send_buf.shape().Size(); - PSKV& pskv = EncodeKey(key, size); + // convert to ps keys + size_t size = send_buf.shape().Size(); + PSKV& pskv = EncodeKey(key, size); #if MKL_EXPERIMENTAL == 1 - mkl_set_tblob_eager_mode(send_buf.data()); + mkl_set_tblob_eager_mode(send_buf.data()); #endif - real_t* data = static_cast(send_buf.data().dptr_); - // do push. false means no delete - ps::SArray vals(data, size, false); - CHECK_NOTNULL(ps_worker_)->ZPush( - pskv.keys, vals, pskv.lens, 0, [cb]() { cb(); }); - }; - Engine::Get()->PushAsync( - push_to_servers, - pinned_ctx_, - {send_buf.var()}, - {}, - FnProperty::kNormal, - priority, - PROFILER_MESSAGE("KVStoreDistPush")); + real_t* data = static_cast(send_buf.data().dptr_); + // do push. false means no delete + ps::SArray vals(data, size, false); + CHECK_NOTNULL(ps_worker_)->ZPush( + pskv.keys, vals, pskv.lens, 0, [cb]() { cb(); }); + }; + Engine::Get()->PushAsync( + push_to_servers, + pinned_ctx_, + {send_buf.var()}, + {}, + FnProperty::kNormal, + priority, + PROFILER_MESSAGE("KVStoreDistDefaultPush")); + } else if (storage_type == kRowSparseStorage) { + PushRowSparse(key, send_buf, priority); + } else { + LOG(FATAL) << "unknown storage type"; + } } } + // pull row sparse weight into `recv_buf` based on indices given by `indices` + void PullRowSparse_(int key, NDArray *recv_buf, const NDArray& indices, int priority) { + using namespace rowsparse; + auto pull_from_servers = [this, key, recv_buf, indices] + (RunContext rctx, Engine::CallbackOnComplete cb) { + // allocate memory for the buffer + size_t num_rows = indices.shape().Size(); + recv_buf->CheckAndAlloc({mshadow::Shape1(num_rows)}); +#if MKL_EXPERIMENTAL == 1 + mkl_set_tblob_eager_mode(recv_buf->data()); +#endif + real_t* data = static_cast(recv_buf->data().dptr_); + auto indices_data = indices.data(); + const auto offsets = indices_data.dptr(); + const auto unit_len = recv_buf->shape().ProdShape(1, recv_buf->shape().ndim()); + const int64_t size = num_rows * unit_len; + // convert to ps keys in row sparse format + PSKV& pskv = EncodeRowSparseKey(key, size, num_rows, offsets, + unit_len, recv_buf->shape()[0]); + if (this->log_verbose_) { + LOG(INFO) << "worker " << get_rank() << " pull lens: " << pskv.lens << " keys: " + << pskv.keys << " size: " << size; + } + auto vals = new ps::SArray(data, size, false); + CHECK_NOTNULL(ps_worker_)->ZPull(pskv.keys, vals, &pskv.lens, kRowSparsePushPull, + [vals, cb]() { delete vals; cb(); }); + // copy indices to recv_buf + mshadow::Copy(recv_buf->aux_data(kIdx).FlatTo1D(), + indices_data.FlatTo1D()); + }; + CHECK_NOTNULL(Engine::Get())->PushAsync( + pull_from_servers, + pinned_ctx_, + {indices.var()}, + {recv_buf->var()}, + FnProperty::kNormal, + priority, + PROFILER_MESSAGE("KVStoreDistRowSparsePull")); + } + + // push row sparse gradient + void PushRowSparse(int key, const NDArray &send_buf, int priority) { + using namespace rowsparse; + auto push_to_servers = [this, key, &send_buf] + (RunContext rctx, Engine::CallbackOnComplete cb) { +#if MKL_EXPERIMENTAL == 1 + mkl_set_tblob_eager_mode(send_buf.data()); +#endif + real_t* data = static_cast(send_buf.data().dptr_); + bool init = send_buf.storage_initialized(); + const int64_t num_rows = init ? send_buf.aux_shape(kIdx)[0] : 0; + const auto offsets = init ? send_buf.aux_data(kIdx).dptr() : nullptr; + const auto unit_len = send_buf.shape().ProdShape(1, send_buf.shape().ndim()); + const int64_t size = num_rows * unit_len; + + // convert to ps keys in row sparse format + PSKV& pskv = EncodeRowSparseKey(key, size, num_rows, offsets, + unit_len, send_buf.shape()[0]); + if (this->log_verbose_) { + LOG(INFO) << "worker " << get_rank() << " push lens: " << pskv.lens << " keys: " + << pskv.keys << " size: " << size; + } + ps::SArray vals(data, size, false); + CHECK_NOTNULL(ps_worker_)->ZPush(pskv.keys, vals, pskv.lens, kRowSparsePushPull, [cb]() { + cb(); + }); + }; + Engine::Get()->PushAsync( + push_to_servers, + pinned_ctx_, + {send_buf.var()}, + {}, + FnProperty::kNormal, + priority, + PROFILER_MESSAGE("KVStoreDistRowSparsePush")); + } + /** * \brief check if the keys are all unique */ @@ -282,7 +421,7 @@ class KVStoreDist : public KVStoreLocal { std::unordered_map ps_kv_; /** - * \brief serizelize EncodeKey + * \brief serizelize EncodeRowSparseKey and EncodeKey */ std::mutex mu_; @@ -329,6 +468,64 @@ class KVStoreDist : public KVStoreLocal { return pskv; } + // TODO(haibin) this encoding method for row sparse keys doesn't allow cross-layer batching + inline PSKV& EncodeRowSparseKey(const int key, const int64_t size, const int64_t num_rows, + const int64_t *offsets, const size_t unit_len, + const int64_t total_num_rows) { + using namespace common; + mu_.lock(); + PSKV& pskv = ps_kv_[key]; + mu_.unlock(); + pskv.keys.clear(); + pskv.lens.clear(); + // TODO(haibin) cache this information + auto krs = ps::Postoffice::Get()->GetServerKeyRanges(); + int num_servers = krs.size(); + CHECK_GT(num_servers, 0); + + if (total_num_rows * unit_len >= bigarray_bound_) { + pskv.size = 0; + int64_t start_row = 0; + // parition it to all servers + for (int i = 0; i < num_servers; ++i) { + // calculate partition ranges + int64_t part_num_rows = + llround(static_cast(total_num_rows) / num_servers * (i + 1)) - + llround(static_cast(total_num_rows) / num_servers * i); + auto end_row = start_row + part_num_rows; + auto lb = std::lower_bound(offsets, offsets + num_rows, start_row); + auto ub = std::upper_bound(offsets, offsets + num_rows, end_row - 1); + ps::Key master_key = krs[i].begin() + key; + pskv.keys.push_back(master_key); + pskv.lens.push_back(0); + for (auto offset = lb; offset < ub; offset++) { + ps::Key ps_key = krs[i].begin() + key + (*offset - start_row); + CHECK_LT(ps_key, krs[i].end()); + pskv.keys.push_back(ps_key); + pskv.lens.push_back(unit_len); + pskv.size += unit_len; + } + start_row = end_row; + } + CHECK_EQ(static_cast(pskv.size), size); + } else { + // send it to a single random picked server + int server = (key * 9973) % num_servers; + ps::Key master_key = krs[server].begin() + key; + pskv.keys.push_back(master_key); + pskv.lens.push_back(0); + for (int64_t i = 0; i < num_rows; i++) { + ps::Key ps_key = krs[server].begin() + key + offsets[i]; + CHECK_LT(ps_key, krs[server].end()); + pskv.keys.push_back(ps_key); + pskv.lens.push_back(unit_len); + } + pskv.size = size; + } + return pskv; + } + + /** * \brief for worker to push and pull data */ @@ -343,6 +540,7 @@ class KVStoreDist : public KVStoreLocal { size_t bigarray_bound_; /// \brief send & recver buffer std::unordered_map comm_buf_; + bool log_verbose_; }; } // namespace kvstore diff --git a/src/kvstore/kvstore_dist_server.h b/src/kvstore/kvstore_dist_server.h index 4e9f887173c5..43a10b034ca6 100644 --- a/src/kvstore/kvstore_dist_server.h +++ b/src/kvstore/kvstore_dist_server.h @@ -33,10 +33,14 @@ #include #include "ps/ps.h" #include "mxnet/kvstore.h" +#include "../operator/tensor/elemwise_binary_op.h" +#include "../operator/tensor/init_op.h" namespace mxnet { namespace kvstore { +static const int kRowSparsePushPull = 1; +static const int kDefaultPushPull = 0; static const int kStopServer = -1; static const int kSyncMode = -2; @@ -110,8 +114,9 @@ class KVStoreDistServer { static_cast(ps_server_)->set_request_handle( std::bind(&KVStoreDistServer::CommandHandle, this, _1, _2)); ps_server_->set_request_handle( - std::bind(&KVStoreDistServer::DataHandle, this, _1, _2, _3)); + std::bind(&KVStoreDistServer::DataHandleEx, this, _1, _2, _3)); sync_mode_ = false; + log_verbose_ = dmlc::GetEnv("MXNET_KVSTORE_DIST_ROW_SPARSE_VERBOSE", false); } ~KVStoreDistServer() { @@ -136,6 +141,11 @@ class KVStoreDistServer { } private: + struct MergeBuf { + std::vector request; + NDArray array; + }; + void CommandHandle(const ps::SimpleData& recved, ps::SimpleApp* app) { if (recved.head == kStopServer) { exec_.Stop(); @@ -151,9 +161,205 @@ class KVStoreDistServer { app->Response(recved); } - void DataHandle(const ps::KVMeta& req_meta, - const ps::KVPairs& req_data, - ps::KVServer* server) { + void DataHandleEx(const ps::KVMeta& req_meta, + const ps::KVPairs& req_data, + ps::KVServer* server) { + if (req_meta.cmd == kRowSparsePushPull) { + DataHandleRowSparse(req_meta, req_data, server); + } else { + DataHandleDefault(req_meta, req_data, server); + } + return; + } + + inline void ApplyUpdates(const int key, MergeBuf *merged, NDArray *stored, + ps::KVServer* server) { + if (merged->request.size() == (size_t) ps::NumWorkers()) { + // let the main thread to execute updater_, which is necessary for python + if (updater_) { + exec_.Exec([this, key, merged, stored](){ + CHECK(updater_); + updater_(key, merged->array, stored); + }); + } else { + // if no updater, just copy + CopyFromTo(merged->array, stored); + } + if (log_verbose_) { + LOG(INFO) << "sync response to " << merged->request.size() << " workers"; + } + for (const auto& req : merged->request) { + server->Response(req); + } + merged->request.clear(); + stored->WaitToRead(); + } else { + merged->array.WaitToRead(); + } + } + + void DecodeRowIds(const ps::SArray &keys, int64_t *indices, + const int64_t master_key, const int64_t num_rows) { + indices[0] = 0; + for (int64_t i = 1; i <= num_rows; i++) { + int key = DecodeKey(keys[i]); + auto row_id = key - master_key; + indices[i - 1] = row_id; + } + } + + void DataHandleRowSparse(const ps::KVMeta& req_meta, + const ps::KVPairs& req_data, + ps::KVServer* server) { + int master_key = DecodeKey(req_data.keys[0]); + auto num_rows = req_data.keys.size() - 1; + auto& stored = store_[master_key]; + if (req_meta.push) { + CHECK_GT(req_data.lens.size(), 0) << "req_data.lens cannot be empty"; + CHECK_EQ(req_data.lens[0], 0); + real_t* data = req_data.vals.data(); + if (stored.is_none()) { + if (log_verbose_) LOG(INFO) << "initial push: " << master_key; + // initialization + CHECK_GT(num_rows, 0) << "init with empty data is not supported"; + auto unit_len = req_data.lens[1]; + CHECK_GT(unit_len, 0); + size_t ds[] = {num_rows, (size_t) unit_len}; + TShape dshape(ds, ds + 2); + CHECK_EQ(req_data.vals.size(), num_rows * unit_len); + TBlob recv_blob(data, dshape, cpu::kDevMask); // NOLINT(*) + NDArray recved = NDArray(recv_blob, 0); + stored = NDArray(kRowSparseStorage, dshape, Context()); + Engine::Get()->PushSync([recved, stored](RunContext ctx) { + NDArray rsp = stored; + stored.CheckAndAlloc({mshadow::Shape1(recved.shape()[0])}); + mshadow::Stream *s = ctx.get_stream(); + op::PopulateFullIdxRspImpl(s, &rsp); + mshadow::Copy(rsp.data().FlatTo1D(), + recved.data().FlatTo1D(), s); + }, recved.ctx(), {recved.var()}, {stored.var()}, + FnProperty::kNormal, 0, PROFILER_MESSAGE_FUNCNAME); + stored.WaitToRead(); + server->Response(req_meta); + return; + } + // synced push + if (sync_mode_) { + if (log_verbose_) LOG(INFO) << "sync push: " << master_key << " " << req_data.keys; + auto& merged = merge_buf_[master_key]; + if (merged.array.is_none()) { + merged.array = NDArray(kRowSparseStorage, stored.shape(), Context()); + } + if (num_rows == 0) { + // reset to zeros + if (merged.request.size() == 0) { + merged.array = NDArray(kRowSparseStorage, stored.shape(), Context()); + } else { + // nothing to aggregate + } + merged.request.push_back(req_meta); + ApplyUpdates(master_key, &merged, &stored, server); + return; + } + auto unit_len = req_data.lens[1]; + CHECK_GT(unit_len, 0); + // indices + std::vector indices(num_rows); + DecodeRowIds(req_data.keys, indices.data(), master_key, num_rows); + // data + TBlob idx_blob(indices.data(), mshadow::Shape1(num_rows), cpu::kDevMask); + size_t ds[] = {(size_t) num_rows, (size_t) unit_len}; + TShape dshape(ds, ds + 2); + TBlob recv_blob(data, dshape, cpu::kDevMask); // NOLINT(*) + // row_sparse NDArray + NDArray recved(kRowSparseStorage, stored.shape(), recv_blob, {idx_blob}, 0); + + if (merged.request.size() == 0) { + CopyFromTo(recved, &merged.array, 0); + } else { + NDArray out(kRowSparseStorage, stored.shape(), Context()); + std::vector const_vars; + const_vars.push_back(recved.var()); + const_vars.push_back(merged.array.var()); + // accumulate row_sparse gradients + // TODO(haibin) override + operator for row_sparse NDArray + // instead of calling BinaryComputeRspRsp directly + using namespace mshadow; + Engine::Get()->PushSync([recved, merged, out](RunContext ctx) { + std::vector inputs, outputs; + inputs.push_back(recved); + inputs.push_back(merged.array); + outputs.push_back(out); + op::BinaryComputeRspRspImpl({}, {}, inputs, {kWriteTo}, outputs); + }, recved.ctx(), const_vars, {out.var()}, + FnProperty::kNormal, 0, PROFILER_MESSAGE_FUNCNAME); + CopyFromTo(out, &merged.array, 0); + } + merged.request.push_back(req_meta); + ApplyUpdates(master_key, &merged, &stored, server); + } else { + // async push + if (log_verbose_) LOG(INFO) << "async push: " << master_key; + if (num_rows == 0) { + server->Response(req_meta); + return; + } + auto unit_len = req_data.lens[1]; + CHECK_GT(unit_len, 0); + // indices + std::vector indices(num_rows); + DecodeRowIds(req_data.keys, indices.data(), master_key, num_rows); + TBlob idx_blob(indices.data(), mshadow::Shape1(num_rows), cpu::kDevMask); + size_t ds[] = {(size_t) num_rows, (size_t) unit_len}; + TShape dshape(ds, ds + 2); + TBlob recv_blob(data, dshape, cpu::kDevMask); // NOLINT(*) + NDArray recved(kRowSparseStorage, stored.shape(), recv_blob, {idx_blob}, 0); + exec_.Exec([this, master_key, &recved, &stored](){ + CHECK(updater_); + updater_(master_key, recved, &stored); + }); + server->Response(req_meta); + stored.WaitToRead(); + } + } else { + // pull + if (log_verbose_) LOG(INFO) << "pull: " << master_key; + ps::KVPairs response; + if (num_rows == 0) { + std::vector lens(req_data.keys.size(), 0); + response.keys = req_data.keys; + response.lens.CopyFrom(lens.begin(), lens.end()); + server->Response(req_meta, response); + return; + } + CHECK(!stored.is_none()) << "init " << master_key << " first"; + auto shape = stored.shape(); + auto unit_len = shape.ProdShape(1, shape.ndim()); + const float* data = stored.data().dptr(); + auto len = unit_len * num_rows; + // concat values + response.vals.resize(len); + for (size_t i = 1; i <= num_rows; i++) { + int key = DecodeKey(req_data.keys[i]); + int64_t row_id = key - master_key; + const auto src = data + row_id * unit_len; + auto begin = (i - 1) * unit_len; + auto end = i * unit_len; + response.vals.segment(begin, end).CopyFrom(src, unit_len); + } + // setup response + response.keys = req_data.keys; + std::vector lens(req_data.keys.size(), unit_len); + lens[0] = 0; + response.lens.CopyFrom(lens.begin(), lens.end()); + server->Response(req_meta, response); + } + } + + void DataHandleDefault(const ps::KVMeta& req_meta, + const ps::KVPairs &req_data, + ps::KVServer* server) { + CHECK_EQ(req_meta.cmd, kDefaultPushPull); // do some check CHECK_EQ(req_data.keys.size(), (size_t)1); if (req_meta.push) { @@ -185,35 +391,13 @@ class KVStoreDistServer { if (merged.array.is_none()) { merged.array = NDArray(dshape, Context()); } - if (merged.request.size() == 0) { CopyFromTo(recved, &merged.array, 0); } else { merged.array += recved; } - merged.request.push_back(req_meta); - - if (merged.request.size() == (size_t)ps::NumWorkers()) { - // let the main thread to execute updater_, which is necessary for - // python - if (updater_) { - exec_.Exec([this, key, &merged, &stored](){ - CHECK(updater_); - updater_(key, merged.array, &stored); - }); - } else { - // if no updater, just copy - CopyFromTo(merged.array, &stored); - } - for (const auto& req : merged.request) { - server->Response(req); - } - merged.request.clear(); - stored.WaitToRead(); - } else { - merged.array.WaitToRead(); - } + ApplyUpdates(key, &merged, &stored, server); } else { // async push exec_.Exec([this, key, &recved, &stored](){ @@ -227,7 +411,7 @@ class KVStoreDistServer { // pull ps::KVPairs response; CHECK(!stored.is_none()) << "init " << key << " first"; - int len = stored.shape()[0]; + auto len = stored.shape().Size(); response.keys = req_data.keys; response.lens = {len}; // TODO(mli) try to remove this CopyFrom @@ -249,16 +433,13 @@ class KVStoreDistServer { KVStore::Updater updater_; std::unordered_map store_; - - struct MergeBuf { - std::vector request; - NDArray array; - }; std::unordered_map merge_buf_; Executor exec_; - ps::KVServer* ps_server_; + + // whether to LOG verbose information + bool log_verbose_; }; } // namespace kvstore diff --git a/src/kvstore/kvstore_local.h b/src/kvstore/kvstore_local.h index 536a89b46e13..11d4b644346e 100644 --- a/src/kvstore/kvstore_local.h +++ b/src/kvstore/kvstore_local.h @@ -30,6 +30,7 @@ #include #include #include +#include #include #include "./comm.h" @@ -62,7 +63,7 @@ class KVStoreLocal : public KVStore { CHECK(local_.find(keys[i]) == local_.end()) << "duplicate init of key " << keys[i]; local_[keys[i]] = values[i].Copy(pinned_ctx_); - comm_->Init(keys[i], values[i].shape(), values[i].dtype()); + comm_->Init(keys[i], values[i].storage_type(), values[i].shape(), values[i].dtype()); } } @@ -85,7 +86,7 @@ class KVStoreLocal : public KVStore { int priority) override { std::vector uniq_keys; std::vector > grouped_vals; - GroupKVPairs(keys, values, &uniq_keys, &grouped_vals); + GroupKVPairsPush(keys, values, &uniq_keys, &grouped_vals); for (size_t i = 0; i < uniq_keys.size(); ++i) { int key = uniq_keys[i]; @@ -100,7 +101,11 @@ class KVStoreLocal : public KVStore { } updater_(key, merged, &local); } else { - local = merged; + if (merged.storage_type() != local.storage_type()) { + local = merged.Copy(local.ctx()); + } else { + local = merged; + } } } } @@ -110,7 +115,7 @@ class KVStoreLocal : public KVStore { int priority) override { std::vector uniq_keys; std::vector > grouped_vals; - GroupKVPairs(keys, values, &uniq_keys, &grouped_vals); + GroupKVPairsPull(keys, values, &uniq_keys, &grouped_vals); for (size_t i = 0; i < uniq_keys.size(); ++i) { int key = uniq_keys[i]; @@ -120,6 +125,30 @@ class KVStoreLocal : public KVStore { } } + void PullRowSparse(const std::vector& keys, + const std::vector>& val_rowids, + int priority = 0) override { + std::vector uniq_keys; + std::vector>> grouped_val_rowids; + GroupKVPairsPullRsp(keys, val_rowids, &uniq_keys, &grouped_val_rowids); + for (size_t i = 0; i < uniq_keys.size(); ++i) { + int key = uniq_keys[i]; + const NDArray& local = local_[key]; + CHECK(!local.is_none()) << "key " << key << " has not been inited"; + CHECK_EQ(local.storage_type(), kRowSparseStorage) + << "PullRowSparse expects row_sparse src NDArray"; + auto &target_val_rowids = grouped_val_rowids[i]; + const size_t num_vals = target_val_rowids.size(); + for (size_t i = 0; i < num_vals; i++) { + auto &row_id = target_val_rowids[i].second; + NDArray indices = row_id.Copy(pinned_ctx_); + Unique(&indices, priority); + target_val_rowids[i].second = indices; + } + comm_->BroadcastRowSparse(key, local, grouped_val_rowids[i], false, priority); + } + } + void Push(const std::vector& str_keys, const std::vector& values, int priority) override { @@ -136,15 +165,85 @@ class KVStoreLocal : public KVStore { Pull(keys, values, priority); } + void PullRowSparse(const std::vector& str_keys, + const std::vector>& val_rowids, + const int priority = 0) override { + std::vector keys(str_keys.size()); + LookupKeys(str_keys, &keys); + PullRowSparse(keys, val_rowids, priority); + } + protected: /** - * \brief group values on keys + * \brief group values on keys for push */ - template + void GroupKVPairsPush(const std::vector& keys, + const std::vector& values, + std::vector *uniq_keys, + std::vector> *grouped_vals) { + // check if the storage type of a value is valid + auto validator = [this](const int key, const NDArray& nd) -> bool { + auto stype = nd.storage_type(); + // valid NDArray + if (stype == kDefaultStorage || stype == kRowSparseStorage) return true; + // invalid NDArray, abort + LOG(FATAL) << "Unexpected storage type detected during kvstore push: " << stype; + return false; + }; + GroupKVPairs(keys, values, uniq_keys, grouped_vals, validator); + } + /** + * \brief group values on keys for pull + */ + void GroupKVPairsPull(const std::vector& keys, + const std::vector& values, + std::vector *uniq_keys, + std::vector> *grouped_vals) { + // check if the storage type of a value is valid + auto validator = [this](const int key, const NDArray* nd) -> bool { + // valid + if (nd->storage_type() == kDefaultStorage) return true; + // invalid, print warning messages once + if (this->warnings_printed_.find(key) == this->warnings_printed_.end()) { + LOG(INFO) << "Warning: non-default weights detected during kvstore pull. " + << "Please make sure to use row_sparse_pull with row_ids instead."; + this->warnings_printed_.insert(key); + } + return false; + }; + GroupKVPairs(keys, values, uniq_keys, grouped_vals, validator); + } + /** + * \brief group values on keys for row_sparse_pull + */ + void GroupKVPairsPullRsp(const std::vector& keys, + const std::vector>& values, + std::vector *uniq_keys, + std::vector>> *grouped_vals) { + // check if the storage type of a value is valid + auto validator = [this](const int key, const std::pair& val_rowid) -> bool { + auto val_stype = val_rowid.first->storage_type(); + auto rowid_stype = val_rowid.second.storage_type(); + // check storage types + CHECK_EQ(val_stype, kRowSparseStorage) << "Expected row_sparse storage type for " + << "row_sparse_pull values, but detected storage type " << val_stype; + CHECK_EQ(rowid_stype, kDefaultStorage) << "Expected default storage type for " + << "row_sparse_pull rowids, but detected storage type " << rowid_stype; + return true; + }; + GroupKVPairs(keys, values, uniq_keys, grouped_vals, validator); + } + + /** + * \brief group values on keys with validation. + * A value `v` is not included in the result if is_valid(v) returns false. + */ + template void GroupKVPairs(const std::vector& keys, const std::vector& values, std::vector* uniq_keys, - std::vector >* grouped_vals) { + std::vector >* grouped_vals, + const FValidate& is_valid) { CHECK_EQ(keys.size(), values.size()); // TODO(mli) check if already sorted as an optimization using Idx = std::pair; @@ -158,12 +257,14 @@ class KVStoreLocal : public KVStore { int pre_key = idx[0].first - 1; for (auto i : idx) { - if (i.first != pre_key) { - uniq_keys->push_back(i.first); - grouped_vals->push_back({values[i.second]}); - pre_key = i.first;; - } else { - grouped_vals->back().push_back(values[i.second]); + if (is_valid(i.first, values[i.second])) { + if (i.first != pre_key) { + uniq_keys->push_back(i.first); + grouped_vals->push_back({values[i.second]}); + pre_key = i.first; + } else { + grouped_vals->back().push_back(values[i.second]); + } } } } @@ -178,6 +279,28 @@ class KVStoreLocal : public KVStore { } } + /** + * \brief sort and get unique values. Output is expected to be on cpu_pinned context + */ + void Unique(NDArray *out, int priority = 0) { + CHECK_EQ(out->ctx().dev_mask(), pinned_ctx_.dev_mask()) + << "Unique expects input with `pinned_ctx_`"; + Engine::Get()->PushSync([out](RunContext rctx) { + NDArray *output = out; + CHECK_EQ(out->shape().ndim(), 1) << "Unique expects 1D inputs"; + const auto size = out->shape()[0]; + auto out_data = output->data(); + MSHADOW_IDX_TYPE_SWITCH(out_data.type_flag_, IType, { + auto dptr = output->data().dptr(); + common::ParallelSort(dptr, dptr + size, omp_get_max_threads()); + auto num_unique_idx = std::unique(dptr, dptr + size) - dptr; + *output = output->Reshape(mshadow::Shape1(num_unique_idx)); + }); + }, pinned_ctx_, {}, {out->var()}, + FnProperty::kCPUPrioritized, priority, PROFILER_MESSAGE("KVStoreUnique")); + out->WaitToRead(); + } + /// reducer and broadcaster Comm* comm_; /// pinned context @@ -188,6 +311,8 @@ class KVStoreLocal : public KVStore { std::unordered_map str_key_dict_; /// the next available integer for string->int key mapping int next_str_key_ = 0; + /// whether printed warning due to mismatch stype in each key + std::unordered_set warnings_printed_; }; } // namespace kvstore } // namespace mxnet diff --git a/src/ndarray/ndarray.cc b/src/ndarray/ndarray.cc index 8e71df729b73..0d2968626d79 100644 --- a/src/ndarray/ndarray.cc +++ b/src/ndarray/ndarray.cc @@ -30,6 +30,9 @@ #include #include #include "./ndarray_function.h" +#include "../common/utils.h" +#include "../operator/tensor/matrix_op-inl.h" +#include "../operator/tensor/init_op.h" #include "./autograd.h" #if MXNET_USE_OPENCV @@ -52,6 +55,8 @@ NDArray NDArray::grad() const { NDArray NDArray::Reshape(const TShape &shape) const { using namespace autograd; + CHECK(storage_type() == kDefaultStorage) << "Reshape for storage type " << + storage_type() << " is not implemented yet"; if (AutogradRuntime::Get()->IsTraining()) { CHECK_GE(shape_.Size(), shape.Size()) << "NDArray.Reshape: target shape must have must have the same size as " @@ -82,13 +87,15 @@ NDArray NDArray::Reshape(const TShape &shape) const { } } - NDArray NDArray::Slice(index_t begin, index_t end) const { using namespace autograd; - NDArray ret = *this; + using namespace mshadow; CHECK(!is_none()) << "NDArray is not initialized"; CHECK_LT(begin, end) << "Invalid slicing range [" << begin << ", " << end << ")"; CHECK_GE(shape_[0], end) << "Slice end index out of range"; + CHECK_EQ(storage_type(), kDefaultStorage); + NDArray ret = *this; + auto stype = storage_type(); size_t length = shape_.ProdShape(1, shape_.ndim()); MSHADOW_TYPE_SWITCH(ret.dtype(), DType, { ret.byte_offset_ += begin * length * sizeof(DType); @@ -115,8 +122,9 @@ NDArray NDArray::Slice(index_t begin, index_t end) const { } } - NDArray NDArray::At(index_t idx) const { + CHECK(storage_type() == kDefaultStorage) << "Storage type " + << storage_type() << " doesn't support At()"; NDArray ret = this->Slice(idx, idx+1); if (shape_.ndim() > 1) { return ret.Reshape(TShape(shape_.data()+1, shape_.data()+shape_.ndim())); @@ -125,6 +133,24 @@ NDArray NDArray::At(index_t idx) const { } } +/*! + * \brief Return deep copy of the current ndarry's aux_data(i) + * as an NDArray of default storage type. This function blocks. + */ +NDArray NDArray::aux_ndarray(size_t i) const { + CHECK_NE(storage_type(), kDefaultStorage); + CHECK(i < ptr_->aux_shapes.size()); + // create a delay_alloc default ndarray as output + NDArray ret(TShape(), ctx(), true, aux_type(i)); + ret.SyncCopyFromNDArray(*this, i); + return ret; +} + +NDArray NDArray::data_ndarray() const { + NDArray ret(TShape(), ctx(), true, dtype_); + ret.SyncCopyFromNDArray(*this); + return ret; +} bool NDArray::fresh_out_grad() const { if (entry_.ag_node != nullptr) return entry_.ag_node->fresh_out_grad; @@ -239,11 +265,11 @@ void BinaryOp(const NDArray &lhs, // redirect everything to mshadow operations switch (lhs.ctx().dev_mask()) { case cpu::kDevMask: { - Engine::Get()->PushSync([lhs, rhs, ret](RunContext ctx) { - TBlob tmp = ret.data(); - ndarray::Eval(lhs.data(), rhs.data(), &tmp, ctx); - }, lhs.ctx(), const_vars, {ret.var()}, - FnProperty::kNormal, 0, PROFILER_MESSAGE_FUNCNAME); + Engine::Get()->PushSync([lhs, rhs, ret](RunContext ctx) { + TBlob tmp = ret.data(); + ndarray::Eval(lhs.data(), rhs.data(), &tmp, ctx); + }, lhs.ctx(), const_vars, {ret.var()}, + FnProperty::kNormal, 0, PROFILER_MESSAGE_FUNCNAME); break; } #if MXNET_USE_CUDA @@ -269,6 +295,7 @@ void SetValueOp(const real_t &rhs, NDArray *out) { switch (ret.ctx().dev_mask()) { case cpu::kDevMask: { Engine::Get()->PushSync([rhs, ret](RunContext ctx) { + CHECK(ret.storage_type() == kDefaultStorage); TBlob tmp = ret.data(); ndarray::Eval(rhs, &tmp, ctx); }, ret.ctx(), {}, {ret.var()}, @@ -340,6 +367,134 @@ void ScalarOp(const NDArray &lhs, } } +size_t num_aux_data(NDArrayStorageType stype) { + size_t num = 0; + switch (stype) { + case kDefaultStorage: num = 0; break; + case kCSRStorage: num = 2; break; + case kRowSparseStorage: num = 1; break; + default: LOG(FATAL) << "Unknown storage type" << stype; break; + } + return num; +} + +// Make a copy of a CSR NDArray +template +inline void CopyFromToCsrImpl(const NDArray from, NDArray *to, RunContext ctx) { + using namespace mshadow; + CHECK_EQ(from.storage_type(), to->storage_type()) << "Copying with different storage type"; + // if source storage is not initialized, fill destination with zeros + auto s = ctx.get_stream(); + if (!from.storage_initialized()) { + op::FillZerosCsrImpl(s, to); + return; + } + // Allocate storage + to->CheckAndAllocAuxData(csr::kIndPtr, from.aux_shape(csr::kIndPtr)); + to->CheckAndAllocAuxData(csr::kIdx, from.aux_shape(csr::kIdx)); + to->CheckAndAllocData(from.aux_shape(csr::kIdx)); + TBlob val = to->data(); + TBlob indptr = to->aux_data(csr::kIndPtr); + TBlob idx = to->aux_data(csr::kIdx); + ndarray::Copy(from.data(), &val, + from.ctx(), to->ctx(), ctx); + ndarray::Copy(from.aux_data(csr::kIndPtr), &indptr, + from.ctx(), to->ctx(), ctx); + ndarray::Copy(from.aux_data(csr::kIdx), &idx, + from.ctx(), to->ctx(), ctx); +} + +// Make a copy of a row-sparse NDArray +template +inline void CopyFromToRspImpl(const NDArray from, NDArray *to, RunContext ctx) { + using namespace mshadow; + CHECK_EQ(from.storage_type(), to->storage_type()) << "Copying with different storage type"; + // if source is zeros, fill destination with zeros, too + auto s = ctx.get_stream(); + if (!from.storage_initialized()) { + op::FillZerosRspImpl(s, to); + return; + } + auto aux_shape = from.aux_shape(rowsparse::kIdx); + to->CheckAndAlloc({aux_shape}); + TBlob val = to->data(); + TBlob idx = to->aux_data(rowsparse::kIdx); + ndarray::Copy(from.data(), &val, + from.ctx(), to->ctx(), ctx); + ndarray::Copy(from.aux_data(rowsparse::kIdx), &idx, + from.ctx(), to->ctx(), ctx); +} + +// Make a copy of a dense NDArray +template +inline void CopyFromToDnsImpl(const NDArray from, NDArray *to, RunContext ctx) { + using namespace mshadow; + CHECK_EQ(from.storage_type(), to->storage_type()) << "Copying with different storage type"; + TBlob tmp = to->data(); + ndarray::Copy(from.data(), &tmp, + from.ctx(), to->ctx(), ctx); +} + +// Make a copy of an NDArray based on storage type +template +void CopyFromToImpl(const NDArray from, NDArray *to, RunContext rctx) { + using namespace std; + using namespace mshadow; + // if storage type doesn't match, cast the storage first + auto from_stype = from.storage_type(); + auto to_stype = to->storage_type(); + CHECK(from_stype == kDefaultStorage + || to_stype == kDefaultStorage + || from_stype == to_stype) + << "Copying ndarray of stype = " << from_stype + << " to stype = " << to_stype << " is not supported"; + const auto from_ctx = from.ctx(); + const auto to_ctx = to->ctx(); + auto s = rctx.get_stream(); + bool is_train = mxnet::autograd::AutogradRuntime::Get()->IsTraining(); + std::vector requested; + if (is_same::value && from_stype != to_stype) { + requested.push_back(ResourceManager::Get()->Request(from_ctx, + ResourceRequest(ResourceRequest::kTempSpace))); + } + OpContext opctx{is_train, + rctx, + engine::CallbackOnComplete(), + requested}; + if (from_ctx == to_ctx && from_stype != to_stype) { + // same ctx, different stypes, use cast op directly without copying + common::CastStorageDispatch(opctx, from, *to); + } else { + NDArray casted_nd; // an intermediate result before copying from to to + if (from_stype == to_stype) { + casted_nd = from; // same stype, no need to cast from + } else { // different stypes on different ctx needs an temporary casted_nd + TShape shape = from.shape(); + if (to_stype == kDefaultStorage) { + casted_nd = NDArray(shape, from_ctx); + } else { + casted_nd = NDArray(to_stype, shape, from_ctx); + } + // convert from_nd to the same stype as to_nd + common::CastStorageDispatch(opctx, from, casted_nd); + } + + if (to_stype == kDefaultStorage) { + CopyFromToDnsImpl(casted_nd, to, rctx); + } else if (to_stype == kRowSparseStorage) { + CopyFromToRspImpl(casted_nd, to, rctx); + } else if (to_stype == kCSRStorage) { + CopyFromToCsrImpl(casted_nd, to, rctx); + } else { + LOG(FATAL) << "unknown storage type" << to_stype; + } + } + if (is_same::value || is_same::value) { + // Wait GPU kernel to complete + rctx.get_stream()->Wait(); + } +} + void CopyFromTo(const NDArray &from, NDArray *to, int priority) { if (from.var() == to->var()) { // skip to copy to itself @@ -354,44 +509,33 @@ void CopyFromTo(const NDArray &from, NDArray *to, int priority) { NDArray ret = *to; int a = from.ctx().dev_mask(); int b = to->ctx().dev_mask(); - std::vector const_vars; if (from.var() != ret.var()) const_vars.push_back(from.var()); if (a == cpu::kDevMask && b == cpu::kDevMask) { Engine::Get()->PushSync([from, ret](RunContext ctx) { - TBlob tmp = ret.data(); - ndarray::Copy(from.data(), &tmp, - from.ctx(), ret.ctx(), ctx); + NDArray nd(ret); + CopyFromToImpl(from, &nd, ctx); }, from.ctx(), const_vars, {ret.var()}, FnProperty::kNormal, priority, PROFILER_MESSAGE("CopyCPU2CPU")); } else { #if MXNET_USE_CUDA if (a == cpu::kDevMask && b == gpu::kDevMask) { Engine::Get()->PushSync([from, ret](RunContext ctx) { - TBlob tmp = ret.data(); - ndarray::Copy(from.data(), &tmp, - from.ctx(), ret.ctx(), ctx); - // Wait GPU kernel to complete - ctx.get_stream()->Wait(); + NDArray nd(ret); + CopyFromToImpl(from, &nd, ctx); }, ret.ctx(), const_vars, {ret.var()}, FnProperty::kCopyToGPU, priority, PROFILER_MESSAGE("CopyCPU2GPU")); } else if (a == gpu::kDevMask && b == cpu::kDevMask) { Engine::Get()->PushSync([from, ret](RunContext ctx) { - TBlob tmp = ret.data(); - ndarray::Copy(from.data(), &tmp, - from.ctx(), ret.ctx(), ctx); - // Wait GPU kernel to complete - ctx.get_stream()->Wait(); + NDArray nd(ret); + CopyFromToImpl(from, &nd, ctx); }, from.ctx(), const_vars, {ret.var()}, FnProperty::kCopyFromGPU, priority, PROFILER_MESSAGE("CopyGPU2CPU")); } else if (a == gpu::kDevMask && b == gpu::kDevMask) { Engine::Get()->PushSync([from, ret](RunContext ctx) { - TBlob tmp = ret.data(); - ndarray::Copy(from.data(), &tmp, - from.ctx(), ret.ctx(), ctx); - // Wait GPU kernel to complete - ctx.get_stream()->Wait(); + NDArray nd(ret); + CopyFromToImpl(from, &nd, ctx); }, from.ctx(), const_vars, {ret.var()}, from.dtype() != ret.dtype() ? FnProperty::kNormal : FnProperty::kCopyFromGPU, priority, PROFILER_MESSAGE("CopyGPU2GPU")); @@ -665,34 +809,76 @@ NDArray &NDArray::operator/=(const real_t &src) { /* magic number for ndarray version 1, with int64_t TShape */ static const uint32_t NDARRAY_V1_MAGIC = 0xF993fac8; +/* magic number for ndarray version 2, with storage type */ +static const uint32_t NDARRAY_V2_MAGIC = 0xF993fac9; + void NDArray::Save(dmlc::Stream *strm) const { - strm->Write(NDARRAY_V1_MAGIC); + // write magic number to mark this version + // for storage type + strm->Write(NDARRAY_V2_MAGIC); + + // save storage type + int32_t stype = storage_type(); + strm->Write(&stype, sizeof(stype)); + + const int32_t nad = num_aux_data(storage_type()); + // save storage shape if ndarray is sparse + if (nad > 0) { + storage_shape().Save(strm); + } + + // save shape shape_.Save(strm); if (is_none()) return; + // save context Context ctx = this->ctx(); ctx.Save(strm); TBlob save_data; - NDArray temp; + NDArray nd_cpu; // a copy of *this on cpu if (ctx.dev_mask() != cpu::kDevMask) { - temp = this->Copy(Context::CPU()); - temp.WaitToRead(); - save_data = temp.data(); + nd_cpu = this->Copy(Context::CPU()); + nd_cpu.WaitToRead(); + save_data = nd_cpu.data(); } else { this->WaitToRead(); save_data = this->data(); + nd_cpu = *this; } + // save type flag int32_t type_flag = save_data.type_flag_; strm->Write(&type_flag, sizeof(type_flag)); + + // save aux_types and aux_shapes + if (nad > 0) { + for (int i = 0; i < nad; ++i) { + int32_t aux_type_flag = aux_type(i); + strm->Write(&aux_type_flag, sizeof(aux_type_flag)); + aux_shape(i).Save(strm); + } + } + + // save data CHECK(save_data.CheckContiguous()); size_t type_size = mshadow::mshadow_sizeof(type_flag); - strm->Write(save_data.dptr_, type_size * shape_.Size()); + // save data could be values of sparse tensors + // must use save_data.shape_ instead of this->shape_ + strm->Write(save_data.dptr_, type_size * save_data.shape_.Size()); + + // save aux data + if (nad > 0) { + for (int i = 0; i < nad; ++i) { + TBlob save_data = nd_cpu.aux_data(i); + // save aux_data + CHECK(save_data.CheckContiguous()); + size_t aux_type_size = mshadow::mshadow_sizeof(aux_type(i)); + strm->Write(save_data.dptr_, aux_type_size * save_data.Size()); + } + } } -bool LegacyTShapeLoad(dmlc::Stream *strm, TShape *shape) { - uint32_t magic; - if (strm->Read(&magic, sizeof(uint32_t)) != sizeof(uint32_t)) return false; +bool LegacyTShapeLoad(dmlc::Stream *strm, TShape *shape, const uint32_t magic) { switch (magic) { case NDARRAY_V1_MAGIC: return shape->Load(strm); @@ -708,10 +894,10 @@ bool LegacyTShapeLoad(dmlc::Stream *strm, TShape *shape) { } } -bool NDArray::Load(dmlc::Stream *strm) { +bool NDArray::LegacyLoad(dmlc::Stream *strm, const uint32_t magic) { // load shape TShape shape; - if (!LegacyTShapeLoad(strm, &shape)) return false; + if (!LegacyTShapeLoad(strm, &shape, magic)) return false; if (shape.ndim() == 0) { *this = NDArray(); return true; } @@ -739,6 +925,88 @@ bool NDArray::Load(dmlc::Stream *strm) { } } +bool NDArray::Load(dmlc::Stream *strm) { + uint32_t magic; + if (strm->Read(&magic, sizeof(uint32_t)) != sizeof(uint32_t)) return false; + if (magic != NDARRAY_V2_MAGIC) { + return LegacyLoad(strm, magic); + } + + // load storage type + int32_t stype; + if (strm->Read(&stype, sizeof(stype)) != sizeof(stype)) return false; + const int32_t nad = num_aux_data(static_cast(stype)); + + // load storage shape + TShape sshape; + if (nad > 0) { + if (!sshape.Load(strm)) return false; + } + + // load shape + TShape shape; + if (!shape.Load(strm)) return false; + if (shape.ndim() == 0) { + *this = NDArray(); return true; + } + + // load context + Context ctx; + if (!ctx.Load(strm)) return false; + + // load type flag + int32_t type_flag; + if (strm->Read(&type_flag, sizeof(type_flag)) != sizeof(type_flag)) return false; + + // load aux_types and aux_shapes + std::vector aux_types; + std::vector aux_shapes; + if (nad > 0) { + aux_types.resize(nad); + aux_shapes.resize(nad); + for (int i = 0; i < nad; ++i) { + // load aux_type(i) + if (strm->Read(&aux_types[i], sizeof(aux_types[i])) != sizeof(aux_types[i])) return false; + // load aux_shapes(i) + if (!aux_shapes[i].Load(strm)) return false; + } + } + + // load data into CPU + NDArray temp; + if (0 == nad) { + temp = NDArray(shape, Context::CPU(), false, type_flag); + } else { + temp = NDArray(static_cast(stype), shape, + Context::CPU(), false, type_flag, + aux_types, aux_shapes, sshape); + } + // load data + TBlob load_data = temp.data(); + size_t type_size = mshadow::mshadow_sizeof(type_flag); + size_t nread = type_size * load_data.Size(); + if (strm->Read(load_data.dptr_, nread) != nread) return false; + + // load aux_data + if (nad > 0) { + for (int i = 0; i < nad; ++i) { + load_data = temp.aux_data(i); + type_size = mshadow::mshadow_sizeof(load_data.type_flag_); + nread = type_size * load_data.Size(); + if (strm->Read(load_data.dptr_, nread) != nread) return false; + } + } + + if (ctx.dev_mask() == cpu::kDevMask) { + *this = std::move(temp); return true; + } else { +#if MXNET_USE_CUDA + *this = temp.Copy(ctx); return true; +#else + *this = std::move(temp); return true; +#endif + } +} const uint64_t kMXAPINDArrayListMagic = 0x112; @@ -771,7 +1039,16 @@ void NDArray::Load(dmlc::Stream* fi, } NDArray NDArray::Copy(Context ctx) const { - NDArray ret(shape(), ctx, true, dtype_); + NDArray ret; + if (kDefaultStorage == storage_type()) { + ret = NDArray(shape(), ctx, true, dtype_); + } else if (kUndefinedStorage != storage_type()) { + ret = NDArray(storage_type(), shape(), ctx, true, dtype_, + ptr_->aux_types, ptr_->aux_shapes, storage_shape()); + } else { + LOG(FATAL) << "NDArray::Copy cannot copy undefined storage-type ndarray to ctx.dev_type=" + << ctx.dev_type << ", ctx.dev_id=" << ctx.dev_id; + } CopyFromTo(*this, &ret); return ret; } @@ -804,6 +1081,101 @@ void NDArray::SyncCopyFromCPU(const void *data, size_t size) const { } } +/*! + * \brief Copy src.data()/aux_data(i) to dst->data()/aux_data(j). + */ +void NDArray::SyncCopyFromNDArray(const NDArray& src, int i, int j) { + if (i >= 0) { + CHECK_NE(src.storage_type(), kDefaultStorage); + } else { + CHECK(!src.is_none()) << "src dense ndarray must have been initialized"; + } + if (j >= 0) { + CHECK_NE(storage_type(), kDefaultStorage); + } else { + CHECK(!this->is_none()) << "dst dense ndarray must have been initialized"; + } + + if (src.var() == var()) { + // skip to copy to itself + LOG(WARNING) << "SyncCopyFromNDArray does not support copying to self"; + return; + } + const int src_dev_mask = src.ctx().dev_mask(); + const int dst_dev_mask = ctx().dev_mask(); + std::vector const_vars; + const_vars.push_back(src.var()); + + // get or create a dst tblob for copying src to it + // if dst is a dense format and has not been allocated, allocate memory for it + // else if dst is not initialized, allocate corresponding data blob for it + auto get_dst_data = [&](const TShape& src_shape) { + if (this->storage_type() == kDefaultStorage) { + this->ReshapeAndAlloc(src_shape); + } else if (!this->storage_initialized()) { + if (j < 0) { + this->CheckAndAllocData(src_shape); + } else { + this->CheckAndAllocAuxData(j, src_shape); + } + } + TBlob dst_data = (j >= 0? this->aux_data(j) : this->data()); + CHECK_LE(src_shape.Size(), dst_data.shape_.Size()); + return dst_data; + }; + + if (src_dev_mask == cpu::kDevMask && dst_dev_mask == cpu::kDevMask) { + Engine::Get()->PushSync([&](RunContext rctx) { + const TBlob src_data = (i >= 0? src.aux_data(i) : src.data()); + TBlob dst_data = get_dst_data(src_data.shape_); + ndarray::Copy(src_data, &dst_data, src.ctx(), this->ctx(), rctx); + }, this->ctx(), const_vars, {this->var()}, + FnProperty::kNormal, 0, PROFILER_MESSAGE("SyncCopyFromNDArrayCPU2CPU")); + } else { +#if MXNET_USE_CUDA + if (src_dev_mask == cpu::kDevMask && dst_dev_mask == gpu::kDevMask) { + Engine::Get()->PushSync([&](RunContext rctx) { + const TBlob src_data = (i >= 0? src.aux_data(i) : src.data()); + TBlob dst_data = get_dst_data(src_data.shape_); + ndarray::Copy(src_data, &dst_data, src.ctx(), this->ctx(), rctx); + rctx.get_stream()->Wait(); + }, this->ctx(), const_vars, {this->var()}, + FnProperty::kCopyToGPU, 0, PROFILER_MESSAGE("SyncCopyFromNDArrayCPU2GPU")); + } else if (src_dev_mask == gpu::kDevMask && dst_dev_mask == cpu::kDevMask) { + Engine::Get()->PushSync([&](RunContext rctx) { + const TBlob src_data = (i >= 0? src.aux_data(i) : src.data()); + TBlob dst_data = get_dst_data(src_data.shape_); + ndarray::Copy(src_data, &dst_data, src.ctx(), this->ctx(), rctx); + rctx.get_stream()->Wait(); + }, this->ctx(), const_vars, {this->var()}, + FnProperty::kCopyFromGPU, 0, PROFILER_MESSAGE("SyncCopyFromNDArrayGPU2CPU")); + } else if (src_dev_mask == gpu::kDevMask && dst_dev_mask == gpu::kDevMask) { + Engine::Get()->PushSync([&](RunContext rctx) { + const TBlob src_data = (i >= 0? src.aux_data(i) : src.data()); + TBlob dst_data = get_dst_data(src_data.shape_); + ndarray::Copy(src_data, &dst_data, src.ctx(), this->ctx(), rctx); + rctx.get_stream()->Wait(); + }, this->ctx(), const_vars, {this->var()}, + src.dtype() != this->dtype() ? FnProperty::kNormal : FnProperty::kCopyFromGPU, + 0, PROFILER_MESSAGE("SyncCopyFromNDArrayGPU2GPU")); + } else { + LOG(FATAL) << "unknown device mask"; + } +#else + LOG(FATAL) << MXNET_GPU_NOT_ENABLED_ERROR; +#endif + } + // The copy operation was pushed to engine to execute. + // Need to wait here for it being completed. + // The reason for pushing the copy operation to engine + // is because when copying data from a sparse tensor + // to the current one, that sparse ndarray's storage_shape/aux_shape + // may not be ready or changed and we need to ensure + // thread safty for reading the correct shape info to allocate + // memory for the current ndarray. + WaitToRead(); +} + void NDArray::SyncCopyToCPU(void *data, size_t size) const { TShape dshape = this->shape(); CHECK_EQ(dshape.Size(), size) diff --git a/src/ndarray/ndarray_function-inl.h b/src/ndarray/ndarray_function-inl.h index 2be55f50f934..b284e0378647 100644 --- a/src/ndarray/ndarray_function-inl.h +++ b/src/ndarray/ndarray_function-inl.h @@ -30,27 +30,28 @@ // macro to help specialize evaluation function #ifndef DECL_TERNARY -#define DECL_TERNARY(XPU, OP, FUN) \ - template<> \ - void Eval(const TBlob &lhs, const TBlob &mhs, \ - const TBlob &rhs, TBlob *ret, RunContext ctx) { \ - FUN(lhs, mhs, rhs, ret, ctx); \ +#define DECL_TERNARY(XPU, OP, FUN) \ + template<> \ + void Eval(const TBlob &lhs, const TBlob &mhs, \ + const TBlob &rhs, TBlob *ret, RunContext ctx) { \ + FUN(lhs, mhs, rhs, ret, ctx); \ } #endif #ifndef DECL_BINARY -#define DECL_BINARY(XPU, OP, FUN) \ - template<> \ +#define DECL_BINARY(XPU, OP, FUN) \ + template<> \ void Eval(const TBlob &lhs, const TBlob &rhs, TBlob *ret, RunContext ctx) { \ - FUN(lhs, rhs, ret, ctx); \ + FUN(lhs, rhs, ret, ctx); \ } #endif #ifndef DECL_SCALAR -#define DECL_SCALAR(XPU, OP, FUN, REVERSE) \ - template<> \ - void Eval(const TBlob &lhs, const real_t &rhs, TBlob *ret, RunContext ctx) { \ - FUN(lhs, rhs, ret, ctx); \ +#define DECL_SCALAR(XPU, OP, FUN, REVERSE) \ + template<> \ + void Eval(const TBlob &lhs, const real_t &rhs, \ + TBlob *ret, RunContext ctx) { \ + FUN(lhs, rhs, ret, ctx); \ } #endif @@ -62,10 +63,11 @@ namespace mxnet { namespace ndarray { + // true implementation template -inline void EvalBinary_(const TBlob &lhs, const TBlob &rhs, - TBlob *ret, RunContext ctx) { +void EvalBinary_(const TBlob &lhs, const TBlob &rhs, + TBlob *ret, RunContext ctx) { using namespace mshadow::expr; mshadow::Stream *s = ctx.get_stream(); CHECK_EQ(ret->type_flag_, lhs.type_flag_) @@ -79,10 +81,9 @@ inline void EvalBinary_(const TBlob &lhs, const TBlob &rhs, }); } - template -inline void EvalOneHot_(const TBlob &index, const TBlob &rhs, - TBlob *ret, RunContext ctx) { +void EvalOneHot_(const TBlob &index, const TBlob &rhs, + TBlob *ret, RunContext ctx) { LOG(INFO) << "The operator onehot_encode is deprecated; use one_hot instead."; using namespace mshadow::expr; mshadow::Stream *s = ctx.get_stream(); @@ -99,8 +100,8 @@ inline void EvalOneHot_(const TBlob &index, const TBlob &rhs, } template -inline void EvalMatChooseRowElem_(const TBlob &lhs, const TBlob &rhs, - TBlob *ret, RunContext ctx) { +void EvalMatChooseRowElem_(const TBlob &lhs, const TBlob &rhs, + TBlob *ret, RunContext ctx) { using namespace mshadow::expr; mshadow::Stream *s = ctx.get_stream(); // TODO(eric): support mixed type choose, i.e. int index and float rhs. @@ -116,8 +117,8 @@ inline void EvalMatChooseRowElem_(const TBlob &lhs, const TBlob &rhs, } template -inline void EvalMatFillRowElem_(const TBlob &lhs, const TBlob &mhs, const TBlob &rhs, - TBlob *ret, RunContext ctx) { +void EvalMatFillRowElem_(const TBlob &lhs, const TBlob &mhs, const TBlob &rhs, + TBlob *ret, RunContext ctx) { using namespace mshadow::expr; mshadow::Stream *s = ctx.get_stream(); ret->get(s) @@ -127,8 +128,8 @@ inline void EvalMatFillRowElem_(const TBlob &lhs, const TBlob &mhs, const TBlob } template -inline void EvalScalar_(const TBlob &lhs, const real_t &rhs, - TBlob *ret, RunContext ctx) { +void EvalScalar_(const TBlob &lhs, const real_t &rhs, + TBlob *ret, RunContext ctx) { using namespace mshadow::expr; mshadow::Stream *s = ctx.get_stream(); CHECK_EQ(ret->type_flag_, lhs.type_flag_) @@ -148,7 +149,7 @@ inline void EvalScalar_(const TBlob &lhs, const real_t &rhs, template<> void EvalClip(const TBlob &src, const real_t &a_min, const real_t &a_max, - TBlob *ret, RunContext ctx) { + TBlob *ret, RunContext ctx) { typedef DEVICE xpu; using namespace mshadow::expr; mshadow::Stream *s = ctx.get_stream(); @@ -163,12 +164,11 @@ void EvalClip(const TBlob &src, const real_t &a_min, const real_t &a_max } template<> -void EvalRandom( - const real_t &a, - const real_t &b, - const Resource &resource, - TBlob *ret, - RunContext ctx) { +void EvalRandom(const real_t &a, + const real_t &b, + const Resource &resource, + TBlob *ret, + RunContext ctx) { typedef DEVICE xpu; mshadow::Stream *s = ctx.get_stream(); switch (ret->type_flag_) { @@ -444,6 +444,7 @@ DECL_SCALAR(DEVICE, Plus, EvalScalar_, true) DECL_SCALAR(DEVICE, Minus, EvalScalar_, true) DECL_SCALAR(DEVICE, Mul, EvalScalar_, true) DECL_SCALAR(DEVICE, Div, EvalScalar_, true) + // for reverse seq DECL_SCALAR(DEVICE, Plus, EvalScalar_, false) DECL_SCALAR(DEVICE, Minus, EvalScalar_, false) diff --git a/src/ndarray/ndarray_function.cc b/src/ndarray/ndarray_function.cc index e4af86d2c824..5cea7942efa6 100644 --- a/src/ndarray/ndarray_function.cc +++ b/src/ndarray/ndarray_function.cc @@ -25,6 +25,7 @@ // this will be invoked by gcc and compile CPU version #include "./ndarray_function.h" #include "./ndarray_function-inl.h" +#include "../common/utils.h" namespace mxnet { namespace ndarray { @@ -44,5 +45,138 @@ void Copy(const TBlob &from, TBlob *to, } }) } + +template +void ElementwiseSumRspImpl(mshadow::Stream* s, + const std::vector& nds, + const std::vector& uniq_row_idx, + NDArray* out, + const int nthreads = 4) { +#pragma omp parallel num_threads(nthreads) + { + const size_t nnr = uniq_row_idx.size(); + const int num_threads = omp_get_num_threads(); + size_t row_block_len = (nnr + num_threads - 1) / num_threads; + const size_t row_block_start = omp_get_thread_num() * row_block_len; + if (row_block_start < nnr) { + const size_t row_block_end = std::min(row_block_start+row_block_len, nnr); + + const size_t row_length = out->data().shape_.ProdShape(1, out->data().shape_.ndim()); + auto out_values = out->data().get_with_shape( + mshadow::Shape2(out->storage_shape()[0], row_length), s); + auto out_indices = out->aux_data(rowsparse::kIdx).FlatTo1D(); + for (size_t i = row_block_start; i < row_block_end; ++i) { + out_indices[i] = uniq_row_idx[i]; + } + for (const auto& nd : nds) { + if (nd.storage_initialized()) { + const auto nd_indices = nd.aux_data(rowsparse::kIdx).FlatTo1D(); + const auto nd_values = nd.data().get_with_shape( + mshadow::Shape2(nd.storage_shape()[0], row_length), s); + const auto nd_num_rows = nd.aux_shape(rowsparse::kIdx).Size(); + const IType* nd_indices_start = &nd_indices[0]; + const IType* nd_indices_end = nd_indices_start + nd_num_rows; + const IType* row_idx_ptr = std::lower_bound(nd_indices_start, nd_indices_end, + out_indices[row_block_start]); + // skip this nd if all of its row indices are smaller than out_indices[row_block_start] + // or current row block is not covered by [*row_idx_ptr, nd_indices_end). + if (nd_indices_end == row_idx_ptr || *row_idx_ptr > out_indices[row_block_end-1]) { + continue; + } + for (size_t irow = row_block_start; + irow < row_block_end && row_idx_ptr != nd_indices_end;) { + if (out_indices[irow] == *row_idx_ptr) { + auto out_value_cur_row = out_values[irow]; + const auto offset = row_idx_ptr - nd_indices_start; + auto nd_value_cur_row = nd_values[offset]; + for (size_t j = 0; j < nd_value_cur_row.shape_[0]; ++j) { + out_value_cur_row[j] += nd_value_cur_row[j]; + } + ++irow; + ++row_idx_ptr; + } else if (out_indices[irow] < *row_idx_ptr) { + ++irow; + } else { + ++row_idx_ptr; + } + } + } + } + } + } +} + +/*! + * \brief Given a vector of ndarrays, generate a index vector containing + * all the unique row indices of the ndarrays. + */ +template +void GetUniqueRspRowIdx(const std::vector& nds, + std::vector* uniq_row_idx) { + using namespace rowsparse; + size_t total_num_rows = 0; + for (const auto& nd : nds) { + CHECK_EQ(nd.storage_type(), kRowSparseStorage); + if (nd.storage_initialized()) { + total_num_rows += nd.aux_shape(kIdx).Size(); + } + } + + uniq_row_idx->resize(total_num_rows); + int nthreads = omp_get_max_threads(); + int offset = 0; + for (const auto& nd : nds) { + if (nd.storage_initialized()) { + const IType* nd_row_idx = nd.aux_data(kIdx).dptr(); + const int num_rows = nd.aux_shape(kIdx).Size(); +#pragma omp parallel for num_threads(nthreads) + for (int i = 0; i < num_rows; ++i) { + (*uniq_row_idx)[offset+i] = nd_row_idx[i]; + } + offset += num_rows; + } + } + + common::ParallelSort(uniq_row_idx->begin(), uniq_row_idx->end(), nthreads); + auto it = std::unique(uniq_row_idx->begin(), uniq_row_idx->end()); + uniq_row_idx->resize(it - uniq_row_idx->begin()); +} + +void ElementwiseSumRsp(mshadow::Stream* s, const std::vector& nds, NDArray* out) { + if (nds.empty()) return; + using namespace rowsparse; + CHECK_EQ(out->storage_type(), kRowSparseStorage) + << "Expected row sparse storage type (" + << out->storage_type() << " given)"; + + MSHADOW_TYPE_SWITCH(out->dtype(), DType, { + MSHADOW_IDX_TYPE_SWITCH(out->aux_type(kIdx), IType, { + std::vector uniq_row_idx; + GetUniqueRspRowIdx(nds, &uniq_row_idx); + out->CheckAndAlloc({mshadow::Shape1(uniq_row_idx.size())}); + out->data().FlatTo2D() = static_cast(0); + ElementwiseSumRspImpl(s, nds, uniq_row_idx, out, omp_get_max_threads()); + }); + }); +} + +/*! + * \brief Parallel cpu impl of elemwise sum for sparse tensors. + * Currently only support row sparse sum. + */ +template<> +void ElementwiseSum(mshadow::Stream* s, + const std::vector& nds, + NDArray* out) { + if (nds.empty()) return; + + if (nds[0].storage_type() == kRowSparseStorage) { + ElementwiseSumRsp(s, nds, out); + } else { + LOG(FATAL) << "ElementwiseSum has not been implemented for storage_type = << " + << nds[0].storage_type(); + } +} + } // namespace ndarray } // namespace mxnet diff --git a/src/ndarray/ndarray_function.h b/src/ndarray/ndarray_function.h index b1ed58db3e74..65c59185f691 100644 --- a/src/ndarray/ndarray_function.h +++ b/src/ndarray/ndarray_function.h @@ -28,6 +28,7 @@ #include #include #include +#include #include #include "../operator/mshadow_op.h" @@ -168,6 +169,14 @@ void ElementwiseSum(const std::vector source, TBlob *out, RunContext ctx); +/*! + * \brief Interface for parallel impl of elemwise sum for sparse matrices + */ +template +void ElementwiseSum(mshadow::Stream* s, + const std::vector& nds, + NDArray* out); + // broadcasting template void EvalBroadcast(TBlob const& src, TBlob* ret, int size, RunContext ctx); diff --git a/src/nnvm/legacy_op_util.cc b/src/nnvm/legacy_op_util.cc index 2bba5f1c3655..6e601780080b 100644 --- a/src/nnvm/legacy_op_util.cc +++ b/src/nnvm/legacy_op_util.cc @@ -60,19 +60,20 @@ class OperatorState { opr_ = opr; fwd_init_ = bwd_init_ = false; - in_data_.resize(prop->ListArguments().size()); + in_data_fwd_.resize(prop->ListArguments().size()); + in_data_bwd_.resize(prop->ListArguments().size()); out_data_.resize(prop->NumOutputs()); aux_data_.resize(prop->ListAuxiliaryStates().size()); - in_grad_.resize(in_data_.size()); + in_grad_.resize(in_data_fwd_.size()); out_grad_.resize(prop->NumVisibleOutputs()); std::vector out_grad_ptr(out_grad_.size()); for (size_t i = 0; i < out_grad_.size(); ++i) { out_grad_ptr[i] = &out_grad_[i]; } - std::vector in_data_ptr(in_data_.size()); - for (size_t i = 0; i < in_data_.size(); ++i) { - in_data_ptr[i] = &in_data_[i]; + std::vector in_data_ptr(in_data_fwd_.size()); + for (size_t i = 0; i < in_data_fwd_.size(); ++i) { + in_data_ptr[i] = &in_data_bwd_[i]; } std::vector out_data_ptr(out_data_.size()); for (size_t i = 0; i < out_data_.size(); ++i) { @@ -89,16 +90,19 @@ class OperatorState { const std::vector& req, const std::vector& outputs) { if (!fwd_init_) { - CHECK_EQ(inputs.size(), in_data_.size() + aux_data_.size()); + CHECK_EQ(inputs.size(), in_data_fwd_.size() + aux_data_.size()); CHECK_EQ(outputs.size(), out_data_.size()); - for (size_t i = 0; i < in_data_.size(); ++i) in_data_[i] = inputs[i]; + // in_data_bwd_ has the same tblobs as the ones in in_data_fwd_, except that the ones + // referred by arg_data_ptr_ will be overriden + for (size_t i = 0; i < in_data_fwd_.size(); ++i) in_data_fwd_[i] = inputs[i]; + for (size_t i = 0; i < in_data_fwd_.size(); ++i) in_data_bwd_[i] = inputs[i]; for (size_t i = 0; i < aux_data_.size(); ++i) { - aux_data_[i] = inputs[i + in_data_.size()]; + aux_data_[i] = inputs[i + in_data_fwd_.size()]; } for (size_t i = 0; i < out_data_.size(); ++i) out_data_[i] = outputs[i]; fwd_init_ = true; } - opr_->Forward(ctx, in_data_, req, out_data_, aux_data_); + opr_->Forward(ctx, in_data_fwd_, req, out_data_, aux_data_); } void Backward(const OpContext &ctx, @@ -108,6 +112,8 @@ class OperatorState { if (!bwd_init_) { CHECK(fwd_init_); CHECK_EQ(arg_data_ptr_.size() + aux_data_.size(), inputs.size()); + // override tblobs pointed by arg_data_ptr_ since they might not contain + // initialized data during forward pass. for (size_t i = 0; i < arg_data_ptr_.size(); ++i) { *arg_data_ptr_[i] = inputs[i]; } @@ -118,13 +124,19 @@ class OperatorState { for (size_t i = 0; i < outputs.size(); ++i) in_grad_[i] = outputs[i]; bwd_init_ = true; } - opr_->Backward(ctx, out_grad_, in_data_, out_data_, req, in_grad_, aux_data_); + opr_->Backward(ctx, out_grad_, in_data_bwd_, out_data_, req, in_grad_, aux_data_); } private: Operator *opr_; bool fwd_init_, bwd_init_; - std::vector in_data_, aux_data_, out_data_, in_grad_, out_grad_; + // input data blobs for forward and backward + // in_data_fwd_ and in_data_bwd_ will hold different tblobs when StorageFallbackOpExecutor + // performs storage fallback on a non-default input NDArray. The one in in_data_fwd_ is + // generated when setting up forward executor, while the one in in_data_bwd_ is generated + // when setting up backward executor. + std::vector in_data_fwd_, in_data_bwd_; + std::vector aux_data_, out_data_, in_grad_, out_grad_; std::vector arg_data_ptr_; }; diff --git a/src/operator/batch_norm.cc b/src/operator/batch_norm.cc index 86f47dd6163f..866b7fe619cb 100644 --- a/src/operator/batch_norm.cc +++ b/src/operator/batch_norm.cc @@ -230,7 +230,7 @@ void BatchNormOp::DoBackward(mshadow::Stream *, #pragma omp parallel for for (int channel = 0; channel < static_cast(channelCount); ++channel) { const AccReal *weight = weights.dptr(); - const AccReal w = weight ? weight[channel] : AccReal(1); + const AccReal w = !param_.fix_gamma ? weight[channel] : AccReal(1); AccReal mean, invstd; if (is_train_and_not_global_stats) { mean = saveMeanDataPtr[channel]; diff --git a/src/operator/batch_norm.cu b/src/operator/batch_norm.cu index 64f7d9373823..9a8b576a16ee 100644 --- a/src/operator/batch_norm.cu +++ b/src/operator/batch_norm.cu @@ -283,7 +283,7 @@ __global__ void BatchNormalizationUpdateOutputKernel( } // Write normalized and update the output - const AccReal gamma = weight.numElements() > 0 + const AccReal gamma = ((flags & FIX_GAMMA_FLAG) == 0 && weight.numElements() > 0) ? ScalarConvert::to(weight[plane]) : ScalarConvert::to(1); const AccReal beta = bias.numElements() > 0 ? ScalarConvert::to(bias[plane]) @@ -332,7 +332,7 @@ static __global__ void BatchNormalizationBackwardKernel( invstd = VARIANCE_TO_INVSTD(tensors.runningVar[plane], eps); } - const AccReal weightVal = tensors.weight.numElements() > 0 ? + const AccReal weightVal = ((flags & FIX_GAMMA_FLAG) == 0 && tensors.weight.numElements() > 0) ? ScalarConvert::to(tensors.weight[plane]) : AccReal(1); const AccReal norm = AccReal(1) / N; diff --git a/src/operator/deconvolution-inl.h b/src/operator/deconvolution-inl.h index 43530138b8ea..9db94a8c5986 100644 --- a/src/operator/deconvolution-inl.h +++ b/src/operator/deconvolution-inl.h @@ -256,7 +256,7 @@ class DeconvolutionOp : public Operator { if (!param_.no_bias) { // add bias, broadcast bias to dim 1: channel Tensor bias = in_data[deconv::kBias].get(s); - out += broadcast<1>(bias, out.shape_); + out += mshadow::expr::broadcast<1>(bias, out.shape_); } } diff --git a/src/operator/elemwise_op_common.h b/src/operator/elemwise_op_common.h index 9b398f947e30..f60bb590a2e6 100644 --- a/src/operator/elemwise_op_common.h +++ b/src/operator/elemwise_op_common.h @@ -80,6 +80,42 @@ inline bool ElemwiseAttr(const nnvm::NodeAttrs& attrs, return true; } +// Only inferring output storage types from input for now +template +inline bool ElemwiseStorageAttr(const nnvm::NodeAttrs& attrs, + std::vector *in_attrs, + std::vector *out_attrs) { + auto deduce = [&](std::vector *vec, const char *name, AttrType& result, + bool fallback) { + auto &v = *vec; + for (size_t i = 0; i < vec->size(); ++i) { + if (v[i] == kUndefinedStorage) { + // if input type is unknown, assume it's default storage + CHECK(assign(&v[i], kDefaultStorage)); + } else if (assign(&result, v[i]) == false && fallback) { + result = kDefaultStorage; + } + } + }; + AttrType dattr = kUndefinedStorage; + deduce(in_attrs, "input", dattr, enable_fallback); + if (reverse_infer) { + LOG(FATAL) << "not implemented yet"; + } + auto write = [&](std::vector *vec, const char *name) { + for (size_t i = 0; i < vec->size(); ++i) { + CHECK(assign(&(*vec)[i], dattr)) + << "Incompatible attr in node " << attrs.name << " at " << i << "-th " + << name << ": " << "expected " << dattr << ", got " << (*vec)[i]; + } + }; + if (is_none(dattr)) dattr = kDefaultStorage; + write(out_attrs, "output"); + return true; +} + template inline bool ElemwiseShape(const nnvm::NodeAttrs& attrs, std::vector *in_attrs, @@ -108,6 +144,18 @@ inline bool ElemwiseType(const nnvm::NodeAttrs& attrs, attrs, in_attrs, out_attrs, -1); } +template +inline bool ElemwiseStorageType(const nnvm::NodeAttrs& attrs, + const Context& ctx, + std::vector *in_attrs, + std::vector *out_attrs) { + // TODO(junwu): add ctx info into storage inference logic + CHECK_EQ(in_attrs->size(), static_cast(n_in)) << " in operator " << attrs.name; + CHECK_EQ(out_attrs->size(), static_cast(n_out)) << " in operator " << attrs.name; + return ElemwiseStorageAttr( + attrs, in_attrs, out_attrs); +} + // Transfer gradient and input to FGradient function struct ElemwiseGradUseIn { const char *op_name; diff --git a/src/operator/leaky_relu-inl.h b/src/operator/leaky_relu-inl.h index 828930a0e405..d228e3e67d03 100644 --- a/src/operator/leaky_relu-inl.h +++ b/src/operator/leaky_relu-inl.h @@ -111,7 +111,7 @@ class LeakyReLUOp : public Operator { case leakyrelu::kPReLU: { weight = in_data[leakyrelu::kGamma].get(s); Assign(out, req[leakyrelu::kOut], - F(data, broadcast<1>(weight, out.shape_))); + F(data, mshadow::expr::broadcast<1>(weight, out.shape_))); break; } case leakyrelu::kRReLU: { @@ -177,7 +177,8 @@ class LeakyReLUOp : public Operator { weight = in_data[leakyrelu::kGamma].get(s); grad_weight = in_grad[leakyrelu::kGamma].get(s); grad_weight = sumall_except_dim<1>(F(data) * grad); - gdata = F(data, broadcast<1>(weight, data.shape_)) * grad; + gdata = F(data, mshadow::expr::broadcast<1>(weight, data.shape_)) + * grad; break; } case leakyrelu::kRReLU: { diff --git a/src/operator/mxnet_op.h b/src/operator/mxnet_op.h index 0af7d026d9d5..3162ab6b7b16 100644 --- a/src/operator/mxnet_op.h +++ b/src/operator/mxnet_op.h @@ -25,8 +25,12 @@ #ifndef MXNET_OPERATOR_MXNET_OP_H_ #define MXNET_OPERATOR_MXNET_OP_H_ +#include #include #include +#ifdef __CUDACC__ +#include "../common/cuda_utils.h" +#endif // __CUDACC__ namespace mxnet { namespace op { @@ -40,6 +44,8 @@ const float PI = 3.14159265358979323846; using std::isnan; #endif +template +int get_num_threads(const int N); #ifdef __CUDACC__ #define CUDA_KERNEL_LOOP(i, n) \ @@ -47,6 +53,13 @@ using std::isnan; i < (n); \ i += blockDim.x * gridDim.x) +inline cudaDeviceProp cuda_get_device_prop() { + int device; + CUDA_CALL(cudaGetDevice(&device)); + cudaDeviceProp deviceProp; + CUDA_CALL(cudaGetDeviceProperties(&deviceProp, device)); + return deviceProp; +} /*! * \brief Get the number of blocks for cuda kernel given N @@ -55,8 +68,18 @@ inline int cuda_get_num_blocks(const int N) { using namespace mshadow::cuda; return std::min(kMaxGridNum, (N + kBaseThreadNum - 1) / kBaseThreadNum); } + +template<> +inline int get_num_threads(const int N) { + using namespace mshadow::cuda; + return kBaseThreadNum * cuda_get_num_blocks(N); +} #endif // __CUDACC__ +template<> +inline int get_num_threads(const int N) { + return omp_get_max_threads(); +} /*! \brief operator request type switch */ #define MXNET_ASSIGN_REQ_SWITCH(req, ReqType, ...) \ @@ -216,7 +239,6 @@ __global__ void mxnet_generic_kernel(int N, Args... args) { } } - template struct Kernel { template diff --git a/src/operator/operator_common.h b/src/operator/operator_common.h index 2d46bd3230ce..dc53e1a7d232 100644 --- a/src/operator/operator_common.h +++ b/src/operator/operator_common.h @@ -29,12 +29,15 @@ #include #include #include +#include +#include #include #include #include #include #include #include "../common/cuda_utils.h" +#include "../common/utils.h" namespace mxnet { namespace op { @@ -125,6 +128,19 @@ inline std::string type_string(const int& x) { return "unknown"; } +/*! \brief get string representation of storage_type */ +inline std::string stype_string(const int& x) { + switch (x) { + case kDefaultStorage: + return "default"; + case kCSRStorage: + return "csr"; + case kRowSparseStorage: + return "row_sparse"; + } + return "unknown"; +} + /*! * \brief Assign x to y. Checks for compatiblity when y is not empty. * Allow missing dim in both x and y (as 0). @@ -201,6 +217,24 @@ inline bool type_assign(int *y, const int& x) { } \ } +/*! + * \brief macro assign type to out if out is unknown (-1) otherwise check consistency + * Use macro so we can see the error file more clearly + * \param type_array the storage type array to store the result + * \param index the index of in the array + * \param type the inferred storage type + */ +#define STORAGE_TYPE_ASSIGN_CHECK(type_array, index, type) \ + { \ + if (!type_assign(&(type_array)[index], type)) { \ + std::ostringstream os; \ + os << "Storage type inconsistent, Provided=" \ + << stype_string((type_array)[index]) << ',' \ + << " inferred storage type=" << stype_string(type); \ + throw ::mxnet::op::InferTypeError(os.str(), index); \ + } \ + } + // helper macro to implement bind dispatch #if MXNET_USE_CUDA #define DO_BIND_DISPATCH(Method, ...) \ @@ -333,6 +367,54 @@ inline void ParamParser(nnvm::NodeAttrs* attrs) { attrs->parsed = std::move(param); } +/*! \brief Perform storage fallback to invoke fcompute. + * \param attrs attributes of the operator + * \param ctx operator context + * \param inputs inputs of fcompute + * \param req req of fcompute + * \param outputs outputs of fcompute + * \param fcompute + * \param fname name of the operator + * \param mutate_idx the indices of mutable inputs + */ +template +void FCompExFallback(const nnvm::NodeAttrs& attrs, + const OpContext& ctx, + const std::vector& inputs, + const std::vector& req, + const std::vector& outputs, + FCompute fcompute, + const std::string& fname, + std::vector mutate_idx = {}) { + using namespace mxnet::common; + std::vector in_blobs, out_blobs; + std::vector pre_temp_src, pre_temp_dst, post_temp_dst, post_temp_src; + // mapping from index in input_blobs to index in pre_temp_dst + std::unordered_map in_temp_idx_map; + SetupDefaultBlobs(inputs, &in_blobs, &pre_temp_src, &pre_temp_dst, &in_temp_idx_map); + SetupDefaultBlobs(outputs, &out_blobs, &post_temp_dst, &post_temp_src); + for (const auto idx : mutate_idx) { + auto map_iter = in_temp_idx_map.find(idx); + if (map_iter != in_temp_idx_map.end()) { + post_temp_src.push_back(pre_temp_dst[map_iter->second]); + post_temp_dst.push_back(inputs[idx]); + } + } + CastNonDefaultStorage(pre_temp_src, pre_temp_dst, ctx, true); + fcompute(attrs, ctx, in_blobs, req, out_blobs); + CastNonDefaultStorage(post_temp_src, post_temp_dst, ctx, true); +} + +#define CHECK_RSP_ALL_ROWS_NON_ZERO(rsp, func, param) \ + { \ + CHECK(rsp.storage_shape()[0] == rsp.shape()[0]) << func \ + << " for RowSparse " << param << " is only implemented for " \ + << "RowSparse " << param << " with all rows containing non-zeros. " \ + << "Expects " << param << ".values.shape[0] (" << rsp.storage_shape()[0] \ + << ") == " << param << ".shape[0] (" << rsp.shape()[0] << ")."; \ + } + + } // namespace op } // namespace mxnet #endif // MXNET_OPERATOR_OPERATOR_COMMON_H_ diff --git a/src/operator/optimizer_op-inl.h b/src/operator/optimizer_op-inl.h index 70759b15251a..28707aae4ce8 100644 --- a/src/operator/optimizer_op-inl.h +++ b/src/operator/optimizer_op-inl.h @@ -36,6 +36,7 @@ #include "./mshadow_op.h" #include "./elemwise_op_common.h" #include "mxnet_op.h" +#include "./tensor/init_op.h" namespace mxnet { namespace op { @@ -102,6 +103,167 @@ inline void SGDUpdate(const nnvm::NodeAttrs& attrs, }); } +/*! \brief kernel for sparse sgd + */ +template +struct SGDDnsRspKernel { + // DType is the output data type + // IType is row sparse idx type + // i is the ith row in row sparse gradient + template + MSHADOW_XINLINE static void Map(int i, const index_t row_length, DType* out, const DType* weight, + const IType* grad_idx, const DType *grad_val, + const DType clip_gradient, const DType lr, + const DType wd, const DType rescale_grad) { + for (index_t j = 0; j < row_length; j++) { + index_t data_i = grad_idx[i] * row_length + j; + index_t grad_i = i * row_length + j; + if (clip_gradient >= 0.0f) { + KERNEL_ASSIGN(out[data_i], req, (1.f - lr * wd) * weight[data_i] - + (lr) * mshadow_op::clip::Map(rescale_grad * grad_val[grad_i], clip_gradient)); + } else { + KERNEL_ASSIGN(out[data_i], req, (1.f - lr * wd) * weight[data_i] - + (lr * rescale_grad) * grad_val[grad_i]); + } + } + } +}; + +template +inline void SGDUpdateDnsRspImpl(const SGDParam& param, + const OpContext &ctx, + const TBlob& weight, + const NDArray& grad, + const OpReqType& req, + TBlob *out) { + using namespace mshadow; + using namespace mshadow::expr; + using namespace mshadow_op; + using namespace mxnet_op; + Stream* s = ctx.get_stream(); + CHECK_EQ(grad.storage_type(), kRowSparseStorage); + // if gradients are zeros, no weights are updated + if (!grad.storage_initialized() || req == kNullOp) return; + CHECK_EQ(req, kWriteInplace) << "kWriteInplace is expected for sparse sgd_mom_update"; + CHECK_GT(weight.shape_.Size(), 0); + + MSHADOW_REAL_TYPE_SWITCH(weight.type_flag_, DType, { + MSHADOW_IDX_TYPE_SWITCH(grad.aux_type(rowsparse::kIdx), IType, { + MXNET_ASSIGN_REQ_SWITCH(req, req_type, { + DType* weight_data = weight.dptr(); + IType* grad_idx = grad.aux_data(rowsparse::kIdx).dptr(); + DType* grad_val = grad.data().dptr(); + index_t num_rows = grad.aux_shape(rowsparse::kIdx)[0]; + auto row_length = weight.shape_.ProdShape(1, weight.ndim()); + Kernel, xpu>::Launch(s, num_rows, row_length, + out->dptr(), weight_data, grad_idx, grad_val, + static_cast(param.clip_gradient), + static_cast(param.lr), static_cast(param.wd), + static_cast(param.rescale_grad)); + }); + }); + }); +} + +/*! \brief kernel for sparse sgd + */ +template +struct SGDRspDnsKernel { + template + MSHADOW_XINLINE static void Map(int i, const index_t num_cols, DType* out, const DType* weight, + const DType *grad, const DType clip_gradient, const DType lr, + const DType wd, const DType rescale_grad) { + bool contains_non_zeros = false; + index_t j = 0; + index_t offset = i * num_cols; + for (; j < num_cols; ++j) { + if (grad[offset + j] != 0) { + contains_non_zeros = true; + break; + } + } + if (!contains_non_zeros) return; + const DType rate = 1.f - lr * wd; + for (index_t j = 0; j < num_cols; j++) { + auto index = offset + j; + if (clip_gradient >= 0.0f) { + KERNEL_ASSIGN(out[index], req, rate * weight[index] - + lr * mshadow_op::clip::Map(rescale_grad * grad[index], clip_gradient)); + } else { + KERNEL_ASSIGN(out[index], req, rate * weight[index] - + lr * rescale_grad * grad[index]); + } + } + } +}; + +template +inline void SGDUpdateRspDnsImpl(const SGDParam& param, + const OpContext &ctx, + const NDArray& weight, + const TBlob& grad, + const OpReqType req, + NDArray *out) { + using namespace mshadow; + using namespace mxnet_op; + using namespace rowsparse; + CHECK_RSP_ALL_ROWS_NON_ZERO(weight, "SGDUpdate", "weights"); + CHECK_EQ(weight.storage_type(), kRowSparseStorage); + if (req == kNullOp) return; + CHECK_EQ(req, kWriteInplace) << "kWriteInplace is expected for sparse sgd_update"; + CHECK(weight.storage_initialized()); + Stream* s = ctx.get_stream(); + MSHADOW_REAL_TYPE_SWITCH(weight.dtype(), DType, { + MXNET_ASSIGN_REQ_SWITCH(req, req_type, { + DType* weight_data = weight.data().dptr(); + DType* grad_data = grad.dptr(); + index_t num_rows = weight.aux_shape(kIdx)[0]; + auto num_cols = weight.shape().ProdShape(1, weight.shape().ndim()); + Kernel, xpu>::Launch(s, num_rows, num_cols, + out->data().dptr(), weight_data, grad_data, + static_cast(param.clip_gradient), + static_cast(param.lr), static_cast(param.wd), + static_cast(param.rescale_grad)); + }); + }); +} + +template +inline void SGDUpdateRspRspImpl(const SGDParam& param, + const OpContext& ctx, + const NDArray& weight, + const NDArray& grad, + const OpReqType& req, + NDArray *out) { + CHECK_RSP_ALL_ROWS_NON_ZERO(weight, "SGDUpdate", "weights"); + // reuse dns rsp implementation when storage_shape == shape + TBlob out_blob = out->data(); + SGDUpdateDnsRspImpl(param, ctx, weight.data(), grad, req, &out_blob); +} + +template +inline void SGDUpdateEx(const nnvm::NodeAttrs& attrs, + const OpContext &ctx, + const std::vector &inputs, + const std::vector &req, + const std::vector &outputs) { + using namespace mshadow; + using namespace mshadow::expr; + using namespace mshadow_op; + const SGDParam& param = nnvm::get(attrs.parsed); + auto weight_stype = inputs[0].storage_type(); + auto grad_stype = inputs[1].storage_type(); + if (weight_stype == kRowSparseStorage && grad_stype == kRowSparseStorage) { + NDArray out = outputs[0]; + SGDUpdateRspRspImpl(param, ctx, inputs[0], inputs[1], req[0], &out); + } else if (weight_stype == kRowSparseStorage && grad_stype == kDefaultStorage) { + NDArray out = outputs[0]; + SGDUpdateRspDnsImpl(param, ctx, inputs[0], inputs[1].data(), req[0], &out); + } else { + FCompExFallback(attrs, ctx, inputs, req, outputs, SGDUpdate, "SGDUpdate"); + } +} + struct SGDMomParam : public dmlc::Parameter { float lr; float momentum; @@ -275,6 +437,196 @@ inline void MP_SGDMomUpdate(const nnvm::NodeAttrs& attrs, }); } +template +struct SGDMomDnsRspDnsKernel { + template + MSHADOW_XINLINE static void Map(int i, index_t row_length, DType* out_data, + DType* mom_data, const DType* weight_data, const IType* grad_idx, + const DType* grad_data, const DType clip_gradient, const DType momentum, + const DType lr, const DType wd, const DType rescale_grad) { + const DType rate = lr * wd; + for (index_t j = 0; j < row_length; j++) { + index_t data_i = grad_idx[i] * row_length + j; + index_t grad_i = i * row_length + j; + if (clip_gradient >= 0.0f) { + mom_data[data_i] = momentum * mom_data[data_i] + - rate * weight_data[data_i] + - lr * + mshadow_op::clip::Map(rescale_grad * grad_data[grad_i], + clip_gradient); + } else { + mom_data[data_i] = momentum * mom_data[data_i] + - rate * weight_data[data_i] + - lr * rescale_grad * grad_data[grad_i]; + } + KERNEL_ASSIGN(out_data[data_i], req, weight_data[data_i] + mom_data[data_i]); + } + } +}; + +template +inline void SGDMomUpdateDnsRspDnsImpl(const SGDMomParam& param, + const OpContext& ctx, + const TBlob& weight, + const NDArray& grad, + const TBlob& mom, + const OpReqType& req, + TBlob *out) { + using namespace mxnet_op; + using namespace rowsparse; + Stream* s = ctx.get_stream(); + if (!grad.storage_initialized() || req == kNullOp) return; + CHECK_EQ(req, kWriteInplace) << "kWriteInplace is expected for sparse sgd_mom_update"; + CHECK_GT(weight.shape_.Size(), 0); + CHECK_GT(mom.shape_.Size(), 0); + + MSHADOW_REAL_TYPE_SWITCH(weight.type_flag_, DType, { + MSHADOW_IDX_TYPE_SWITCH(grad.aux_type(kIdx), IType, { + MXNET_ASSIGN_REQ_SWITCH(req, req_type, { + DType* weight_data = weight.dptr(); + IType* grad_idx = grad.aux_data(kIdx).dptr(); + DType* grad_val = grad.data().dptr(); + DType* mom_data = mom.dptr(); + DType* out_data = out->dptr(); + index_t num_rows = grad.aux_shape(kIdx)[0]; + auto row_length = weight.shape_.ProdShape(1, weight.ndim()); + Kernel, xpu>::Launch(s, num_rows, row_length, + out_data, mom_data, weight_data, grad_idx, grad_val, + static_cast(param.clip_gradient), static_cast(param.momentum), + static_cast(param.lr), static_cast(param.wd), + static_cast(param.rescale_grad)); + }); + }); + }); +} + +template +struct SGDMomRspDnsKernel { + template + MSHADOW_XINLINE static void Map(int i, index_t num_cols, DType* out, DType* mom, + const DType* weight, const DType *grad, + const DType clip_gradient, const DType momentum, + const DType lr, const DType wd, const DType rescale_grad) { + bool contains_non_zeros = false; + index_t j = 0; + index_t offset = i * num_cols; + for (; j < num_cols; ++j) { + if (grad[offset + j] != 0) { + contains_non_zeros = true; + break; + } + } + if (!contains_non_zeros) return; + const DType rate = lr * wd; + for (index_t j = 0; j < num_cols; j++) { + auto index = offset + j; + if (clip_gradient >= 0.0f) { + mom[index] = momentum * mom[index] - rate * weight[index] + - lr * mshadow_op::clip::Map(rescale_grad * grad[index], clip_gradient); + } else { + mom[index] = momentum * mom[index] - rate * weight[index] + - lr * rescale_grad * grad[index]; + } + KERNEL_ASSIGN(out[index], req, weight[index] + mom[index]); + } + } +}; + +template +inline void SGDMomUpdateRspDnsImpl(const SGDMomParam& param, + const OpContext &ctx, + const NDArray& weight, + const TBlob& grad, + const NDArray& mom, + const OpReqType req, + NDArray *out) { + using namespace mshadow; + using namespace mxnet_op; + using namespace rowsparse; + CHECK_RSP_ALL_ROWS_NON_ZERO(weight, "SGDMomUpdate", "weights"); + Stream* s = ctx.get_stream(); + CHECK_EQ(weight.storage_type(), kRowSparseStorage); + if (req == kNullOp) return; + CHECK_EQ(req, kWriteInplace) << "kWriteInplace is expected for sparse sgd_mom_update"; + CHECK(weight.storage_initialized()); + // fill mom with zero values if not initialized yet + if (!mom.storage_initialized()) { + NDArray mom_zeros = mom; + FillDnsZerosRspImpl(s, &mom_zeros); + } + MSHADOW_REAL_TYPE_SWITCH(weight.dtype(), DType, { + MXNET_ASSIGN_REQ_SWITCH(req, req_type, { + DType* weight_data = weight.data().dptr(); + DType* grad_data = grad.dptr(); + DType* mom_data = mom.data().dptr(); + index_t num_rows = weight.aux_shape(kIdx)[0]; + auto num_cols = weight.shape().ProdShape(1, weight.shape().ndim()); + Kernel, xpu>::Launch(s, num_rows, num_cols, + out->data().dptr(), mom_data, weight_data, grad_data, + static_cast(param.clip_gradient), static_cast(param.momentum), + static_cast(param.lr), static_cast(param.wd), + static_cast(param.rescale_grad)); + }); + }); +} + + +template +inline void SGDMomUpdateRspRspRspImpl(const SGDMomParam& param, + const OpContext& ctx, + const NDArray& weight, + const NDArray& grad, + const NDArray& mom, + const OpReqType& req, + NDArray *out) { + using namespace mshadow; + using namespace mshadow::expr; + using namespace mxnet_op; + using namespace rowsparse; + CHECK_RSP_ALL_ROWS_NON_ZERO(weight, "SGDMomUpdate", "weights"); + Stream* s = ctx.get_stream(); + // fill mom with zero values in order to reuse the sgd mom dns impl + if (!mom.storage_initialized()) { + NDArray mom_zeros = mom; + FillDnsZerosRspImpl(s, &mom_zeros); + } + TBlob out_blob = out->data(); + // reuse dns rsp implementation when storage_shape == shape + SGDMomUpdateDnsRspDnsImpl(param, ctx, weight.data(), grad, + mom.data(), req, &out_blob); +} + +template +inline void SGDMomUpdateEx(const nnvm::NodeAttrs& attrs, + const OpContext &ctx, + const std::vector &inputs, + const std::vector &req, + const std::vector &outputs) { + using namespace mxnet_op; + const SGDMomParam& param = nnvm::get(attrs.parsed); + auto &weight = inputs[0]; + auto &grad = inputs[1]; + auto &mom = inputs[2]; + auto weight_stype = weight.storage_type(); + auto grad_stype = grad.storage_type(); + auto mom_stype = mom.storage_type(); + CHECK_EQ(weight_stype, mom_stype) << "Inconsistent storage type detected between mom.stype = " + << mom_stype << " and weight.stype = " << weight_stype; + if (weight_stype == kRowSparseStorage && grad_stype == kRowSparseStorage && + mom_stype == kRowSparseStorage) { + NDArray out = outputs[0]; + SGDMomUpdateRspRspRspImpl(param, ctx, weight, grad, mom, req[0], &out); + } else if (weight_stype == kRowSparseStorage && grad_stype == kDefaultStorage && + mom_stype == kRowSparseStorage) { + NDArray out = outputs[0]; + SGDMomUpdateRspDnsImpl(param, ctx, weight, grad.data(), mom, req[0], &out); + } else { + // inputs[2] is a mutable input + FCompExFallback(attrs, ctx, inputs, req, outputs, + SGDMomUpdate, "SGDMomUpdate", {2}); + } +} + struct AdamParam : public dmlc::Parameter { float lr; float beta1; @@ -348,6 +700,147 @@ inline void AdamUpdate(const nnvm::NodeAttrs& attrs, }); } +/*! + * Note: this kernel performs sparse adam update. For each row-slice in row_sparse + * gradient, it finds the corresponding elements in weight, mean and var and performs + * the update. + * The kernel assumes dense weight/mean/var, and row_sparse gradient + */ +template +struct AdamDnsRspDnsKernel { + template + MSHADOW_XINLINE static void Map(int i, const nnvm::dim_t row_length, DType* out_data, + DType* mean_data, DType* var_data, const DType* weight_data, const IType* grad_idx, + const DType* grad_data, const DType clip_gradient, const DType beta1, const DType beta2, + const DType lr, const DType wd, const DType epsilon, const DType rescale_grad) { + using nnvm::dim_t; + using namespace mshadow_op; + const dim_t row_offset = grad_idx[i] * row_length; + for (dim_t j = 0; j < row_length; j++) { + // index in data/mean/var + const dim_t data_i = row_offset + j; + // index in grad + const dim_t grad_i = i * row_length + j; + const DType grad_rescaled = grad_data[grad_i] * rescale_grad + weight_data[data_i] * wd; + if (clip_gradient >= 0.0f) { + mean_data[data_i] = beta1 * mean_data[data_i] + (1.f - beta1) * + clip::Map(grad_rescaled, clip_gradient); + var_data[data_i] = beta2 * var_data[data_i] + (1.f - beta2) * square::Map( + clip::Map(grad_rescaled, clip_gradient)); + } else { + mean_data[data_i] = beta1 * mean_data[data_i] + (1.f - beta1) * grad_rescaled; + var_data[data_i] = beta2 * var_data[data_i] + + (1.f - beta2) * grad_rescaled * grad_rescaled; + } + KERNEL_ASSIGN(out_data[data_i], req, weight_data[data_i] - lr * mean_data[data_i] / + (square_root::Map(var_data[data_i]) + epsilon)); + } + } +}; + + +template +inline void AdamUpdateDnsRspDnsImpl(const AdamParam& param, + const OpContext& ctx, + const TBlob& weight, + const NDArray& grad, + const TBlob& mean, + const TBlob& var, + const OpReqType& req, + TBlob *out) { + using namespace mxnet_op; + using namespace rowsparse; + Stream* s = ctx.get_stream(); + if (!grad.storage_initialized() || req == kNullOp) return; + CHECK_EQ(req, kWriteInplace) << "kWriteInplace is expected for sparse adam_update"; + CHECK_GT(weight.shape_.Size(), 0); + CHECK_GT(mean.shape_.Size(), 0); + CHECK_GT(var.shape_.Size(), 0); + + MSHADOW_REAL_TYPE_SWITCH(weight.type_flag_, DType, { + MSHADOW_IDX_TYPE_SWITCH(grad.aux_type(kIdx), IType, { + MXNET_ASSIGN_REQ_SWITCH(req, req_type, { + const DType* weight_data = weight.dptr(); + const IType* grad_idx = grad.aux_data(kIdx).dptr(); + const DType* grad_val = grad.data().dptr(); + DType* mean_data = mean.dptr(); + DType* var_data = var.dptr(); + DType* out_data = out->dptr(); + nnvm::dim_t num_rows = grad.aux_shape(kIdx)[0]; + const auto row_length = weight.shape_.ProdShape(1, weight.ndim()); + Kernel, xpu>::Launch(s, num_rows, row_length, + out_data, mean_data, var_data, weight_data, grad_idx, grad_val, + static_cast(param.clip_gradient), static_cast(param.beta1), + static_cast(param.beta2), static_cast(param.lr), + static_cast(param.wd), static_cast(param.epsilon), + static_cast(param.rescale_grad)); + }); + }); + }); +} + +template +inline void AdamUpdateRspRspRspImpl(const AdamParam& param, + const OpContext& ctx, + const NDArray& weight, + const NDArray& grad, + const NDArray& mean, + const NDArray& var, + const OpReqType& req, + NDArray *out) { + using namespace mshadow; + using namespace mshadow::expr; + using namespace mxnet_op; + using namespace rowsparse; + CHECK_RSP_ALL_ROWS_NON_ZERO(weight, "AdamUpdate", "weights"); + Stream* s = ctx.get_stream(); + // fill mean and variance with zero values in order to reuse the sgd mom dns impl + if (!mean.storage_initialized()) { + NDArray mean_zeros = mean; + FillDnsZerosRspImpl(s, &mean_zeros); + } + if (!var.storage_initialized()) { + NDArray var_zeros = var; + FillDnsZerosRspImpl(s, &var_zeros); + } + TBlob out_blob = out->data(); + // reuse dns rsp implementation when storage_shape == shape + AdamUpdateDnsRspDnsImpl(param, ctx, weight.data(), grad, mean.data(), + var.data(), req, &out_blob); +} + + +template +inline void AdamUpdateEx(const nnvm::NodeAttrs& attrs, + const OpContext &ctx, + const std::vector &inputs, + const std::vector &req, + const std::vector &outputs) { + const AdamParam& param = nnvm::get(attrs.parsed); + mshadow::Stream* s = ctx.get_stream(); + const auto weight_stype = inputs[0].storage_type(); + const auto grad_stype = inputs[1].storage_type(); + const auto mean_stype = inputs[2].storage_type(); + const auto var_stype = inputs[3].storage_type(); + + const auto out_stype = outputs[0].storage_type(); + CHECK_EQ(mean_stype, weight_stype) << "Inconsistent storage type detected between " + << " mean.stype = " << mean_stype << " and weight.stype = " << weight_stype; + CHECK_EQ(var_stype, weight_stype) << "Inconsistent storage type detected between " + << " var.stype = " << var_stype << " and weight.stype = " << weight_stype; + if (weight_stype == kRowSparseStorage && mean_stype == kRowSparseStorage && + var_stype == kRowSparseStorage && grad_stype == kRowSparseStorage && + out_stype == kRowSparseStorage) { + NDArray out = outputs[0]; + AdamUpdateRspRspRspImpl(param, ctx, inputs[0], inputs[1], inputs[2], + inputs[3], req[0], &out); + } else { + LOG(FATAL) << "Unexpected storage types: weight.stype = " << weight_stype + << ", var.stype = " << var_stype << ", mean.stype = " << mean_stype + << ", grad.stype = " << grad_stype; + } +} + // This RMSProp code follows the version in // http://arxiv.org/pdf/1308.0850v5.pdf Eq(38) - Eq(45) // by Alex Graves, 2013. diff --git a/src/operator/optimizer_op.cc b/src/operator/optimizer_op.cc index b26c333edaef..9b2b088c5095 100644 --- a/src/operator/optimizer_op.cc +++ b/src/operator/optimizer_op.cc @@ -40,6 +40,9 @@ It updates the weights using:: weight = weight - learning_rate * gradient +If weight is stored with `row_sparse` storage type, +only the row slices whose indices appear in grad.indices are updated. + )code" ADD_FILELINE) .set_num_inputs(2) .set_num_outputs(1) @@ -47,6 +50,7 @@ It updates the weights using:: .set_attr("FInferShape", ElemwiseShape<2, 1>) .set_attr("FInferType", ElemwiseType<2, 1>) .set_attr("FCompute", SGDUpdate) +.set_attr("FComputeEx", SGDUpdateEx) .add_argument("weight", "NDArray-or-Symbol", "Weight") .add_argument("grad", "NDArray-or-Symbol", "Gradient") .add_arguments(SGDParam::__FIELDS__()); @@ -70,6 +74,9 @@ It updates the weights using:: Where the parameter ``momentum`` is the decay rate of momentum estimates at each epoch. +If weights are stored with `row_sparse` storage type, +only the row slices whose indices appear in grad.indices are updated (for both weight and momentum). + )code" ADD_FILELINE) .set_num_inputs(3) .set_num_outputs(1) @@ -81,6 +88,7 @@ Where the parameter ``momentum`` is the decay rate of momentum estimates at each return std::vector{2}; }) .set_attr("FCompute", SGDMomUpdate) +.set_attr("FComputeEx", SGDMomUpdateEx) .add_argument("weight", "NDArray-or-Symbol", "Weight") .add_argument("grad", "NDArray-or-Symbol", "Gradient") .add_argument("mom", "NDArray-or-Symbol", "Momentum") @@ -152,6 +160,7 @@ It updates the weights using:: return std::vector{2, 3}; }) .set_attr("FCompute", AdamUpdate) +.set_attr("FComputeEx", AdamUpdateEx) .add_argument("weight", "NDArray-or-Symbol", "Weight") .add_argument("grad", "NDArray-or-Symbol", "Gradient") .add_argument("mean", "NDArray-or-Symbol", "Moving mean") diff --git a/src/operator/optimizer_op.cu b/src/operator/optimizer_op.cu index 0e74e303dbc9..fe45f4be8c66 100644 --- a/src/operator/optimizer_op.cu +++ b/src/operator/optimizer_op.cu @@ -28,10 +28,12 @@ namespace mxnet { namespace op { NNVM_REGISTER_OP(sgd_update) -.set_attr("FCompute", SGDUpdate); +.set_attr("FCompute", SGDUpdate) +.set_attr("FComputeEx", SGDUpdateEx); NNVM_REGISTER_OP(sgd_mom_update) -.set_attr("FCompute", SGDMomUpdate); +.set_attr("FCompute", SGDMomUpdate) +.set_attr("FComputeEx", SGDMomUpdateEx); NNVM_REGISTER_OP(mp_sgd_update) .set_attr("FCompute", MP_SGDUpdate); @@ -40,7 +42,8 @@ NNVM_REGISTER_OP(mp_sgd_mom_update) .set_attr("FCompute", MP_SGDMomUpdate); NNVM_REGISTER_OP(adam_update) -.set_attr("FCompute", AdamUpdate); +.set_attr("FCompute", AdamUpdate) +.set_attr("FComputeEx", AdamUpdateEx); NNVM_REGISTER_OP(rmsprop_update) .set_attr("FCompute", RMSPropUpdate); diff --git a/src/operator/random/sample_op.cc b/src/operator/random/sample_op.cc index 8d87d2b99d14..363163cbc697 100644 --- a/src/operator/random/sample_op.cc +++ b/src/operator/random/sample_op.cc @@ -61,7 +61,8 @@ Example:: [ 0.54488319, 0.84725171]] )code" ADD_FILELINE) -.set_attr("FCompute", SampleUniform_); +.set_attr("FCompute", SampleUniform_) +.set_attr("FComputeEx", SampleUniformEx_); // Add "normal" alias for backward compatibility MXNET_OPERATOR_REGISTER_SAMPLE(random_normal, SampleNormalParam) @@ -78,7 +79,8 @@ Example:: random_normal(loc=0, scale=1, shape=(2,2)) = [[ 1.89171135, -1.16881478], [-1.23474145, 1.55807114]] )code" ADD_FILELINE) -.set_attr("FCompute", SampleNormal_); +.set_attr("FCompute", SampleNormal_) +.set_attr("FComputeEx", SampleNormalEx_); MXNET_OPERATOR_REGISTER_SAMPLE(random_gamma, SampleGammaParam) .add_alias("_sample_gamma") @@ -91,7 +93,8 @@ Example:: random_gamma(alpha=9, beta=0.5, shape=(2,2)) = [[ 7.10486984, 3.37695289], [ 3.91697288, 3.65933681]] )code" ADD_FILELINE) -.set_attr("FCompute", SampleGamma_); +.set_attr("FCompute", SampleGamma_) +.set_attr("FComputeEx", SampleGammaEx_); MXNET_OPERATOR_REGISTER_SAMPLE(random_exponential, SampleExponentialParam) .add_alias("_sample_exponential") diff --git a/src/operator/random/sample_op.cu b/src/operator/random/sample_op.cu index 0d4b2e5a8270..7bdb9faf334e 100644 --- a/src/operator/random/sample_op.cu +++ b/src/operator/random/sample_op.cu @@ -28,21 +28,20 @@ namespace op { // GPU versions of uniform and normal distribution. template<> -void SampleUniform_(const nnvm::NodeAttrs& attrs, - const OpContext& ctx, - const std::vector& inputs, - const std::vector& req, - const std::vector& outputs) { +void SampleUniformDnsImpl(const nnvm::NodeAttrs& attrs, + const OpContext& ctx, + const OpReqType& req, + TBlob* output) { using namespace mxnet::op; using namespace mshadow::expr; typedef gpu xpu; mshadow::Stream *s = ctx.get_stream(); const SampleUniformParam& param = nnvm::get(attrs.parsed); mshadow::Random *prnd = ctx.requested[0].get_random(s); - if (outputs[0].type_flag_ != mshadow::kFloat32) { - MSHADOW_REAL_TYPE_SWITCH(outputs[0].type_flag_, DType, { + if (output->type_flag_ != mshadow::kFloat32) { + MSHADOW_REAL_TYPE_SWITCH(output->type_flag_, DType, { // Not float32: use workspace and copy to output - mshadow::Tensor out = outputs[0].FlatTo2D(s); + mshadow::Tensor out = output->FlatTo2D(s); mshadow::Tensor workspace = ctx.requested[1].get_space_typed (mshadow::Shape1(out.shape_.Size()), s); @@ -51,27 +50,36 @@ void SampleUniform_(const nnvm::NodeAttrs& attrs, }); } else { // float32: write directly into output - mshadow::Tensor out = outputs[0].FlatTo2D(s); + mshadow::Tensor out = output->FlatTo2D(s); prnd->SampleUniform(&out, param.low, param.high); } } template<> -void SampleNormal_(const nnvm::NodeAttrs& attrs, - const OpContext& ctx, - const std::vector& inputs, - const std::vector& req, - const std::vector& outputs) { +void SampleUniform_(const nnvm::NodeAttrs& attrs, + const OpContext& ctx, + const std::vector& inputs, + const std::vector& req, + const std::vector& outputs) { + TBlob out = outputs[0]; + SampleUniformDnsImpl(attrs, ctx, req[0], &out); +} + +template<> +void SampleNormalDnsImpl(const nnvm::NodeAttrs& attrs, + const OpContext& ctx, + const OpReqType& req, + TBlob* output) { using namespace mxnet::op; using namespace mshadow::expr; typedef gpu xpu; mshadow::Stream *s = ctx.get_stream(); const SampleNormalParam& param = nnvm::get(attrs.parsed); mshadow::Random *prnd = ctx.requested[0].get_random(s); - if (outputs[0].type_flag_ != mshadow::kFloat32) { - MSHADOW_REAL_TYPE_SWITCH(outputs[0].type_flag_, DType, { + if (output->type_flag_ != mshadow::kFloat32) { + MSHADOW_REAL_TYPE_SWITCH(output->type_flag_, DType, { // Not float32: use workspace and copy to output - mshadow::Tensor out = outputs[0].FlatTo2D(s); + mshadow::Tensor out = output->FlatTo2D(s); mshadow::Tensor workspace = ctx.requested[1].get_space_typed (mshadow::Shape1(out.shape_.Size()), s); @@ -80,16 +88,28 @@ void SampleNormal_(const nnvm::NodeAttrs& attrs, }); } else { // float32: write directly into output - mshadow::Tensor out = outputs[0].FlatTo2D(s); + mshadow::Tensor out = output->FlatTo2D(s); prnd->SampleGaussian(&out, param.loc, param.scale); } } +template<> +void SampleNormal_(const nnvm::NodeAttrs& attrs, + const OpContext& ctx, + const std::vector& inputs, + const std::vector& req, + const std::vector& outputs) { + TBlob out = outputs[0]; + SampleNormalDnsImpl(attrs, ctx, req[0], &out); +} + NNVM_REGISTER_OP(random_uniform) -.set_attr("FCompute", SampleUniform_); +.set_attr("FCompute", SampleUniform_) +.set_attr("FComputeEx", SampleUniformEx_); NNVM_REGISTER_OP(random_normal) -.set_attr("FCompute", SampleNormal_); +.set_attr("FCompute", SampleNormal_) +.set_attr("FComputeEx", SampleNormalEx_); } // namespace op } // namespace mxnet diff --git a/src/operator/random/sample_op.h b/src/operator/random/sample_op.h index a1a6a2345b1b..0cd3f6bc2efb 100644 --- a/src/operator/random/sample_op.h +++ b/src/operator/random/sample_op.h @@ -232,29 +232,75 @@ struct SampleGenNegBinomialParam : public dmlc::Parameter; + template -void SampleUniform_(const nnvm::NodeAttrs& attrs, - const OpContext& ctx, - const std::vector& inputs, - const std::vector& req, - const std::vector& outputs) { +void SampleComputeEx_(const nnvm::NodeAttrs& attrs, + const OpContext& ctx, + const std::vector& inputs, + const std::vector& req, + const std::vector& outputs, + FSampleCompute fcomp) { + NDArray output = outputs[0]; + mshadow::Stream *s = ctx.get_stream(); + if (output.storage_type() == kRowSparseStorage) { + // indices + nnvm::dim_t nnr = output.shape()[0]; + output.CheckAndAlloc({mshadow::Shape1(nnr)}); + PopulateFullIdxRspImpl(s, &output); + // data + TBlob out_blob = output.data(); + fcomp(attrs, ctx, req[0], &out_blob); + } else { + LOG(FATAL) << "Unexpected storage type for SampleComputeEx_: " + << output.storage_type(); + } +} + +template +void SampleUniformDnsImpl(const nnvm::NodeAttrs& attrs, + const OpContext& ctx, + const OpReqType& req, + TBlob* output) { using namespace mxnet::op; using namespace mshadow::expr; mshadow::Stream *s = ctx.get_stream(); const SampleUniformParam& param = nnvm::get(attrs.parsed); - MSHADOW_REAL_TYPE_SWITCH(outputs[0].type_flag_, DType, { + MSHADOW_REAL_TYPE_SWITCH(output->type_flag_, DType, { mshadow::Random *prnd = ctx.requested[0].get_random(s); - mshadow::Tensor out = outputs[0].FlatTo2D(s); + mshadow::Tensor out = output->FlatTo2D(s); prnd->SampleUniform(&out, param.low, param.high); }); } template -void SampleNormal_(const nnvm::NodeAttrs& attrs, - const OpContext& ctx, - const std::vector& inputs, - const std::vector& req, - const std::vector& outputs) { +void SampleUniform_(const nnvm::NodeAttrs& attrs, + const OpContext& ctx, + const std::vector& inputs, + const std::vector& req, + const std::vector& outputs) { + TBlob out = outputs[0]; + SampleUniformDnsImpl(attrs, ctx, req[0], &out); +} + + +template +void SampleUniformEx_(const nnvm::NodeAttrs& attrs, + const OpContext& ctx, + const std::vector& inputs, + const std::vector& req, + const std::vector& outputs) { + SampleComputeEx_(attrs, ctx, inputs, req, outputs, SampleUniformDnsImpl); +} + +template +void SampleNormalDnsImpl(const nnvm::NodeAttrs& attrs, + const OpContext& ctx, + const OpReqType& req, + TBlob* outputs) { using namespace mxnet::op; using namespace mshadow::expr; mshadow::Stream *s = ctx.get_stream(); @@ -268,11 +314,29 @@ void SampleNormal_(const nnvm::NodeAttrs& attrs, } template -void SampleGamma_(const nnvm::NodeAttrs& attrs, +void SampleNormal_(const nnvm::NodeAttrs& attrs, const OpContext& ctx, const std::vector& inputs, const std::vector& req, const std::vector& outputs) { + TBlob out = outputs[0]; + SampleNormalDnsImpl(attrs, ctx, req[0], &out); +} + +template +void SampleNormalEx_(const nnvm::NodeAttrs& attrs, + const OpContext& ctx, + const std::vector& inputs, + const std::vector& req, + const std::vector& outputs) { + SampleComputeEx_(attrs, ctx, inputs, req, outputs, SampleNormalDnsImpl); +} + +template +void SampleGammaDnsImpl(const nnvm::NodeAttrs& attrs, + const OpContext& ctx, + const OpReqType& req, + TBlob* outputs) { using namespace mxnet::op; using namespace mshadow::expr; mshadow::Stream *s = ctx.get_stream(); @@ -286,6 +350,25 @@ void SampleGamma_(const nnvm::NodeAttrs& attrs, }); } +template +void SampleGamma_(const nnvm::NodeAttrs& attrs, + const OpContext& ctx, + const std::vector& inputs, + const std::vector& req, + const std::vector& outputs) { + TBlob out = outputs[0]; + SampleGammaDnsImpl(attrs, ctx, req[0], &out); +} + +template +void SampleGammaEx_(const nnvm::NodeAttrs& attrs, + const OpContext& ctx, + const std::vector& inputs, + const std::vector& req, + const std::vector& outputs) { + SampleComputeEx_(attrs, ctx, inputs, req, outputs, SampleGammaDnsImpl); +} + template void SampleExponential_(const nnvm::NodeAttrs& attrs, const OpContext& ctx, diff --git a/src/operator/tensor/cast_storage-inl.cuh b/src/operator/tensor/cast_storage-inl.cuh new file mode 100644 index 000000000000..afef53e979ea --- /dev/null +++ b/src/operator/tensor/cast_storage-inl.cuh @@ -0,0 +1,589 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * Copyright (c) 2017 by Contributors + * \file cast_storage-inl.cuh + * \brief implementation of cast_storage op on GPU + */ +#ifndef MXNET_OPERATOR_TENSOR_CAST_STORAGE_INL_CUH_ +#define MXNET_OPERATOR_TENSOR_CAST_STORAGE_INL_CUH_ + +#include +#include +#include +#include +#include "./util/tensor_util-inl.cuh" + +namespace mxnet { +namespace op { + +/*! + * \brief GPU Kernel for filling the value array of the rsp tensor. + * Parallelized by rsp tensor elements: 1 thread/element + */ +struct CastDnsRspValsKernel { + /*! + * \brief + * \param tid global thread id + * \param rsp_val value array of rsp tensor to store data + * \param row_idx indices of non-zero rows + * \param dns dense matrix data + * \param nnr number of non-zero rows + * \param row_length number of elements per row + */ + template + __device__ __forceinline__ static void Map(int tid, + DType* rsp_val, + const RType* row_idx, + const DType* dns, + const nnvm::dim_t nnr, + const nnvm::dim_t row_length) { + using nnvm::dim_t; + if (tid < nnr*row_length) { + const dim_t row_id = tid / row_length; + const dim_t row_el = tid % row_length; + const dim_t dns_idx = row_idx[row_id] * row_length + row_el; + rsp_val[tid] = dns[dns_idx]; + } + } +}; + +template +inline mshadow::Tensor AllocateTempDataForCast(const OpContext& op_ctx, + const mshadow::Shape& shape) { + Resource rsc = ResourceManager::Get()->Request(op_ctx.run_ctx.ctx, + ResourceRequest(ResourceRequest::kTempSpace)); + mshadow::Stream *stream = op_ctx.run_ctx.get_stream(); + return rsc.get_space_typed(shape, stream); +}; + +/*! + * \brief GPU implementation of casting a dns tensor to rsp type. + */ +inline void CastStorageDnsRspImpl(const OpContext& ctx, + const gpu& gpu_dev, + const TBlob& dns, + NDArray* rsp) { + CHECK(rsp != nullptr); + CHECK_EQ(rsp->storage_type(), kRowSparseStorage); + CHECK_EQ(dns.shape_, rsp->shape()); + using mshadow::Shape1; + using mxnet_op::Kernel; + using nnvm::dim_t; + mshadow::Stream* s = ctx.get_stream(); + MSHADOW_TYPE_SWITCH(dns.type_flag_, DType, { // data type + MSHADOW_IDX_TYPE_SWITCH(rsp->aux_type(rowsparse::kIdx), RType, { // row idx type + const dim_t num_rows = dns.shape_[0]; + const dim_t row_length = dns.shape_.ProdShape(1, dns.shape_.ndim()); + const dim_t threads_per_warp = mxnet_op::cuda_get_device_prop().warpSize; + const dim_t threads_per_block = mshadow::cuda::kBaseThreadNum; + const dim_t min_num_warps = 512; + dim_t num_threads; + // TODO: remove kernel dependency on warpSize=32 + if (threads_per_warp != 32) { + LOG(FATAL) << "CastStorageDnsRspImpl GPU kernels expect warpSize=32"; + } + // Determine temporary device storage requirements + dim_t* row_flg = NULL; + void* d_temp_storage = NULL; + size_t temp_storage_bytes = 0; + cub::DeviceScan::InclusiveSum(d_temp_storage, + temp_storage_bytes, + row_flg, + row_flg, + num_rows, + mshadow::Stream::GetStream(s)); + + // Allocate temp storage for marking non-zero rows and for cub's prefix sum + auto workspace = AllocateTempDataForCast(ctx, Shape1(num_rows*sizeof(dim_t) + + temp_storage_bytes)); + row_flg = reinterpret_cast(workspace.dptr_); + d_temp_storage = workspace.dptr_ + num_rows*sizeof(dim_t); + + // Mark non-zero rows as 'one' in row_flg + // Different kernel versions are optimized for different matrix instances + // (1) 'Thread kernel' (one thread computing one row) + // (2) 'Warp kernel' (one warp computing one row) + // (3) 'Block kernel' (one thread block computing one row) + const int kernel_version = 0; + switch (kernel_version) { + case 1: + num_threads = num_rows; + Kernel::Launch(s, num_threads, + row_flg, dns.dptr(), num_rows, row_length); + break; + case 2: + num_threads = num_rows * threads_per_warp; + Kernel::Launch(s, num_threads, + row_flg, dns.dptr(), num_rows, row_length); + break; + case 3: + num_threads = num_rows * threads_per_block; + Kernel::Launch(s, num_threads, + row_flg, dns.dptr(), num_rows, row_length); + break; + default: + if (row_length < threads_per_warp) { + num_threads = num_rows; + Kernel::Launch(s, num_threads, + row_flg, dns.dptr(), num_rows, row_length); + } else if (row_length < threads_per_block || num_rows > min_num_warps) { + num_threads = num_rows * threads_per_warp; + Kernel::Launch(s, num_threads, + row_flg, dns.dptr(), num_rows, row_length); + } else { + num_threads = num_rows * threads_per_block; + Kernel::Launch(s, num_threads, + row_flg, dns.dptr(), num_rows, row_length); + } + break; + } + // Compute non-zero row indices through inclusive prefix sum + cub::DeviceScan::InclusiveSum(d_temp_storage, + temp_storage_bytes, + row_flg, + row_flg, + num_rows, + mshadow::Stream::GetStream(s)); + + // Get total number of non-zero rows from device + dim_t nnr = 0; + CUDA_CALL(cudaMemcpy(&nnr, &row_flg[num_rows-1], sizeof(dim_t), cudaMemcpyDeviceToHost)); + + // Allocate rsp tensor row index array and fill + rsp->CheckAndAllocAuxData(rowsparse::kIdx, Shape1(nnr)); + if (0 == nnr) return; + RType* row_idx = rsp->aux_data(rowsparse::kIdx).dptr(); + num_threads = num_rows; + Kernel::Launch(s, num_threads, + row_idx, row_flg, num_rows); + + // Construct shape of rsp tensor data, allocate, and fill + auto storage_shape = dns.shape_; + storage_shape[0] = nnr; + rsp->CheckAndAllocData(storage_shape); + num_threads = nnr * row_length; + Kernel::Launch(s, num_threads, + rsp->data().dptr(), row_idx, dns.dptr(), nnr, row_length); + }); + }); +} + +/*! + * \brief Thread kernel for initializing the indptr in a csr matrix. + * Parallelized by matrix rows: 1 thread/row + */ +struct CastDnsCsrIndPtrThreadKernel { + /*! + * \brief + * \param tid global thread id + * \param indptr index pointer array of the csr matrix + * \param dns dense matrix + * \param num_rows number of rows of the dense matrix + * \param num_cols number of columns of the dense matrix + */ + template + __device__ __forceinline__ static void Map(int tid, + IType* indptr, + const DType* dns, + const nnvm::dim_t num_rows, + const nnvm::dim_t num_cols) { + using nnvm::dim_t; + if (tid == 0) { + indptr[tid] = 0; + } + if (tid < num_rows) { + dim_t nnz = 0; + const dim_t offset = tid * num_cols; + for (dim_t j = 0; j < num_cols; ++j) { + if (dns[offset+j] != 0) { + nnz++; + } + } + indptr[tid+1] = nnz; + } + } +}; + +/*! + * \brief Thread kernel for initializing the col_idx and value array of the csr matrix. + * Parallelized by matrix rows: 1 thread/row + */ +struct CastDnsCsrColIdxAndValsThreadKernel { + /*! + * \brief + * \param tid global thread id + * \param val data array of the csr matrix + * \param col_idx column index array of the csr matrix + * \param indptr index pointer array of the csr matrix + * \param dns dense matrix + * \param num_rows number of rows of the dense matrix + * \param num_cols number of columns of the dense matrix + */ + template + __device__ __forceinline__ static void Map(int tid, + DType* val, + CType* col_idx, + const IType* indptr, + const DType* dns, + const nnvm::dim_t num_rows, + const nnvm::dim_t num_cols) { + using nnvm::dim_t; + if (tid < num_rows) { + const dim_t offset = tid * num_cols; + dim_t k = indptr[tid]; + for (dim_t j = 0; j < num_cols; ++j) { + if (dns[offset+j] != 0) { + val[k] = dns[offset+j]; + col_idx[k] = j; + ++k; + } + } + } + } +}; + +/*! + * \brief Warp kernel for initializing the indptr in a csr matrix. + * Parallelized by matrix rows: 1 warp/row + */ +struct CastDnsCsrIndPtrWarpKernel { + template + __device__ __forceinline__ static void Map(int tid, + IType* indptr, + const DType* dns, + const nnvm::dim_t num_rows, + const nnvm::dim_t num_cols) { + using nnvm::dim_t; + typedef cub::WarpReduce WarpReduce; + const dim_t warps_per_block = mshadow::cuda::kBaseThreadNum / 32; + __shared__ typename WarpReduce::TempStorage temp_storage[warps_per_block]; + + if (tid == 0) { + indptr[tid] = 0; + } + const dim_t warp_id = tid / 32; // global warp id + const dim_t warp_lane = threadIdx.x / 32; // local warp id within thread block + const dim_t lane = tid & (32-1); // local thread id within warp + if (warp_id < num_rows) { + dim_t lane_nnz = 0; + const dim_t offset = warp_id * num_cols; + for (dim_t j = lane; j < num_cols; j+=32) { + if (dns[offset+j] != 0) { + lane_nnz++; + } + } + dim_t aggr = WarpReduce(temp_storage[warp_lane]).Sum(lane_nnz); + if (lane == 0) { + indptr[warp_id+1] = aggr; + } + } + } +}; + +/*! + * \brief Warp kernel for initializing the col_idx and value array of the csr matrix. + * Parallelized by matrix rows: 1 warp/row + */ +struct CastDnsCsrColIdxAndValsWarpKernel { + template + __device__ __forceinline__ static void Map(int tid, + DType* val, + CType* col_idx, + const IType* indptr, + const DType* dns, + const nnvm::dim_t num_rows, + const nnvm::dim_t num_cols) { + using nnvm::dim_t; + typedef cub::WarpScan WarpScan; + const dim_t warps_per_block = mshadow::cuda::kBaseThreadNum / 32; + __shared__ typename WarpScan::TempStorage temp_storage[warps_per_block]; + __shared__ volatile dim_t warp_nnz[warps_per_block]; + + const dim_t warp_id = tid / 32; // global warp id + const dim_t warp_lane = threadIdx.x / 32; // local warp id within thread block + const dim_t lane = tid & (32-1); // local thread id within warp + if (warp_id < num_rows) { + const dim_t offset = warp_id * num_cols; + dim_t k = indptr[warp_id]; + dim_t nnz; + for (dim_t j = lane; j < num_cols+lane; j+=32) { + nnz = 0; + if (j < num_cols) { + if (dns[offset+j] != 0) { + nnz++; + } + } + if (lane == 31) { + warp_nnz[warp_lane] = nnz; + } + // Compute index each thread has to write to + WarpScan(temp_storage[warp_lane]).ExclusiveSum(nnz, nnz); + if (j < num_cols) { + if (dns[offset+j] != 0) { + val[k+nnz] = dns[offset+j]; + col_idx[k+nnz] = j; + } + } + if (lane == 31) { + warp_nnz[warp_lane] += nnz; + } + __syncwarp(); + k += warp_nnz[warp_lane]; + } + } + } +}; + +/*! + * \brief Block kernel for initializing the indptr in a csr matrix. + * Parallelized by matrix rows: 1 threadBlock/row + */ +struct CastDnsCsrIndPtrBlockKernel { + template + __device__ __forceinline__ static void Map(int tid, + IType* indptr, + const DType* dns, + const nnvm::dim_t num_rows, + const nnvm::dim_t num_cols) { + using mshadow::cuda::kBaseThreadNum; + using nnvm::dim_t; + typedef cub::BlockReduce BlockReduce; + __shared__ typename BlockReduce::TempStorage temp_storage; + + if (tid == 0) { + indptr[tid] = 0; + } + if (blockIdx.x < num_rows) { + dim_t lane_nnz = 0; + const dim_t offset = blockIdx.x * num_cols; + for (dim_t j = threadIdx.x; j < num_cols; j+=kBaseThreadNum) { + if (dns[offset+j] != 0) { + lane_nnz++; + } + } + dim_t aggr = BlockReduce(temp_storage).Sum(lane_nnz); + if (threadIdx.x == 0) { + indptr[blockIdx.x+1] = aggr; + } + } + } +}; + +/*! + * \brief Block kernel for initializing the col_idx and value array of the csr matrix. + * Parallelized by matrix rows: 1 threadBlock/row + */ +struct CastDnsCsrColIdxAndValsBlockKernel { + template + __device__ __forceinline__ static void Map(int tid, + DType* val, + CType* col_idx, + const IType* indptr, + const DType* dns, + const nnvm::dim_t num_rows, + const nnvm::dim_t num_cols) { + using mshadow::cuda::kBaseThreadNum; + using nnvm::dim_t; + typedef cub::BlockScan BlockScan; + __shared__ typename BlockScan::TempStorage temp_storage; + __shared__ volatile dim_t block_nnz; + + if (blockIdx.x < num_rows) { + const dim_t offset = blockIdx.x * num_cols; + dim_t k = indptr[blockIdx.x]; + dim_t nnz; + for (dim_t j = threadIdx.x; j < num_cols+threadIdx.x; j+=kBaseThreadNum) { + nnz = 0; + if (j < num_cols) { + if (dns[offset+j] != 0) { + nnz++; + } + } + if (threadIdx.x == kBaseThreadNum-1) { + block_nnz = nnz; + } + // Compute index each thread has to write to + BlockScan(temp_storage).ExclusiveSum(nnz, nnz); + if (j < num_cols) { + if (dns[offset+j] != 0) { + val[k+nnz] = dns[offset+j]; + col_idx[k+nnz] = j; + } + } + if (threadIdx.x == kBaseThreadNum-1) { + block_nnz += nnz; + } + __syncthreads(); + k += block_nnz; + } + } + } +}; + +/*! + * \brief GPU implementation of casting a dense matrix to csr type. + */ +inline void CastStorageDnsCsrImpl(const OpContext& ctx, + const gpu& gpu_dev, + const TBlob& dns, + NDArray* csr) { + CHECK(csr != nullptr); + CHECK_EQ(csr->storage_type(), kCSRStorage); + CHECK_EQ(dns.shape_.ndim(), 2); + CHECK_EQ(dns.shape_, csr->shape()); + using mshadow::Shape1; + using mxnet_op::Kernel; + using nnvm::dim_t; + mshadow::Stream* s = ctx.get_stream(); + MSHADOW_TYPE_SWITCH(dns.type_flag_, DType, { // data type + MSHADOW_IDX_TYPE_SWITCH(csr->aux_type(csr::kIndPtr), IType, { // indptr type + MSHADOW_IDX_TYPE_SWITCH(csr->aux_type(csr::kIdx), CType, { // col_idx type + const dim_t num_rows = dns.shape_[0]; + const dim_t num_cols = dns.shape_[1]; + const dim_t threads_per_warp = mxnet_op::cuda_get_device_prop().warpSize; + const dim_t threads_per_block = mshadow::cuda::kBaseThreadNum; + const dim_t min_num_warps = 512; + dim_t num_threads; + // TODO: remove kernel dependency on warpSize=32 + if (threads_per_warp != 32) { + LOG(FATAL) << "CastStorageDnsCsrImpl GPU kernels expect warpSize=32"; + } + csr->CheckAndAllocAuxData(csr::kIndPtr, Shape1(num_rows+1)); + IType* indptr = csr->aux_data(csr::kIndPtr).dptr(); + DType* dns_data = dns.dptr(); + + // Different kernel versions are optimized for different matrix instances + // (1) 'Thread kernel' (one thread computing one row) + // (2) 'Warp kernel' (one warp computing one row) + // (3) 'Block kernel' (one thread block computing one row) + const int kernel_version = 0; + switch (kernel_version) { + case 1: + num_threads = num_rows; + Kernel::Launch(s, num_threads, + indptr, dns_data, num_rows, num_cols); + break; + case 2: + num_threads = num_rows * threads_per_warp; + Kernel::Launch(s, num_threads, + indptr, dns_data, num_rows, num_cols); + break; + case 3: + num_threads = num_rows * threads_per_block; + Kernel::Launch(s, num_threads, + indptr, dns_data, num_rows, num_cols); + break; + default: + if (num_cols < threads_per_warp) { + num_threads = num_rows; + Kernel::Launch(s, num_threads, + indptr, dns_data, num_rows, num_cols); + } else if (num_cols < threads_per_block || num_rows > min_num_warps) { + num_threads = num_rows * threads_per_warp; + Kernel::Launch(s, num_threads, + indptr, dns_data, num_rows, num_cols); + } else { + num_threads = num_rows * threads_per_block; + Kernel::Launch(s, num_threads, + indptr, dns_data, num_rows, num_cols); + } + break; + } + + // Determine temporary device storage requirements + void *d_temp_storage = NULL; + size_t temp_storage_bytes = 0; + cub::DeviceScan::InclusiveSum(d_temp_storage, + temp_storage_bytes, + indptr, + indptr, + num_rows+1, + mshadow::Stream::GetStream(s)); + + // Allocate temporary storage + auto workspace = AllocateTempDataForCast(ctx, Shape1(temp_storage_bytes)); + + d_temp_storage = workspace.dptr_; + + // Compute indptr through inclusive prefix sum + cub::DeviceScan::InclusiveSum(d_temp_storage, + temp_storage_bytes, + indptr, + indptr, + num_rows+1, + mshadow::Stream::GetStream(s)); + + // Receive total number of nnz values from device + IType nnz = 0; + CUDA_CALL(cudaMemcpy(&nnz, &(indptr[num_rows]), sizeof(IType), cudaMemcpyDeviceToHost)); + + // Allocate column index array and data array of the csr matrix + csr->CheckAndAllocAuxData(csr::kIdx, Shape1(static_cast(nnz))); + csr->CheckAndAllocData(Shape1(static_cast(nnz))); + + // Compute and fill column index array and data array of the csr matrix + switch (kernel_version) { + case 1: + num_threads = num_rows; + Kernel::Launch(s, num_threads, + csr->data().dptr(), csr->aux_data(csr::kIdx).dptr(), + indptr, dns_data, num_rows, num_cols); + break; + case 2: + num_threads = num_rows * threads_per_warp; + Kernel::Launch(s, num_threads, + csr->data().dptr(), csr->aux_data(csr::kIdx).dptr(), + indptr, dns_data, num_rows, num_cols); + break; + case 3: + num_threads = num_rows * threads_per_block; + Kernel::Launch(s, num_threads, + csr->data().dptr(), csr->aux_data(csr::kIdx).dptr(), + indptr, dns_data, num_rows, num_cols); + break; + default: + if (num_cols < threads_per_warp) { + num_threads = num_rows; + Kernel::Launch(s, num_threads, + csr->data().dptr(), csr->aux_data(csr::kIdx).dptr(), + indptr, dns_data, num_rows, num_cols); + } else if (num_cols < threads_per_block || num_rows > min_num_warps) { + num_threads = num_rows * threads_per_warp; + Kernel::Launch(s, num_threads, + csr->data().dptr(), csr->aux_data(csr::kIdx).dptr(), + indptr, dns_data, num_rows, num_cols); + } else { + num_threads = num_rows * threads_per_block; + Kernel::Launch(s, num_threads, + csr->data().dptr(), csr->aux_data(csr::kIdx).dptr(), + indptr, dns_data, num_rows, num_cols); + } + break; + } + }); + }); + }); +} + +} // namespace op +} // namespace mxnet + +#endif // MXNET_OPERATOR_TENSOR_CAST_STORAGE_INL_CUH_ diff --git a/src/operator/tensor/cast_storage-inl.h b/src/operator/tensor/cast_storage-inl.h new file mode 100644 index 000000000000..acb30a9eff2b --- /dev/null +++ b/src/operator/tensor/cast_storage-inl.h @@ -0,0 +1,392 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file cast_storage-inl.h + * \brief cast_storage implementation for dense and sparse tensors + */ +#ifndef MXNET_OPERATOR_TENSOR_CAST_STORAGE_INL_H_ +#define MXNET_OPERATOR_TENSOR_CAST_STORAGE_INL_H_ + +#include +#include +#include +#include "../mxnet_op.h" +#include "../operator_common.h" +#ifdef __CUDACC__ +#include "./cast_storage-inl.cuh" +#endif // __CUDACC__ + + +namespace mxnet { +namespace op { + +/*! + * \brief CPU Kernel for marking row_idx of a RSP tensor per row. + */ +struct MarkRspRowIdx { + // i represents the row index of the tensor data + template + MSHADOW_CINLINE static void Map(int i, + RType* row_idx, + const DType* data, + const nnvm::dim_t row_length) { + using nnvm::dim_t; + dim_t j = 0; + dim_t offset = i * row_length; + for (; j < row_length; ++j) { + if (data[offset+j] != 0) { + break; + } + } + if (row_length == j) { + row_idx[i] = 0; // mark as zero for zero row + } else { + row_idx[i] = 1; // mark as one for non-zero row + } + } +}; + +/*! + * \brief CPU implementation of casting a dns tensor to rsp type. + */ +inline void CastStorageDnsRspImpl(const OpContext& ctx, + const cpu& cpu_dev, + const TBlob& dns, + NDArray* rsp) { + using namespace rowsparse; + using namespace mshadow; + using nnvm::dim_t; + CHECK(rsp != nullptr); + CHECK_EQ(rsp->storage_type(), kRowSparseStorage); + CHECK_EQ(dns.shape_, rsp->shape()); + mshadow::Stream* s = ctx.get_stream(); + MSHADOW_TYPE_SWITCH(dns.type_flag_, DType, { // data type + MSHADOW_IDX_TYPE_SWITCH(rsp->aux_type(kIdx), RType, { // row idx type + const dim_t num_rows = dns.shape_[0]; + const dim_t row_length = dns.shape_.ProdShape(1, dns.shape_.ndim()); + rsp->CheckAndAllocAuxData(kIdx, Shape1(num_rows)); + TBlob row_idx_blob = rsp->aux_data(kIdx); + RType* row_idx = row_idx_blob.dptr(); + dim_t num_threads = num_rows; + mxnet_op::Kernel::Launch(s, num_threads, + row_idx, dns.dptr(), row_length); + dim_t nnr = 0; + nnr = common::ParallelAccumulate(row_idx, num_rows, nnr); + rsp->set_aux_shape(kIdx, Shape1(nnr)); + if (0 == nnr) return; + auto storage_shape = dns.shape_; + storage_shape[0] = nnr; + rsp->CheckAndAllocData(storage_shape); + auto dns_data = dns.get_with_shape(Shape2(num_rows, row_length), s); + auto rsp_data = rsp->data().get_with_shape(Shape2(nnr, row_length), s); + dim_t idx = 0; + for (dim_t i = 0; i < num_rows; ++i) { + if (row_idx[i] > 0) { + row_idx[idx] = i; + Copy(rsp_data[idx], dns_data[i], s); + ++idx; + } + } + }); + }); +} + +// TODO(haibin) Use memcopy instead will be much faster than assigning each individual element +struct CastStorageRspDnsKernel { + template + MSHADOW_XINLINE static void Map(int i, + const nnvm::dim_t row_length, + const IType* idx, + const DType *data, + DType* dns) { + using nnvm::dim_t; + IType rid = idx[i]; + dim_t dns_offset = rid * row_length; + dim_t rsp_offset = i * row_length; + for (dim_t col = 0; col < row_length; col++) { + dns[dns_offset + col] = data[rsp_offset + col]; + } + } +}; + +/*! + * \brief This function assumes that the memory for dns has been allocated already + * since the shape is known at binding stage. + */ +template +void CastStorageRspDnsImpl(const OpContext& ctx, + const NDArray& rsp, + TBlob* dns) { + mshadow::Stream* s = ctx.get_stream(); + CHECK_EQ(rsp.storage_type(), kRowSparseStorage); + using nnvm::dim_t; + MSHADOW_TYPE_SWITCH(dns->type_flag_, DType, { + MSHADOW_IDX_TYPE_SWITCH(rsp.aux_type(rowsparse::kIdx), IType, { + // assign zeros + mxnet_op::Kernel::Launch(s, dns->Size(), dns->dptr()); + if (rsp.storage_initialized()) { + // copy over row by row + auto in_idx = rsp.aux_data(rowsparse::kIdx).FlatTo1D(s).dptr_; + auto in_data = rsp.data().dptr(); + auto out_data = dns->dptr(); + auto shape = rsp.shape(); + const dim_t num_rows = rsp.aux_shape(rowsparse::kIdx).Size(); + const dim_t row_length = shape.ProdShape(1, shape.ndim()); + const dim_t num_threads = num_rows; + mxnet_op::Kernel::Launch(s, num_threads, + row_length, in_idx, in_data, out_data); + } + }); + }); +} + +/*! + * \brief CPU kernel for initializing the indptr in a csr matrix. + */ +struct FillCsrIndPtr { + /*! + * \brief + * \param i the i-th row of the dns tensor + * \param indptr the indptr of the csr tensor + * \param dns the dns tensor + * \param num_rows number of rows of the dns tensor + * \param num_cols number of columns of the dns tensor + */ + template + MSHADOW_CINLINE static void Map(int i, + IType* indptr, + const DType* dns, + const nnvm::dim_t num_rows, + const nnvm::dim_t num_cols) { + using nnvm::dim_t; + indptr[i+1] = 0; + const dim_t offset = i * num_cols; + for (dim_t j = 0; j < num_cols; ++j) { + if (dns[offset+j] != 0) { + ++indptr[i+1]; + } + } + } +}; + +/*! + * \brief CPU kernel for initializing the col_idx and value array of the csr matrix. + */ +struct FillCsrColIdxAndVals { + /*! + * \brief + * \param i the i-th row of the dns tensor + * \param val value array of the csr tensor + * \param col_idx column idx array of the csr tensor + * \param indptr indptr array of the csr tensor + * \param dns dns tensor + * \param num_rows number of rows of the dns tensor + * \param num_cols number of columns of the dns tensor + */ + template + MSHADOW_CINLINE static void Map(int i, + DType* val, + CType* col_idx, + const IType* indptr, + const DType* dns, + const nnvm::dim_t num_rows, + const nnvm::dim_t num_cols) { + using nnvm::dim_t; + const dim_t offset = i * num_cols; + IType k = indptr[i]; + for (dim_t j = 0; j < num_cols; ++j) { + if (dns[offset+j] != 0) { + val[k] = dns[offset+j]; + col_idx[k] = j; + ++k; + } + } + } +}; + +/*! + * \brief CPU implementation of casting a dns matrix to csr type. + */ +inline void CastStorageDnsCsrImpl(const OpContext& ctx, + const cpu& cpu_dev, + const TBlob& dns, + NDArray* csr) { + CHECK(csr != nullptr); + CHECK_EQ(csr->storage_type(), kCSRStorage); + CHECK_EQ(dns.shape_.ndim(), 2); + CHECK_EQ(dns.shape_, csr->shape()); + using mshadow::Shape1; + using nnvm::dim_t; + mshadow::Stream* s = ctx.get_stream(); + MSHADOW_TYPE_SWITCH(dns.type_flag_, DType, { // data type + MSHADOW_IDX_TYPE_SWITCH(csr->aux_type(csr::kIndPtr), IType, { // indptr type + MSHADOW_IDX_TYPE_SWITCH(csr->aux_type(csr::kIdx), CType, { // col idx type + const dim_t num_rows = dns.shape_[0]; + const dim_t num_cols = dns.shape_[1]; + csr->CheckAndAllocAuxData(csr::kIndPtr, mshadow::Shape1(num_rows+1)); + IType* indptr = csr->aux_data(csr::kIndPtr).dptr(); + DType* dns_data = dns.dptr(); + dim_t num_threads = num_rows; + mxnet_op::Kernel::Launch(s, num_threads, + indptr, dns_data, num_rows, num_cols); + // single thread to accumulate indptr + // indptr[num_rows] indicates the number of non-zero elements + indptr[0] = 0; + for (dim_t i = 0; i < num_rows; ++i) { + indptr[i+1] += indptr[i]; + } + // allocate column idx array and value array + csr->CheckAndAllocAuxData(csr::kIdx, Shape1(static_cast(indptr[num_rows]))); + csr->CheckAndAllocData(Shape1(static_cast(indptr[num_rows]))); + // fill col_idx and value arrays of the csr + mxnet_op::Kernel::Launch(s, num_threads, + csr->data().dptr(), csr->aux_data(csr::kIdx).dptr(), + indptr, dns_data, num_rows, num_cols); + }); + }); + }); +} + +/*! + * \brief This is the kernel for copying csr.data to its corresponding dns matrix. + */ +struct CopyCsrDataToDns { + /*! + * \brief + * \param i the i-th row of the dns tensor + * \param dns_data data blob of the dns tensor + * \param col_idx column idx array of the csr tensor + * \param indptr indptr array of the csr tensor + * \param csr_data data blob of the csr tensor + * \param num_cols number of columns of the dns tensor + */ + template + MSHADOW_XINLINE static void Map(int i, + DType* dns_data, + const CType* col_idx, + const IType* indptr, + const DType* csr_data, + const nnvm::dim_t num_cols) { + const nnvm::dim_t offset = i * num_cols; + for (IType j = indptr[i]; j < indptr[i+1]; ++j) { + dns_data[offset+col_idx[j]] = csr_data[j]; + } + } +}; + +/*! + * \brief Casts a csr matrix to dns format. + */ +template +void CastStorageCsrDnsImpl(const OpContext& ctx, + const NDArray& csr, + TBlob* dns) { + CHECK(dns != nullptr); + CHECK_EQ(csr.storage_type(), kCSRStorage); + CHECK_EQ(dns->shape_.ndim(), 2); + CHECK_EQ(dns->shape_, csr.shape()); + using nnvm::dim_t; + mshadow::Stream* s = ctx.get_stream(); + MSHADOW_TYPE_SWITCH(dns->type_flag_, DType, { // data type + MSHADOW_IDX_TYPE_SWITCH(csr.aux_type(csr::kIndPtr), IType, { // indptr type + MSHADOW_IDX_TYPE_SWITCH(csr.aux_type(csr::kIdx), CType, { // col idx type + const dim_t num_rows = dns->shape_[0]; + const dim_t num_cols = dns->shape_[1]; + DType* dns_data = dns->dptr(); + dim_t num_threads = dns->shape_.Size(); + mxnet_op::Kernel::Launch(s, num_threads, dns_data); + if (!csr.storage_initialized()) return; + const IType* indptr = csr.aux_data(csr::kIndPtr).dptr(); + const CType* col_idx = csr.aux_data(csr::kIdx).dptr(); + const DType* csr_data = csr.data().dptr(); + num_threads = num_rows; + mxnet_op::Kernel::Launch(s, num_threads, + dns_data, col_idx, indptr, csr_data, num_cols); + }); + }); + }); +} + +template +void CastStorageComputeImpl(const OpContext& ctx, + const NDArray& input, + const NDArray& output) { + const auto src_stype = input.storage_type(); + const auto dst_stype = output.storage_type(); + if (src_stype == kRowSparseStorage && dst_stype == kDefaultStorage) { + TBlob ret = output.data(); + CastStorageRspDnsImpl(ctx, input, &ret); + } else if (src_stype == kDefaultStorage && dst_stype == kRowSparseStorage) { + NDArray ret = output; // get rid of the const qualifer + CastStorageDnsRspImpl(ctx, xpu(), input.data(), &ret); + } else if (src_stype == kDefaultStorage && dst_stype == kCSRStorage) { + NDArray ret = output; // get rid of the const qualifer + CastStorageDnsCsrImpl(ctx, xpu(), input.data(), &ret); + } else if (src_stype == kCSRStorage && dst_stype == kDefaultStorage) { + TBlob ret = output.data(); + CastStorageCsrDnsImpl(ctx, input, &ret); + } else { + LOG(FATAL) << "Not implemented"; + } +} + +struct CastStorageParam : public dmlc::Parameter { + int stype; + DMLC_DECLARE_PARAMETER(CastStorageParam) { + DMLC_DECLARE_FIELD(stype) + .add_enum("default", kDefaultStorage) + .add_enum("row_sparse", kRowSparseStorage) + .add_enum("csr", kCSRStorage) + .describe("Output storage type."); + } +}; + +inline bool CastStorageInferStorageType(const nnvm::NodeAttrs& attrs, + const Context& ctx, + std::vector *in_attrs, + std::vector *out_attrs) { + CHECK_EQ(in_attrs->size(), 1U); + CHECK_EQ(out_attrs->size(), 1U); + CHECK_NE(in_attrs->at(0), kUndefinedStorage) + << "src ndarray's storage type must be specified"; + const CastStorageParam& param = nnvm::get(attrs.parsed); + CHECK_NE(param.stype, kUndefinedStorage) + << "dst ndarray's storage type must be specified"; + TYPE_ASSIGN_CHECK(*out_attrs, 0, param.stype); + return true; +} + +template +void CastStorageComputeEx(const nnvm::NodeAttrs& attrs, + const OpContext& ctx, + const std::vector& inputs, + const std::vector& req, + const std::vector& outputs) { + CHECK_EQ(inputs.size(), 1); + CHECK_EQ(outputs.size(), 1); + if (req[0] == kNullOp) return; + CHECK_EQ(req[0], kWriteTo) << "CastStorageComputeEx expects req[0] == kWriteTo"; + CastStorageComputeImpl(ctx, inputs[0], outputs[0]); +} + +} // namespace op +} // namespace mxnet + +#endif // MXNET_OPERATOR_TENSOR_CAST_STORAGE_INL_H_ diff --git a/src/operator/tensor/cast_storage.cc b/src/operator/tensor/cast_storage.cc new file mode 100644 index 000000000000..b5de8d0f08bd --- /dev/null +++ b/src/operator/tensor/cast_storage.cc @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file cast_storage.cc + * \brief CPU Implementation of cast_storage operator. + */ + +#include "./cast_storage-inl.h" +#include "../elemwise_op_common.h" +#include "../tensor/elemwise_unary_op.h" + +namespace mxnet { +namespace op { + +DMLC_REGISTER_PARAMETER(CastStorageParam); +NNVM_REGISTER_OP(cast_storage) +.add_alias("_sparse_cast_storage") +.describe(R"code(Casts tensor storage type to the new type. + +When an NDArray with default storage type is cast to csr or row_sparse storage, +the result is compact, which means: + +- for csr, zero values will not be retained +- for row_sparse, row slices of all zeros will not be retained + +The storage type of ``cast_storage`` output depends on stype parameter: + +- cast_storage(csr, 'default') = default +- cast_storage(row_sparse, 'default') = default +- cast_storage(default, 'csr') = csr +- cast_storage(default, 'row_sparse') = row_sparse + +Example:: + + dense = [[ 0., 1., 0.], + [ 2., 0., 3.], + [ 0., 0., 0.], + [ 0., 0., 0.]] + + # cast to row_sparse storage type + rsp = cast_storage(default, 'default') + rsp.indices = [0, 1] + rsp.values = [[ 0., 1., 0.], + [ 2., 0., 3.]] + + # cast to row_sparse storage type + csr = cast_storage(default, 'default') + csr.indices = [1, 0, 2] + csr.values = [ 1., 2., 3.] + csr.indptr = [0, 1, 3, 3, 3] + +)code" ADD_FILELINE) +.set_num_inputs(1) +.set_num_outputs(1) +.set_attr_parser(ParamParser) +.set_attr("FInferShape", ElemwiseShape<1, 1>) +.set_attr("FInferType", ElemwiseType<1, 1>) +.set_attr("FInferStorageType", CastStorageInferStorageType) +.set_attr("FResourceRequest", + [](const NodeAttrs& attrs) { + return std::vector{ResourceRequest::kTempSpace}; + }) +.set_attr("FCompute", IdentityCompute) +.set_attr("FComputeEx", CastStorageComputeEx) +.set_attr("FGradient", ElemwiseGradUseNone{"_copy"}) +.add_argument("data", "NDArray-or-Symbol", "The input.") +.add_arguments(CastStorageParam::__FIELDS__()); + +} // namespace op +} // namespace mxnet diff --git a/src/operator/tensor/cast_storage.cu b/src/operator/tensor/cast_storage.cu new file mode 100644 index 000000000000..1be5f79ae297 --- /dev/null +++ b/src/operator/tensor/cast_storage.cu @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file cast_storage.cu + * \brief GPU Implementation of cast_storage operator. + */ +#include "./cast_storage-inl.h" +#include "../tensor/elemwise_unary_op.h" + +namespace mxnet { +namespace op { + +NNVM_REGISTER_OP(cast_storage) +.set_attr("FCompute", IdentityCompute) +.set_attr("FComputeEx", CastStorageComputeEx); + +} // namespace op +} // namespace mxnet diff --git a/src/operator/tensor/dot-inl.cuh b/src/operator/tensor/dot-inl.cuh new file mode 100644 index 000000000000..41c3faaf419f --- /dev/null +++ b/src/operator/tensor/dot-inl.cuh @@ -0,0 +1,883 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * Copyright (c) 2017 by Contributors + * \file dot-inl.cuh + * \brief implementation of matrix dot op on GPU + */ +#ifndef MXNET_OPERATOR_TENSOR_DOT_INL_CUH_ +#define MXNET_OPERATOR_TENSOR_DOT_INL_CUH_ + +#include +#include +#include "./util/tensor_util-inl.cuh" + +namespace mxnet { +namespace op { + +/*! + * \brief GPU scalar kernel of dot(csr, dns1) = dns2 + * Parallelization by output matrix elements: 1 thread/element + */ +template +struct DotCsrDnsDnsScalarKernel { + /*! + * \brief This function represents performing an inner product between a row of lhs + * and a column of rhs and then assigning the value to out[tid]. + * \param tid global thread id + * \param out output matrix data + * \param data_l csr matrix data + * \param indptr_l csr matrix row index pointer + * \param col_idx_l csr matrix column indices + * \param data_r dns1 matrix data of rhs + * \param num_cols_r dns1 matrix number of columns + */ + template + __device__ __forceinline__ static void Map(int tid, + DType* out, + const DType* data_l, + const IType* indptr_l, + const CType* col_idx_l, + const DType* data_r, + const nnvm::dim_t num_cols_r) { + const nnvm::dim_t irow = tid / num_cols_r; // row id of the lhs + const nnvm::dim_t icol = tid % num_cols_r; // col id of the rhs + DType sum = 0; + for (IType j = indptr_l[irow]; j < indptr_l[irow+1]; ++j) { + const CType cur_col = col_idx_l[j]; // corresponding row id of the rhs + sum += data_l[j] * data_r[cur_col*num_cols_r+icol]; + } + KERNEL_ASSIGN(out[tid], req, sum); + } +}; + +/*! + * \brief GPU vector kernel of dot(csr, dns1) = dns2 + * Parallelization by output matrix elements: 1 warp/element + */ +template +struct DotCsrDnsDnsVectorKernel { + /*! + * \brief see DotCsrDnsDnsScalarKernel Map for documentation. + */ + template + __device__ __forceinline__ static void Map(int tid, + DType* out, + const DType* data_l, + const IType* indptr_l, + const CType* col_idx_l, + const DType* data_r, + const nnvm::dim_t num_cols_r) { + using nnvm::dim_t; + __shared__ volatile DType vals[mshadow::cuda::kBaseThreadNum]; + const dim_t warp_id = tid / 32; // global warp id + const dim_t lane = tid & (32-1); // local thread id within warp + const dim_t irow = warp_id / num_cols_r; // lhs row that this warp computes + const dim_t kcol = warp_id % num_cols_r; // rhs column that this warp computes + + // Range of nnz elements in this row + const dim_t low = static_cast(indptr_l[irow]); + const dim_t high = static_cast(indptr_l[irow+1]); + + // Compute running sum per thread + DType sum = 0; + for (dim_t j = low+lane; j < high; j+=32) { + sum += data_l[j] * data_r[col_idx_l[j]*num_cols_r + kcol]; + } + vals[threadIdx.x] = sum; __syncwarp(); + + // Parallel reduction in shared memory + if (lane < 16) {vals[threadIdx.x] += vals[threadIdx.x+16];} __syncwarp(); + if (lane < 8) {vals[threadIdx.x] += vals[threadIdx.x+ 8];} __syncwarp(); + if (lane < 4) {vals[threadIdx.x] += vals[threadIdx.x+ 4];} __syncwarp(); + if (lane < 2) {vals[threadIdx.x] += vals[threadIdx.x+ 2];} __syncwarp(); + if (lane < 1) {vals[threadIdx.x] += vals[threadIdx.x+ 1];} __syncwarp(); + + if (lane == 0) { + KERNEL_ASSIGN(out[irow*num_cols_r+kcol], req, vals[threadIdx.x]); + } + } +}; + +/*! + * \brief GPU scalar kernel of dot(csr.T, dns1) = dns2 + * Parallelization by output matrix elements: 1 thread/element + */ +template +struct DotCsrTransDnsDnsScalarKernel { + /*! + * \brief This function represents performing an inner product between a column of lhs + * and a column of rhs and then assigning the value to out[tid]. + * \param tid global thread id + * \param out output matrix + * \param data_l csr matrix data + * \param indptr_l csr matrix row index pointer + * \param col_idx_l csr matrix column indices + * \param data_r dns1 matrix data of rhs + * \param num_rows_l csr matrix number of rows (= number of columns of csr.T) + * \param num_cols_r dns1 matrix number of columns + */ + template + __device__ __forceinline__ static void Map(int tid, + DType* out, + const DType* data_l, + const IType* indptr_l, + const CType* col_idx_l, + const DType* data_r, + const nnvm::dim_t num_rows_l, + const nnvm::dim_t num_cols_r) { + using nnvm::dim_t; + const dim_t irow = tid / num_cols_r; // col id of the lhs + const dim_t icol = tid % num_cols_r; // col id of the rhs + DType sum = 0; + + // Each thread scans each column with binary search to find nnz elements in its row + for (dim_t k = 0; k < num_rows_l; ++k) { + const dim_t low = static_cast(indptr_l[k]); + const dim_t high = static_cast(indptr_l[k+1]); + if (low == high || irow < col_idx_l[low] || irow > col_idx_l[high-1]) continue; + dim_t j = high, l = low, r = high - 1; + while (l <= r) { + dim_t m = l + (r - l) / 2; + if (col_idx_l[m] == irow) { + j = m; break; + } + if (col_idx_l[m] < irow) { + l = m + 1; + } else { + r = m - 1; + } + } + if (j < high) { + sum += data_l[j] * data_r[k*num_cols_r+icol]; + } + } + KERNEL_ASSIGN(out[tid], req, sum); + } +}; + +/*! + * \brief GPU warp kernel of dot(csr.T, dns1) = dns2 + * Parallelization by columns: 1 warp computes one lhs column for one rhs column + */ +struct DotCsrTransDnsDnsWarpKernel { + /*! + * \brief see DotCsrTransDnsDnsScalarKernel Map for documentation. + */ + template + __device__ __forceinline__ static void Map(int tid, + DType* out, + const DType* data_l, + const IType* indptr_l, + const CType* col_idx_l, + const DType* data_r, + const nnvm::dim_t num_cols_r) { + using nnvm::dim_t; + const dim_t warp_id = tid / 32; // global warp id + const dim_t lane = tid & (32-1); // local thread id within warp + const dim_t icol = warp_id / num_cols_r; // lhs column that this warp computes + const dim_t kcol = warp_id % num_cols_r; // rhs column that this warp computes + + // Compute range of nnz elements in this column + const dim_t low = static_cast(indptr_l[icol]); + const dim_t high = static_cast(indptr_l[icol+1]); + + // Iterate through the nnz elements in this column + for (dim_t j = low+lane; j < high; j+=32) { + const dim_t irow = static_cast(col_idx_l[j]); + const DType val = data_l[j]*data_r[icol*num_cols_r+kcol]; + atomicAdd(static_cast(&(out[irow*num_cols_r+kcol])), val); + } + } +}; + +/*! + * \brief GPU thread block kernel of dot(csr.T, dns1) = dns2 + * Parallelization by columns: 1 thread block computes one lhs column for all rhs columns + */ +struct DotCsrTransDnsDnsThreadBlockKernel { + /*! + * \brief see DotCsrTransDnsDnsScalarKernel Map for documentation. + */ + template + __device__ __forceinline__ static void Map(int tid, + DType* out, + const DType* data_l, + const IType* indptr_l, + const CType* col_idx_l, + const DType* data_r, + const nnvm::dim_t num_cols_r) { + using nnvm::dim_t; + const dim_t warps_per_block = blockDim.x / 32; // number of warps in this thread block + const dim_t warp_id = tid / 32; // global warp id + const dim_t lane = tid & (32-1); // local thread id within warp + const dim_t icol = blockIdx.x; // lhs column that this thread block computes + const dim_t kcol = warp_id % warps_per_block; // rhs column where warp starts computing (offset) + + // Compute range of nnz elements in this lhs column + const dim_t low = static_cast(indptr_l[icol]); + const dim_t high = static_cast(indptr_l[icol+1]); + + // Iterate through the nnz elements in this lhs column + for (dim_t j = low+lane; j < high; j+=32) { + const dim_t irow = static_cast(col_idx_l[j]); + const DType datum_l = data_l[j]; + // Iterate over rhs columns that this warp computes + for (dim_t k = kcol; k < num_cols_r; k+=warps_per_block) { + const DType val = datum_l*data_r[icol*num_cols_r+k]; + atomicAdd(static_cast(&(out[irow*num_cols_r+k])), val); + } + } + } +}; + +/*! + * \brief GPU warp block kernel of dot(csr.T, dns1) = dns2 + * Parallelization by columns: 1 warp computes one lhs column for all rhs columns + */ +struct DotCsrTransDnsDnsWarpBlockKernel { + /*! + * \brief see DotCsrTransDnsDnsScalarKernel Map for documentation. + */ + template + __device__ __forceinline__ static void Map(int tid, + DType* out, + const DType* data_l, + const IType* indptr_l, + const CType* col_idx_l, + const DType* data_r, + const nnvm::dim_t num_cols_r) { + using nnvm::dim_t; + const dim_t warp_id = tid / 32; // global warp id + const dim_t lane = tid & (32-1); // local thread id within warp + const dim_t icol = warp_id; // lhs column that this warp computes + + // Compute range of nnz elements in this column + const dim_t low = static_cast(indptr_l[icol]); + const dim_t high = static_cast(indptr_l[icol+1]); + + // Iterate through the nnz elements in lhs column + for (dim_t j = low+lane; j < high; j+=32) { + const dim_t irow = static_cast(col_idx_l[j]); + const DType datum_l = data_l[j]; + // Iterate over all rhs columns + for (dim_t k = 0; k < num_cols_r; k++) { + const DType val = datum_l*data_r[icol*num_cols_r+k]; + atomicAdd(static_cast(&(out[irow*num_cols_r+k])), val); + } + } + } +}; + +/*! + * \brief GPU warp kernel of dot(csr.T, dns) = rsp + * Parallelization by columns: 1 warp computes one lhs column for one rhs column + */ +struct DotCsrTransDnsRspWarpKernel { + /*! + * \brief + * \param tid global thread id + * \param out output rsp matrix data + * \param row_flg_sum_out inclusive prefix sum array over 0/1 marked row flag array + * \param data_l csr matrix data + * \param indptr_l csr matrix row index pointer + * \param col_idx_l csr matrix column indices + * \param data_r dns matrix data + * \param num_cols_r dns matrix number of columns + */ + template + __device__ __forceinline__ static void Map(int tid, + DType* out, + const nnvm::dim_t* row_flg_sum_out, + const DType* data_l, + const IType* indptr_l, + const CType* col_idx_l, + const DType* data_r, + const nnvm::dim_t num_cols_r) { + using nnvm::dim_t; + const dim_t warp_id = tid / 32; // global warp id + const dim_t lane = tid & (32-1); // local thread id within warp + const dim_t icol = warp_id / num_cols_r; // lhs column that this warp computes + const dim_t kcol = warp_id % num_cols_r; // rhs column that this warp computes + + // Compute range of nnz elements in this column + const dim_t low = static_cast(indptr_l[icol]); + const dim_t high = static_cast(indptr_l[icol+1]); + + // Iterate through the nnz elements in this column + for (dim_t j = low+lane; j < high; j+=32) { + const dim_t irow = static_cast(col_idx_l[j]); + const dim_t rsp_row = row_flg_sum_out[irow]-1; + const DType val = data_l[j]*data_r[icol*num_cols_r+kcol]; + atomicAdd(static_cast(&(out[rsp_row*num_cols_r+kcol])), val); + } + } +}; + +/*! + * \brief GPU Kernel of dot(csr.T, rsp1) = rsp2 + * Parallelization by rows: 1 thread/row + * TODO: write a faster kernel optimized for GPU + */ +struct DotCsrTransRspRspByRowsKernel { + /*! + * \brief + * \param tid global thread id + * \param out output rsp matrix data + * \param row_idx_out output rsp matrix non-zero row indices + * \param data_l csr matrix data + * \param indptr_l csr matrix row index pointer + * \param col_idx_l csr matrix column indices + * \param data_r rsp1 matrix data + * \param row_idx_r rsp1 matrix non-zero row indices + * \param num_cols_r rsp1 matrix number of cols + * \param nnr_r rsp1 matrix number of non-zero rows + * \param nnr_out output rsp matrix number of non-zero rows + */ + template + __device__ __forceinline__ static void Map(int tid, + DType* out, + const RType* row_idx_out, + const DType* data_l, + const IType* indptr_l, + const CType* col_idx_l, + const DType* data_r, + const RType* row_idx_r, + const nnvm::dim_t num_cols_r, + const nnvm::dim_t nnr_r, + const nnvm::dim_t nnr_out) { + using nnvm::dim_t; + // This thread computes non-zero row 'tid' of the output matrix + // The actual row id corresponding to the lhs row is row_idx_out[tid] + if (tid < nnr_out) { + const dim_t offset_out = tid * num_cols_r; + // Iterate over rhs matrix rows (or, equivalently, lhs columns worthy taking a look at) + for (dim_t i = 0; i < nnr_r; i++) { + const RType j = row_idx_r[i]; // j is the actual rhs row id (= lhs column id) + if (indptr_l[j] == indptr_l[j+1]) continue; + const dim_t offset_r = i * num_cols_r; + // Iterate over lhs column j to find possible non-zero value in this row + // TODO: remove sequential search, this is a bottleneck + for (IType k = indptr_l[j]; k < indptr_l[j+1]; k++) { + const CType col_idx = col_idx_l[k]; + if (col_idx == row_idx_out[tid]) { + for (dim_t l = 0; l < num_cols_r; l++) { + out[offset_out+l] += data_l[k] * data_r[offset_r+l]; + } + } else if (col_idx > row_idx_out[tid]) { + break; + } + } + } + } + } +}; + +/*! + * \brief GPU Kernel of dot(csr, rsp) = dns + * Parallelization by output elements: 1 thread/element + */ +struct DotCsrRspDnsScalarKernel { + /*! + * \brief + * \param tid global thread id + * \param out output dns matrix data + * \param data_l csr matrix data + * \param indptr_l csr matrix row index pointer + * \param col_idx_l csr matrix column indices + * \param data_r rsp matrix data + * \param row_idx_r rsp matrix non-zero row indices + * \param row_flg_r rsp matrix auxiliary array holding storage indices of non-zero rows + * \param nnr_r rsp matrix number of non-zero rows + * \param num_rows output dns matrix number of rows + * \param num_cols output dns matrix number of columns + */ + template + __device__ __forceinline__ static void Map(int tid, + DType* out, + const DType* data_l, + const IType* indptr_l, + const CType* col_idx_l, + const DType* data_r, + const RType* row_idx_r, + const RType* row_flg_r, + const nnvm::dim_t nnr_r, + const nnvm::dim_t num_rows, + const nnvm::dim_t num_cols) { + using nnvm::dim_t; + if (tid < num_rows*num_cols) { + const dim_t i = static_cast(tid) / num_cols; // i = row this thread computes + const dim_t k = static_cast(tid) % num_cols; // k = col this thread computes + // Compute inner product of i-th row and k-th col + DType sum = 0; + for (IType j = indptr_l[i]; j < indptr_l[i+1]; j++) { + const dim_t csr_col = col_idx_l[j]; + const dim_t rsp_row_idx = row_flg_r[csr_col]; + if (rsp_row_idx > 0) { + sum += data_l[j] * data_r[(rsp_row_idx-1)*num_cols+k]; + } + } + if (sum != 0) { + out[i*num_cols+k] += sum; + } + } + } +}; + +/*! + * \brief GPU Impl of dot(csr, dns1) = dns2 and dot(csr.T, dns1) = dns2 + */ +inline void DotCsrDnsDnsImpl(const OpContext& ctx, + const gpu& gpu_dev, + const NDArray& lhs, + const TBlob& rhs, + const OpReqType req, + const bool trans_lhs, + TBlob* ret) { + if (kNullOp == req) return; + CHECK_EQ(lhs.storage_type(), kCSRStorage); + if (!lhs.storage_initialized()) return; + + using mshadow::cuda::kBaseThreadNum; + using mxnet_op::Kernel; + using mxnet_op::set_zero; + using nnvm::dim_t; + mshadow::Stream* s = ctx.get_stream(); + + const dim_t num_rows_l = lhs.shape()[0]; + const dim_t num_cols_r = rhs.shape_[1]; + const dim_t threads_per_warp = mxnet_op::cuda_get_device_prop().warpSize; + const dim_t threads_per_block = kBaseThreadNum; + dim_t num_threads; + // TODO: remove kernel dependency on warpSize=32 + if (threads_per_warp != 32) { + LOG(FATAL) << "DotCsrDnsDnsImpl GPU kernels expect warpSize=32"; + } + + const TBlob data_l = lhs.data(); + const TBlob indptr_l = lhs.aux_data(csr::kIndPtr); + const TBlob col_idx_l = lhs.aux_data(csr::kIdx); + const TBlob& data_r = rhs; + const TBlob data_out = *ret; + + MSHADOW_SGL_DBL_TYPE_SWITCH(data_l.type_flag_, DType, { // data type + MSHADOW_IDX_TYPE_SWITCH(indptr_l.type_flag_, IType, { // indptr type + MSHADOW_IDX_TYPE_SWITCH(col_idx_l.type_flag_, CType, { // col idx type + if (kWriteTo == req) { + num_threads = data_out.Size(); + Kernel::Launch(s, num_threads, data_out.dptr()); + } + if (trans_lhs) { + // Different kernel versions are optimized for different matrix instances + // TODO: switch between kernel versions depending on input + // (1) 'Scalar kernel' (one thread computing one output element ) + // (2) 'Warp kernel' (one warp computing one lhs column for one rhs column ) + // (3) 'Thread block kernel' (one thread block computing one lhs column for all rhs columns) + // (4) 'Warp block kernel' (one warp computing one lhs column for all rhs columns) + const int kernel_version = 0; + switch (kernel_version) { + case 1: + num_threads = data_out.Size(); + MXNET_ASSIGN_REQ_SWITCH(req, ReqType, { + Kernel, gpu>::Launch(s, num_threads, + data_out.dptr(), data_l.dptr(), indptr_l.dptr(), + col_idx_l.dptr(), data_r.dptr(), num_rows_l, num_cols_r); + }); + break; + case 2: + num_threads = threads_per_warp * num_rows_l * num_cols_r; + Kernel::Launch(s, num_threads, + data_out.dptr(), data_l.dptr(), indptr_l.dptr(), + col_idx_l.dptr(), data_r.dptr(), num_cols_r); + break; + case 3: + num_threads = threads_per_block * num_rows_l; + Kernel::Launch(s, num_threads, + data_out.dptr(), data_l.dptr(), indptr_l.dptr(), + col_idx_l.dptr(), data_r.dptr(), num_cols_r); + break; + case 4: + num_threads = threads_per_warp * num_rows_l; + Kernel::Launch(s, num_threads, + data_out.dptr(), data_l.dptr(), indptr_l.dptr(), + col_idx_l.dptr(), data_r.dptr(), num_cols_r); + break; + default: + num_threads = threads_per_warp * num_rows_l * num_cols_r; + Kernel::Launch(s, num_threads, + data_out.dptr(), data_l.dptr(), indptr_l.dptr(), + col_idx_l.dptr(), data_r.dptr(), num_cols_r); + break; + } + } else { + // Different kernel versions are optimized for different matrix instances + // (1) 'Scalar kernel' (one thread computing one output element) + // (2) 'Vector kernel' (one warp computing one output element) + const int kernel_version = 0; + switch (kernel_version) { + case 1: + num_threads = data_out.Size(); + MXNET_ASSIGN_REQ_SWITCH(req, ReqType, { + Kernel, gpu>::Launch(s, num_threads, + data_out.dptr(), data_l.dptr(), indptr_l.dptr(), + col_idx_l.dptr(), data_r.dptr(), num_cols_r); + }); + break; + case 2: + num_threads = threads_per_warp * num_rows_l * num_cols_r; + MXNET_ASSIGN_REQ_SWITCH(req, ReqType, { + Kernel, gpu>::Launch(s, num_threads, + data_out.dptr(), data_l.dptr(), indptr_l.dptr(), + col_idx_l.dptr(), data_r.dptr(), num_cols_r); + }); + break; + default: + if (num_cols_r > 4) { + num_threads = data_out.Size(); + MXNET_ASSIGN_REQ_SWITCH(req, ReqType, { + Kernel, gpu>::Launch(s, num_threads, + data_out.dptr(), data_l.dptr(), indptr_l.dptr(), + col_idx_l.dptr(), data_r.dptr(), num_cols_r); + }); + } else { + num_threads = threads_per_warp * num_rows_l * num_cols_r; + MXNET_ASSIGN_REQ_SWITCH(req, ReqType, { + Kernel, gpu>::Launch(s, num_threads, + data_out.dptr(), data_l.dptr(), indptr_l.dptr(), + col_idx_l.dptr(), data_r.dptr(), num_cols_r); + }); + } + break; + } + } + }); + }); + }); +} + +/*! + * \brief GPU Impl of dot(csr, dns) = rsp and dot(csr.T, dns) = rsp + */ +inline void DotCsrDnsRspImpl(const OpContext& ctx, + const gpu& gpu_dev, + const NDArray& lhs, + const TBlob& rhs, + const OpReqType req, + const bool trans_lhs, + NDArray* ret) { + if (kNullOp == req) return; + CHECK_EQ(lhs.storage_type(), kCSRStorage); + CHECK_EQ(ret->storage_type(), kRowSparseStorage); + CHECK_EQ(req, kWriteTo); + if (!lhs.storage_initialized()) return; + + using mshadow::Shape1; + using mxnet_op::Kernel; + using mxnet_op::set_zero; + using nnvm::dim_t; + mshadow::Stream* s = ctx.get_stream(); + + const TBlob data_l = lhs.data(); + const TBlob indptr_l = lhs.aux_data(csr::kIndPtr); + const TBlob col_idx_l = lhs.aux_data(csr::kIdx); + const TBlob& data_r = rhs; + + const dim_t num_rows_l = lhs.shape()[0]; + const dim_t num_cols_l = lhs.shape()[1]; + const dim_t num_cols_r = rhs.shape_[1]; + const dim_t threads_per_warp = mxnet_op::cuda_get_device_prop().warpSize; + dim_t num_threads; + // TODO: remove kernel dependency on warpSize=32 + if (threads_per_warp != 32) { + LOG(FATAL) << "DotCsrDnsRspImpl GPU kernels expect warpSize=32"; + } + + MSHADOW_SGL_DBL_TYPE_SWITCH(data_l.type_flag_, DType, { // data type + MSHADOW_IDX_TYPE_SWITCH(indptr_l.type_flag_, IType, { // indptr type + MSHADOW_IDX_TYPE_SWITCH(col_idx_l.type_flag_, CType, { // col idx type + if (trans_lhs) { + // Compute number of non-zero rows (nnr) of output matrix + // - alloc temp storage for row_flg array and for cub's prefix sum + // - mark non-zero columns of csr matrix in row_flg + // - compute inclusive prefix sum over marked array + // - copy last value (nnr_out) from device to host + dim_t* row_flg_out = NULL; + void* d_temp_storage = NULL; + size_t temp_storage_bytes = 0; + cub::DeviceScan::InclusiveSum(d_temp_storage, + temp_storage_bytes, + row_flg_out, + row_flg_out, + num_cols_l, + mshadow::Stream::GetStream(s)); + mshadow::Tensor workspace = ctx.requested[0] + .get_space_typed(Shape1(num_cols_l * sizeof(dim_t) + + temp_storage_bytes), s); + row_flg_out = reinterpret_cast(workspace.dptr_); + d_temp_storage = workspace.dptr_ + num_cols_l*sizeof(dim_t); + num_threads = num_cols_l; + Kernel::Launch(s, num_threads, row_flg_out); + num_threads = num_rows_l * threads_per_warp; + Kernel::Launch(s, num_threads, + row_flg_out, col_idx_l.dptr(), indptr_l.dptr(), + num_rows_l, num_cols_l); + cub::DeviceScan::InclusiveSum(d_temp_storage, + temp_storage_bytes, + row_flg_out, + row_flg_out, + num_cols_l, + mshadow::Stream::GetStream(s)); + dim_t nnr_out = 0; + CUDA_CALL(cudaMemcpy(&nnr_out, &row_flg_out[num_cols_l-1], sizeof(dim_t), + cudaMemcpyDeviceToHost)); + + // Allocate output matrix space + ret->CheckAndAlloc({Shape1(nnr_out)}); + const TBlob data_out_blob = ret->data(); + const TBlob row_idx_out_blob = ret->aux_data(rowsparse::kIdx); + MSHADOW_IDX_TYPE_SWITCH(row_idx_out_blob.type_flag_, RType, { // row idx type + DType* data_out = data_out_blob.dptr(); + RType* row_idx_out = row_idx_out_blob.dptr(); + num_threads = nnr_out * num_cols_r; + Kernel::Launch(s, num_threads, data_out); + num_threads = nnr_out; + Kernel::Launch(s, num_threads, row_idx_out); + + // Fill row_idx array of output matrix, using the row_flg values + num_threads = num_cols_l; + Kernel::Launch(s, num_threads, + row_idx_out, row_flg_out, num_cols_l); + + // Perform matrix-matrix multiply + num_threads = threads_per_warp * num_rows_l * num_cols_r; + Kernel::Launch(s, num_threads, + data_out, row_flg_out, + data_l.dptr(), indptr_l.dptr(), col_idx_l.dptr(), + data_r.dptr(), num_cols_r); + }); + } else { + LOG(FATAL) << "DotCsrDnsRspImpl has not implemented dot(csr, dns) = rsp yet."; + } + }); + }); + }); +} + +/*! + * \brief GPU Impl of dot(csr, rsp1) = rsp2 and dot(csr.T, rsp1) = rsp2 + * TODO: Optimize for GPU; this is a baseline implementation providing + * the operator functionality, it is not yet fully optimized for GPU. + */ +inline void DotCsrRspRspImpl(const OpContext& ctx, + const gpu& gpu_dev, + const NDArray& lhs, + const NDArray& rhs, + const OpReqType req, + const bool trans_lhs, + NDArray* ret) { + if (kNullOp == req) return; + // Reuse dot(csr, dns) implementation if rhs rsp matrix is in fact dense + if (rhs.storage_shape()[0] == rhs.shape()[0]) { + DotCsrDnsRspImpl(ctx, gpu_dev, lhs, rhs.data(), req, trans_lhs, ret); + return; + } + CHECK_EQ(lhs.storage_type(), kCSRStorage); + CHECK_EQ(rhs.storage_type(), kRowSparseStorage); + CHECK_EQ(ret->storage_type(), kRowSparseStorage); + if (!lhs.storage_initialized() || !rhs.storage_initialized()) return; + CHECK_EQ(req, kWriteTo); + + using mshadow::Shape1; + using mxnet_op::Kernel; + using mxnet_op::set_zero; + using nnvm::dim_t; + mshadow::Stream* s = ctx.get_stream(); + + const TBlob data_l = lhs.data(); + const TBlob indptr_l = lhs.aux_data(csr::kIndPtr); + const TBlob col_idx_l = lhs.aux_data(csr::kIdx); + const TBlob data_r = rhs.data(); + const TBlob row_idx_r = rhs.aux_data(rowsparse::kIdx); + + const dim_t num_rows_l = lhs.shape()[0]; + const dim_t num_cols_l = lhs.shape()[1]; + const dim_t num_cols_r = rhs.shape()[1]; + const dim_t nnr_r = rhs.storage_shape()[0]; + const dim_t threads_per_warp = mxnet_op::cuda_get_device_prop().warpSize; + dim_t num_threads; + // TODO: remove kernel dependency on warpSize=32 + if (threads_per_warp != 32) { + LOG(FATAL) << "DotCsrRspRspImpl GPU kernels expect warpSize=32"; + } + + MSHADOW_SGL_DBL_TYPE_SWITCH(data_l.type_flag_, DType, { // data type + MSHADOW_IDX_TYPE_SWITCH(indptr_l.type_flag_, IType, { // indptr type + MSHADOW_IDX_TYPE_SWITCH(col_idx_l.type_flag_, CType, { // col idx type + MSHADOW_IDX_TYPE_SWITCH(row_idx_r.type_flag_, RType, { // row idx type + if (trans_lhs) { + // Compute number of non-zero rows (nnr) of output matrix + // - alloc temp storage for row_flg array and for cub's prefix sum + // - mark non-zero columns of csr matrix in row_flg + // - compute inclusive prefix sum over marked array + // - copy last value (nnr_out) from device to host + dim_t* row_flg_out = NULL; + void* d_temp_storage = NULL; + size_t temp_storage_bytes = 0; + cub::DeviceScan::InclusiveSum(d_temp_storage, + temp_storage_bytes, + row_flg_out, + row_flg_out, + num_cols_l, + mshadow::Stream::GetStream(s)); + mshadow::Tensor workspace = ctx.requested[0] + .get_space_typed(Shape1(num_cols_l * sizeof(dim_t) + + temp_storage_bytes), s); + row_flg_out = reinterpret_cast(workspace.dptr_); + d_temp_storage = workspace.dptr_ + num_cols_l*sizeof(dim_t); + num_threads = num_cols_l; + Kernel::Launch(s, num_threads, row_flg_out); + num_threads = num_rows_l * threads_per_warp; + Kernel::Launch(s, num_threads, + row_flg_out, col_idx_l.dptr(), indptr_l.dptr(), + num_rows_l, num_cols_l); + cub::DeviceScan::InclusiveSum(d_temp_storage, + temp_storage_bytes, + row_flg_out, + row_flg_out, + num_cols_l, + mshadow::Stream::GetStream(s)); + dim_t nnr_out = 0; + CUDA_CALL(cudaMemcpy(&nnr_out, &row_flg_out[num_cols_l-1], sizeof(dim_t), + cudaMemcpyDeviceToHost)); + + // Allocate output matrix space + ret->CheckAndAlloc({mshadow::Shape1(nnr_out)}); + const TBlob data_out_blob = ret->data(); + const TBlob row_idx_out_blob = ret->aux_data(rowsparse::kIdx); + DType* data_out = data_out_blob.dptr(); + RType* row_idx_out = row_idx_out_blob.dptr(); + num_threads = nnr_out * num_cols_r; + Kernel::Launch(s, num_threads, data_out); + num_threads = nnr_out; + Kernel::Launch(s, num_threads, row_idx_out); + + // Fill row_idx array of output matrix, using the row_flg values + num_threads = num_cols_l; + Kernel::Launch(s, num_threads, + row_idx_out, row_flg_out, num_cols_l); + + // Perform matrix-matrix multiply + num_threads = nnr_out; + Kernel::Launch(s, num_threads, + data_out, row_idx_out, + data_l.dptr(), indptr_l.dptr(), col_idx_l.dptr(), + data_r.dptr(), row_idx_r.dptr(), + num_cols_r, nnr_r, nnr_out); + } else { + LOG(FATAL) << "DotCsrRspRspImpl has not implemented dot(csr, rsp1) = rsp2 yet."; + } + }); + }); + }); + }); +} + +/*! + * \brief GPU Impl of dot(csr, rsp) = dns and dot(csr.T, rsp) = dns + */ +inline void DotCsrRspDnsImpl(const OpContext& ctx, + const gpu& gpu_dev, + const NDArray& lhs, + const NDArray& rhs, + const OpReqType req, + const bool trans_lhs, + TBlob* ret) { + // Reuse dot(csr, dns) implementation if rhs rsp matrix is in fact dense + if (rhs.storage_shape()[0] == rhs.shape()[0]) { + DotCsrDnsDnsImpl(ctx, gpu_dev, lhs, rhs.data(), req, trans_lhs, ret); + return; + } + if (kNullOp == req) return; + CHECK_EQ(lhs.storage_type(), kCSRStorage); + CHECK_EQ(rhs.storage_type(), kRowSparseStorage); + + using mxnet_op::Kernel; + using mxnet_op::set_zero; + mshadow::Stream* s = ctx.get_stream(); + if (!lhs.storage_initialized() || !rhs.storage_initialized()) { + if (kWriteTo == req) { + MSHADOW_TYPE_SWITCH(ret->type_flag_, DType, { // data type + Kernel::Launch(s, ret->Size(), ret->dptr()); + }); + } + return; + } + + using nnvm::dim_t; + const dim_t num_rows = ret->shape_[0]; + const dim_t num_cols = ret->shape_[1]; + const dim_t nnr_r = rhs.storage_shape()[0]; + dim_t num_threads; + + const TBlob data_l = lhs.data(); + const TBlob indptr_l = lhs.aux_data(csr::kIndPtr); + const TBlob col_idx_l = lhs.aux_data(csr::kIdx); + const TBlob data_r = rhs.data(); + const TBlob row_idx_r = rhs.aux_data(rowsparse::kIdx); + + MSHADOW_SGL_DBL_TYPE_SWITCH(data_l.type_flag_, DType, { // data type + MSHADOW_IDX_TYPE_SWITCH(indptr_l.type_flag_, IType, { // indptr type + MSHADOW_IDX_TYPE_SWITCH(col_idx_l.type_flag_, CType, { // col idx type + MSHADOW_IDX_TYPE_SWITCH(row_idx_r.type_flag_, RType, { // row idx type + if (kWriteTo == req) { + num_threads = num_rows*num_cols; + Kernel::Launch(s, num_threads, ret->dptr()); + } + if (trans_lhs) { + LOG(FATAL) << "DotCsrRspDnsImpl has not implemented dot(csr.T, rsp) = dns yet."; + } else { + // TODO: Consider implementing a vector kernel for SpMV (similar to DotCsrDnsDns) + // Alloc temp storage for row_flg array + RType* row_flg_r = ctx.requested[0] + .get_space_typed(mshadow::Shape1(rhs.shape()[0]), s).dptr_; + num_threads = rhs.shape()[0]; + Kernel::Launch(s, num_threads, row_flg_r); + // Set row_flg index array + num_threads = nnr_r; + Kernel::Launch(s, num_threads, + row_flg_r, row_idx_r.dptr(), nnr_r); + // Perform sparse matrix-matrix multiply + num_threads = num_rows*num_cols; + Kernel::Launch(s, num_threads, + ret->dptr(), + data_l.dptr(), indptr_l.dptr(), col_idx_l.dptr(), + data_r.dptr(), row_idx_r.dptr(), row_flg_r, rhs.storage_shape()[0], + num_rows, num_cols); + } + }); + }); + }); + }); +} + +} // namespace op +} // namespace mxnet + +#endif // MXNET_OPERATOR_TENSOR_DOT_INL_CUH_ diff --git a/src/operator/tensor/dot-inl.h b/src/operator/tensor/dot-inl.h new file mode 100644 index 000000000000..aaf242e26fe1 --- /dev/null +++ b/src/operator/tensor/dot-inl.h @@ -0,0 +1,1007 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file dot-inl.h + * \brief Function definition of matrix dot operator + */ + +#ifndef MXNET_OPERATOR_TENSOR_DOT_INL_H_ +#define MXNET_OPERATOR_TENSOR_DOT_INL_H_ + +#include +#include +#include +#include +#include +#include "../mshadow_op.h" +#include "../elemwise_op_common.h" +#include "../mxnet_op.h" +#ifdef __CUDACC__ +#include "./dot-inl.cuh" +#endif // __CUDACC__ + +namespace mxnet { +namespace op { + +struct DotParam : public dmlc::Parameter { + bool transpose_a; + bool transpose_b; + DMLC_DECLARE_PARAMETER(DotParam) { + DMLC_DECLARE_FIELD(transpose_a) + .describe("If true then transpose the first input before dot.") + .set_default(false); + DMLC_DECLARE_FIELD(transpose_b) + .describe("If true then transpose the second input before dot.") + .set_default(false); + } +}; + +template +void DotForward_(const nnvm::NodeAttrs& attrs, + const OpContext& ctx, + const std::vector& inputs, + const std::vector& req, + const std::vector& outputs) { + using namespace mshadow; + using namespace mshadow::expr; + const DotParam& param = nnvm::get(attrs.parsed); + Stream *s = ctx.get_stream(); + CHECK_EQ(outputs[0].type_flag_, inputs[0].type_flag_) + << "Binary function only support input/output with the same type"; + CHECK_EQ(outputs[0].type_flag_, inputs[1].type_flag_) + << "Binary function only support input/output with the same type"; + CHECK(outputs[0].type_flag_ == kFloat32 || outputs[0].type_flag_ == kFloat64) + << "dot only supports float32 and float64"; + MSHADOW_SGL_DBL_TYPE_SWITCH(outputs[0].type_flag_, DType, { + if (inputs[0].ndim() == 1 && inputs[1].ndim() == 1) { + CHECK_NE(req[0], kAddTo) << "AddTo not yet suported"; + Tensor out = outputs[0].get(s); + VectorDot(out, + inputs[0].get(s), + inputs[1].get(s)); + } else { + int ma, na, mb, nb, m, n; + if (param.transpose_a) { + ma = inputs[0].size(0); + na = inputs[0].Size()/ma; + m = na; + } else { + na = inputs[0].size(inputs[0].ndim()-1); + ma = inputs[0].Size()/na; + m = ma; + } + if (param.transpose_b) { + nb = inputs[1].size(inputs[1].ndim()-1); + mb = inputs[1].Size()/nb; + n = mb; + } else { + mb = inputs[1].size(0); + nb = inputs[1].Size()/mb; + n = nb; + } + Tensor input0 = + inputs[0].get_with_shape(Shape2(ma, na), s); + Tensor input1 = + inputs[1].get_with_shape(Shape2(mb, nb), s); + Tensor out = + outputs[0].get_with_shape(Shape2(m, n), s); + if (param.transpose_a && param.transpose_b) { + ASSIGN_DISPATCH(out, req[0], dot(input0.T(), input1.T())); + } else if (!param.transpose_a && param.transpose_b) { + ASSIGN_DISPATCH(out, req[0], dot(input0, input1.T())); + } else if (param.transpose_a && !param.transpose_b) { + ASSIGN_DISPATCH(out, req[0], dot(input0.T(), input1)); + } else { + ASSIGN_DISPATCH(out, req[0], dot(input0, input1)); + } + } + }); +} + +template +void DotBackward_(const nnvm::NodeAttrs& attrs, + const OpContext& ctx, + const std::vector& inputs, + const std::vector& req, + const std::vector& outputs) { + using namespace mshadow; + using namespace mshadow::expr; + const DotParam& param = nnvm::get(attrs.parsed); + Stream *s = ctx.get_stream(); + CHECK_NE(req[0], kWriteInplace); + CHECK_NE(req[1], kWriteInplace); + MSHADOW_SGL_DBL_TYPE_SWITCH(outputs[0].type_flag_, DType, { + if (inputs[1].ndim() == 1 && inputs[2].ndim() == 1) { + Tensor mout_grad = inputs[0].get(s); + Tensor mlhs_data = inputs[1].get(s); + Tensor mrhs_data = inputs[2].get(s); + Tensor mlhs_grad = outputs[0].get(s); + Tensor mrhs_grad = outputs[1].get(s); + ASSIGN_DISPATCH(mrhs_grad, req[1], + broadcast_scalar(mout_grad, mlhs_data.shape_) * mlhs_data); + ASSIGN_DISPATCH(mlhs_grad, req[0], + broadcast_scalar(mout_grad, mlhs_data.shape_) * mrhs_data); + } else { + int ma, na, mb, nb, m, n; + if (param.transpose_a) { + ma = outputs[0].size(0); + na = outputs[0].Size()/ma; + m = na; + } else { + na = outputs[0].size(outputs[0].ndim()-1); + ma = outputs[0].Size()/na; + m = ma; + } + if (param.transpose_b) { + nb = outputs[1].size(outputs[1].ndim()-1); + mb = outputs[1].Size()/nb; + n = mb; + } else { + mb = outputs[1].size(0); + nb = outputs[1].Size()/mb; + n = nb; + } + Tensor mout_grad = + inputs[0].get_with_shape(Shape2(m, n), s); + Tensor mlhs_data = + inputs[1].get_with_shape(Shape2(ma, na), s); + Tensor mrhs_data = + inputs[2].get_with_shape(Shape2(mb, nb), s); + Tensor mlhs_grad = + outputs[0].get_with_shape(Shape2(ma, na), s); + Tensor mrhs_grad = + outputs[1].get_with_shape(Shape2(mb, nb), s); + if (param.transpose_a && param.transpose_b) { + // Gradient of z = dot(x.T, y.T) + // dy = dot(x, dz).T = dot(dz.T, x.T) + // dx = dot(dz, y).T = dot(y.T, dz.T) + ASSIGN_DISPATCH(mrhs_grad, req[1], dot(mout_grad.T(), mlhs_data.T())); + ASSIGN_DISPATCH(mlhs_grad, req[0], dot(mrhs_data.T(), mout_grad.T())); + } else if (!param.transpose_a && param.transpose_b) { + // Gradient of z = dot(x, y.T) + // dy = dot(x.T, dz).T = dot(dz.T, x) + // dx = dot(dz, y) + ASSIGN_DISPATCH(mrhs_grad, req[1], dot(mout_grad.T(), mlhs_data)); + ASSIGN_DISPATCH(mlhs_grad, req[0], dot(mout_grad, mrhs_data)); + } else if (param.transpose_a && !param.transpose_b) { + // Gradient of z = dot(x.T, y) + // dy = dot(x, dz) + // dx = dot(dz, y.T).T = dot(y, dz.T) + ASSIGN_DISPATCH(mrhs_grad, req[1], dot(mlhs_data, mout_grad)); + ASSIGN_DISPATCH(mlhs_grad, req[0], dot(mrhs_data, mout_grad.T())); + } else { + // Gradient of z = dot(x, y) + // dy = dot(x.T, dz) + // dx = dot(dz, y.T) + ASSIGN_DISPATCH(mrhs_grad, req[1], dot(mlhs_data.T(), mout_grad)); + ASSIGN_DISPATCH(mlhs_grad, req[0], dot(mout_grad, mrhs_data.T())); + } + } + }); +} + +inline bool DotForwardInferStorageType(const nnvm::NodeAttrs& attrs, + const Context& ctx, + std::vector *in_attrs, + std::vector *out_attrs) { + CHECK_EQ(in_attrs->size(), 2U); + CHECK_EQ(out_attrs->size(), 1U); + const DotParam& param = nnvm::get(attrs.parsed); + // csr has many zero columns, so the result of dot(csr.T, matrix) should be rsp + // TODO(stefan/haibin/jun): check type_assign return value + if (param.transpose_a && kCSRStorage == (*in_attrs)[0]) { + type_assign(&((*out_attrs)[0]), kRowSparseStorage); + } else { + type_assign(&((*out_attrs)[0]), kDefaultStorage); + } + return true; +} + +inline bool DotBackwardInferStorageType(const nnvm::NodeAttrs& attrs, + const Context& ctx, + std::vector *in_attrs, + std::vector *out_attrs) { + CHECK_EQ(in_attrs->size(), 3U); + CHECK_EQ(out_attrs->size(), 2U); + const DotParam& param = nnvm::get(attrs.parsed); + type_assign(&((*out_attrs)[0]), kDefaultStorage); + if (!param.transpose_a && kCSRStorage == (*in_attrs)[1]) { + type_assign(&((*out_attrs)[1]), kRowSparseStorage); + } else { + type_assign(&((*out_attrs)[1]), kDefaultStorage); + } + return true; +} + +/*! + * \brief CPU Kernel of dot(csr, dns1) = dns2 + * Parallelization by row blocks + */ +struct DotCsrDnsDnsByRowBlocks { + /*! + * \brief + * \param i the i-th thread + */ + template + MSHADOW_CINLINE static void Map(int i, + DType* out, + const DType* data_l, + const IType* indptr_l, + const CType* col_idx_l, + const DType* data_r, + const nnvm::dim_t seg_len, + const nnvm::dim_t num_rows, + const nnvm::dim_t num_cols) { + using nnvm::dim_t; + const dim_t seg_start = i * seg_len; + if (seg_start >= num_rows) return; + const dim_t seg_end = std::min(seg_start + seg_len, num_rows); + for (dim_t j = seg_start; j < seg_end; ++j) { + if (indptr_l[j] == indptr_l[j+1]) continue; + const dim_t offset_out = j * num_cols; + for (IType k = indptr_l[j]; k < indptr_l[j+1]; ++k) { + const DType val = data_l[k]; + const dim_t offset_r = col_idx_l[k] * num_cols; + for (dim_t l = 0; l < num_cols; ++l) { + out[offset_out+l] += data_r[offset_r+l] * val; + } + } + } + } +}; + +/*! + * \brief CPU Kernel of dot(csr.T(), dns1) = dns2 + * Parallelization by row blocks + */ +struct DotCsrTransDnsDnsByRowBlocks { + /*! + * \brief + * \param i the i-th thread + */ + template + MSHADOW_CINLINE static void Map(int i, + DType* out, + const DType* data_l, + const IType* indptr_l, + const CType* col_idx_l, + const DType* data_r, + const nnvm::dim_t seg_len, + const nnvm::dim_t num_rows_l, + const nnvm::dim_t num_rows, + const nnvm::dim_t num_cols) { + using nnvm::dim_t; + const dim_t seg_start = i * seg_len; + if (seg_start >= num_rows) return; + const dim_t seg_end = (i + 1) * seg_len; + for (dim_t j = 0; j < num_rows_l; ++j) { + if (indptr_l[j] == indptr_l[j+1]) continue; + const dim_t offset_r = j * num_cols; + for (IType k = indptr_l[j]; k < indptr_l[j+1]; ++k) { + const CType col_idx = col_idx_l[k]; + if (col_idx < seg_start || col_idx >= seg_end) continue; + const dim_t offset_out = col_idx * num_cols; + const DType val = data_l[k]; + for (dim_t l = 0; l < num_cols; ++l) { + out[offset_out+l] += data_r[offset_r+l] * val; + } + } + } + } +}; + +/*! + * \brief CPU Kernel of dot(csr.T(), dns) = rsp + * Parallelization by row blocks. + * This kernel fills up the row_idx array of the rsp + * with 1 for nonzero rows and 0 for zero rows. + * The matrix will be compacted after this kernel call. + */ +struct DotCsrTransDnsRspByRowBlocks { + /*! + * \brief + * \param i the i-th thread + */ + template + MSHADOW_CINLINE static void Map(int i, + DType* out, + RType* row_idx, + const DType* data_l, + const IType* indptr_l, + const CType* col_idx_l, + const DType* data_r, + const nnvm::dim_t seg_len, + const nnvm::dim_t num_rows_l, + const nnvm::dim_t num_rows, + const nnvm::dim_t num_cols) { + using nnvm::dim_t; + const dim_t seg_start = i * seg_len; + if (seg_start >= num_rows) return; + const dim_t seg_end = (i + 1) * seg_len; + for (dim_t j = 0; j < num_rows_l; ++j) { + if (indptr_l[j] == indptr_l[j+1]) continue; + const dim_t offset_r = j * num_cols; + for (IType k = indptr_l[j]; k < indptr_l[j+1]; ++k) { + const CType col_idx = col_idx_l[k]; + if (col_idx < seg_start || col_idx >= seg_end) continue; + const dim_t offset_out = col_idx * num_cols; + row_idx[col_idx] = 1; + const DType val = data_l[k]; + for (dim_t l = 0; l < num_cols; ++l) { + out[offset_out+l] += data_r[offset_r+l] * val; + } + } + } + } +}; + +/*! + * \brief CPU Kernel of dot(csr, rsp) = dns + * Parallelization by row blocks + */ +struct DotCsrRspDnsByRowBlocks { + /*! + * \brief + * \param i the i-th thread + * \param nnr_r storage_shape[0] of the rsp + * \param num_rows dns.shape[0] + * \param num_cols dns.shape[1] + */ + template + MSHADOW_CINLINE static void Map(int i, + DType* out, + const DType* data_l, + const IType* indptr_l, + const CType* col_idx_l, + const DType* data_r, + const RType* row_idx_r, + const nnvm::dim_t nnr_r, + const nnvm::dim_t num_rows, + const nnvm::dim_t num_cols, + const nnvm::dim_t seg_len) { + using nnvm::dim_t; + const dim_t seg_start = i * seg_len; + if (seg_start >= num_rows) return; + const dim_t seg_end = std::min(seg_start + seg_len, num_rows); + for (dim_t j = seg_start; j < seg_end; ++j) { + if (indptr_l[j] == indptr_l[j+1]) continue; + const dim_t offset_out = j * num_cols; + // Use binary search to find the lower_bound of val in row_idx array + const RType* first = row_idx_r; + const RType* last = row_idx_r + nnr_r; + const CType val = col_idx_l[indptr_l[j]]; + const RType* it; + int count = last - first, step; + while (count > 0) { + it = first; + step = count / 2; + it += step; + if (*it < val) { + first = ++it; + count -= step + 1; + } else { + count = step; + } + } + const RType* row_idx_ptr = first; + // end of binary search + if (row_idx_ptr == row_idx_r+nnr_r || *row_idx_ptr > col_idx_l[indptr_l[j+1]-1]) continue; + for (IType k = indptr_l[j]; k < indptr_l[j+1] && row_idx_ptr != row_idx_r+nnr_r;) { + if (col_idx_l[k] == *row_idx_ptr) { + const dim_t offset_r = (row_idx_ptr - row_idx_r) * num_cols; + for (dim_t l = 0; l < num_cols; ++l) { + out[offset_out+l] += data_l[k] * data_r[offset_r+l]; + } + ++k; + ++row_idx_ptr; + } else if (col_idx_l[k] < *row_idx_ptr) { + ++k; + } else { + ++row_idx_ptr; + } + } + } + } +}; + +/*! + * \brief CPU Kernel of dot(csr.T(), rsp1) = rsp2, with row_idx marked for non-zero rows + * Parallelization by row blocks + */ +struct DotCsrTransRspRspByRowBlocks { + /*! + * \brief + * \param i the i-th thread + * \param num_rows_l number of rows of lhs matrix + * \param nnr_r number of non-zero rows of rhs matrix + * \param num_rows number of rows of out matrix + * \param num_cols number of cols of out matrix + */ + template + MSHADOW_CINLINE static void Map(int i, + DType* out, + RType* row_idx_out, + const DType* data_l, + const IType* indptr_l, + const CType* col_idx_l, + const DType* data_r, + const RType* row_idx_r, + const nnvm::dim_t num_rows_l, + const nnvm::dim_t nnr_r, + const nnvm::dim_t num_rows, + const nnvm::dim_t num_cols, + const nnvm::dim_t seg_len) { + using nnvm::dim_t; + const dim_t seg_start = i * seg_len; + if (seg_start >= num_rows) return; + const dim_t seg_end = (i + 1) * seg_len; + for (dim_t rid = 0; rid < nnr_r; ++rid) { + const RType j = row_idx_r[rid]; + if (indptr_l[j] == indptr_l[j+1]) continue; + const dim_t offset_r = rid * num_cols; + for (IType k = indptr_l[j]; k < indptr_l[j+1]; ++k) { + const CType col_idx = col_idx_l[k]; + if (col_idx < seg_start || col_idx >= seg_end) continue; + row_idx_out[col_idx] = 1; // mark nonzero row as 1 + const dim_t offset_out = col_idx * num_cols; + for (dim_t l = 0; l < num_cols; ++l) { + out[offset_out+l] += data_r[offset_r+l] * data_l[k]; + } + } + } + } +}; + +/*! + * \brief CPU Impl of dot(csr, dns1) = dns2 and dot(csr.T, dns1) = dns2 + */ +inline void DotCsrDnsDnsImpl(const OpContext& ctx, + const cpu& cpu_dev, + const NDArray& lhs, + const TBlob& rhs, + const OpReqType req, + const bool trans_lhs, + TBlob* ret) { + if (kNullOp == req) return; + CHECK_EQ(lhs.storage_type(), kCSRStorage); + if (!lhs.storage_initialized()) return; + + using nnvm::dim_t; + + mshadow::Stream* s = ctx.get_stream(); + const TBlob data_l = lhs.data(); + const TBlob indptr_l = lhs.aux_data(csr::kIndPtr); + const TBlob col_idx_l = lhs.aux_data(csr::kIdx); + const TBlob& data_r = rhs; + const TBlob data_out = *ret; + + MSHADOW_SGL_DBL_TYPE_SWITCH(data_l.type_flag_, DType, { // data type + MSHADOW_IDX_TYPE_SWITCH(indptr_l.type_flag_, IType, { // indptr type + MSHADOW_IDX_TYPE_SWITCH(col_idx_l.type_flag_, CType, { // col idx type + dim_t num_threads; + if (kWriteTo == req) { + num_threads = data_out.Size(); + mxnet_op::Kernel::Launch( + s, num_threads, data_out.dptr()); + } + num_threads = mxnet_op::get_num_threads(data_out.shape_[0]); + dim_t seg_len = (data_out.shape_[0] + num_threads - 1) / num_threads; + if (trans_lhs) { + mxnet_op::Kernel::Launch(s, num_threads, + data_out.dptr(), data_l.dptr(), indptr_l.dptr(), + col_idx_l.dptr(), data_r.dptr(), seg_len, + lhs.shape()[0], data_out.shape_[0], data_out.shape_[1]); + } else { + mxnet_op::Kernel::Launch(s, num_threads, + data_out.dptr(), data_l.dptr(), indptr_l.dptr(), + col_idx_l.dptr(), data_r.dptr(), seg_len, + data_out.shape_[0], data_out.shape_[1]); + } + }); + }); + }); +} + +/*! + * \brief CPU Impl of dot(csr.T, dns) = rsp + */ +inline void DotCsrDnsRspImpl(const OpContext& ctx, + const cpu& cpu_dev, + const NDArray& lhs, + const TBlob& rhs, + const OpReqType req, + const bool trans_lhs, + NDArray* ret) { + if (kNullOp == req) return; + CHECK_EQ(lhs.storage_type(), kCSRStorage); + CHECK_EQ(ret->storage_type(), kRowSparseStorage); + if (!lhs.storage_initialized()) return; + CHECK_EQ(req, kWriteTo); + + using mxnet_op::set_zero; + using nnvm::dim_t; + + mshadow::Stream* s = ctx.get_stream(); + const TBlob data_l = lhs.data(); + const TBlob indptr_l = lhs.aux_data(csr::kIndPtr); + const TBlob col_idx_l = lhs.aux_data(csr::kIdx); + const TBlob& data_r = rhs; + + // pre-allocate spaces for ret using the dense dimension size + ret->CheckAndAlloc({mshadow::Shape1(lhs.shape()[1])}); + const TBlob data_out = ret->data(); + const TBlob row_idx_out = ret->aux_data(rowsparse::kIdx); + + MSHADOW_SGL_DBL_TYPE_SWITCH(data_l.type_flag_, DType, { // data type + MSHADOW_IDX_TYPE_SWITCH(indptr_l.type_flag_, IType, { // indptr type + MSHADOW_IDX_TYPE_SWITCH(col_idx_l.type_flag_, CType, { // col idx type + MSHADOW_IDX_TYPE_SWITCH(row_idx_out.type_flag_, RType, { // row idx type + dim_t num_threads = data_out.Size(); + mxnet_op::Kernel::Launch(s, num_threads, data_out.dptr()); + RType* row_idx = row_idx_out.dptr(); + num_threads = row_idx_out.Size(); + mxnet_op::Kernel::Launch(s, num_threads, row_idx); + num_threads = mxnet_op::get_num_threads(data_out.shape_[0]); + dim_t seg_len = (data_out.shape_[0] + num_threads - 1) / num_threads; + if (trans_lhs) { + mxnet_op::Kernel::Launch(s, num_threads, + data_out.dptr(), row_idx, data_l.dptr(), + indptr_l.dptr(), col_idx_l.dptr(), data_r.dptr(), + seg_len, lhs.shape()[0], data_out.shape_[0], data_out.shape_[1]); + dim_t nnr = 0; + nnr = mxnet::common::ParallelAccumulate(row_idx, ret->shape()[0], nnr); + ret->set_aux_shape(rowsparse::kIdx, mshadow::Shape1(nnr)); + if (0 == nnr) return; + mshadow::Tensor rsp_data = data_out.FlatTo2D(s); + dim_t idx = 0; + for (index_t i = 0; i < ret->shape()[0]; ++i) { + if (row_idx[i] > 0) { + row_idx[idx] = i; + mshadow::Copy(rsp_data[idx], rsp_data[i], s); + ++idx; + } + } + } else { + LOG(FATAL) << "DotCsrDnsRspImpl has not implemented dot(csr, dns)=rsp yet."; + } + }); + }); + }); + }); +} + +/*! + * \brief CPU Impl of dot(csr, rsp) = dns + */ +inline void DotCsrRspDnsImpl(const OpContext& ctx, + const cpu& cpu_dev, + const NDArray& lhs, + const NDArray& rhs, + const OpReqType req, + const bool trans_lhs, + TBlob* ret) { + if (kNullOp == req) return; + // reuse csr dns implementation when storage_shape == shape for rhs + if (rhs.storage_shape()[0] == rhs.shape()[0]) { // if rsp is actually dense + DotCsrDnsDnsImpl(ctx, cpu_dev, lhs, rhs.data(), req, trans_lhs, ret); + return; + } + + CHECK_EQ(lhs.storage_type(), kCSRStorage); + CHECK_EQ(rhs.storage_type(), kRowSparseStorage); + mshadow::Stream* s = ctx.get_stream(); + if (!lhs.storage_initialized() || !rhs.storage_initialized()) { + if (kWriteTo == req) { + MSHADOW_SGL_DBL_TYPE_SWITCH(ret->type_flag_, DType, { // data type + mxnet_op::Kernel::Launch( + s, ret->Size(), ret->dptr()); + }); + } + return; + } + using nnvm::dim_t; + + const TBlob data_l = lhs.data(); + const TBlob indptr_l = lhs.aux_data(csr::kIndPtr); + const TBlob col_idx_l = lhs.aux_data(csr::kIdx); + const TBlob data_r = rhs.data(); + const TBlob row_idx_r = rhs.aux_data(rowsparse::kIdx); + + MSHADOW_SGL_DBL_TYPE_SWITCH(data_l.type_flag_, DType, { // data type + MSHADOW_IDX_TYPE_SWITCH(indptr_l.type_flag_, IType, { // indptr type + MSHADOW_IDX_TYPE_SWITCH(col_idx_l.type_flag_, CType, { // col idx type + MSHADOW_IDX_TYPE_SWITCH(row_idx_r.type_flag_, RType, { // row idx type + dim_t num_threads; + if (kWriteTo == req) { + num_threads = ret->Size(); + mxnet_op::Kernel::Launch(s, num_threads, + ret->dptr()); + } + num_threads = mxnet_op::get_num_threads(ret->shape_[0]); + dim_t seg_len = (ret->shape_[0] + num_threads - 1) / num_threads; + if (trans_lhs) { + LOG(FATAL) << "DotCsrRspDnsImpl has not implemented dot(csr.T, rsp) = dns yet"; + } else { + mxnet_op::Kernel::Launch(s, num_threads, + ret->dptr(), data_l.dptr(), + indptr_l.dptr(), col_idx_l.dptr(), data_r.dptr(), + row_idx_r.dptr(), rhs.storage_shape()[0], + ret->shape_[0], ret->shape_[1], seg_len); + } + }); + }); + }); + }); +} + +/*! + * \brief CPU Impl of dot(csr.T, rsp1) = rsp2 + */ +inline void DotCsrRspRspImpl(const OpContext& ctx, + const cpu& cpu_dev, + const NDArray& lhs, + const NDArray& rhs, + const OpReqType req, + const bool trans_lhs, + NDArray* ret) { + if (kNullOp == req) return; + // reuse csr dns implementation when storage_shape == shape for rhs + if (rhs.storage_shape()[0] == rhs.shape()[0]) { // if rsp is actually dense + DotCsrDnsRspImpl(ctx, cpu_dev, lhs, rhs.data(), req, trans_lhs, ret); + return; + } + + CHECK_EQ(lhs.storage_type(), kCSRStorage); + CHECK_EQ(rhs.storage_type(), kRowSparseStorage); + CHECK_EQ(ret->storage_type(), kRowSparseStorage); + if (!lhs.storage_initialized() || !rhs.storage_initialized()) return; + CHECK_EQ(req, kWriteTo); + + using mxnet_op::set_zero; + using nnvm::dim_t; + + mshadow::Stream* s = ctx.get_stream(); + const TBlob data_l = lhs.data(); + const TBlob indptr_l = lhs.aux_data(csr::kIndPtr); + const TBlob col_idx_l = lhs.aux_data(csr::kIdx); + const TBlob data_r = rhs.data(); + const TBlob row_idx_r = rhs.aux_data(rowsparse::kIdx); + + // pre-allocate spaces for ret using the dense dimension size + if (ret->storage_type() == kRowSparseStorage) { + ret->CheckAndAlloc({mshadow::Shape1(lhs.shape()[1])}); + } + const TBlob data_out = ret->data(); + const TBlob row_idx_out = ret->aux_data(rowsparse::kIdx); + + MSHADOW_SGL_DBL_TYPE_SWITCH(data_l.type_flag_, DType, { // data type + MSHADOW_IDX_TYPE_SWITCH(indptr_l.type_flag_, IType, { // indptr type + MSHADOW_IDX_TYPE_SWITCH(col_idx_l.type_flag_, CType, { // col idx type + MSHADOW_IDX_TYPE_SWITCH(row_idx_r.type_flag_, RType, { // row idx type + dim_t num_threads = data_out.Size(); + mxnet_op::Kernel::Launch(s, num_threads, data_out.dptr()); + num_threads = mxnet_op::get_num_threads(data_out.shape_[0]); + dim_t seg_len = (data_out.shape_[0] + num_threads - 1) / num_threads; + if (trans_lhs) { + RType* row_idx = row_idx_out.dptr(); + num_threads = row_idx_out.Size(); + mxnet_op::Kernel::Launch(s, num_threads, row_idx); + mxnet_op::Kernel::Launch(s, num_threads, + data_out.dptr(), row_idx, data_l.dptr(), + indptr_l.dptr(), col_idx_l.dptr(), data_r.dptr(), + row_idx_r.dptr(), lhs.shape()[0], rhs.storage_shape()[0], + ret->shape()[0], ret->shape()[1], seg_len); + dim_t nnr = 0; + nnr = mxnet::common::ParallelAccumulate(row_idx, ret->shape()[0], nnr); + ret->set_aux_shape(rowsparse::kIdx, mshadow::Shape1(nnr)); + if (0 == nnr) return; + mshadow::Tensor rsp_data = data_out.FlatTo2D(s); + dim_t idx = 0; + for (index_t i = 0; i < ret->shape()[0]; ++i) { + if (row_idx[i] > 0) { + row_idx[idx] = i; + mshadow::Copy(rsp_data[idx], rsp_data[i], s); + ++idx; + } + } + } else { + LOG(FATAL) << "DotCsrRspRspImpl has not implemented dot(csr, rsp) = rsp2 yet"; + } + }); + }); + }); + }); +} + +inline bool DotShape(const nnvm::NodeAttrs& attrs, + std::vector *in_attrs, + std::vector *out_attrs) { + const DotParam& param = nnvm::get(attrs.parsed); + CHECK_EQ(in_attrs->size(), 2U); + CHECK_EQ(out_attrs->size(), 1U); + TShape& lshape = (*in_attrs)[0]; + TShape& rshape = (*in_attrs)[1]; + if (lshape.ndim() == 1 && rshape.ndim() == 1) { + CHECK(!param.transpose_a && !param.transpose_b) << "Cannot transpose vectors"; + CHECK_EQ(lshape[0], rshape[0]) << "dot shape error: " << lshape << " X " << rshape; + SHAPE_ASSIGN_CHECK(*out_attrs, 0, mshadow::Shape1(1)); + } else { + bool Ta = param.transpose_a, Tb = param.transpose_b; + TShape L[2], R[2]; + if (Ta) { + L[0] = mshadow::Shape1(lshape[0]); + L[1] = lshape.ndim() > 1 ? TShape(&lshape[1], &lshape[lshape.ndim()]) : TShape(1); + } else { + L[0] = lshape.ndim() > 1 ? TShape(&lshape[0], &lshape[lshape.ndim()-1]) : TShape(1); + L[1] = mshadow::Shape1(lshape[lshape.ndim()-1]); + } + if (Tb) { + R[0] = rshape.ndim() > 1 ? TShape(&rshape[0], &rshape[rshape.ndim()-1]) : TShape(1); + R[1] = mshadow::Shape1(rshape[rshape.ndim()-1]); + } else { + R[0] = mshadow::Shape1(rshape[0]); + R[1] = rshape.ndim() > 1 ? TShape(&rshape[1], &rshape[rshape.ndim()]) : TShape(1); + } + + if (L[!Ta].Size() != 0 && R[Tb].Size() != 0) { + CHECK_EQ(L[!Ta].Size(), R[Tb].Size()) + << "dot shape error: " << lshape << " X " << rshape; + } + std::vector buf; + if (lshape.ndim() > 1) buf.insert(buf.end(), &L[Ta][0], &L[Ta][L[Ta].ndim()]); + if (rshape.ndim() > 1) buf.insert(buf.end(), &R[!Tb][0], &R[!Tb][R[!Tb].ndim()]); + TShape oshape(buf.begin(), buf.end()); + SHAPE_ASSIGN_CHECK(*out_attrs, 0, oshape); + } + return true; +} + +template +void DotForwardEx(const nnvm::NodeAttrs& attrs, + const OpContext& ctx, + const std::vector& inputs, + const std::vector& req, + const std::vector& outputs) { + CHECK_EQ(inputs.size(), 2U); + CHECK_EQ(outputs.size(), 1U); + CHECK_EQ(req.size(), 1U); + const DotParam& param = nnvm::get(attrs.parsed); + CHECK(!param.transpose_b) << "transposing rhs of the sparse dot op is not supported"; + CHECK_EQ(inputs[0].shape().ndim(), 2) << "sparse dot only supports 2 dimensional lhs"; + CHECK_EQ(inputs[1].shape().ndim(), 2) << "sparse dot only supports 2 dimensional rhs"; + auto lhs_stype = inputs[0].storage_type(); + auto rhs_stype = inputs[1].storage_type(); + auto out_stype = outputs[0].storage_type(); + if (lhs_stype == kCSRStorage && rhs_stype == kDefaultStorage && out_stype == kDefaultStorage) { + TBlob ret = outputs[0].data(); + DotCsrDnsDnsImpl(ctx, xpu(), inputs[0], inputs[1].data(), req[0], param.transpose_a, &ret); + } else if (lhs_stype == kCSRStorage && rhs_stype == kRowSparseStorage + && out_stype == kDefaultStorage) { + TBlob ret = outputs[0].data(); + DotCsrRspDnsImpl(ctx, xpu(), inputs[0], inputs[1], req[0], param.transpose_a, &ret); + } else if (lhs_stype == kCSRStorage && rhs_stype == kDefaultStorage + && out_stype == kRowSparseStorage) { + NDArray out = outputs[0]; + DotCsrDnsRspImpl(ctx, xpu(), inputs[0], inputs[1].data(), req[0], param.transpose_a, &out); + } else if (lhs_stype == kCSRStorage && rhs_stype == kRowSparseStorage + && out_stype == kRowSparseStorage) { + NDArray ret = outputs[0]; + DotCsrRspRspImpl(ctx, xpu(), inputs[0], inputs[1], req[0], param.transpose_a, &ret); + } else { + FCompExFallback(attrs, ctx, inputs, req, outputs, DotForward_, "DotForward_"); + } +} + +template +void DotBackwardEx(const nnvm::NodeAttrs& attrs, + const OpContext& ctx, + const std::vector& inputs, + const std::vector& req, + const std::vector& outputs) { + CHECK_EQ(inputs.size(), 3U); + CHECK_EQ(outputs.size(), 2U); + CHECK_EQ(req.size(), 2U); + CHECK_EQ(kNullOp, req[0]) + << "sparse dot does not support computing the gradient of the csr/lhs"; + CHECK_NE(req[1], kWriteInplace) << "DotBackwardEx does not support WriteInplace"; + + const DotParam& param = nnvm::get(attrs.parsed); + CHECK(!param.transpose_b) << "sparse dot only supports dot(A, X) and dot(A.T(), X)"; + CHECK_EQ(inputs[0].shape().ndim(), 2) << "sparse dot only supports 2 dimensional lhs"; + CHECK_EQ(inputs[1].shape().ndim(), 2) << "sparse dot only supports 2 dimensional rhs"; + const auto ograd_stype = inputs[0].storage_type(); + const auto lhs_stype = inputs[1].storage_type(); + const auto rhs_stype = inputs[2].storage_type(); + const auto grad_rhs_stype = outputs[1].storage_type(); + if (ograd_stype == kDefaultStorage // ograd dns format + && lhs_stype == kCSRStorage // csr input lhs of the op + && grad_rhs_stype == kDefaultStorage) { // grad(rhs) dns format + TBlob ret = outputs[1].data(); + DotCsrDnsDnsImpl(ctx, xpu(), inputs[1], inputs[0].data(), req[1], !param.transpose_a, &ret); + } else if (ograd_stype == kDefaultStorage + && lhs_stype == kCSRStorage + && grad_rhs_stype == kRowSparseStorage) { + NDArray ret = outputs[1]; + DotCsrDnsRspImpl(ctx, xpu(), inputs[1], inputs[0].data(), req[1], !param.transpose_a, &ret); + } else { + FCompExFallback(attrs, ctx, inputs, req, outputs, DotBackward_, "DotBackward_"); + } +} + +template +void BatchDotForward_(const nnvm::NodeAttrs& attrs, + const OpContext& ctx, + const std::vector& inputs, + const std::vector& req, + const std::vector& outputs) { + using namespace mshadow; + using namespace mshadow::expr; + mshadow::Stream *s = ctx.get_stream(); + const DotParam& param = nnvm::get(attrs.parsed); + CHECK_EQ(outputs[0].type_flag_, inputs[0].type_flag_) + << "Binary function only support input/output with the same type"; + CHECK_EQ(outputs[0].type_flag_, inputs[1].type_flag_) + << "Binary function only support input/output with the same type"; + CHECK(outputs[0].type_flag_ == kFloat32 || outputs[0].type_flag_ == kFloat64) + << "dot only supports float32 and float64"; + MSHADOW_SGL_DBL_TYPE_SWITCH(outputs[0].type_flag_, DType, { + mshadow::Tensor out = outputs[0].get(s); + mshadow::Tensor mlhs = inputs[0].get(s); + mshadow::Tensor mrhs = inputs[1].get(s); + mshadow::Tensor workspace = + ctx.requested[0].get_space_typed(mshadow::Shape1(3 * out.size(0)), s); + if (kNullOp != req[0]) { + if (param.transpose_a && param.transpose_b) { + mshadow::BatchGEMM(out, mlhs, mrhs, (DType)1.0f, + (kAddTo == req[0]) ? (DType)1.0f : (DType)0.0f, + workspace); + } else if (!param.transpose_a && param.transpose_b) { + mshadow::BatchGEMM(out, mlhs, mrhs, (DType)1.0f, + (kAddTo == req[0]) ? (DType)1.0f : (DType)0.0f, + workspace); + } else if (param.transpose_a && !param.transpose_b) { + mshadow::BatchGEMM(out, mlhs, mrhs, (DType)1.0f, + (kAddTo == req[0]) ? (DType)1.0f : (DType)0.0f, + workspace); + } else { + mshadow::BatchGEMM(out, mlhs, mrhs, (DType)1.0f, + (kAddTo == req[0]) ? (DType)1.0f : (DType)0.0f, + workspace); + } + } + }); +} + +template +void BatchDotBackward_(const nnvm::NodeAttrs& attrs, + const OpContext& ctx, + const std::vector& inputs, + const std::vector& req, + const std::vector& outputs) { + using namespace mshadow; + using namespace mshadow::expr; + mshadow::Stream *s = ctx.get_stream(); + const DotParam& param = nnvm::get(attrs.parsed); + CHECK_NE(req[1], kWriteInplace); + CHECK_NE(req[0], kWriteInplace); + CHECK(outputs[0].type_flag_ == kFloat32 || outputs[0].type_flag_ == kFloat64) + << "dot only supports float32 and float64"; + MSHADOW_SGL_DBL_TYPE_SWITCH(outputs[0].type_flag_, DType, { + mshadow::Tensor mout_grad = inputs[0].get(s); + mshadow::Tensor mlhs_data = inputs[1].get(s); + mshadow::Tensor mrhs_data = inputs[2].get(s); + mshadow::Tensor mlhs_grad = outputs[0].get(s); + mshadow::Tensor mrhs_grad = outputs[1].get(s); + mshadow::Tensor workspace = + ctx.requested[0].get_space_typed( + mshadow::Shape2(2, 3 * mout_grad.size(0)), s); + mshadow::Tensor rhs_workspace = workspace[0]; + mshadow::Tensor lhs_workspace = workspace[1]; + if (param.transpose_a && param.transpose_b) { + // Gradient of z = dot(x.T, y.T) + // dy = dot(x, dz).T = dot(dz.T, x.T) + // dx = dot(dz, y).T = dot(y.T, dz.T) + if (kNullOp != req[1]) { + mshadow::BatchGEMM(mrhs_grad, mout_grad, mlhs_data, (DType)1.0f, + (kAddTo == req[1]) ? (DType)1.0f : (DType)0.0f, + rhs_workspace); + } + if (kNullOp != req[0]) { + mshadow::BatchGEMM(mlhs_grad, mrhs_data, mout_grad, (DType)1.0f, + (kAddTo == req[0]) ? (DType)1.0f : (DType)0.0f, + lhs_workspace); + } + } else if (!param.transpose_a && param.transpose_b) { + // Gradient of z = dot(x, y.T) + // dy = dot(x.T, dz).T = dot(dz.T, x) + // dx = dot(dz, y) + if (kNullOp != req[1]) { + mshadow::BatchGEMM(mrhs_grad, mout_grad, mlhs_data, (DType)1.0f, + (kAddTo == req[1]) ? (DType)1.0f : (DType)0.0f, + rhs_workspace); + } + if (kNullOp != req[0]) { + mshadow::BatchGEMM(mlhs_grad, mout_grad, mrhs_data, (DType)1.0f, + (kAddTo == req[0]) ? (DType)1.0f : (DType)0.0f, + lhs_workspace); + } + } else if (param.transpose_a && !param.transpose_b) { + // Gradient of z = dot(x.T, y) + // dy = dot(x, dz) + // dx = dot(dz, y.T).T = dot(y, dz.T) + if (kNullOp != req[1]) { + mshadow::BatchGEMM(mrhs_grad, mlhs_data, mout_grad, (DType)1.0f, + (kAddTo == req[1]) ? (DType)1.0f : (DType)0.0f, + rhs_workspace); + } + if (kNullOp != req[0]) { + mshadow::BatchGEMM(mlhs_grad, mrhs_data, mout_grad, (DType)1.0f, + (kAddTo == req[0]) ? (DType)1.0f : (DType)0.0f, + lhs_workspace); + } + } else { + // Gradient of z = dot(x, y) + // dy = dot(x.T, dz) + // dx = dot(dz, y.T) + if (kNullOp != req[1]) { + mshadow::BatchGEMM(mrhs_grad, mlhs_data, mout_grad, (DType)1.0f, + (kAddTo == req[1]) ? (DType)1.0f : (DType)0.0f, + rhs_workspace); + } + if (kNullOp != req[0]) { + mshadow::BatchGEMM(mlhs_grad, mout_grad, mrhs_data, (DType)1.0f, + (kAddTo == req[0]) ? (DType)1.0f : (DType)0.0f, + lhs_workspace); + } + } + }); +} + +inline bool BatchDotShape(const nnvm::NodeAttrs& attrs, + std::vector *in_attrs, + std::vector *out_attrs) { + CHECK_EQ(in_attrs->size(), 2U); + CHECK_EQ(out_attrs->size(), 1U); + const DotParam& param = nnvm::get(attrs.parsed); + TShape& lshape = (*in_attrs)[0]; + TShape& rshape = (*in_attrs)[1]; + if (lshape.ndim() == 3 && rshape.ndim() == 3) { + CHECK(lshape[0] == rshape[0]) + << "batch_dot shape error(batch_size must be equal): " << lshape << " X " << rshape + << " trans_a=" << param.transpose_a << " trans_b=" << param.transpose_b; + index_t out_m = param.transpose_a ? lshape[2] : lshape[1]; + index_t lshape_k = param.transpose_a ? lshape[1] : lshape[2]; + index_t out_n = param.transpose_b ? rshape[1] : rshape[2]; + index_t rshape_k = param.transpose_b ? rshape[2] : rshape[1]; + CHECK(lshape_k == rshape_k) + << "batch_dot shape error(shape mismatch): " << lshape << " X " << rshape + << " trans_a=" << param.transpose_a << " trans_b=" << param.transpose_b; + SHAPE_ASSIGN_CHECK(*out_attrs, 0, mshadow::Shape3(lshape[0], out_m, out_n)); + } else { + LOG(FATAL) << "batch_dot currently only support 3D*3D array" + << lshape << " v.s. " << rshape; + } + return true; +} + +} // namespace op +} // namespace mxnet + +#endif // MXNET_OPERATOR_TENSOR_DOT_INL_H_ diff --git a/src/operator/tensor/dot.cc b/src/operator/tensor/dot.cc new file mode 100644 index 000000000000..a7fa2c7933a5 --- /dev/null +++ b/src/operator/tensor/dot.cc @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file dot.cc + * \brief CPU Implementation of matrix dot + */ + +#include "./dot-inl.h" + +namespace mxnet { +namespace op { +DMLC_REGISTER_PARAMETER(DotParam); + +NNVM_REGISTER_OP(dot) +.add_alias("_sparse_dot") // alias for op registration under mxnet.ndarray.sparse +.describe(R"doc(Dot product of two arrays. + +``dot``'s behavior depends on the input array dimensions: + +- 1-D arrays: inner product of vectors +- 2-D arrays: matrix multiplication +- N-D arrays: a sum product over the last axis of the first input and the first + axis of the second input + + For example, given 3-D ``x`` with shape `(n,m,k)` and ``y`` with shape `(k,r,s)`, the + result array will have shape `(n,m,r,s)`. It is computed by:: + + dot(x,y)[i,j,a,b] = sum(x[i,j,:]*y[:,a,b]) + + Example:: + + x = reshape([0,1,2,3,4,5,6,7], shape=(2,2,2)) + y = reshape([7,6,5,4,3,2,1,0], shape=(2,2,2)) + dot(x,y)[0,0,1,1] = 0 + sum(x[0,0,:]*y[:,1,1]) = 0 + +The storage type of ``dot`` output depends on storage types of inputs and transpose options: + +- dot(csr, default) = default +- dot(csr.T, default) = row_sparse +- dot(csr, row_sparse) = default +- otherwise, ``dot`` generates output with default storage + +)doc" ADD_FILELINE) +.set_num_inputs(2) +.set_num_outputs(1) +.set_attr_parser(ParamParser) +.set_attr("FListInputNames", + [](const NodeAttrs& attrs) { + return std::vector{"lhs", "rhs"}; + }) +.set_attr("FInferShape", DotShape) +.set_attr("FInferType", ElemwiseType<2, 1>) +.set_attr("FInferStorageType", DotForwardInferStorageType) +.set_attr("FResourceRequest", + [](const NodeAttrs& attrs) { + return std::vector{ResourceRequest::kTempSpace}; + }) +.set_attr("FCompute", DotForward_) +.set_attr("FComputeEx", DotForwardEx) +.set_attr("FGradient", ElemwiseGradUseIn{"_backward_dot"}) +.add_argument("lhs", "NDArray-or-Symbol", "The first input") +.add_argument("rhs", "NDArray-or-Symbol", "The second input") +.add_arguments(DotParam::__FIELDS__()); + +NNVM_REGISTER_OP(_backward_dot) +.set_num_inputs(3) +.set_num_outputs(2) +.set_attr_parser(ParamParser) +.set_attr("TIsBackward", true) +.set_attr("FInferStorageType", DotBackwardInferStorageType) +.set_attr("FResourceRequest", + [](const NodeAttrs& attrs) { + return std::vector{ResourceRequest::kTempSpace}; + }) +.set_attr("FCompute", DotBackward_) +.set_attr("FComputeEx", DotBackwardEx) +.add_arguments(DotParam::__FIELDS__()); + +NNVM_REGISTER_OP(batch_dot) +.describe(R"doc(Batchwise dot product. + +``batch_dot`` is used to compute dot product of ``x`` and ``y`` when ``x`` and +``y`` are data in batch, namely 3D arrays in shape of `(batch_size, :, :)`. + +For example, given ``x`` with shape `(batch_size, n, m)` and ``y`` with shape +`(batch_size, m, k)`, the result array will have shape `(batch_size, n, k)`, +which is computed by:: + + batch_dot(x,y)[i,:,:] = dot(x[i,:,:], y[i,:,:]) + +)doc" ADD_FILELINE) +.set_num_inputs(2) +.set_num_outputs(1) +.set_attr_parser(ParamParser) +.set_attr("FListInputNames", + [](const NodeAttrs& attrs) { + return std::vector{"lhs", "rhs"}; + }) +.set_attr("FInferShape", BatchDotShape) +.set_attr("FInferType", ElemwiseType<2, 1>) +.set_attr("FResourceRequest", + [](const NodeAttrs& attrs) { + return std::vector{ResourceRequest::kTempSpace}; + }) +.set_attr("FCompute", BatchDotForward_) +.set_attr("FGradient", ElemwiseGradUseIn{"_backward_batch_dot"}) +.add_argument("lhs", "NDArray-or-Symbol", "The first input") +.add_argument("rhs", "NDArray-or-Symbol", "The second input") +.add_arguments(DotParam::__FIELDS__()); + +NNVM_REGISTER_OP(_backward_batch_dot) +.set_num_inputs(3) +.set_num_outputs(2) +.set_attr_parser(ParamParser) +.set_attr("FResourceRequest", + [](const NodeAttrs& attrs) { + return std::vector{ResourceRequest::kTempSpace}; + }) +.set_attr("TIsBackward", true) +.set_attr("FCompute", BatchDotBackward_); + +} // namespace op +} // namespace mxnet diff --git a/src/operator/tensor/dot.cu b/src/operator/tensor/dot.cu new file mode 100644 index 000000000000..8ee2e2832fbb --- /dev/null +++ b/src/operator/tensor/dot.cu @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file dot.cu + * \brief GPU Implementation of matrix dot + */ + +#include "./dot-inl.h" + +namespace mxnet { +namespace op { + +NNVM_REGISTER_OP(dot) +.set_attr("FCompute", DotForward_) +.set_attr("FComputeEx", DotForwardEx); + +NNVM_REGISTER_OP(_backward_dot) +.set_attr("FCompute", DotBackward_) +.set_attr("FComputeEx", DotBackwardEx); + +NNVM_REGISTER_OP(batch_dot) +.set_attr("FCompute", BatchDotForward_); + +NNVM_REGISTER_OP(_backward_batch_dot) +.set_attr("FCompute", BatchDotBackward_); + +} // namespace op +} // namespace mxnet diff --git a/src/operator/tensor/elemwise_binary_broadcast_op_basic.cc b/src/operator/tensor/elemwise_binary_broadcast_op_basic.cc index c80d46a883ea..8c97849e20dc 100644 --- a/src/operator/tensor/elemwise_binary_broadcast_op_basic.cc +++ b/src/operator/tensor/elemwise_binary_broadcast_op_basic.cc @@ -123,6 +123,7 @@ Example:: .set_attr("FCompute", BinaryBroadcastCompute) .set_attr("FGradient", ElemwiseGradUseIn{"_backward_broadcast_mul"}); + NNVM_REGISTER_OP(_backward_broadcast_mul) .set_num_inputs(3) .set_num_outputs(2) diff --git a/src/operator/tensor/elemwise_binary_op.h b/src/operator/tensor/elemwise_binary_op.h index 87b0d46a63c9..ddcad5e61ba0 100644 --- a/src/operator/tensor/elemwise_binary_op.h +++ b/src/operator/tensor/elemwise_binary_op.h @@ -28,10 +28,12 @@ #include #include #include +#include #include "../mxnet_op.h" #include "../mshadow_op.h" #include "../elemwise_op_common.h" -#include "../mxnet_op.h" +#include "./init_op.h" +#include "../../common/utils.h" namespace mxnet { namespace op { @@ -141,6 +143,120 @@ void BinaryBackwardUseNone_(const nnvm::NodeAttrs& attrs, } } +// TODO(haibin) This is a single-thread inefficient implementation +// This implementation only works on CPU +template +void BinaryComputeRspRspImpl(const nnvm::NodeAttrs& attrs, + const OpContext& ctx, + const std::vector& inputs, + const std::vector& req, + const std::vector& outputs) { + if (req[0] == kNullOp) return; + CHECK(req[0] == kWriteTo) << "only kWriteTo is supported for rowsparse elemwise_add"; + using namespace rowsparse; + using namespace mshadow; + auto &lhs = inputs[0]; + auto &rhs = inputs[1]; + auto &output = outputs[0]; + + bool init_l = lhs.storage_initialized(); + bool init_r = rhs.storage_initialized(); + Stream *s = ctx.get_stream(); + // both inputs are zeros + if (!init_l && !init_r) { + NDArray out = output; + FillZerosRspImpl(s, &out); + return; + } + // Memory Estimation: This is (roughly) the number of result rows. We still + // need to subtract the number of common rows + unsigned int num_rows_l = lhs.aux_shape(kIdx)[0]; + unsigned int num_rows_r = rhs.aux_shape(kIdx)[0]; + unsigned int num_rows_total = num_rows_l + num_rows_r; + auto row_len = output.shape().ProdShape(1, output.shape().ndim()); + output.CheckAndAlloc({Shape1(num_rows_total)}); + CHECK_GT(row_len, 0); + MSHADOW_TYPE_SWITCH(output.dtype(), DType, { + MSHADOW_TYPE_SWITCH(lhs.aux_type(kIdx), IType, { + // Indices + auto indices_l = lhs.aux_data(kIdx).dptr(); + auto indices_r = rhs.aux_data(kIdx).dptr(); + auto indices_out = output.aux_data(kIdx).dptr(); + // Data + auto data_l = lhs.data().get_with_shape(Shape2(num_rows_l, row_len), s); + auto data_r = rhs.data().get_with_shape(Shape2(num_rows_r, row_len), s); + auto out = output.data().get_with_shape(Shape2(num_rows_total, row_len), s); + + // TODO(haibin) A more appropriate way: Copy to output, then apply ops + size_t iter_l = 0; + size_t iter_r = 0; + size_t iter_out = 0; + int32_t num_common_rows = 0; + while (iter_l < num_rows_l && iter_r < num_rows_r) { + auto idx_l = indices_l[iter_l]; + auto idx_r = indices_r[iter_r]; + if (idx_l == idx_r) { + // Same row + indices_out[iter_out] = idx_l; + Copy(out[iter_out], data_l[iter_l++], s); + out[iter_out] += data_r[iter_r++]; + num_common_rows++; + } else if (idx_l < idx_r) { + // Left only + indices_out[iter_out] = idx_l; + Copy(out[iter_out], data_l[iter_l++], s); + } else { + // Right only + indices_out[iter_out] = idx_r; + Copy(out[iter_out], data_r[iter_r++], s); + } + iter_out++; + } + // Copying over the rest of the rows + while (iter_l < num_rows_l) { + indices_out[iter_out] = indices_l[iter_l]; + Copy(out[iter_out++], data_l[iter_l++], s); + } + while (iter_r < num_rows_r) { + indices_out[iter_out] = indices_r[iter_r]; + Copy(out[iter_out++], data_r[iter_r++], s); + } + auto new_sshape = TShape(output.aux_shape(rowsparse::kIdx)); + CHECK_GT(new_sshape[0], num_common_rows); + new_sshape[0] -= num_common_rows; + output.set_aux_shape(rowsparse::kIdx, new_sshape); + }); + }); +} + +template +void BinaryComputeEx(const nnvm::NodeAttrs& attrs, + const OpContext& ctx, + const std::vector& inputs, + const std::vector& req, + const std::vector& outputs) { + using namespace mshadow; + using namespace mshadow::expr; + Stream *s = ctx.get_stream(); + CHECK_EQ(inputs.size(), 2); + CHECK_EQ(outputs.size(), 1); + if (typeid(OP) == typeid(mshadow::op::plus)) { + // If any input is dense, fallback to FCompute + // TODO(haibin) implement dns + rsp in a separate kernel + if (common::ContainsDefaultStorage(inputs)) { + FCompExFallback(attrs, ctx, inputs, req, outputs, + BinaryCompute, "BinaryCompute"); + return; + } + CHECK_EQ(inputs[0].storage_type(), kRowSparseStorage) << "Sparse type not supported yet"; + CHECK_EQ(inputs[1].storage_type(), kRowSparseStorage) << "Sparse type not supported yet"; + BinaryComputeRspRspImpl(attrs, ctx, inputs, req, outputs); + return; + } else { + LOG(FATAL) << "Not implemented"; + } +} + template void BinaryBackwardUseNone(const nnvm::NodeAttrs& attrs, const OpContext& ctx, @@ -152,6 +268,55 @@ void BinaryBackwardUseNone(const nnvm::NodeAttrs& attrs, }); } +// Only implemented for _backward_add for now +template +void BinaryBackwardUseNoneRsp(const nnvm::NodeAttrs& attrs, + const OpContext& ctx, + const std::vector& inputs, + const std::vector& req, + const std::vector& outputs) { + using namespace mshadow; + using namespace mshadow::expr; + Stream *s = ctx.get_stream(); + CHECK_EQ(inputs[0].storage_type(), kRowSparseStorage); + CHECK_EQ(outputs[0].storage_type(), kRowSparseStorage); + CHECK_EQ(outputs[1].storage_type(), kRowSparseStorage); + CHECK(typeid(LOP) == typeid(mshadow_op::identity)); + CHECK(typeid(ROP) == typeid(mshadow_op::identity)); + TShape shape = inputs[0].aux_shape(rowsparse::kIdx); + outputs[0].CheckAndAlloc({shape}); + outputs[1].CheckAndAlloc({shape}); + MSHADOW_TYPE_SWITCH(outputs[0].dtype(), DType, { + MSHADOW_TYPE_SWITCH(outputs[0].aux_type(rowsparse::kIdx), IType, { + auto lgrad_idx = outputs[0].aux_data(rowsparse::kIdx).FlatTo1D(s); + auto rgrad_idx = outputs[1].aux_data(rowsparse::kIdx).FlatTo1D(s); + auto ograd_idx = inputs[0].aux_data(rowsparse::kIdx).FlatTo1D(s); + auto lgrad = outputs[0].data().FlatTo1D(s); + Tensor rgrad = outputs[1].data().FlatTo1D(s); + Tensor ograd = inputs[0].data().FlatTo1D(s); + ASSIGN_DISPATCH(lgrad, req[0], F(ograd)); + ASSIGN_DISPATCH(rgrad, req[1], F(ograd)); + ASSIGN_DISPATCH(lgrad_idx, req[0], F(ograd_idx)); + ASSIGN_DISPATCH(rgrad_idx, req[1], F(ograd_idx)); + }); + }); +} +// Only implemented for _backward_add for now +template +void BinaryBackwardUseNoneEx(const nnvm::NodeAttrs& attrs, + const OpContext& ctx, + const std::vector& inputs, + const std::vector& req, + const std::vector& outputs) { + using namespace mshadow; + using namespace mshadow::expr; + Stream *s = ctx.get_stream(); + auto stype = inputs[0].storage_type(); + CHECK_EQ(stype, kRowSparseStorage) << "Not implemented yet"; + BinaryBackwardUseNoneRsp(attrs, ctx, inputs, req, outputs); + // TODO(haibin) fallback for kDefaultStorage +} + template void BinaryBackwardUseNoneWithHalf2(const nnvm::NodeAttrs& attrs, const OpContext& ctx, @@ -232,7 +397,7 @@ void BinaryBackwardUseInWithHalf2(const nnvm::NodeAttrs& attrs, [](const NodeAttrs& attrs){ \ return std::vector >{{0, 0}, {1, 0}}; \ }) \ - .add_argument("lhs", "NDArray-or-Symbol", "first input") \ + .add_argument("lhs", "NDArray-or-Symbol", "first input") \ .add_argument("rhs", "NDArray-or-Symbol", "second input") } // namespace op diff --git a/src/operator/tensor/elemwise_binary_op_basic.cc b/src/operator/tensor/elemwise_binary_op_basic.cc index 65d4ca9aadd6..a40d86fdfcd6 100644 --- a/src/operator/tensor/elemwise_binary_op_basic.cc +++ b/src/operator/tensor/elemwise_binary_op_basic.cc @@ -27,10 +27,19 @@ namespace mxnet { namespace op { MXNET_OPERATOR_REGISTER_BINARY(elemwise_add) -.add_alias("_add").add_alias("_plus").add_alias("_Plus") -.describe("Adds arguments element-wise.") +.add_alias("_add").add_alias("_plus").add_alias("_Plus").add_alias("_sparse_elemwise_add") +.describe(R"code(Adds arguments element-wise. + +The storage type of ``elemwise_add`` output depends on storage types of inputs + +- elemwise_add(row_sparse, row_sparse) = row_sparse +- otherwise, ``elemwise_add`` generates output with default storage + +)code") .set_attr("FCompute", BinaryCompute) -.set_attr("FGradient", CloneGradient{"_backward_add"}); +.set_attr("FGradient", CloneGradient{"_backward_add"}) +.set_attr("FComputeEx", BinaryComputeEx) +.set_attr("FInferStorageType", ElemwiseStorageType<2, 1>); // specialized gradient add function to do add to optimization // this must differ from elemwise_add to prevent add to optimization in forward pass. @@ -46,7 +55,10 @@ NNVM_REGISTER_OP(_backward_add) return std::vector >{{0, 0}, {0, 1}}; }) .set_attr("FCompute", BinaryBackwardUseNone); + mshadow_op::identity>) +.set_attr("FComputeEx", + BinaryBackwardUseNoneEx) +.set_attr("FInferStorageType", ElemwiseStorageType<1, 2>); MXNET_OPERATOR_REGISTER_BINARY(_sub) .add_alias("_minus").add_alias("_Minus") diff --git a/src/operator/tensor/elemwise_sum.cc b/src/operator/tensor/elemwise_sum.cc index 652be72f3fab..f6b6859505f8 100644 --- a/src/operator/tensor/elemwise_sum.cc +++ b/src/operator/tensor/elemwise_sum.cc @@ -22,6 +22,7 @@ * \brief elementwise sum operator */ #include "./elemwise_sum.h" +#include "../../ndarray/ndarray_function.h" namespace mxnet { namespace op { @@ -54,14 +55,69 @@ std::vector ElementWiseSumGrad( return ret; } +bool ElementWiseSumShape(const nnvm::NodeAttrs& attrs, + std::vector *in_attrs, + std::vector *out_attrs) { + CHECK_EQ(out_attrs->size(), 1); + return ElemwiseAttr( + attrs, in_attrs, out_attrs, TShape()); +} + +bool ElementWiseSumType(const nnvm::NodeAttrs& attrs, + std::vector *in_attrs, + std::vector *out_attrs) { + CHECK_EQ(out_attrs->size(), 1); + return ElemwiseAttr( + attrs, in_attrs, out_attrs, -1); +} + +bool ElementWiseSumForwardInferStorageType(const nnvm::NodeAttrs& attrs, + const Context& ctx, + std::vector *in_attrs, + std::vector *out_attrs) { + CHECK(!in_attrs->empty()); + CHECK_EQ(out_attrs->size(), 1U); + return ElemwiseStorageAttr( + attrs, in_attrs, out_attrs); +} + +void ElementWiseSumComputeExCPU(const nnvm::NodeAttrs& attrs, + const OpContext& ctx, + const std::vector& inputs, + const std::vector& req, + const std::vector& outputs) { + CHECK(!inputs.empty()); + CHECK_EQ(outputs.size(), 1U); + CHECK_EQ(req.size(), 1U); + if (req[0] == kNullOp) return; + CHECK_EQ(req[0], kWriteTo) << "ElementWiseSumComputeExCPU only supports req = kWriteTo"; + using namespace mshadow; + Stream* s = ctx.get_stream(); + NDArray out_nd = outputs[0]; + if (inputs[0].storage_type() == kRowSparseStorage) { + mxnet::ndarray::ElementwiseSum(s, inputs, &out_nd); + } else { + FCompExFallback(attrs, ctx, inputs, req, outputs, + ElementWiseSumCompute, "ElementWiseSumCompute"); + } +} + NNVM_REGISTER_OP(add_n) .add_alias("ElementWiseSum") +.add_alias("_sparse_add_n") +.add_alias("_sparse_ElementWiseSum") .describe(R"doc(Adds all input arguments element-wise. .. math:: add\_n(a_1, a_2, ..., a_n) = a_1 + a_2 + ... + a_n ``add_n`` is potentially more efficient than calling ``add`` by `n` times. + +The storage type of ``add_n`` output depends on storage types of inputs + +- add_n(row_sparse, row_sparse, ..) = row_sparse +- otherwise, ``add_n`` generates output with default storage + )doc" ADD_FILELINE) .set_attr_parser(ParamParser) .set_num_inputs([](const nnvm::NodeAttrs& attrs) { @@ -79,16 +135,16 @@ NNVM_REGISTER_OP(add_n) }) .set_attr("key_var_num_args", "num_args") .set_attr("FCompute", ElementWiseSumCompute) +.set_attr("FComputeEx", ElementWiseSumComputeExCPU) .set_attr( "FInplaceOption", [](const NodeAttrs& attrs) { return std::vector >{{0, 0}}; }) -.set_attr("FInferShape", ElemwiseShape<-1, 1>) -.set_attr("FInferType", ElemwiseType<-1, 1>) -.set_attr("FGradient", CloneGradient{"_backward_add_n"}) +.set_attr("FInferShape", ElementWiseSumShape) +.set_attr("FInferType", ElementWiseSumType) +.set_attr("FInferStorageType", ElementWiseSumForwardInferStorageType) +.set_attr("FGradient", ElementWiseSumGrad) .add_argument("args", "NDArray-or-Symbol[]", "Positional input arguments"); - - } // namespace op } // namespace mxnet diff --git a/src/operator/tensor/elemwise_unary_op.cc b/src/operator/tensor/elemwise_unary_op.cc index defe72d3738c..e94b8bfb9fea 100644 --- a/src/operator/tensor/elemwise_unary_op.cc +++ b/src/operator/tensor/elemwise_unary_op.cc @@ -70,7 +70,9 @@ MXNET_OPERATOR_REGISTER_UNARY(_copy) [](const NodeAttrs& attrs){ return std::vector{true}; }) +.set_attr("FInferStorageType", ElemwiseStorageType<1, 1>) .set_attr("FCompute", IdentityCompute) +.set_attr("FComputeEx", IdentityComputeEx) .set_attr("FGradient", ElemwiseGradUseNone{"_copy"}); NNVM_REGISTER_OP(_backward_copy) @@ -85,7 +87,9 @@ NNVM_REGISTER_OP(_backward_copy) [](const NodeAttrs& attrs){ return std::vector{true}; }) -.set_attr("FCompute", IdentityCompute); +.set_attr("FInferStorageType", ElemwiseStorageType<1, 1>) +.set_attr("FCompute", IdentityCompute) +.set_attr("FComputeEx", IdentityComputeEx); MXNET_OPERATOR_REGISTER_UNARY(BlockGrad) .add_alias("stop_gradient") @@ -162,7 +166,9 @@ NNVM_REGISTER_OP(_identity_with_attr_like_rhs) .set_attr("FIgnoreInputs", [](const NodeAttrs& attrs) { return std::vector(1, 1); }) .set_attr("FCompute", IdentityCompute) +.set_attr("FComputeEx", IdentityLikeRhsComputeEx) .set_attr("FInferShape", ElemwiseShape<2, 1>) +.set_attr("FInferStorageType", IdentityAttrLikeRhsStorageType) .set_attr( "FGradient", [](const nnvm::NodePtr& n, const std::vector& ograds) { @@ -219,6 +225,7 @@ NNVM_REGISTER_OP(_backward_cast) }) .set_attr("FCompute", CastCompute); + // negative MXNET_OPERATOR_REGISTER_UNARY(negative) .MXNET_DESCRIBE("Numerical negative of the argument, element-wise.") diff --git a/src/operator/tensor/elemwise_unary_op.cu b/src/operator/tensor/elemwise_unary_op.cu index 4211ea305b4e..f5d711c01a29 100644 --- a/src/operator/tensor/elemwise_unary_op.cu +++ b/src/operator/tensor/elemwise_unary_op.cu @@ -40,7 +40,8 @@ NNVM_REGISTER_OP(_backward_sigmoid) // copy NNVM_REGISTER_OP(_copy) -.set_attr("FCompute", IdentityCompute); +.set_attr("FCompute", IdentityCompute) +.set_attr("FComputeEx", IdentityComputeEx); NNVM_REGISTER_OP(_backward_copy) .set_attr("FCompute", IdentityCompute); @@ -53,7 +54,9 @@ NNVM_REGISTER_OP(make_loss) // identity output as first input, but attributes are constrainted to be like rhs NNVM_REGISTER_OP(_identity_with_attr_like_rhs) -.set_attr("FCompute", IdentityCompute); +.set_attr("FCompute", IdentityCompute) +.set_attr("FComputeEx", IdentityLikeRhsComputeEx); + NNVM_REGISTER_OP(Cast) .set_attr("FCompute", CastCompute); diff --git a/src/operator/tensor/elemwise_unary_op.h b/src/operator/tensor/elemwise_unary_op.h index b6994844e0fe..16477b1973d3 100644 --- a/src/operator/tensor/elemwise_unary_op.h +++ b/src/operator/tensor/elemwise_unary_op.h @@ -31,15 +31,17 @@ #include "../mshadow_op.h" #include "../elemwise_op_common.h" #include "../special_functions-inl.h" +#include "./broadcast_reduce-inl.h" +#include "./init_op.h" namespace mxnet { namespace op { template void UnaryLaunch(const nnvm::NodeAttrs& attrs, - const OpContext& ctx, - const std::vector& inputs, - const std::vector& req, - const std::vector& outputs) { + const OpContext& ctx, + const std::vector& inputs, + const std::vector& req, + const std::vector& outputs) { using namespace mshadow; using namespace mxnet_op; Stream *s = ctx.get_stream(); @@ -95,6 +97,108 @@ void IdentityCompute(const nnvm::NodeAttrs& attrs, }); } +template +void IdentityComputeRspRspImpl(const nnvm::NodeAttrs& attrs, + mshadow::Stream *s, + const NDArray& input, + const OpReqType req, + NDArray* output) { + using namespace mshadow; + using namespace mshadow::expr; + using namespace rowsparse; + if (req == kNullOp) return; + CHECK_EQ(req, kWriteTo) << "kWriteTo is expected for IdentityComputeRspRspImpl"; + if (!input.storage_initialized()) { + FillZerosRspImpl(s, output); + return; + } + TShape shape = input.aux_shape(kIdx); + output->CheckAndAlloc({shape}); + MSHADOW_TYPE_SWITCH(output->dtype(), DType, { + MSHADOW_TYPE_SWITCH(output->aux_type(kIdx), AuxType, { + auto out_d = output->data().FlatTo1D(s); + auto out_aux = output->aux_data(kIdx).FlatTo1D(s); + auto in_aux = input.aux_data(kIdx).FlatTo1D(s); + ASSIGN_DISPATCH(out_d, req, + F(input.data().FlatTo1D(s))); + ASSIGN_DISPATCH(out_aux, req, F(in_aux)); + }); + }); +} + +template +void IdentityComputeEx(const nnvm::NodeAttrs& attrs, + const OpContext& ctx, + const std::vector& inputs, + const std::vector& req, + const std::vector& outputs) { + CHECK_EQ(inputs.size(), 1U); + CHECK_EQ(outputs.size(), 1U); + CHECK_EQ(req.size(), 1U); + const auto in_stype = inputs[0].storage_type(); + const auto out_stype = outputs[0].storage_type(); + mshadow::Stream *s = ctx.get_stream(); + if (req[0] == kNullOp) return; + if (in_stype == out_stype) { + if (in_stype == kDefaultStorage) { // dense ndarray + IdentityCompute(attrs, ctx, {inputs[0].data()}, req, {outputs[0].data()}); + } else if (in_stype == kRowSparseStorage || in_stype == kCSRStorage) { // sparse ndarray + if (!inputs[0].storage_initialized()) { + FillComputeZerosEx(attrs, ctx, inputs, req, outputs); + return; + } + CHECK_NE(req[0], kAddTo) << "kAddTo is not supported for IdentityComputeEx"; + const size_t n = mxnet::num_aux_data(out_stype); + outputs[0].CheckAndAlloc(inputs[0].aux_shapes()); + IdentityCompute(attrs, ctx, {inputs[0].data()}, req, {outputs[0].data()}); + for (size_t i = 0; i < n; ++i) { + IdentityCompute(attrs, ctx, {inputs[0].aux_data(i)}, req, {outputs[0].aux_data(i)}); + } + } else { + LOG(FATAL) << "IdentityComputeEx does not support input stype = " << in_stype; + } + } else { + FCompExFallback(attrs, ctx, inputs, req, outputs, IdentityCompute, "IdentityCompute"); + } +} + +inline bool IdentityAttrLikeRhsStorageType(const nnvm::NodeAttrs& attrs, + const Context& ctx, + std::vector *in_attrs, + std::vector *out_attrs) { + // TODO(junwu): add ctx info into storage inference logic + CHECK_EQ(in_attrs->size(), static_cast(2)) << " in operator " << attrs.name; + CHECK_EQ(out_attrs->size(), static_cast(1)) << " in operator " << attrs.name; + auto &in = *in_attrs; + auto &out = *out_attrs; + CHECK_NE(in[1], kUndefinedStorage) << "rhs storage type must be known"; + if (in[0] == kUndefinedStorage) STORAGE_TYPE_ASSIGN_CHECK(in, 0, in[1]); + if (out[0] == kUndefinedStorage) STORAGE_TYPE_ASSIGN_CHECK(out, 0, in[1]); + return true; +} + +template +void IdentityLikeRhsComputeEx(const nnvm::NodeAttrs& attrs, + const OpContext& ctx, + const std::vector& inputs, + const std::vector& req, + const std::vector& outputs) { + using namespace mshadow; + using namespace mshadow::expr; + CHECK_EQ(inputs.size(), 2); + CHECK_EQ(outputs.size(), 1); + Stream *s = ctx.get_stream(); + const auto in_stype = inputs[0].storage_type(); + const auto out_stype = outputs[0].storage_type(); + if (in_stype == out_stype) { + std::vector in{inputs[0]}; + IdentityComputeEx(attrs, ctx, in, req, outputs); + } else { + LOG(FATAL) << "IdentityLikeRhsComputeEx not implemented for in_stype = " << in_stype + << " out_stype = " << out_stype; + } +} + struct CastParam : public dmlc::Parameter { // use int for enumeration int dtype; @@ -186,4 +290,5 @@ struct relu_grad { } // namespace op } // namespace mxnet + #endif // MXNET_OPERATOR_TENSOR_ELEMWISE_UNARY_OP_H_ diff --git a/src/operator/tensor/indexing_op.cc b/src/operator/tensor/indexing_op.cc index e5cb41088e22..8c5d4f5411f8 100644 --- a/src/operator/tensor/indexing_op.cc +++ b/src/operator/tensor/indexing_op.cc @@ -104,7 +104,6 @@ NNVM_REGISTER_OP(_backward_Embedding) .set_attr("TIsBackward", true) .set_attr("FCompute", EmbeddingOpBackward); - NNVM_REGISTER_OP(take) .describe(R"code(Takes elements from an input array along the given axis. diff --git a/src/operator/tensor/indexing_op.h b/src/operator/tensor/indexing_op.h index ef42b01fb5b6..a9ee408082d4 100644 --- a/src/operator/tensor/indexing_op.h +++ b/src/operator/tensor/indexing_op.h @@ -40,6 +40,9 @@ #include "../elemwise_op_common.h" #include "../mxnet_op.h" #include "./sort_op.h" +#include "./dot-inl.h" +#include "./init_op.h" +#include "./matrix_op-inl.h" namespace mxnet { namespace op { diff --git a/src/operator/tensor/init_op.cc b/src/operator/tensor/init_op.cc index 8dac22a64966..9f333d2d5efe 100644 --- a/src/operator/tensor/init_op.cc +++ b/src/operator/tensor/init_op.cc @@ -39,6 +39,7 @@ NNVM_REGISTER_OP(_zeros) .set_attr("FInferShape", InitShape) .set_attr("FInferType", InitType) .set_attr("FCompute", FillCompute) +.set_attr("FComputeEx", FillComputeZerosEx) .add_arguments(InitOpParam::__FIELDS__()); NNVM_REGISTER_OP(_ones) diff --git a/src/operator/tensor/init_op.cu b/src/operator/tensor/init_op.cu index 6e2b65cc8519..cbee203c2b31 100644 --- a/src/operator/tensor/init_op.cu +++ b/src/operator/tensor/init_op.cu @@ -27,7 +27,8 @@ namespace mxnet { namespace op { NNVM_REGISTER_OP(_zeros) -.set_attr("FCompute", FillCompute); +.set_attr("FCompute", FillCompute) +.set_attr("FComputeEx", FillComputeZerosEx); NNVM_REGISTER_OP(_ones) .set_attr("FCompute", FillCompute); diff --git a/src/operator/tensor/init_op.h b/src/operator/tensor/init_op.h index 30a5a3a3af1b..12999b943be4 100644 --- a/src/operator/tensor/init_op.h +++ b/src/operator/tensor/init_op.h @@ -33,6 +33,8 @@ #include #include #include "../elemwise_op_common.h" +#include "../mxnet_op.h" + namespace mxnet { namespace op { @@ -129,7 +131,6 @@ inline bool InitType(const nnvm::NodeAttrs& attrs, return true; } - template void FillCompute(const nnvm::NodeAttrs& attrs, const OpContext& ctx, @@ -145,6 +146,91 @@ void FillCompute(const nnvm::NodeAttrs& attrs, }); } +// Fill in the indices and values of a RowSparse NDArray to represent a zeros NDArray, +// instead of the usual compact representation. +template +inline void FillDnsZerosRspImpl(mshadow::Stream *s, NDArray *dst) { + using namespace rowsparse; + using namespace mshadow::expr; + using namespace mshadow; + using namespace mxnet_op; + CHECK_EQ(dst->storage_type(), kRowSparseStorage); + MSHADOW_REAL_TYPE_SWITCH(dst->dtype(), DType, { + MSHADOW_IDX_TYPE_SWITCH(dst->aux_type(kIdx), IType, { + auto num_rows = dst->shape()[0]; + dst->CheckAndAlloc({Shape1(num_rows)}); + auto idx = dst->aux_data(kIdx).FlatTo1D(s); + auto val = dst->data(); + Kernel::Launch(s, val.Size(), val.dptr()); + ASSIGN_DISPATCH(idx, kWriteTo, range(0, num_rows, 1, 1)); + }); + }); +} + +struct PopulateFullIdxRspKernel { + template + MSHADOW_XINLINE static void Map(int i, IType* out) { + KERNEL_ASSIGN(out[i], kWriteTo, i); + } +}; + +// Fill full indices NDArray with zeros by updating the aux shape. +template +void PopulateFullIdxRspImpl(mshadow::Stream *s, NDArray *dst) { + using namespace rowsparse; + CHECK_EQ(dst->storage_type(), kRowSparseStorage); + nnvm::dim_t nnr = dst->shape()[0]; + dst->CheckAndAllocAuxData(kIdx, mshadow::Shape1(nnr)); + MSHADOW_IDX_TYPE_SWITCH(dst->aux_type(kIdx), IType, { + IType* idx = dst->aux_data(kIdx).dptr(); + mxnet_op::Kernel::Launch(s, nnr, idx); + }); +} + +// Fill a rsp NDArray with zeros by updating the aux shape. +template +void FillZerosRspImpl(mshadow::Stream *s, NDArray *dst) { + if (!dst->storage_initialized()) return; + // reset the shapes if it's not zeros + auto storage_shape = dst->storage_shape(); + storage_shape[0] = 0; + dst->set_aux_shape(rowsparse::kIdx, TShape(mshadow::Shape1(0))); +} + +// Fill a CSR NDArray with zeros by updating the aux shape. +template +void FillZerosCsrImpl(mshadow::Stream *s, NDArray *dst) { + if (!dst->storage_initialized()) return; + // reset the shapes if it's not zeros + TShape new_shape(mshadow::Shape1(0)); + dst->set_aux_shape(csr::kIndPtr, new_shape); + dst->set_aux_shape(csr::kIdx, new_shape); +} + +template +void FillComputeZerosEx(const nnvm::NodeAttrs& attrs, + const OpContext& ctx, + const std::vector& inputs, + const std::vector& req, + const std::vector& outputs) { + using namespace mshadow; + using namespace mshadow::expr; + Stream *s = ctx.get_stream(); + CHECK_EQ(outputs.size(), 1); + auto stype = outputs[0].storage_type(); + if (req[0] == kNullOp) return; + CHECK_EQ(req[0], kWriteTo) << "kWriteTo is expected for FillComputeZerosEx"; + if (stype == kRowSparseStorage) { + NDArray nd(outputs[0]); + FillZerosRspImpl(s, &nd); + } else if (stype == kCSRStorage) { + NDArray nd(outputs[0]); + FillZerosCsrImpl(s, &nd); + } else { + // no fallback is required since the output doesn't depend on input + LOG(FATAL) << "storage type " << stype << " not implemented."; + } +} template void RangeCompute(const nnvm::NodeAttrs& attrs, diff --git a/src/operator/tensor/matrix_op-inl.h b/src/operator/tensor/matrix_op-inl.h index af0de593c1be..4654b37ab2bc 100644 --- a/src/operator/tensor/matrix_op-inl.h +++ b/src/operator/tensor/matrix_op-inl.h @@ -28,6 +28,7 @@ #include #include #include +#include #include "../mshadow_op.h" #include "../elemwise_op_common.h" #include "../channel_op_common.h" @@ -368,364 +369,6 @@ inline bool ExpandDimShape(const nnvm::NodeAttrs& attrs, return true; } -struct DotParam : public dmlc::Parameter { - bool transpose_a; - bool transpose_b; - DMLC_DECLARE_PARAMETER(DotParam) { - DMLC_DECLARE_FIELD(transpose_a) - .describe("If true then transpose the first input before dot.") - .set_default(false); - DMLC_DECLARE_FIELD(transpose_b) - .describe("If true then transpose the second input before dot.") - .set_default(false); - } -}; - -template -void DotForward_(const nnvm::NodeAttrs& attrs, - const OpContext& ctx, - const std::vector& inputs, - const std::vector& req, - const std::vector& outputs) { - using namespace mshadow; - using namespace mshadow::expr; - const DotParam& param = nnvm::get(attrs.parsed); - Stream *s = ctx.get_stream(); - CHECK_EQ(outputs[0].type_flag_, inputs[0].type_flag_) - << "Binary function only support input/output with the same type"; - CHECK_EQ(outputs[0].type_flag_, inputs[1].type_flag_) - << "Binary function only support input/output with the same type"; - CHECK(outputs[0].type_flag_ == kFloat32 || outputs[0].type_flag_ == kFloat64) - << "dot only supports float32 and float64"; - MSHADOW_TYPE_SWITCH(outputs[0].type_flag_, DType, { - if (inputs[0].ndim() == 1 && inputs[1].ndim() == 1) { - CHECK_NE(req[0], kAddTo) << "AddTo not yet suported"; - Tensor out = outputs[0].get(s); - VectorDot(out, - inputs[0].get(s), - inputs[1].get(s)); - } else { - int ma, na, mb, nb, m, n; - if (param.transpose_a) { - ma = inputs[0].size(0); - na = inputs[0].Size()/ma; - m = na; - } else { - na = inputs[0].size(inputs[0].ndim()-1); - ma = inputs[0].Size()/na; - m = ma; - } - if (param.transpose_b) { - nb = inputs[1].size(inputs[1].ndim()-1); - mb = inputs[1].Size()/nb; - n = mb; - } else { - mb = inputs[1].size(0); - nb = inputs[1].Size()/mb; - n = nb; - } - Tensor input0 = - inputs[0].get_with_shape(Shape2(ma, na), s); - Tensor input1 = - inputs[1].get_with_shape(Shape2(mb, nb), s); - Tensor out = - outputs[0].get_with_shape(Shape2(m, n), s); - if (param.transpose_a && param.transpose_b) { - ASSIGN_DISPATCH(out, req[0], dot(input0.T(), input1.T())); - } else if (!param.transpose_a && param.transpose_b) { - ASSIGN_DISPATCH(out, req[0], dot(input0, input1.T())); - } else if (param.transpose_a && !param.transpose_b) { - ASSIGN_DISPATCH(out, req[0], dot(input0.T(), input1)); - } else { - ASSIGN_DISPATCH(out, req[0], dot(input0, input1)); - } - } - }); -} - -template -void DotBackward_(const nnvm::NodeAttrs& attrs, - const OpContext& ctx, - const std::vector& inputs, - const std::vector& req, - const std::vector& outputs) { - using namespace mshadow; - using namespace mshadow::expr; - const DotParam& param = nnvm::get(attrs.parsed); - Stream *s = ctx.get_stream(); - CHECK_NE(req[0], kWriteInplace); - CHECK_NE(req[1], kWriteInplace); - CHECK(outputs[0].type_flag_ == kFloat32 || outputs[0].type_flag_ == kFloat64) - << "dot only supports float32 and float64"; - MSHADOW_TYPE_SWITCH(outputs[0].type_flag_, DType, { - if (inputs[1].ndim() == 1 && inputs[2].ndim() == 1) { - Tensor mout_grad = inputs[0].get(s); - Tensor mlhs_data = inputs[1].get(s); - Tensor mrhs_data = inputs[2].get(s); - Tensor mlhs_grad = outputs[0].get(s); - Tensor mrhs_grad = outputs[1].get(s); - ASSIGN_DISPATCH(mrhs_grad, req[1], - broadcast_scalar(mout_grad, mlhs_data.shape_) * mlhs_data); - ASSIGN_DISPATCH(mlhs_grad, req[0], - broadcast_scalar(mout_grad, mlhs_data.shape_) * mrhs_data); - } else { - int ma, na, mb, nb, m, n; - if (param.transpose_a) { - ma = outputs[0].size(0); - na = outputs[0].Size()/ma; - m = na; - } else { - na = outputs[0].size(outputs[0].ndim()-1); - ma = outputs[0].Size()/na; - m = ma; - } - if (param.transpose_b) { - nb = outputs[1].size(outputs[1].ndim()-1); - mb = outputs[1].Size()/nb; - n = mb; - } else { - mb = outputs[1].size(0); - nb = outputs[1].Size()/mb; - n = nb; - } - Tensor mout_grad = - inputs[0].get_with_shape(Shape2(m, n), s); - Tensor mlhs_data = - inputs[1].get_with_shape(Shape2(ma, na), s); - Tensor mrhs_data = - inputs[2].get_with_shape(Shape2(mb, nb), s); - Tensor mlhs_grad = - outputs[0].get_with_shape(Shape2(ma, na), s); - Tensor mrhs_grad = - outputs[1].get_with_shape(Shape2(mb, nb), s); - if (param.transpose_a && param.transpose_b) { - // Gradient of z = dot(x.T, y.T) - // dy = dot(x, dz).T = dot(dz.T, x.T) - // dx = dot(dz, y).T = dot(y.T, dz.T) - ASSIGN_DISPATCH(mrhs_grad, req[1], dot(mout_grad.T(), mlhs_data.T())); - ASSIGN_DISPATCH(mlhs_grad, req[0], dot(mrhs_data.T(), mout_grad.T())); - } else if (!param.transpose_a && param.transpose_b) { - // Gradient of z = dot(x, y.T) - // dy = dot(x.T, dz).T = dot(dz.T, x) - // dx = dot(dz, y) - ASSIGN_DISPATCH(mrhs_grad, req[1], dot(mout_grad.T(), mlhs_data)); - ASSIGN_DISPATCH(mlhs_grad, req[0], dot(mout_grad, mrhs_data)); - } else if (param.transpose_a && !param.transpose_b) { - // Gradient of z = dot(x.T, y) - // dy = dot(x, dz) - // dx = dot(dz, y.T).T = dot(y, dz.T) - ASSIGN_DISPATCH(mrhs_grad, req[1], dot(mlhs_data, mout_grad)); - ASSIGN_DISPATCH(mlhs_grad, req[0], dot(mrhs_data, mout_grad.T())); - } else { - // Gradient of z = dot(x, y) - // dy = dot(x.T, dz) - // dx = dot(dz, y.T) - ASSIGN_DISPATCH(mrhs_grad, req[1], dot(mlhs_data.T(), mout_grad)); - ASSIGN_DISPATCH(mlhs_grad, req[0], dot(mout_grad, mrhs_data.T())); - } - } - }); -} - -inline bool DotShape(const nnvm::NodeAttrs& attrs, - std::vector *in_attrs, - std::vector *out_attrs) { - const DotParam& param = nnvm::get(attrs.parsed); - CHECK_EQ(in_attrs->size(), 2U); - CHECK_EQ(out_attrs->size(), 1U); - TShape& lshape = (*in_attrs)[0]; - TShape& rshape = (*in_attrs)[1]; - if (lshape.ndim() == 1 && rshape.ndim() == 1) { - CHECK(!param.transpose_a && !param.transpose_b) << "Cannot transpose vectors"; - CHECK_EQ(lshape[0], rshape[0]) << "dot shape error: " << lshape << " X " << rshape; - SHAPE_ASSIGN_CHECK(*out_attrs, 0, mshadow::Shape1(1)); - } else { - bool Ta = param.transpose_a, Tb = param.transpose_b; - TShape L[2], R[2]; - if (Ta) { - L[0] = mshadow::Shape1(lshape[0]); - L[1] = lshape.ndim() > 1 ? TShape(&lshape[1], &lshape[lshape.ndim()]) : TShape(1); - } else { - L[0] = lshape.ndim() > 1 ? TShape(&lshape[0], &lshape[lshape.ndim()-1]) : TShape(1); - L[1] = mshadow::Shape1(lshape[lshape.ndim()-1]); - } - if (Tb) { - R[0] = rshape.ndim() > 1 ? TShape(&rshape[0], &rshape[rshape.ndim()-1]) : TShape(1); - R[1] = mshadow::Shape1(rshape[rshape.ndim()-1]); - } else { - R[0] = mshadow::Shape1(rshape[0]); - R[1] = rshape.ndim() > 1 ? TShape(&rshape[1], &rshape[rshape.ndim()]) : TShape(1); - } - - if (L[!Ta].Size() != 0 && R[Tb].Size() != 0) { - CHECK_EQ(L[!Ta].Size(), R[Tb].Size()) - << "dot shape error: " << lshape << " X " << rshape; - } - std::vector buf; - if (lshape.ndim() > 1) buf.insert(buf.end(), &L[Ta][0], &L[Ta][L[Ta].ndim()]); - if (rshape.ndim() > 1) buf.insert(buf.end(), &R[!Tb][0], &R[!Tb][R[!Tb].ndim()]); - TShape oshape(buf.begin(), buf.end()); - SHAPE_ASSIGN_CHECK(*out_attrs, 0, oshape); - } - return true; -} - -template -void BatchDotForward_(const nnvm::NodeAttrs& attrs, - const OpContext& ctx, - const std::vector& inputs, - const std::vector& req, - const std::vector& outputs) { - using namespace mshadow; - using namespace mshadow::expr; - mshadow::Stream *s = ctx.get_stream(); - const DotParam& param = nnvm::get(attrs.parsed); - CHECK_EQ(outputs[0].type_flag_, inputs[0].type_flag_) - << "Binary function only support input/output with the same type"; - CHECK_EQ(outputs[0].type_flag_, inputs[1].type_flag_) - << "Binary function only support input/output with the same type"; - CHECK(outputs[0].type_flag_ == kFloat32 || outputs[0].type_flag_ == kFloat64) - << "dot only supports float32 and float64"; - MSHADOW_TYPE_SWITCH(outputs[0].type_flag_, DType, { - mshadow::Tensor out = outputs[0].get(s); - mshadow::Tensor mlhs = inputs[0].get(s); - mshadow::Tensor mrhs = inputs[1].get(s); - mshadow::Tensor workspace = - ctx.requested[0].get_space_typed(mshadow::Shape1(3 * out.size(0)), s); - if (kNullOp != req[0]) { - if (param.transpose_a && param.transpose_b) { - mshadow::BatchGEMM(out, mlhs, mrhs, (DType)1.0f, - (kAddTo == req[0]) ? (DType)1.0f : (DType)0.0f, - workspace); - } else if (!param.transpose_a && param.transpose_b) { - mshadow::BatchGEMM(out, mlhs, mrhs, (DType)1.0f, - (kAddTo == req[0]) ? (DType)1.0f : (DType)0.0f, - workspace); - } else if (param.transpose_a && !param.transpose_b) { - mshadow::BatchGEMM(out, mlhs, mrhs, (DType)1.0f, - (kAddTo == req[0]) ? (DType)1.0f : (DType)0.0f, - workspace); - } else { - mshadow::BatchGEMM(out, mlhs, mrhs, (DType)1.0f, - (kAddTo == req[0]) ? (DType)1.0f : (DType)0.0f, - workspace); - } - } - }); -} - -template -void BatchDotBackward_(const nnvm::NodeAttrs& attrs, - const OpContext& ctx, - const std::vector& inputs, - const std::vector& req, - const std::vector& outputs) { - using namespace mshadow; - using namespace mshadow::expr; - mshadow::Stream *s = ctx.get_stream(); - const DotParam& param = nnvm::get(attrs.parsed); - CHECK_NE(req[1], kWriteInplace); - CHECK_NE(req[0], kWriteInplace); - CHECK(outputs[0].type_flag_ == kFloat32 || outputs[0].type_flag_ == kFloat64) - << "dot only supports float32 and float64"; - MSHADOW_TYPE_SWITCH(outputs[0].type_flag_, DType, { - mshadow::Tensor mout_grad = inputs[0].get(s); - mshadow::Tensor mlhs_data = inputs[1].get(s); - mshadow::Tensor mrhs_data = inputs[2].get(s); - mshadow::Tensor mlhs_grad = outputs[0].get(s); - mshadow::Tensor mrhs_grad = outputs[1].get(s); - mshadow::Tensor workspace = - ctx.requested[0].get_space_typed( - mshadow::Shape2(2, 3 * mout_grad.size(0)), s); - mshadow::Tensor rhs_workspace = workspace[0]; - mshadow::Tensor lhs_workspace = workspace[1]; - if (param.transpose_a && param.transpose_b) { - // Gradient of z = dot(x.T, y.T) - // dy = dot(x, dz).T = dot(dz.T, x.T) - // dx = dot(dz, y).T = dot(y.T, dz.T) - if (kNullOp != req[1]) { - mshadow::BatchGEMM(mrhs_grad, mout_grad, mlhs_data, (DType)1.0f, - (kAddTo == req[1]) ? (DType)1.0f : (DType)0.0f, - rhs_workspace); - } - if (kNullOp != req[0]) { - mshadow::BatchGEMM(mlhs_grad, mrhs_data, mout_grad, (DType)1.0f, - (kAddTo == req[0]) ? (DType)1.0f : (DType)0.0f, - lhs_workspace); - } - } else if (!param.transpose_a && param.transpose_b) { - // Gradient of z = dot(x, y.T) - // dy = dot(x.T, dz).T = dot(dz.T, x) - // dx = dot(dz, y) - if (kNullOp != req[1]) { - mshadow::BatchGEMM(mrhs_grad, mout_grad, mlhs_data, (DType)1.0f, - (kAddTo == req[1]) ? (DType)1.0f : (DType)0.0f, - rhs_workspace); - } - if (kNullOp != req[0]) { - mshadow::BatchGEMM(mlhs_grad, mout_grad, mrhs_data, (DType)1.0f, - (kAddTo == req[0]) ? (DType)1.0f : (DType)0.0f, - lhs_workspace); - } - } else if (param.transpose_a && !param.transpose_b) { - // Gradient of z = dot(x.T, y) - // dy = dot(x, dz) - // dx = dot(dz, y.T).T = dot(y, dz.T) - if (kNullOp != req[1]) { - mshadow::BatchGEMM(mrhs_grad, mlhs_data, mout_grad, (DType)1.0f, - (kAddTo == req[1]) ? (DType)1.0f : (DType)0.0f, - rhs_workspace); - } - if (kNullOp != req[0]) { - mshadow::BatchGEMM(mlhs_grad, mrhs_data, mout_grad, (DType)1.0f, - (kAddTo == req[0]) ? (DType)1.0f : (DType)0.0f, - lhs_workspace); - } - } else { - // Gradient of z = dot(x, y) - // dy = dot(x.T, dz) - // dx = dot(dz, y.T) - if (kNullOp != req[1]) { - mshadow::BatchGEMM(mrhs_grad, mlhs_data, mout_grad, (DType)1.0f, - (kAddTo == req[1]) ? (DType)1.0f : (DType)0.0f, - rhs_workspace); - } - if (kNullOp != req[0]) { - mshadow::BatchGEMM(mlhs_grad, mout_grad, mrhs_data, (DType)1.0f, - (kAddTo == req[0]) ? (DType)1.0f : (DType)0.0f, - lhs_workspace); - } - } - }); -} - -inline bool BatchDotShape(const nnvm::NodeAttrs& attrs, - std::vector *in_attrs, - std::vector *out_attrs) { - CHECK_EQ(in_attrs->size(), 2U); - CHECK_EQ(out_attrs->size(), 1U); - const DotParam& param = nnvm::get(attrs.parsed); - TShape& lshape = (*in_attrs)[0]; - TShape& rshape = (*in_attrs)[1]; - if (lshape.ndim() == 3 && rshape.ndim() == 3) { - CHECK(lshape[0] == rshape[0]) - << "batch_dot shape error(batch_size must be equal): " << lshape << " X " << rshape - << " trans_a=" << param.transpose_a << " trans_b=" << param.transpose_b; - index_t out_m = param.transpose_a ? lshape[2] : lshape[1]; - index_t lshape_k = param.transpose_a ? lshape[1] : lshape[2]; - index_t out_n = param.transpose_b ? rshape[1] : rshape[2]; - index_t rshape_k = param.transpose_b ? rshape[2] : rshape[1]; - CHECK(lshape_k == rshape_k) - << "batch_dot shape error(shape mismatch): " << lshape << " X " << rshape - << " trans_a=" << param.transpose_a << " trans_b=" << param.transpose_b; - SHAPE_ASSIGN_CHECK(*out_attrs, 0, mshadow::Shape3(lshape[0], out_m, out_n)); - } else { - LOG(FATAL) << "batch_dot currently only support 3D*3D array" - << lshape << " v.s. " << rshape; - } - return true; -} - struct SliceParam : public dmlc::Parameter { nnvm::Tuple > begin, end; DMLC_DECLARE_PARAMETER(SliceParam) { @@ -845,6 +488,96 @@ void Slice(const nnvm::NodeAttrs& attrs, }); } +// slice the indptr of a csr +struct SliceCsrIndPtr { + template + MSHADOW_XINLINE static void Map(int i, IType* out, const IType* in, const IType* base) { + KERNEL_ASSIGN(out[i], kWriteTo, in[i] - *base); + } +}; + +/* + * a wrapper to launch SliceCsrIndPtr kernel. + * slice [src[begin] .. src[end]) and store in dst[0, end - begin) + */ +template +void SliceCsrIndPtrImpl(const int begin, const int end, RunContext ctx, + const IType* src, IType* dst) { + using namespace mshadow; + using namespace mxnet_op; + Stream *s = ctx.get_stream(); + int indptr_len = end - begin + 1; + Kernel::Launch(s, indptr_len, dst, src + begin, src + begin); +} + +/* + * Slice a CSR NDArray + * Only implemented for CPU + */ +template +void SliceCsrImpl(const SliceParam ¶m, const OpContext& ctx, + const NDArray &in, OpReqType req, const NDArray &out) { + using namespace mshadow; + using namespace mxnet_op; + using namespace csr; + CHECK((std::is_same::value)) << "Slice for CSR input only implemented for CPU"; + if (req == kNullOp) return; + CHECK_NE(req, kAddTo) << "kAddTo for Slice on CSR input is not supported"; + CHECK_NE(req, kWriteInplace) << "kWriteInplace for Slice on CSR input is not supported"; + Stream *s = ctx.get_stream(); + int begin = *param.begin[0]; + int end = *param.end[0]; + int indptr_len = end - begin + 1; + out.CheckAndAllocAuxData(kIndPtr, Shape1(indptr_len)); + if (!in.storage_initialized()) { + out.set_aux_shape(kIndPtr, Shape1(0)); + return; + } + // assume idx indptr share the same type + MSHADOW_IDX_TYPE_SWITCH(in.aux_type(kIndPtr), RType, { + MSHADOW_IDX_TYPE_SWITCH(in.aux_type(kIdx), IType, { + MSHADOW_TYPE_SWITCH(in.dtype(), DType, { + auto in_indptr = in.aux_data(kIndPtr).dptr(); + auto out_indptr = out.aux_data(kIndPtr).dptr(); + SliceCsrIndPtrImpl(begin, end, ctx.run_ctx, in_indptr, out_indptr); + + // retrieve nnz (CPU implementation) + int nnz = out_indptr[indptr_len - 1]; + // copy indices and values + out.CheckAndAllocAuxData(kIdx, Shape1(nnz)); + out.CheckAndAllocData(Shape1(nnz)); + auto in_idx = in.aux_data(kIdx).dptr(); + auto out_idx = out.aux_data(kIdx).dptr(); + auto in_data = in.data().dptr(); + auto out_data = out.data().dptr(); + int offset = in_indptr[begin]; + // this is also a CPU-only implementation + memcpy(out_idx, in_idx + offset, nnz * sizeof(IType)); + memcpy(out_data, in_data + offset, nnz * sizeof(DType)); + }); + }); + }); +} + +template +void SliceEx(const nnvm::NodeAttrs& attrs, + const OpContext& ctx, + const std::vector& inputs, + const std::vector& req, + const std::vector& outputs) { + CHECK_EQ(inputs.size(), 1); + CHECK_EQ(outputs.size(), 1); + const SliceParam& param = nnvm::get(attrs.parsed); + auto in_stype = inputs[0].storage_type(); + CHECK_NE(in_stype, kDefaultStorage) + << "SliceEx is not expected to execute for input with default storage type"; + if (in_stype == kCSRStorage) { + SliceCsrImpl(param, ctx, inputs[0], req[0], outputs[0]); + } else { + LOG(FATAL) << "Slice not implemented for storage type" << in_stype; + } +} + inline bool SliceAssignShape(const nnvm::NodeAttrs& attrs, std::vector *in_attrs, std::vector *out_attrs) { diff --git a/src/operator/tensor/matrix_op.cc b/src/operator/tensor/matrix_op.cc index e7e8f5548a1c..d409b9ec6056 100644 --- a/src/operator/tensor/matrix_op.cc +++ b/src/operator/tensor/matrix_op.cc @@ -34,7 +34,6 @@ DMLC_REGISTER_PARAMETER(ClipParam); DMLC_REGISTER_PARAMETER(SimpleCropAssignScalarParam); DMLC_REGISTER_PARAMETER(SliceParam); DMLC_REGISTER_PARAMETER(SliceAxisParam); -DMLC_REGISTER_PARAMETER(DotParam); DMLC_REGISTER_PARAMETER(RepeatParam); DMLC_REGISTER_PARAMETER(TileParam); DMLC_REGISTER_PARAMETER(ReverseParam); @@ -263,6 +262,9 @@ and ``end=(e_1, e_2, ... e_n)`` indices will result in an array with the shape The resulting array's *k*-th dimension contains elements from the *k*-th dimension of the input array with the open range ``[b_k, e_k)``. +For an input array of non-default storage type(e.g. `csr` or `row_sparse`), it only supports +slicing on the first dimension. + Example:: x = [[ 1., 2., 3., 4.], @@ -276,8 +278,10 @@ Example:: .set_attr_parser(ParamParser) .set_attr("FInferShape", SliceShape) .set_attr("FInferType", ElemwiseType<1, 1>) +.set_attr("FInferStorageType", ElemwiseStorageType<1, 1>) .set_attr("FGradient", ElemwiseGradUseNone{"_backward_slice"}) .set_attr("FCompute", Slice) +.set_attr("FComputeEx", SliceEx) .add_argument("data", "NDArray-or-Symbol", "Source input") .add_arguments(SliceParam::__FIELDS__()); @@ -370,94 +374,6 @@ NNVM_REGISTER_OP(_backward_slice_axis) .set_attr("TIsBackward", true) .set_attr("FCompute", SliceAxisGrad_); -NNVM_REGISTER_OP(dot) -.describe(R"doc(Dot product of two arrays. - -``dot``'s behavior depends on the input array dimensions: - -- 1-D arrays: inner product of vectors -- 2-D arrays: matrix multiplication -- N-D arrays: a sum product over the last axis of the first input and the first - axis of the second input - - For example, given 3-D ``x`` with shape `(n,m,k)` and ``y`` with shape `(k,r,s)`, the - result array will have shape `(n,m,r,s)`. It is computed by:: - - dot(x,y)[i,j,a,b] = sum(x[i,j,:]*y[:,a,b]) - - Example:: - - x = reshape([0,1,2,3,4,5,6,7], shape=(2,2,2)) - y = reshape([7,6,5,4,3,2,1,0], shape=(2,2,2)) - dot(x,y)[0,0,1,1] = 0 - sum(x[0,0,:]*y[:,1,1]) = 0 -)doc" ADD_FILELINE) -.set_num_inputs(2) -.set_num_outputs(1) -.set_attr_parser(ParamParser) -.set_attr("FListInputNames", - [](const NodeAttrs& attrs) { - return std::vector{"lhs", "rhs"}; - }) -.set_attr("FInferShape", DotShape) -.set_attr("FInferType", ElemwiseType<2, 1>) -.set_attr("FCompute", DotForward_) -.set_attr("FGradient", ElemwiseGradUseIn{"_backward_dot"}) -.add_argument("lhs", "NDArray-or-Symbol", "The first input") -.add_argument("rhs", "NDArray-or-Symbol", "The second input") -.add_arguments(DotParam::__FIELDS__()); - -NNVM_REGISTER_OP(_backward_dot) -.set_num_inputs(3) -.set_num_outputs(2) -.set_attr_parser(ParamParser) -.set_attr("TIsBackward", true) -.set_attr("FCompute", DotBackward_) -.add_arguments(DotParam::__FIELDS__()); - -NNVM_REGISTER_OP(batch_dot) -.describe(R"doc(Batchwise dot product. - -``batch_dot`` is used to compute dot product of ``x`` and ``y`` when ``x`` and -``y`` are data in batch, namely 3D arrays in shape of `(batch_size, :, :)`. - -For example, given ``x`` with shape `(batch_size, n, m)` and ``y`` with shape -`(batch_size, m, k)`, the result array will have shape `(batch_size, n, k)`, -which is computed by:: - - batch_dot(x,y)[i,:,:] = dot(x[i,:,:], y[i,:,:]) - -)doc" ADD_FILELINE) -.set_num_inputs(2) -.set_num_outputs(1) -.set_attr_parser(ParamParser) -.set_attr("FListInputNames", - [](const NodeAttrs& attrs) { - return std::vector{"lhs", "rhs"}; - }) -.set_attr("FInferShape", BatchDotShape) -.set_attr("FInferType", ElemwiseType<2, 1>) -.set_attr("FResourceRequest", - [](const NodeAttrs& attrs) { - return std::vector{ResourceRequest::kTempSpace}; - }) -.set_attr("FCompute", BatchDotForward_) -.set_attr("FGradient", ElemwiseGradUseIn{"_backward_batch_dot"}) -.add_argument("lhs", "NDArray-or-Symbol", "The first input") -.add_argument("rhs", "NDArray-or-Symbol", "The second input") -.add_arguments(DotParam::__FIELDS__()); - -NNVM_REGISTER_OP(_backward_batch_dot) -.set_num_inputs(3) -.set_num_outputs(2) -.set_attr_parser(ParamParser) -.set_attr("FResourceRequest", - [](const NodeAttrs& attrs) { - return std::vector{ResourceRequest::kTempSpace}; - }) -.set_attr("TIsBackward", true) -.set_attr("FCompute", BatchDotBackward_); - NNVM_REGISTER_OP(clip) .describe(R"code(Clips (limits) the values in an array. diff --git a/src/operator/tensor/matrix_op.cu b/src/operator/tensor/matrix_op.cu index ca40419a9367..3cf2a7a753d0 100644 --- a/src/operator/tensor/matrix_op.cu +++ b/src/operator/tensor/matrix_op.cu @@ -57,18 +57,6 @@ NNVM_REGISTER_OP(slice_axis) NNVM_REGISTER_OP(_backward_slice_axis) .set_attr("FCompute", SliceAxisGrad_); -NNVM_REGISTER_OP(dot) -.set_attr("FCompute", DotForward_); - -NNVM_REGISTER_OP(_backward_dot) -.set_attr("FCompute", DotBackward_); - -NNVM_REGISTER_OP(batch_dot) -.set_attr("FCompute", BatchDotForward_); - -NNVM_REGISTER_OP(_backward_batch_dot) -.set_attr("FCompute", BatchDotBackward_); - NNVM_REGISTER_OP(clip) .set_attr("FCompute", Clip); diff --git a/src/operator/tensor/sparse_retain-inl.h b/src/operator/tensor/sparse_retain-inl.h new file mode 100644 index 000000000000..5add57c83b24 --- /dev/null +++ b/src/operator/tensor/sparse_retain-inl.h @@ -0,0 +1,396 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file sparse_retain-inl.h + * \brief +*/ +#ifndef MXNET_OPERATOR_TENSOR_SPARSE_RETAIN_INL_H_ +#define MXNET_OPERATOR_TENSOR_SPARSE_RETAIN_INL_H_ + +#include +#include +#include +#include "./init_op.h" +#include "../mshadow_op.h" +#include "../elemwise_op_common.h" +#include "../mxnet_op.h" + +namespace mxnet { +namespace op { + +/*! + * \brief sparse retain namespace + */ +namespace sr { +enum SparseRetainOpInputs {kArr, kIdx}; +enum SparseRetainOpOutputs {kOut}; +} // namespace sr + +inline bool SparseRetainOpShape(const nnvm::NodeAttrs& attrs, + std::vector *in_attrs, + std::vector *out_attrs) { + CHECK_EQ(in_attrs->size(), 2U) + << "sparse_retain operator takes 2 arguments (" << in_attrs->size() << " given)"; + CHECK_EQ(out_attrs->size(), 1U); + + TShape tshape((*in_attrs)[sr::kArr]); + shape_assign(&tshape, (*out_attrs)[sr::kOut]); + SHAPE_ASSIGN_CHECK(*in_attrs, sr::kArr, tshape); + SHAPE_ASSIGN_CHECK(*out_attrs, sr::kOut, tshape); + return true; +} + +inline bool SparseRetainOpType(const nnvm::NodeAttrs& attrs, + std::vector *in_attrs, + std::vector *out_attrs) { + CHECK_EQ(in_attrs->size(), 2U); + CHECK_EQ(out_attrs->size(), 1U); + CHECK_NE((*in_attrs)[sr::kIdx], -1) << "Index type must be set for sparse_retain operator"; + + TYPE_ASSIGN_CHECK(*out_attrs, 0, (*in_attrs)[sr::kArr]); + TYPE_ASSIGN_CHECK(*in_attrs, 0, (*out_attrs)[sr::kOut]); + return (*in_attrs)[0] != -1; +} + +inline bool SparseRetainForwardInferStorageType(const nnvm::NodeAttrs& attrs, + const Context& ctx, + std::vector *in_attrs, + std::vector *out_attrs) { + CHECK_EQ(in_attrs->size(), 2U); + CHECK_EQ(out_attrs->size(), 1U); + type_assign(&(in_attrs->at(sr::kArr)), kRowSparseStorage); + type_assign(&(in_attrs->at(sr::kIdx)), kDefaultStorage); + type_assign(&(out_attrs->at(sr::kOut)), kRowSparseStorage); + return true; +} + +inline bool SparseRetainBackwardInferStorageType(const nnvm::NodeAttrs& attrs, + const Context& ctx, + std::vector *in_attrs, + std::vector *out_attrs) { + CHECK_EQ(in_attrs->size(), 2U); + CHECK_EQ(out_attrs->size(), 2U); + + type_assign(&(in_attrs->at(sr::kOut)), kDefaultStorage); + type_assign(&(in_attrs->at(sr::kIdx)), kDefaultStorage); + type_assign(&(out_attrs->at(sr::kArr)), kRowSparseStorage); + type_assign(&(out_attrs->at(sr::kIdx)), kDefaultStorage); + return true; +} + +/*! + * \brief Each thread searches for a user input index in the input + * row sparse ndarray alternatively. This ensures each thread + * has the almost the same workload. The overhead is the binary + * search. If all the indices of the idx array are contained + * in the in_idx, one should use SparseRetainRspRowBlockKernel instead, + * where each thread only perform binary search once. + */ +struct SparseRetainRspThreadKernel { + template + MSHADOW_XINLINE static void Map(int i, DType* out_data, RType* out_idx, + const DType* in_data, const RType* in_idx, + const IType* idx, const size_t nnr, + const size_t row_length) { + const RType irow = idx[i]; + int j = -1, left = 0, right = nnr - 1; + while (left <= right) { + int m = left + (right - left) / 2; + const auto in_idx_m = in_idx[m]; + if (in_idx_m == irow) { + j = m; + break; + } else if (in_idx_m < irow) { + left = m + 1; + } else { + right = m - 1; + } + } + out_idx[i] = idx[i]; + if (j >= 0) { + const size_t in_offset = j * row_length; + const size_t out_offset = i * row_length; + for (size_t k = 0; k < row_length; ++k) { + out_data[out_offset+k] = in_data[in_offset+k]; + } + } + } +}; + +/*! + * \brief This kernel should be invoked when the row indices + * to be retained are all in the input rsp. + * Each thread searches for a subarray of indices of + * the user-input idx array for retain. The first index + * in the subarray will be searched for using binary search. + * The rest of the indices will be searched for starting from + * the lower bound of the binary search. This kernel assumes + * that idx has been sorted in ascending order. + */ +struct SparseRetainRspRowBlockKernel { + template + MSHADOW_XINLINE static void Map(int i, DType* out_data, RType* out_idx, + const DType* in_data, const RType* in_idx, + const IType* idx, const size_t num_indices, + const size_t nnr, const size_t row_length, + const size_t seg_len) { + const size_t seg_start = i * seg_len; + if (seg_start >= num_indices) return; + const size_t seg_end = (seg_start+seg_len < num_indices? seg_start+seg_len : num_indices); + for (size_t j = seg_start; j < seg_end; ++j) { + out_idx[j] = idx[j]; + } + // use binary search to find the lower bound of idx[seg_start] in in_idx + const RType* first = in_idx; + const RType* last = in_idx + nnr; + const auto val = idx[seg_start]; + const RType* it; + int count = last - first, step; + while (count > 0) { + it = first; + step = count / 2; + it += step; + if (*it < val) { + first = ++it; + count -= step + 1; + } else { + count = step; + } + } + size_t cur_row_idx = first - in_idx; + // end of binary search + if (cur_row_idx == nnr || in_idx[cur_row_idx] > idx[seg_end-1]) { + return; + } + size_t cur_idx = seg_start; + while (cur_row_idx < nnr && cur_idx < seg_end) { + if (in_idx[cur_row_idx] == idx[cur_idx]) { + const size_t in_offset = cur_row_idx * row_length; + const size_t out_offset = cur_idx * row_length; + for (size_t k = 0; k < row_length; ++k) { + out_data[out_offset+k] = in_data[in_offset+k]; + } + ++cur_row_idx; + ++cur_idx; + } else if (in_idx[cur_row_idx] < idx[cur_idx]) { + ++cur_row_idx; + } else { + ++cur_idx; + } + } + } +}; + +/*! + * Copy input indices to output indices. + * Only used when input rsp is dense. + */ +struct SparseRetainCopyIndices { + template + MSHADOW_XINLINE static void Map(int i, RType* out_idx, IType* idx) { + out_idx[i] = idx[i]; + } +}; + +/*! + * Copy input retained rows to output rows. + * Only used when input rsp is dense. + * This kernel is only used when ctx is on GPU. + * So it's parallelized by out_rows' elements, + * instead of rows. + * For CPU ctx, we simply call mshadow::Copy. + */ +struct SparseRetainCopyRetainedRowsFromDns { + template + MSHADOW_XINLINE static void Map(int i, DType* out_rows, const DType* in_rows, + const RType* in_row_idx, const IType* idx, + const size_t row_length) { + const size_t irow = i / row_length; + const size_t icol = i % row_length; + out_rows[i] = in_rows[static_cast(idx[irow]) * row_length + icol]; + } +}; + +template +void SparseRetainOpForwardRspImpl(mshadow::Stream *s, + const NDArray& input_nd, + const TBlob& idx_data, + const OpReqType req, + NDArray* output_nd) { + if (req == kNullOp) return; + CHECK_EQ(req, kWriteTo) << "SparseRetainOpForwardRspImpl only support req = kWriteTo now"; + CHECK_EQ(input_nd.storage_type(), kRowSparseStorage) + << "SparseRetainOpForwardRspImpl operator only takes row sparse NDArray as input"; + CHECK_EQ(output_nd->storage_type(), kRowSparseStorage) + << "SparseRetainOpForwardRspImpl operator only outputs row sparse NDArray"; + + if (!input_nd.storage_initialized() + || idx_data.Size() == 0U + || input_nd.shape()[0] == 0) { + FillZerosRspImpl(s, output_nd); + return; + } + + const TBlob input_data = input_nd.data(); + const TBlob input_idx = input_nd.aux_data(rowsparse::kIdx); + + output_nd->CheckAndAlloc({mshadow::Shape1(idx_data.Size())}); + TBlob output_data = output_nd->data(); + TBlob output_idx = output_nd->aux_data(rowsparse::kIdx); + const auto row_length = input_data.shape_.ProdShape(1, input_data.shape_.ndim()); + + using namespace mxnet_op; + MSHADOW_TYPE_SWITCH(output_data.type_flag_, DType, { // output data type + Kernel::Launch(s, output_data.Size(), output_data.dptr()); + MSHADOW_IDX_TYPE_SWITCH(output_idx.type_flag_, RType, { // row index data type + MSHADOW_TYPE_SWITCH(idx_data.type_flag_, IType, { // index array data type + if (input_idx.Size() == input_nd.shape()[0]) { // input rsp is dense + using namespace mshadow; + // copy indices + Tensor output_idx_tensor = output_idx.FlatTo1D(s); + const size_t num_rows_retained = output_idx.Size(); + if (output_idx.type_flag_ == idx_data.type_flag_) { // same type, use Copy + const Tensor idx_tensor = idx_data.FlatTo1D(s); + Copy(output_idx_tensor, idx_tensor, s); + } else { // different index types, use Kernel::Launch + Kernel::Launch(s, num_rows_retained, + output_idx.dptr(), idx_data.dptr()); + } + // copy data + if (std::is_same::value) { // For cpu, we can access output_idx_tensor[i] + const Tensor input_tensor = + input_data.get_with_shape(Shape2(input_data.shape_[0], row_length), s); + Tensor output_tensor = + output_data.get_with_shape(Shape2(output_data.shape_[0], row_length), + s); + for (size_t i = 0; i < num_rows_retained; ++i) { + Copy(output_tensor[i], input_tensor[output_idx_tensor[i]], s); + } + } else { // For gpu, have to kernel launch + Kernel::Launch(s, output_data.Size(), + output_data.dptr(), input_data.dptr(), input_idx.dptr(), + idx_data.dptr(), row_length); + } + } else { // input rsp is not dense + Kernel::Launch(s, idx_data.Size(), + output_data.dptr(), output_idx.dptr(), input_data.dptr(), + input_idx.dptr(), idx_data.dptr(), input_data.shape_[0], row_length); + } + }); + }); + }); +} + +template +void SparseRetainOpForwardEx(const nnvm::NodeAttrs& attrs, + const OpContext& ctx, + const std::vector& inputs, + const std::vector& req, + const std::vector& outputs) { + CHECK_EQ(inputs.size(), 2U); + CHECK_EQ(outputs.size(), 1U); + CHECK_EQ(req.size(), 1U); + if (req[sr::kOut] == kNullOp) return; + CHECK_EQ(req[sr::kOut], kWriteTo) << "sparse_retain only supports req=\'write\'"; + CHECK_EQ(inputs[sr::kIdx].storage_type(), kDefaultStorage) + << "sparse_retain operator only takes default NDArray as its index array"; + if (inputs[sr::kArr].storage_type() == kRowSparseStorage) { + NDArray output_nd = outputs[sr::kOut]; + SparseRetainOpForwardRspImpl(ctx.get_stream(), inputs[sr::kArr], + inputs[sr::kIdx].data(), req[sr::kOut], &output_nd); + } else { + LOG(FATAL) << "sparse_retain op only supports row-sparse ndarrays as input"; + } +} + +template +struct SparseRetainRspGradKernel { + template + MSHADOW_XINLINE static void Map(int i, DType* in_grad, RType* in_grad_idx, + const DType* out_grad, const IType* idx, + const size_t row_length) { + const RType irow = idx[i]; + in_grad_idx[i] = irow; + const size_t out_offset = irow * row_length; + const size_t in_offset = i * row_length; + for (size_t j = 0; j < row_length; ++j) { + KERNEL_ASSIGN(in_grad[in_offset+j], req, out_grad[out_offset+j]); + } + } +}; + +template +void SparseRetainOpBackwardEx(const nnvm::NodeAttrs& attrs, + const OpContext& ctx, + const std::vector& inputs, + const std::vector& req, + const std::vector& outputs) { + CHECK_EQ(req.size(), 2U); + CHECK_EQ(req[sr::kIdx], kNullOp); + if (req[sr::kArr] == kNullOp) return; + CHECK_EQ(req[sr::kArr], kWriteTo); + + CHECK_EQ(inputs.size(), 2U); + CHECK_EQ(outputs.size(), 2U) + << "sparse_retain does not support calculating gradients of indices"; + + CHECK_EQ(inputs[sr::kOut].storage_type(), kDefaultStorage) + << "sparse_retain backward only takes default NDArray as ograd"; + CHECK_EQ(inputs[sr::kIdx].storage_type(), kDefaultStorage) + << "sparse_retain backward only takes default NDArray as its index array"; + CHECK_EQ(outputs[sr::kArr].storage_type(), kRowSparseStorage) + << "sparse_retain backward only outputs row sparse NDArray as grad of input"; + + using namespace mxnet_op; + using namespace mshadow; + Stream *s = ctx.get_stream(); + const TBlob idx_data = inputs[sr::kIdx].data(); + if (idx_data.Size() == 0U) { + NDArray output = outputs[sr::kArr]; + FillZerosRspImpl(s, &output); + return; + } + + const TBlob out_grad_data = inputs[sr::kOut].data(); + + NDArray in_grad_nd = outputs[sr::kArr]; + in_grad_nd.CheckAndAlloc({mshadow::Shape1(idx_data.Size())}); + TBlob in_grad_data = in_grad_nd.data(); + TBlob in_grad_idx = in_grad_nd.aux_data(rowsparse::kIdx); + const auto row_length = out_grad_data.shape_.ProdShape(1, out_grad_data.shape_.ndim()); + + MSHADOW_TYPE_SWITCH(out_grad_data.type_flag_, DType, { // output data type + MSHADOW_IDX_TYPE_SWITCH(in_grad_idx.type_flag_, RType, { // row index data type + MSHADOW_TYPE_SWITCH(idx_data.type_flag_, IType, { // index array data type + MXNET_ASSIGN_REQ_SWITCH(req[sr::kArr], req_type, { + Kernel, xpu>::Launch( + s, in_grad_idx.Size(), in_grad_data.dptr(), in_grad_idx.dptr(), + out_grad_data.dptr(), idx_data.dptr(), row_length); + }); + }); + }); + }); +} + + +} // namespace op +} // namespace mxnet + +#endif // MXNET_OPERATOR_TENSOR_SPARSE_RETAIN_INL_H_ diff --git a/src/operator/tensor/sparse_retain.cc b/src/operator/tensor/sparse_retain.cc new file mode 100644 index 000000000000..f8fc325c0534 --- /dev/null +++ b/src/operator/tensor/sparse_retain.cc @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file sparse_retain.cc + * \brief +*/ + +#include "./sparse_retain-inl.h" +namespace mxnet { +namespace op { + +// Add prefix "_sparse_" to prevent it from being registered +// under mxnet.ndarray in python frontend as this op only +// accepts row-sparse format ndarrays. It will be registered +// under mxnet.ndarray.sparse with name retain. +NNVM_REGISTER_OP(_sparse_retain) +.describe(R"code(pick rows specified by user input index array from a row sparse matrix +and save them in the output sparse matrix. + +Example:: + + data = [[1, 2], [3, 4], [5, 6]] + indices = [0, 1, 3] + shape = (4, 2) + rsp_in = row_sparse(data, indices) + to_retain = [0, 3] + rsp_out = retain(rsp_in, to_retain) + rsp_out.values = [[1, 2], [5, 6]] + rsp_out.indices = [0, 3] + +The storage type of ``retain`` output depends on storage types of inputs + +- retain(row_sparse, default) = row_sparse +- otherwise, ``retain`` is not supported + +)code" ADD_FILELINE) +.set_num_inputs(2) +.set_num_outputs(1) +.set_attr("FListInputNames", + [](const NodeAttrs& attrs) { + return std::vector{"data", "indices"}; + }) +.set_attr("FInferShape", SparseRetainOpShape) +.set_attr("FInferType", SparseRetainOpType) +.set_attr("FInferStorageType", SparseRetainForwardInferStorageType) +.set_attr("FComputeEx", SparseRetainOpForwardEx) +.set_attr("FGradient", + [](const nnvm::NodePtr& n, const std::vector& ograds) { + return MakeNonlossGradNode("_backward_sparse_retain", n, ograds, + {n->inputs[sr::kIdx]}, n->attrs.dict); + }) +.add_argument("data", "NDArray-or-Symbol", "The input array for sparse_retain operator.") +.add_argument("indices", "NDArray-or-Symbol", "The index array of rows ids that will be retained."); + +NNVM_REGISTER_OP(_backward_sparse_retain) +.set_num_inputs(2) +.set_num_outputs(2) +.set_attr("TIsBackward", true) +.set_attr("FInferStorageType", SparseRetainBackwardInferStorageType) +.set_attr("FComputeEx", SparseRetainOpBackwardEx); + +} // namespace op +} // namespace mxnet diff --git a/src/operator/tensor/sparse_retain.cu b/src/operator/tensor/sparse_retain.cu new file mode 100644 index 000000000000..6b4ac1bdf1a1 --- /dev/null +++ b/src/operator/tensor/sparse_retain.cu @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file sparse_retain.cu + * \brief +*/ + +#include "./sparse_retain-inl.h" +namespace mxnet { +namespace op { + +NNVM_REGISTER_OP(_sparse_retain) +.set_attr("FComputeEx", SparseRetainOpForwardEx); + +NNVM_REGISTER_OP(_backward_sparse_retain) +.set_attr("FComputeEx", SparseRetainOpBackwardEx); + +} // namespace op +} // namespace mxnet diff --git a/src/operator/tensor/square_sum-inl.h b/src/operator/tensor/square_sum-inl.h new file mode 100644 index 000000000000..beb77c37b8d2 --- /dev/null +++ b/src/operator/tensor/square_sum-inl.h @@ -0,0 +1,456 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file square_sum-inl.h + * \brief This is a temporary solution for fusing operators + * square and sum together as a composite op for row sparse tensors. + * The purpose for fusing square and sum for row sparse tensors + * is that the gradient of the fused operator depends on the input + * ndarray and thus its gradient is a row-sparse ndarray too. + * This fused op will become deprecated after the functionality + * of fusing operators is finished in the future. + */ + +#ifndef MXNET_OPERATOR_TENSOR_SQUARE_SUM_INL_H_ +#define MXNET_OPERATOR_TENSOR_SQUARE_SUM_INL_H_ + +#include +#include +#include +#include "../mxnet_op.h" +#include "./broadcast_reduce_op.h" + +namespace mxnet { +namespace op { + +inline bool SquareSumForwardInferStorageType(const nnvm::NodeAttrs& attrs, + const Context& ctx, + std::vector* in_attrs, + std::vector* out_attrs) { + CHECK_EQ(in_attrs->size(), 1U); + CHECK_EQ(out_attrs->size(), 1U); + const ReduceAxesParam& param = nnvm::get(attrs.parsed); + if (in_attrs->at(0) == kRowSparseStorage) { // current impl + if (param.axis[0] == 1 && param.keepdims) { // sum per row and keep dims + STORAGE_TYPE_ASSIGN_CHECK(*out_attrs, 0, kRowSparseStorage); + } else { + STORAGE_TYPE_ASSIGN_CHECK(*out_attrs, 0, kDefaultStorage); + } + } else { // fallback + type_assign(&((*in_attrs)[0]), kDefaultStorage); + type_assign(&((*out_attrs)[0]), kDefaultStorage); + } + return true; +} + +inline bool SquareSumBackwardInferStorageType(const nnvm::NodeAttrs& attrs, + const Context& ctx, + std::vector* in_attrs, + std::vector* out_attrs) { + CHECK_EQ(in_attrs->size(), 2U); + CHECK_EQ(out_attrs->size(), 1U); + const ReduceAxesParam& param = nnvm::get(attrs.parsed); + if (in_attrs->at(0) == kDefaultStorage || in_attrs->at(0) == kRowSparseStorage) { + STORAGE_TYPE_ASSIGN_CHECK(*in_attrs, 1, kRowSparseStorage); + STORAGE_TYPE_ASSIGN_CHECK(*out_attrs, 0, kRowSparseStorage); + } else { // fallback + type_assign(&((*in_attrs)[0]), kDefaultStorage); + type_assign(&((*in_attrs)[1]), kDefaultStorage); + type_assign(&((*out_attrs)[0]), kDefaultStorage); + } + return true; +} + +/*! + * \brief square sum of a rsp + * if axis = -1, same as mx.nd.sum(tensor*tensor) + * if axis = 0, same as mx.nd.sum(tensor*tensor, axis=0) + * if axis = 1, same as mx.nd.sum(tensor*tensor, axis=1) + * where tensor*tensor is elemwise multiplication of two ndarrays. + */ +template +struct SquareSumRspKernel; + +/*! + * \brief square sum of a rsp on axis=0 without keeping the dim + */ +template +struct SquareSumRspKernel { + /*! + * \param j the element index in out_data and column id of in_data + */ + template + MSHADOW_XINLINE static void Map(int j, DType* out_data, const DType* in_data, + const int64_t nnr, const int64_t num_cols) { + DType sum = 0; + for (int64_t i = 0; i < nnr; ++i) { + const DType val = in_data[i*num_cols+j]; + sum += val * val; + } + KERNEL_ASSIGN(out_data[j], req, sum); + } +}; + +/*! + * \brief square sum of a rsp on axis=1 without keeping the dim + */ +template +struct SquareSumRspKernel { + /*! + * \param i the i-th non-zero row of in_data + */ + template + MSHADOW_XINLINE static void Map(int i, DType* out_data, const IType* in_row_idx, + const DType* in_data, const int64_t num_cols) { + DType sum = 0; + const int64_t offset = i * num_cols; + for (int64_t j = 0; j < num_cols; ++j) { + const DType val = in_data[offset+j]; + sum += val * val; + } + KERNEL_ASSIGN(out_data[in_row_idx[i]], req, sum); + } +}; + +/*! + * \brief square sum of a rsp on axis=1 keeping the dim + */ +template +struct SquareSumRspKernel { + /*! + * \param i the i-th non-zero row of in_data + */ + template + MSHADOW_XINLINE static void Map(int i, IType* out_row_idx, DType* out_data, + const IType* in_row_idx, const DType* in_data, + const int64_t num_cols) { + DType sum = 0; + out_row_idx[i] = in_row_idx[i]; + const int64_t offset = i * num_cols; + for (int64_t j = 0; j < num_cols; ++j) { + const DType val = in_data[offset+j]; + sum += val * val; + } + KERNEL_ASSIGN(out_data[i], req, sum); + } +}; + +template +struct SquareSumRspGradKernel; + +template +struct SquareSumRspGradKernel { + /*! + * \param i element index in in_grad and in_data + * \param in_grad_row_idx row_idx of the gradient of the op's input + * \param in_grad gradient of the op's input + * \param out_grad gradient of the op's output + * \param in_row_idx row idx of the op's input + * \param in_data op's input + */ + template + MSHADOW_XINLINE static void Map(int i, IType* in_grad_row_idx, DType* in_grad, + const DType* out_grad, const IType* in_row_idx, + const DType* in_data, const int64_t num_cols) { + const int64_t row = i / num_cols; + in_grad_row_idx[row] = in_row_idx[row]; + KERNEL_ASSIGN(in_grad[i], req, 2*in_data[i]*out_grad[i%num_cols]); + } +}; + +template +struct SquareSumRspGradKernel { + /*! + * \param i element index in in_grad and in_data + * \param in_grad_row_idx row_idx of the gradient of the op's input + * \param in_grad gradient of the op's input + * \param out_grad gradient of the op's output + * \param in_row_idx row idx of the op's input + * \param in_data op's input + */ + template + MSHADOW_XINLINE static void Map(int i, IType* in_grad_row_idx, DType* in_grad, + const DType* out_grad, const IType* in_row_idx, + const DType* in_data, const int64_t num_cols) { + const int64_t row = i / num_cols; + in_grad_row_idx[row] = in_row_idx[row]; + KERNEL_ASSIGN(in_grad[i], req, 2*in_data[i]*out_grad[in_row_idx[row]]); + } +}; + +/*! + * Note: This kernel assumes that the ograd and in_data + * are all rsp and have equal row_idx array, or + * in_data is a full rsp. + */ +template +struct SquareSumRspGradKernel { + /*! + * \param i index of igrad.data() + * \param in_grad_row_idx row_idx of the gradient of the op's input + * \param in_grad gradient of the op's input + * \param out_grad_row_idx row_idx of the gradient of the op's output + * \param out_grad gradient of the op's output + * \param in_data op's input + */ + template + MSHADOW_XINLINE static void Map(int i, IType* in_grad_row_idx, DType* in_grad, + const IType* out_grad_row_idx, const DType* out_grad, + const DType* in_data, const int64_t num_cols) { + const int64_t row = i / num_cols; + in_grad_row_idx[row] = out_grad_row_idx[row]; + KERNEL_ASSIGN(in_grad[i], req, 2*in_data[i]*out_grad[row]); + } +}; + +template +void SquareSumRspImpl(const nnvm::NodeAttrs& attrs, + mshadow::Stream* s, + const NDArray& input, + const OpReqType req, + NDArray* output) { + if (req == kNullOp) return; + const ReduceAxesParam& param = nnvm::get(attrs.parsed); + CHECK_EQ(param.axis.ndim(), 1U) << "_square_sum(row_sparse_matrix) only supports axis=0 or 1"; + CHECK(param.axis[0] == 0 || param.axis[0] == 1) + << "_square_sum(row_sparse_matrix) only supports axis=0 or 1"; + CHECK_EQ(input.storage_type(), kRowSparseStorage) + << "_square_sum op only supports row-sparse matrix as input"; + int64_t out_data_size = 0; + if (param.axis[0] == 0) { // axis = 0 + CHECK_EQ(output->storage_type(), kDefaultStorage); + out_data_size = input.storage_shape()[1]; + } else if (param.keepdims) { // axis = 1, keepdims = true + CHECK_EQ(output->storage_type(), kRowSparseStorage); + out_data_size = input.storage_shape()[0]; + } else { // axis = 1, keepdims = false + CHECK_EQ(output->storage_type(), kDefaultStorage); + out_data_size = input.shape()[0]; + } + CHECK_NE(req, kWriteInplace); + + using namespace mxnet_op; + if (!input.storage_initialized()) { + if (req == kWriteTo) { + if (output->storage_type() == kDefaultStorage) { + MSHADOW_TYPE_SWITCH(output->data().type_flag_, DType, { + Kernel::Launch(s, out_data_size, output->data().dptr()); + }) + } else if (output->storage_type() == kRowSparseStorage) { + FillZerosRspImpl(s, output); + } else { + LOG(FATAL) << "SquareSumRspImpl only supports row-sparse/dense output storage type"; + } + } + return; + } + + if (output->storage_type() == kRowSparseStorage) { + output->CheckAndAlloc({input.aux_shape(rowsparse::kIdx)}); + } + const TBlob& out_data = output->data(); + const int64_t nnr = input.storage_shape()[0]; + const int64_t num_cols = input.storage_shape()[1]; + const TBlob& in_data = input.data(); + if (0 == param.axis[0]) { // axis = 0, output is dense + MSHADOW_TYPE_SWITCH(out_data.type_flag_, DType, { + MXNET_ASSIGN_REQ_SWITCH(req, req_type, { + Kernel, xpu>::Launch(s, num_cols, + out_data.dptr(), input.data().dptr(), nnr, num_cols); + }) + }) + } else { // axis = 1 + const TBlob in_row_idx = input.aux_data(rowsparse::kIdx); + if (param.keepdims) { // output is rsp + const TBlob out_row_idx = output->aux_data(rowsparse::kIdx); + MSHADOW_TYPE_SWITCH(out_data.type_flag_, DType, { + MSHADOW_IDX_TYPE_SWITCH(in_row_idx.type_flag_, IType, { + MXNET_ASSIGN_REQ_SWITCH(req, req_type, { + Kernel, xpu>::Launch(s, nnr, + out_row_idx.dptr(), out_data.dptr(), in_row_idx.dptr(), + in_data.dptr(), num_cols); + }) + }) + }) + } else { // output is dense + if (req == kWriteTo) { + MSHADOW_TYPE_SWITCH(out_data.type_flag_, DType, { + Kernel::Launch(s, out_data_size, out_data.dptr()); + }) + } + MSHADOW_TYPE_SWITCH(out_data.type_flag_, DType, { + MSHADOW_IDX_TYPE_SWITCH(in_row_idx.type_flag_, IType, { + MXNET_ASSIGN_REQ_SWITCH(req, req_type, { + Kernel, xpu>::Launch(s, nnr, + out_data.dptr(), in_row_idx.dptr(), in_data.dptr(), num_cols); + }) + }) + }) + } + } +} + +template +void SquareSumRspGradImpl(const nnvm::NodeAttrs& attrs, + mshadow::Stream* s, + const NDArray& ograd, + const NDArray& input, + const OpReqType req, + NDArray* igrad) { + if (req == kNullOp) return; + const ReduceAxesParam& param = nnvm::get(attrs.parsed); + CHECK_EQ(param.axis.ndim(), 1U) << "_square_sum(row_sparse_matrix) only supports axis=0/1"; + CHECK(param.axis[0] == 0 || param.axis[0] == 1) + << "_square_sum(row_sparse_matrix) only supports axis=0 or 1"; + CHECK(ograd.storage_type() == kDefaultStorage || ograd.storage_type() == kRowSparseStorage); + CHECK_EQ(input.storage_type(), kRowSparseStorage); + CHECK_EQ(igrad->storage_type(), kRowSparseStorage); + CHECK_EQ(req, kWriteTo); + if (!input.storage_initialized()) { + FillZerosRspImpl(s, igrad); + return; + } + + using namespace mxnet_op; + // TODO(junwu) change the input of CheckAndAlloc + // if we want to support differen row idx arrays + // for ograd and input when they are both row-sparse ndarrays + igrad->CheckAndAlloc({input.aux_shape(rowsparse::kIdx)}); + const int64_t num_cols = input.storage_shape()[1]; + const TBlob& igrad_data = igrad->data(); + const TBlob igrad_row_idx = igrad->aux_data(rowsparse::kIdx); + const TBlob& ograd_data = ograd.data(); + const TBlob& in_data = input.data(); + const TBlob in_row_idx = input.aux_data(rowsparse::kIdx); + if (ograd.storage_type() == kDefaultStorage) { + if (0 == param.axis[0]) { // forward is sum per column + MSHADOW_TYPE_SWITCH(igrad_data.type_flag_, DType, { + MSHADOW_IDX_TYPE_SWITCH(igrad_row_idx.type_flag_, IType, { + MXNET_ASSIGN_REQ_SWITCH(req, req_type, { + Kernel, xpu>::Launch( + s, igrad_data.Size(), igrad_row_idx.dptr(), + igrad_data.dptr(), ograd_data.dptr(), + in_row_idx.dptr(), in_data.dptr(), num_cols); + }) + }) + }) + } else { // forward is sum per row + MSHADOW_TYPE_SWITCH(igrad_data.type_flag_, DType, { + MSHADOW_IDX_TYPE_SWITCH(igrad_row_idx.type_flag_, IType, { + MXNET_ASSIGN_REQ_SWITCH(req, req_type, { + Kernel, xpu>::Launch( + s, igrad_data.Size(), igrad_row_idx.dptr(), + igrad_data.dptr(), ograd_data.dptr(), + in_row_idx.dptr(), in_data.dptr(), num_cols); + }) + }) + }) + } + } else if (ograd.storage_type() == kRowSparseStorage) { + CHECK_EQ(1, param.axis[0]) << "SquareSumRspGradImpl only supports axis = 1" + " when ograd_stype = kRowSparseStorage"; + CHECK_EQ(ograd.shape().ndim(), 2U); + const TBlob ograd_row_idx = ograd.aux_data(rowsparse::kIdx); + CHECK(ograd_row_idx.Size() == in_row_idx.Size() || in_row_idx.Size() == in_data.shape_[0]); + MSHADOW_IDX_TYPE_SWITCH(igrad_row_idx.type_flag_, IType, { + if (std::is_same::value) { + const IType* first1 = ograd_row_idx.dptr(); + const IType* last1 = first1 + ograd_row_idx.Size(); + const IType* first2 = in_row_idx.dptr(); + // when ograd_row_idx and in_row_idx have the same size and input is not a full rsp + // ograd_row_idx and in_row_idx are expected to have the same elements + if (ograd_row_idx.Size() == in_row_idx.Size() && in_row_idx.Size() != in_data.shape_[0]) { + CHECK(std::equal(first1, last1, first2)) << "SquareSumRspGradImpl only supports" + " equal ograd_row_idx and input_row_idx" + " when ograd and input are both" + " row-sparse"; + } + } else { + LOG(FATAL) << "SquareSumRspGradImpl has not implemented GPU version when" + " ograd and input are both row-sparse"; + } + MSHADOW_TYPE_SWITCH(igrad_data.type_flag_, DType, { + MXNET_ASSIGN_REQ_SWITCH(req, req_type, { + Kernel, xpu>::Launch( + s, igrad_data.Size(), igrad_row_idx.dptr(), + igrad_data.dptr(), ograd_row_idx.dptr(), + ograd_data.dptr(), in_data.dptr(), num_cols); + }) + }) + }) + } else { + LOG(FATAL) << "SquareSumRspGradImpl only supports ograd_stype" + << " = kDefaultStorage/kRowSparseStorage"; + } +} + +template +void SquareSumOpForwardEx(const nnvm::NodeAttrs& attrs, + const OpContext& ctx, + const std::vector& inputs, + const std::vector& req, + const std::vector& outputs) { + CHECK_EQ(inputs.size(), 1U); + CHECK_EQ(outputs.size(), 1U); + CHECK_EQ(req.size(), 1U); + mshadow::Stream* s = ctx.get_stream(); + const NDArrayStorageType istype = inputs[0].storage_type(); + if (istype == kRowSparseStorage) { + CHECK_EQ(inputs[0].shape().ndim(), 2U) << "_square_sum op only supports" + " 2D ndarray as input"; + NDArray output = outputs[0]; + SquareSumRspImpl(attrs, s, inputs[0], req[0], &output); + } else { + LOG(FATAL) << "_square_sum op only supports row-sparse ndarray" + " as input, while input stype = " + << istype; + } +} + +template +void SquareSumOpBackwardEx(const nnvm::NodeAttrs& attrs, + const OpContext& ctx, + const std::vector& inputs, + const std::vector& req, + const std::vector& outputs) { + CHECK_EQ(inputs.size(), 2U); + CHECK_EQ(outputs.size(), 1U); + CHECK_EQ(req.size(), 1U); + mshadow::Stream* s = ctx.get_stream(); + const NDArrayStorageType ograd_stype = inputs[0].storage_type(); + const NDArrayStorageType input_stype = inputs[1].storage_type(); + if (input_stype == kRowSparseStorage + && (ograd_stype == kDefaultStorage || ograd_stype == kRowSparseStorage)) { + CHECK_EQ(inputs[1].shape().ndim(), 2U) << "_square_sum op only supports" + " 2D ndarray as input"; + NDArray output = outputs[0]; + SquareSumRspGradImpl(attrs, s, inputs[0], inputs[1], req[0], &output); + } else { + LOG(FATAL) << "_square_sum op backward only supports dense ndarray as ograd," + " row-sparse ndarray as input and row-sparse ndarray as igrad," + " while ograd_stype = " << ograd_stype + << " input_stype = " << input_stype; + } +} + +} // namespace op +} // namespace mxnet + +#endif // MXNET_OPERATOR_TENSOR_SQUARE_SUM_INL_H_ diff --git a/src/operator/tensor/square_sum.cc b/src/operator/tensor/square_sum.cc new file mode 100644 index 000000000000..e4b49d7f7fcb --- /dev/null +++ b/src/operator/tensor/square_sum.cc @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file square_sum.cc + * \brief CPU Implementation of square_sum op. + */ +#include "./square_sum-inl.h" + +namespace mxnet { +namespace op { +MXNET_OPERATOR_REGISTER_REDUCE(_square_sum) +.describe(R"code(Computes the square sum of array elements over a given axis +for row-sparse matrix. This is a temporary solution for fusing ops square and +sum together for row-sparse matrix to save memory for storing gradients. +It will become deprecated once the functionality of fusing operators is finished +in the future. + +Example:: + + dns = mx.nd.array([[0, 0], [1, 2], [0, 0], [3, 4], [0, 0]]) + rsp = dns.tostype('row_sparse') + sum = mx.nd._internal._square_sum(rsp, axis=1) + sum = [0, 5, 0, 25, 0] +)code" ADD_FILELINE) +.set_attr("FInferStorageType", SquareSumForwardInferStorageType) +.set_attr("FComputeEx", SquareSumOpForwardEx) +.set_attr("FGradient", ElemwiseGradUseIn{"_backward_square_sum"}); + +MXNET_OPERATOR_REGISTER_REDUCE_BACKWARD(_backward_square_sum) +.set_num_inputs(2) +.set_attr("FInferStorageType", SquareSumBackwardInferStorageType) +.set_attr("FComputeEx", SquareSumOpBackwardEx); + +} // namespace op +} // namespace mxnet diff --git a/src/operator/tensor/util/tensor_util-inl.cuh b/src/operator/tensor/util/tensor_util-inl.cuh new file mode 100644 index 000000000000..cf268e7ae9fc --- /dev/null +++ b/src/operator/tensor/util/tensor_util-inl.cuh @@ -0,0 +1,240 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * Copyright (c) 2017 by Contributors + * \file tensor_util-inl.cuh + * \brief commonly utilized tensor operator GPU kernels + */ +#ifndef MXNET_OPERATOR_TENSOR_UTIL_TENSOR_UTIL_INL_CUH_ +#define MXNET_OPERATOR_TENSOR_UTIL_TENSOR_UTIL_INL_CUH_ + +#include +#include +#include + +namespace mxnet { +namespace op { + +/*! + * \brief Thread kernel for marking non-zero rows of a tensor. + * Parallelized by tensor rows: 1 thread/row + */ +struct MarkRspRowThreadKernel { + /*! + * \brief + * \param tid global thread id + * \param row_flg row flag array to mark non-zero rows + * \param dns dense matrix data + * \param num_rows number of rows (size of first dimension of tensor) + * \param row_length number of elements per row + */ + template + __device__ __forceinline__ static void Map(int tid, + nnvm::dim_t* row_flg, + const DType* dns, + const nnvm::dim_t num_rows, + const nnvm::dim_t row_length) { + using nnvm::dim_t; + if (tid < num_rows) { + dim_t j = 0; + dim_t offset = tid * row_length; + for (; j < row_length; ++j) { + if (dns[offset+j] != 0) { + break; + } + } + if (j < row_length) { + row_flg[tid] = 1; // mark as one for non-zero row + } else { + row_flg[tid] = 0; // mark as zero for zero row + } + } + } +}; + +/*! + * \brief Warp kernel for marking non-zero rows of a tensor. + * Parallelized by tensor rows: 1 warp/row + */ +struct MarkRspRowWarpKernel { + template + __device__ __forceinline__ static void Map(int tid, + nnvm::dim_t* row_flg, + const DType* dns, + const nnvm::dim_t num_rows, + const nnvm::dim_t row_length) { + using nnvm::dim_t; + typedef cub::WarpReduce WarpReduce; + const dim_t warps_per_block = mshadow::cuda::kBaseThreadNum / 32; + __shared__ typename WarpReduce::TempStorage temp_storage[warps_per_block]; + + const dim_t warp_id = tid / 32; // global warp id + const dim_t warp_lane = threadIdx.x / 32; // local warp id within thread block + const dim_t lane = tid & (32-1); // local thread id within warp + + if (warp_id < num_rows) { + dim_t flg = 0; + dim_t offset = warp_id * row_length; + for (dim_t j = lane; j < row_length; j+=32) { + if (dns[offset+j] != 0) { + // avoid break: causes slower performance on sparse tensors (<20% density), + // due to thread divergence + flg++; + } + } + dim_t aggr = WarpReduce(temp_storage[warp_lane]).Sum(flg); + if (lane == 0) { + if (aggr > 0) { + row_flg[warp_id] = 1; // mark as one for non-zero row + } else { + row_flg[warp_id] = 0; // mark as zero for zero row + } + } + } + } +}; + +/*! + * \brief Block kernel for marking non-zero rows of a tensor. + * Parallelized by tensor rows: 1 threadBlock/row + */ +struct MarkRspRowBlockKernel { + template + __device__ __forceinline__ static void Map(int tid, + nnvm::dim_t* row_flg, + const DType* dns, + const nnvm::dim_t num_rows, + const nnvm::dim_t row_length) { + using nnvm::dim_t; + using mshadow::cuda::kBaseThreadNum; + typedef cub::BlockReduce BlockReduce; + __shared__ typename BlockReduce::TempStorage temp_storage; + if (blockIdx.x < num_rows) { + dim_t flg = 0; + dim_t offset = blockIdx.x * row_length; + for (dim_t j = threadIdx.x; j < row_length; j+=kBaseThreadNum) { + if (dns[offset+j] != 0) { + // avoid break: causes slower performance on sparse tensors (<20% density), + // due to thread divergence + flg++; + } + } + dim_t aggr = BlockReduce(temp_storage).Sum(flg); + if (threadIdx.x == 0) { + if (aggr > 0) { + row_flg[blockIdx.x] = 1; // mark as one for non-zero row + } else { + row_flg[blockIdx.x] = 0; // mark as zero for zero row + } + } + } + } +}; + +/*! + * \brief GPU kernel to flag non-zero rows of an rsp tensor with indices. + * Parallelized by matrix rows: 1 thread/row + */ +struct SetRspRowFlgKernel { + /*! + * \brief + * \param tid global thread id + * \param row_flg array to flag storage indices of non-zero rows + * \param row_idx rsp matrix row index array storing indices of non-zero rows + * \param nnr rsp matrix number of non-zero rows (storage shape) + */ + template + __device__ __forceinline__ static void Map(int tid, + RType* row_flg, + const RType* row_idx, + const nnvm::dim_t nnr) { + if (tid < nnr) { + row_flg[row_idx[tid]] = tid+1; + } + } +}; + +/*! + * \brief GPU kernel for filling the row index array of an rsp tensor. + * Parallelized by tensor rows: 1 thread/row + */ +struct FillRspRowIdxKernel { + /*! + * \brief + * \param tid global thread id + * \param row_idx row index array to store indices of non-zero rows + * \param row_flg_sum inclusive prefix sum array over 0/1 marked row flag array + * \param num_rows rsp tensor number of rows (shape) + */ + template + __device__ __forceinline__ static void Map(int tid, + RType* row_idx, + const nnvm::dim_t* row_flg_sum, + const nnvm::dim_t num_rows) { + if (tid < num_rows) { + nnvm::dim_t prev = (tid == 0)? 0 : row_flg_sum[tid-1]; + if (row_flg_sum[tid] > prev) { + row_idx[prev] = static_cast(tid); + } + } + } +}; + +/*! + * \brief GPU kernel for marking non-zero columns of a csr matrix. + * Parallelized by matrix rows: 1 warp/row + */ +struct MarkCsrColWarpKernel { + /*! + * \brief + * \param tid global thread id + * \param flg flg array to mark non-zero columns + * \param col_idx csr matrix column indices + * \param indptr csr matrix row index pointer + * \param num_rows csr matrix number of rows + * \param num_cols csr matrix number of columns + */ + template + __device__ __forceinline__ static void Map(int tid, + nnvm::dim_t* flg, + const CType* col_idx, + const IType* indptr, + const nnvm::dim_t num_rows, + const nnvm::dim_t num_cols) { + typedef unsigned long long int uint64_cu; + static_assert(sizeof(uint64_cu) == sizeof(nnvm::dim_t), "unexpected sizeof dim_t"); + + const nnvm::dim_t warp_id = tid / 32; // global warp id + const nnvm::dim_t lane = tid & (32-1); // local thread id within warp + + if (warp_id < num_rows) { + uint64_cu zero = 0; + uint64_cu one = 1; + for (IType j = indptr[warp_id]+lane; j < indptr[warp_id+1]; j+=32) { + atomicCAS(reinterpret_cast(flg+col_idx[j]), zero, one); + } + } + } +}; + +} // namespace op +} // namespace mxnet + +#endif // MXNET_OPERATOR_TENSOR_UTIL_TENSOR_UTIL_INL_CUH_ diff --git a/tests/ci_build/install/ubuntu_install_python.sh b/tests/ci_build/install/ubuntu_install_python.sh index bb67e3401a89..db4e9c4e0c94 100755 --- a/tests/ci_build/install/ubuntu_install_python.sh +++ b/tests/ci_build/install/ubuntu_install_python.sh @@ -24,5 +24,5 @@ apt-get update && apt-get install -y python-dev python3-dev # the version of the pip shipped with ubuntu may be too lower, install a recent version here cd /tmp && wget https://bootstrap.pypa.io/get-pip.py && python3 get-pip.py && python2 get-pip.py -pip2 install nose pylint numpy nose-timer requests h5py -pip3 install nose pylint numpy nose-timer requests h5py +pip2 install nose pylint numpy nose-timer requests h5py scipy +pip3 install nose pylint numpy nose-timer requests h5py scipy diff --git a/tests/cpp/operator/batchnorm_test.cc b/tests/cpp/operator/batchnorm_test.cc index 3fef28f79a0a..cd202ace1686 100644 --- a/tests/cpp/operator/batchnorm_test.cc +++ b/tests/cpp/operator/batchnorm_test.cc @@ -19,7 +19,7 @@ /*! * \file batchnorm_test.cc - * \brief operator unit test utility functions + * \brief batchnorm operator unit test utility functions * \author Chris Olivier */ @@ -892,8 +892,8 @@ TEST(BATCH_NORM, TestIterAll) { kwargs.push_back({ "cudnn_off", "True" }); } for (TShape shape : shapes) { - for (int g1 = 0; g1 < 2U; ++g1) { - for (int g2 = 0; g2 < 2U; ++g2) { + for (int g1 = 0; g1 < 2; ++g1) { + for (int g2 = 0; g2 < 2; ++g2) { for (int type : v2_types) { MSHADOW_REAL_TYPE_SWITCH_EX( type, DType, AccReal, diff --git a/tests/nightly/dist_sync_kvstore.py b/tests/nightly/dist_sync_kvstore.py index 3fbf9f910879..af1ecfc5036f 100644 --- a/tests/nightly/dist_sync_kvstore.py +++ b/tests/nightly/dist_sync_kvstore.py @@ -22,45 +22,155 @@ sys.path.insert(0, "../../python/") import mxnet as mx import numpy as np +import numpy.random as rnd import time -def check_diff_to_scalar(A, x): +def check_diff_to_scalar(A, x, rank=None): """ assert A == x""" - assert(np.sum(np.abs((A - x).asnumpy())) == 0), A.asnumpy() + assert(np.sum(np.abs((A - x).asnumpy())) == 0), (rank, A.asnumpy(), x) # setup -keys = [3, 5, 7] +keys = ['3', '5', '7'] +rsp_keys = ['9', '11', '13'] + rate = 2 -shape = (2, 2) -big_shape = (1200, 1200) # big than BIGARRAY_BOUND +shape = (2, 3) +big_shape = (1200, 1200) # bigger than BIGARRAY_BOUND -kv = mx.kv.create('dist_sync') +def init_kv(): + kv = mx.kv.create('dist_sync') + # init kv dns keys + kv.init(keys, [mx.nd.ones(shape)] * len(keys)) + kv.init('99', mx.nd.ones(big_shape)) + # init kv row_sparse keys + kv.init(rsp_keys, [mx.nd.ones(shape).tostype('row_sparse')] * len(rsp_keys)) + kv.init('100', mx.nd.ones(big_shape).tostype('row_sparse')) + # worker info + my_rank = kv.rank + nworker = kv.num_workers + # init updater on servers + kv.set_optimizer(mx.optimizer.create('test', rescale_grad=rate)) + return kv, my_rank, nworker -# init kv -kv.init(keys, [mx.nd.ones(shape)] * len(keys)) -kv.init(99, mx.nd.ones(big_shape)) -# init updater on servers -kv.set_optimizer(mx.optimizer.create('test', rate)) +def test_sync_push_pull(): + kv, my_rank, nworker = init_kv() + def check_default_keys(kv, my_rank, nworker): + nrepeat = 3 + for i in range(nrepeat): + kv.push('3', mx.nd.ones(shape)*(my_rank+1)) + kv.push('99', mx.nd.ones(big_shape)*(my_rank+1)) -my_rank = kv.rank -nworker = kv.num_workers + num = (nworker + 1) * nworker * rate / 2 * nrepeat + 1 + val = mx.nd.zeros(shape) + kv.pull('3', out=val) + check_diff_to_scalar(val, num) -def test_sync_push_pull(): - nrepeat = 3 - for i in range(nrepeat): - kv.push(3, mx.nd.ones(shape)*(my_rank+1)) - kv.push(99, mx.nd.ones(big_shape)*(my_rank+1)) - - num = (nworker + 1 ) * nworker * rate / 2 * nrepeat + 1 - val = mx.nd.zeros(shape) - kv.pull(3, out = val) - check_diff_to_scalar(val, num) - # print val.asnumpy() - - val2 = mx.nd.zeros(big_shape) - kv.pull(99, out = val2) - check_diff_to_scalar(val2, num) + val2 = mx.nd.zeros(big_shape) + kv.pull('99', out=val2) + check_diff_to_scalar(val2, num) + + def check_row_sparse_keys(kv, my_rank, nworker): + nrepeat = 3 + # prepare gradient + v = mx.nd.zeros(shape) + my_row = my_rank % shape[0] + v[my_row] = my_rank + 1 + # push + for i in range(nrepeat): + kv.push('9', v.tostype('row_sparse')) + # select a random subset of rows this worker is interested in + num_rows = shape[0] + row_ids_np = np.random.randint(num_rows, size=num_rows) + row_ids = mx.nd.array(row_ids_np, dtype='int64') + # perform pull + val = mx.nd.zeros(shape, stype='row_sparse') + kv.row_sparse_pull('9', out=val, row_ids=row_ids) + # prepare updated values + updated_val = mx.nd.ones(shape) + for rank in range(nworker): + row = rank % shape[0] + updated_val[row] += (rank + 1) * rate * nrepeat + # verify subset of updated values + expected = mx.nd.zeros(shape) + for row in row_ids_np: + expected[row] = updated_val[row] + check_diff_to_scalar(val, expected) + + def check_row_sparse_keys_with_zeros(kv, my_rank, nworker): + nrepeat = 3 + # prepare gradient + v = mx.nd.zeros(shape) + big_v = mx.nd.zeros(big_shape) + # push + for i in range(nrepeat): + kv.push('11', v.tostype('row_sparse')) + kv.push('100', big_v.tostype('row_sparse')) + + # pull a subset of rows this worker is interested in + all_row_ids = np.arange(shape[0]) + val = mx.nd.ones(shape).tostype('row_sparse') + big_val = mx.nd.ones(big_shape).tostype('row_sparse') + kv.row_sparse_pull('11', out=val, row_ids=mx.nd.array(all_row_ids, dtype='int64')) + big_num_rows = shape[0] + big_all_row_ids = np.arange(big_shape[0]) + kv.row_sparse_pull('100', out=big_val, row_ids=mx.nd.array(big_all_row_ids, dtype='int64')) + # verify results + check_diff_to_scalar(val, mx.nd.ones(shape)) + check_diff_to_scalar(big_val, mx.nd.ones(big_shape)) + + def check_big_row_sparse_keys(kv, my_rank, nworker): + mx.random.seed(123) + rnd.seed(123) + density = 0.3 + nrepeat = 3 + # prepare gradient + v = mx.nd.zeros(big_shape) + idx_sample = rnd.rand(big_shape[0]) + indices = np.argwhere(idx_sample < density).flatten() + # each worker chooses a subset of the indices to update + update_rows = [] + for rank in range(nworker): + rows = [] + i = 0 + step = (rank + 1) * 2 + while i < len(indices): + rows.append(indices[i]) + i += step + update_rows.append(np.array(rows)) + # rows to update for this worker + for row in update_rows[my_rank]: + v[row] = my_rank + 1 + # push + for i in range(nrepeat): + kv.push('100', v.tostype('row_sparse')) + + # select a random subset of rows this worker is interested in + mx.random.seed(my_rank) + rnd.seed(my_rank) + num_rows = big_shape[0] + row_ids_np = np.random.randint(num_rows, size=num_rows) + row_ids = mx.nd.array(row_ids_np, dtype='int64') + # perform pull + val = mx.nd.zeros(big_shape, stype='row_sparse') + kv.row_sparse_pull('100', out=val, row_ids=row_ids) + # prepare expected result + updated_val = mx.nd.ones(big_shape) + # apply updates from each worker + for rank in range(nworker): + for row in update_rows[rank]: + updated_val[row] += (rank + 1) * rate * nrepeat + + expected = mx.nd.zeros(big_shape) + for row in row_ids_np: + expected[row] = updated_val[row] + check_diff_to_scalar(val, expected, rank=my_rank) + + check_default_keys(kv, my_rank, nworker) + check_row_sparse_keys(kv, my_rank, nworker) + check_row_sparse_keys_with_zeros(kv, my_rank, nworker) + check_big_row_sparse_keys(kv, my_rank, nworker) + print('worker ' + str(my_rank) + ' is done') if __name__ == "__main__": test_sync_push_pull() diff --git a/tests/python/gpu/test_kvstore_gpu.py b/tests/python/gpu/test_kvstore_gpu.py new file mode 100644 index 000000000000..ffc0cc1f93e0 --- /dev/null +++ b/tests/python/gpu/test_kvstore_gpu.py @@ -0,0 +1,68 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# pylint: skip-file +import mxnet as mx +import numpy as np +from mxnet.test_utils import assert_almost_equal, default_context + +shape = (4, 4) +keys = [5, 7, 11] +str_keys = ['b', 'c', 'd'] + + +def init_kv_with_str(stype='default'): + """init kv """ + kv = mx.kv.create() + # single + kv.init('a', mx.nd.zeros(shape, stype=stype)) + # list + kv.init(str_keys, [mx.nd.zeros(shape=shape, stype=stype)] * len(keys)) + return kv + + +def test_row_sparse_pull(): + kv = init_kv_with_str('row_sparse') + kv.init('e', mx.nd.ones(shape).tostype('row_sparse')) + + def check_row_sparse_pull(kv, count, ctx=default_context()): + num_rows = shape[0] + vals = [] + row_ids = [] + all_row_ids = np.arange(num_rows) + for i in range(count): + vals.append(mx.nd.zeros(shape, ctx=ctx).tostype('row_sparse')) + row_id = np.random.randint(num_rows, size=num_rows) + row_ids.append(mx.nd.array(row_id, dtype='int64')) + row_ids_to_pull = row_ids[0] if len(row_ids) == 1 else row_ids + vals_to_pull = vals[0] if len(vals) == 1 else vals + + kv.row_sparse_pull('e', out=vals_to_pull, row_ids=row_ids_to_pull) + for val, row_id in zip(vals, row_ids): + retained = val.asnumpy() + excluded_row_ids = np.setdiff1d(all_row_ids, row_id.asnumpy()) + for row in range(num_rows): + expected_val = np.zeros_like(retained[row]) + expected_val += 0 if row in excluded_row_ids else 1 + assert_almost_equal(retained[row], expected_val) + + check_row_sparse_pull(kv, 1, mx.gpu(0)) + check_row_sparse_pull(kv, 4, mx.gpu(0)) + + +if __name__ == '__main__': + test_row_sparse_pull() diff --git a/tests/python/gpu/test_operator_gpu.py b/tests/python/gpu/test_operator_gpu.py index 81492fe6bbdb..35a20f935573 100644 --- a/tests/python/gpu/test_operator_gpu.py +++ b/tests/python/gpu/test_operator_gpu.py @@ -31,6 +31,9 @@ from test_gluon import * #from test_rnn import * from test_gluon_rnn import * +from test_sparse_operator import test_cast_storage_ex, test_sparse_dot +from test_sparse_operator import test_sparse_nd_zeros, test_sparse_retain +from test_sparse_ndarray import test_create_csr, test_create_row_sparse set_default_context(mx.gpu(0)) del test_support_vector_machine_l1_svm diff --git a/tests/python/unittest/test_autograd.py b/tests/python/unittest/test_autograd.py index 30dd662ff1cc..37bb5626f765 100644 --- a/tests/python/unittest/test_autograd.py +++ b/tests/python/unittest/test_autograd.py @@ -106,29 +106,41 @@ def autograd_assert(*args, **kwargs): assert same(a.asnumpy(), b.asnumpy()) def test_unary_func(): - x = nd.uniform(shape=(4, 5)) - f_exp = lambda x: nd.exp(x) - f_exp_grad = lambda x: [nd.exp(x)] - autograd_assert(x, func=f_exp, grad_func=f_exp_grad) - f_half = lambda x: x/2 - f_half_grad = lambda x: [nd.ones(x.shape) * 0.5] - autograd_assert(x, func=f_half, grad_func=f_half_grad) - f_square = lambda x: x**2 - f_square_grad = lambda x: [2*x] - autograd_assert(x, func=f_square, grad_func=f_square_grad) + def check_unary_func(x): + f_exp = lambda x: nd.exp(x) + f_exp_grad = lambda x: [nd.exp(x)] + autograd_assert(x, func=f_exp, grad_func=f_exp_grad) + f_half = lambda x: x/2 + f_half_grad = lambda x: [nd.ones(x.shape) * 0.5] + autograd_assert(x, func=f_half, grad_func=f_half_grad) + f_square = lambda x: x**2 + f_square_grad = lambda x: [2*x] + autograd_assert(x, func=f_square, grad_func=f_square_grad) + uniform = nd.uniform(shape=(4, 5)) + stypes = ['row_sparse', 'csr', 'default'] + for stype in stypes: + check_unary_func(uniform.tostype(stype)) def test_binary_func(): - x = nd.uniform(shape=(4, 5)) - y = nd.uniform(shape=(4, 5)) - f_add = lambda x, y: x+y - f_add_grad = lambda x, y: [nd.ones(x.shape), nd.ones(y.shape)] - autograd_assert(x, y, func=f_add, grad_func=f_add_grad) - f_mul = lambda x, y: x*y - f_mul_grad = lambda x, y: [y, x] - autograd_assert(x, y, func=f_mul, grad_func=f_mul_grad) - f_compose = lambda x, y: x+x*y - f_compose_grad = lambda x, y: [nd.ones(x.shape) + y, x] - autograd_assert(x, y, func=f_compose, grad_func=f_compose_grad) + def check_binary_func(x, y): + f_add = lambda x, y: x+y + f_add_grad = lambda x, y: [nd.ones(x.shape), nd.ones(y.shape)] + autograd_assert(x, y, func=f_add, grad_func=f_add_grad) + f_mul = lambda x, y: x*y + f_mul_grad = lambda x, y: [y, x] + autograd_assert(x, y, func=f_mul, grad_func=f_mul_grad) + f_compose = lambda x, y: x+x*y + f_compose_grad = lambda x, y: [nd.ones(x.shape) + y, x] + autograd_assert(x, y, func=f_compose, grad_func=f_compose_grad) + uniform_x = nd.uniform(shape=(4, 5)) + uniform_y = nd.uniform(shape=(4, 5)) + stypes = ['row_sparse', 'csr', 'default'] + for stype_x in stypes: + for stype_y in stypes: + x = uniform_x.tostype(stype_x) + y = uniform_y.tostype(stype_y) + check_binary_func(x, y) + def test_operator_with_state(): def f_fc(a, b, weight, bias): @@ -255,14 +267,19 @@ def test_retain_grad(): def test_attach_grad(): - x = mx.nd.zeros((10,)) - assert x.grad is None - x.attach_grad() - with record(): - y = x * 2 - assert y.grad is None - y.backward() - assert (x.grad.asnumpy() == 2).all() + def check_attach_grad(x): + assert x.grad is None + x.attach_grad() + with record(): + y = x * 2 + assert y.grad is None + y.backward() + assert (x.grad.asnumpy() == 2).all() + zeros = mx.nd.zeros((10, 10)) + stypes = ['default', 'row_sparse', 'csr'] + for stype in stypes: + x = zeros.tostype(stype) + check_attach_grad(x) def test_is_train(): diff --git a/tests/python/unittest/test_infer_shape.py b/tests/python/unittest/test_infer_shape.py index d7f52e216659..73654a604135 100644 --- a/tests/python/unittest/test_infer_shape.py +++ b/tests/python/unittest/test_infer_shape.py @@ -52,7 +52,7 @@ def test_backward_infer(): # broadcast add here, not being able to deduce shape correctly wt = mx.sym.broadcast_add(w, wshift) # shape constraint, this is what enables backward shape inference - wt = mx._symbol_internal._identity_with_attr_like_rhs(wt, w) + wt = mx.symbol._internal._identity_with_attr_like_rhs(wt, w) net = mx.sym.FullyConnected(data=data, weight=wt, num_hidden=11, no_bias=True) data_shape = (7, 100) arg_shapes, out_shapes, aux_shapes = net.infer_shape(data=data_shape) @@ -129,6 +129,24 @@ def test_incomplete_infer_concat(): assert arg_shapes['b'] == (2, 5) assert arg_shapes['d'] == (2, 15) +def test_fc_infer_type(): + mx_real_t = mx.base.mx_real_t + data = mx.symbol.Variable('data') + out = mx.symbol.FullyConnected(data=data, name='fc1', num_hidden=1000) + + # infer type + data_type = mx_real_t + arg_types, out_types, aux_types = out.infer_type(data=data_type) + arg_type_dict = dict(zip(out.list_arguments(), arg_types)) + assert len(out_types) == 1 + assert out_types[0] == mx_real_t + true_types = { + 'fc1_bias' : mx_real_t, + 'fc1_weight' : mx_real_t } + for k, v in true_types.items(): + assert arg_type_dict[k] == v + + if __name__ == "__main__": test_mlp2_infer_shape() test_mlp2_infer_error() diff --git a/tests/python/unittest/test_io.py b/tests/python/unittest/test_io.py index c0f2acd4ed47..a543463f3663 100644 --- a/tests/python/unittest/test_io.py +++ b/tests/python/unittest/test_io.py @@ -17,6 +17,7 @@ # pylint: skip-file import mxnet as mx +from mxnet.test_utils import * import numpy as np import os, gzip import pickle as pickle @@ -152,6 +153,109 @@ def test_NDArrayIter_h5py(): else: assert(labelcount[i] == 100) +def test_NDArrayIter_csr(): + import scipy.sparse as sp + # creating toy data + num_rows = rnd.randint(5, 15) + num_cols = rnd.randint(1, 20) + batch_size = rnd.randint(1, num_rows) + shape = (num_rows, num_cols) + csr, _ = rand_sparse_ndarray(shape, 'csr') + dns = csr.asnumpy() + + # make iterators + csr_iter = iter(mx.io.NDArrayIter(csr, csr, batch_size)) + begin = 0 + for batch in csr_iter: + expected = np.zeros((batch_size, num_cols)) + end = begin + batch_size + expected[:num_rows - begin] = dns[begin:end] + if end > num_rows: + expected[num_rows - begin:] = dns[0:end - num_rows] + assert_almost_equal(batch.data[0].asnumpy(), expected) + begin += batch_size + +def test_LibSVMIter(): + def get_data(data_dir, data_name, url, data_origin_name): + if not os.path.isdir(data_dir): + os.system("mkdir " + data_dir) + os.chdir(data_dir) + if (not os.path.exists(data_name)): + if sys.version_info[0] >= 3: + from urllib.request import urlretrieve + else: + from urllib import urlretrieve + zippath = os.path.join(data_dir, data_origin_name) + urlretrieve(url, zippath) + import bz2 + bz_file = bz2.BZ2File(data_origin_name, 'rb') + with open(data_name, 'wb') as fout: + try: + content = bz_file.read() + fout.write(content) + finally: + bz_file.close() + os.chdir("..") + + def check_libSVMIter_synthetic(): + cwd = os.getcwd() + data_path = os.path.join(cwd, 'data.t') + label_path = os.path.join(cwd, 'label.t') + with open(data_path, 'w') as fout: + fout.write('1.0 0:0.5 2:1.2\n') + fout.write('-2.0\n') + fout.write('-3.0 0:0.6 1:2.4 2:1.2\n') + fout.write('4 2:-1.2\n') + + with open(label_path, 'w') as fout: + fout.write('1.0\n') + fout.write('-2.0 0:0.125\n') + fout.write('-3.0 2:1.2\n') + fout.write('4 1:1.0 2:-1.2\n') + + data_dir = os.path.join(cwd, 'data') + data_train = mx.io.LibSVMIter(data_libsvm=data_path, label_libsvm=label_path, + data_shape=(3, ), label_shape=(3, ), batch_size=3) + + first = mx.nd.array([[ 0.5, 0., 1.2], [ 0., 0., 0.], [ 0.6, 2.4, 1.2]]) + second = mx.nd.array([[ 0., 0., -1.2], [ 0.5, 0., 1.2], [ 0., 0., 0.]]) + i = 0 + for batch in iter(data_train): + expected = first.asnumpy() if i == 0 else second.asnumpy() + assert_almost_equal(data_train.getdata().asnumpy(), expected) + i += 1 + + def check_libSVMIter_news_data(): + news_metadata = { + 'name': 'news20.t', + 'origin_name': 'news20.t.bz2', + 'url': "http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multiclass/news20.t.bz2", + 'feature_dim': 62060, + 'num_classes': 20, + 'num_examples': 3993, + } + num_parts = 3 + batch_size = 128 + num_examples = news_metadata['num_examples'] + data_dir = os.path.join(os.getcwd(), 'data') + get_data(data_dir, news_metadata['name'], news_metadata['url'], + news_metadata['origin_name']) + path = os.path.join(data_dir, news_metadata['name']) + data_train = mx.io.LibSVMIter(data_libsvm=path, data_shape=(news_metadata['feature_dim'],), + batch_size=batch_size, num_parts=num_parts, part_index=0) + num_batches = 0 + iterator = iter(data_train) + for batch in iterator: + # check the range of labels + assert(np.sum(batch.label[0].asnumpy() > 20) == 0) + assert(np.sum(batch.label[0].asnumpy() <= 0) == 0) + num_batches += 1 + import math + expected_num_batches = math.ceil(num_examples * 1.0 / batch_size / num_parts) + assert(num_batches == int(expected_num_batches)), (num_batches, expected_num_batches) + + check_libSVMIter_synthetic() + check_libSVMIter_news_data() if __name__ == "__main__": test_NDArrayIter() @@ -159,3 +263,5 @@ def test_NDArrayIter_h5py(): test_NDArrayIter_h5py() test_MNISTIter() test_Cifar10Rec() + test_LibSVMIter() + test_NDArrayIter_csr() diff --git a/tests/python/unittest/test_kvstore.py b/tests/python/unittest/test_kvstore.py index f1e10c757fad..a43b98a635fb 100644 --- a/tests/python/unittest/test_kvstore.py +++ b/tests/python/unittest/test_kvstore.py @@ -18,44 +18,74 @@ # pylint: skip-file import mxnet as mx import numpy as np +from mxnet.test_utils import rand_ndarray, assert_almost_equal shape = (4, 4) keys = [5, 7, 11] str_keys = ['b', 'c', 'd'] -def init_kv(): +def init_kv(stype='default'): """init kv """ kv = mx.kv.create() # single - kv.init(3, mx.nd.zeros(shape)) + kv.init(3, mx.nd.zeros(shape=shape, stype=stype)) # list - kv.init(keys, [mx.nd.zeros(shape)] * len(keys)) + kv.init(keys, [mx.nd.zeros(shape=shape, stype=stype)] * len(keys)) return kv -def init_kv_with_str(): +def init_kv_with_str(stype='default'): """init kv """ kv = mx.kv.create() # single - kv.init('a', mx.nd.zeros(shape)) + kv.init('a', mx.nd.zeros(shape, stype=stype)) # list - kv.init(str_keys, [mx.nd.zeros(shape)] * len(keys)) + kv.init(str_keys, [mx.nd.zeros(shape=shape, stype=stype)] * len(keys)) return kv def check_diff_to_scalar(A, x): """ assert A == x""" assert(np.sum(np.abs((A - x).asnumpy())) == 0) + def test_single_kv_pair(): """single key-value pair push & pull""" def check_single_kv_pair(kv, key): kv.push(key, mx.nd.ones(shape)) val = mx.nd.empty(shape) - kv.pull(key, out = val) + kv.pull(key, out=val) check_diff_to_scalar(val, 1) check_single_kv_pair(init_kv(), 3) check_single_kv_pair(init_kv_with_str(), 'a') +def test_row_sparse_pull(): + kv = init_kv_with_str('row_sparse') + kv.init('e', mx.nd.ones(shape).tostype('row_sparse')) + + def check_row_sparse_pull(kv, count): + num_rows = shape[0] + vals = [] + row_ids = [] + all_row_ids = np.arange(num_rows) + for i in range(count): + vals.append(mx.nd.zeros(shape).tostype('row_sparse')) + row_id = np.random.randint(num_rows, size=num_rows) + row_ids.append(mx.nd.array(row_id, dtype='int64')) + row_ids_to_pull = row_ids[0] if len(row_ids) == 1 else row_ids + vals_to_pull = vals[0] if len(vals) == 1 else vals + + kv.row_sparse_pull('e', out=vals_to_pull, row_ids=row_ids_to_pull) + for val, row_id in zip(vals, row_ids): + retained = val.asnumpy() + excluded_row_ids = np.setdiff1d(all_row_ids, row_id.asnumpy()) + for row in range(num_rows): + expected_val = np.zeros_like(retained[row]) + expected_val += 0 if row in excluded_row_ids else 1 + assert_almost_equal(retained[row], expected_val) + + check_row_sparse_pull(kv, 1) + check_row_sparse_pull(kv, 4) + def test_init(): """test init""" def check_init(kv, key): @@ -72,7 +102,7 @@ def test_list_kv_pair(): def check_list_kv_pair(kv, key): kv.push(key, [mx.nd.ones(shape)*4] * len(key)) val = [mx.nd.empty(shape)] * len(key) - kv.pull(key, out = val) + kv.pull(key, out=val) for v in val: check_diff_to_scalar(v, 4) @@ -92,7 +122,7 @@ def check_aggregator(kv, key, key_list): vals = [mx.nd.ones(shape, d) for d in devs] kv.push(key, vals) - kv.pull(key, out = vals) + kv.pull(key, out=vals) for v in vals: check_diff_to_scalar(v, num_devs) @@ -100,7 +130,7 @@ def check_aggregator(kv, key, key_list): # list vals = [[mx.nd.ones(shape, d)*2.0 for d in devs]] * len(key_list) kv.push(key_list, vals) - kv.pull(key_list, out = vals) + kv.pull(key_list, out=vals) for vv in vals: for v in vv: @@ -110,10 +140,50 @@ def check_aggregator(kv, key, key_list): check_aggregator(init_kv_with_str(), 'a', str_keys) +def test_sparse_aggregator(): + """aggregate sparse ndarray on muliple devices""" + + stype = 'row_sparse' + kv = init_kv_with_str(stype) + + # devices + num_devs = 4 + devs = [mx.Context('cpu', i) for i in range(num_devs)] + + # single + vals = [rand_ndarray(shape, stype).copyto(devs[i]) for i in range(num_devs)] + expected_sum = np.zeros(shape) + for v in vals: + expected_sum += v.asnumpy() + + # prepare row_ids + all_rows = mx.nd.array(np.arange(shape[0]), dtype='int64') + kv.push('a', vals) + kv.row_sparse_pull('a', out=vals, row_ids=[all_rows] * len(vals)) + result_sum = np.zeros(shape) + for v in vals: + result_sum += v.asnumpy() + assert_almost_equal(result_sum, expected_sum * num_devs) + + # list + vals = [[rand_ndarray(shape, stype).copyto(devs[i]) for i in range(num_devs)]] * len(keys) + expected_sum = np.zeros(shape) + for v in vals[0]: + expected_sum += v.asnumpy() + + kv.push(str_keys, vals) + kv.row_sparse_pull(str_keys, out=vals, row_ids=[[all_rows] * num_devs] * len(vals)) + for vv in vals: + result_sum = np.zeros(shape) + for v in vv: + result_sum += v.asnumpy() + assert_almost_equal(result_sum, expected_sum * num_devs) + def updater(key, recv, local): """use updater: +=""" local += recv + def test_updater(dev = 'cpu'): """updater""" @@ -126,7 +196,7 @@ def check_updater(kv, key, key_list): vals = [mx.nd.ones(shape, d) for d in devs] kv.push(key, vals) - kv.pull(key, out = vals) + kv.pull(key, out=vals) for v in vals: check_diff_to_scalar(v, num_devs) @@ -138,7 +208,7 @@ def check_updater(kv, key, key_list): for i in range(num_push): kv.push(key_list, vals) - kv.pull(key_list, out = vals) + kv.pull(key_list, out=vals) for vv in vals: for v in vv: @@ -152,16 +222,54 @@ def check_updater(kv, key, key_list): str_kv._set_updater(updater) check_updater(str_kv, 'a', str_keys) - def test_get_type(): kvtype = 'local_allreduce_cpu' kv = mx.kv.create(kvtype) assert kv.type == kvtype +def test_invalid_pull(): + def check_invalid_single_kv_pair(kv, key): + dns_val = mx.nd.ones(shape) * 2 + rsp_val = dns_val.tostype('row_sparse') + kv.pull(key, out=rsp_val) + # pull should be ignored with no values updated + check_diff_to_scalar(rsp_val, 2) + try: + # row_sparse_pull should be aborted when vals.stype != row_sparse + kv.row_sparse_pull(key, out=dns_val, rowids=mx.nd.array([1])) + assert(False) + except: + pass + + def check_invalid_list_kv_pair(kv, key): + dns_val = [mx.nd.ones(shape) * 2] * len(key) + rsp_val = [val.tostype('row_sparse') for val in dns_val] + kv.pull(key, out=rsp_val) + for v in rsp_val: + # pull should be ignored with no values updated + check_diff_to_scalar(v, 2) + try: + # row_sparse_pull should be aborted when vals.stype != row_sparse + kv.row_sparse_pull(key, out=dns_val, rowids=[mx.nd.array([1])] * len(key)) + assert(False) + except: + pass + + int_kv = init_kv() + str_kv = init_kv_with_str() + + check_invalid_single_kv_pair(int_kv, 3) + check_invalid_single_kv_pair(str_kv, 'a') + + check_invalid_list_kv_pair(int_kv, keys) + check_invalid_list_kv_pair(str_kv, str_keys) + if __name__ == '__main__': test_init() test_get_type() test_single_kv_pair() test_list_kv_pair() + test_sparse_aggregator() test_aggregator() test_updater() + test_row_sparse_pull() diff --git a/tests/python/unittest/test_module.py b/tests/python/unittest/test_module.py index f522f29dae39..9d8d76f5aa92 100644 --- a/tests/python/unittest/test_module.py +++ b/tests/python/unittest/test_module.py @@ -17,12 +17,15 @@ import mxnet as mx import mxnet.ndarray as nd +from mxnet.test_utils import * import numpy as np from functools import reduce from mxnet.module.executor_group import DataParallelExecutorGroup from common import assertRaises from collections import namedtuple +import numpy.random as rnd + def test_module_dtype(): dtype = np.float16 @@ -345,7 +348,6 @@ def mean_abs(x): break assert(mon_result_counts == [2, 2, 1, 6, 6, 4]) - def test_executor_group(): def get_rnn_sym(num_layers, num_words, num_hidden, num_embed, seq_len): stack = mx.rnn.SequentialRNNCell() @@ -458,6 +460,107 @@ def test_shared_exec_group(exec_grp_shared, exec_grp_created, shared_arg_names=N shared_arg_names=shared_arg_names, extra_args=extra_args) +def test_factorization_machine_module(): + """ Test factorization machine model with sparse operators """ + mx.random.seed(11) + rnd.seed(11) + + def fm(factor_size, feature_dim, init): + x = mx.symbol.Variable("data", stype='csr') + v = mx.symbol.Variable("v", shape=(feature_dim, factor_size), + init=init, stype='row_sparse') + + w1_weight = mx.symbol.var('w1_weight', shape=(feature_dim, 1), + init=init, stype='row_sparse') + w1_bias = mx.symbol.var('w1_bias', shape=(1)) + w1 = mx.symbol.broadcast_add(mx.symbol.dot(x, w1_weight), w1_bias) + + v_s = mx.symbol._internal._square_sum(data=v, axis=1, keepdims=True) + x_s = mx.symbol.square(data=x) + bd_sum = mx.sym.dot(x_s, v_s) + + w2 = mx.symbol.dot(x, v) + w2_squared = 0.5 * mx.symbol.square(data=w2) + + w_all = mx.symbol.Concat(w1, w2_squared, dim=1) + sum1 = mx.symbol.sum(data=w_all, axis=1, keepdims=True) + sum2 = 0.5 * mx.symbol.negative(bd_sum) + model = mx.sym.elemwise_add(sum1, sum2) + + y = mx.symbol.Variable("label") + model = mx.symbol.LinearRegressionOutput(data=model, label=y) + return model + + # model + ctx = default_context() + init = mx.initializer.Normal(sigma=0.01) + factor_size = 4 + feature_dim = 10000 + model = fm(factor_size, feature_dim, init) + + # data iter + num_batches = 5 + batch_size = 64 + num_samples = batch_size * num_batches + import scipy.sparse as sp + # generate some random scipy csr data + csr_sp = sp.rand(num_samples, feature_dim, density=0.1, format='csr') + csr_nd = mx.nd.sparse.csr_matrix(csr_sp.data, csr_sp.indptr, csr_sp.indices, + (num_samples, feature_dim)) + label = mx.nd.ones((num_samples,1)) + # the alternative is to use LibSVMIter + train_iter = mx.io.NDArrayIter(data=csr_nd, + label={'label':label}, + batch_size=batch_size) + # create module + mod = mx.mod.Module(symbol=model, data_names=['data'], label_names=['label']) + # allocate memory by given the input data and lable shapes + mod.bind(data_shapes=train_iter.provide_data, label_shapes=train_iter.provide_label) + # initialize parameters by uniform random numbers + mod.init_params(initializer=init) + # use Sparse SGD with learning rate 0.1 to train + adam = mx.optimizer.Adam(clip_gradient=5.0, learning_rate=0.001, rescale_grad=1.0/batch_size) + mod.init_optimizer(optimizer=adam) + # use accuracy as the metric + metric = mx.metric.create('MSE') + # train 10 epoch + for epoch in range(10): + train_iter.reset() + metric.reset() + for batch in train_iter: + mod.forward(batch, is_train=True) # compute predictions + mod.update_metric(metric, batch.label) # accumulate prediction accuracy + mod.backward() # compute gradients + mod.update() # update parameters + # print('Epoch %d, Training %s' % (epoch, metric.get())) + assert(metric.get()[1] < 0.05), metric.get()[1] + + +def test_module_initializer(): + def regression_model(m): + x = mx.symbol.var("data", stype='csr') + v = mx.symbol.var("v", shape=(m, 1), init=mx.init.Uniform(scale=.1), + stype='row_sparse') + model = mx.symbol.dot(lhs=x, rhs=v) + y = mx.symbol.Variable("label") + model = mx.symbol.LinearRegressionOutput(data=model, label=y, name="out") + return model + + n, m = 128, 100 + model = regression_model(m) + + data = mx.nd.zeros(shape=(n, m), stype='csr') + label = mx.nd.zeros((n, 1)) + iterator = mx.io.NDArrayIter(data=data, label={'label':label}, batch_size=n) + + # create module + mod = mx.mod.Module(symbol=model, data_names=['data'], label_names=['label']) + mod.bind(data_shapes=iterator.provide_data, label_shapes=iterator.provide_label) + mod.init_params() + v = mod._arg_params['v'] + assert(v.stype == 'row_sparse') + assert(np.sum(v.asnumpy()) != 0) + def test_forward_reshape(): num_class=10 data1 = mx.sym.Variable('data1') diff --git a/tests/python/unittest/test_multi_device_exec.py b/tests/python/unittest/test_multi_device_exec.py index 6f8eb17ff34e..0a2739d9bb4e 100644 --- a/tests/python/unittest/test_multi_device_exec.py +++ b/tests/python/unittest/test_multi_device_exec.py @@ -16,6 +16,7 @@ # under the License. import os +import numpy as np import mxnet as mx def test_ctx_group(): @@ -49,5 +50,31 @@ def test_ctx_group(): else: assert arr.context == group2ctx['stage2'] +def test_ctx_group_sparse(): + with mx.AttrScope(ctx_group='stage1'): + lhs = mx.symbol.Variable('lhs', stype='csr') + rhs = mx.symbol.Variable('rhs', stype='row_sparse') + dot = mx.symbol.dot(lhs, rhs, name='dot') + + set_stage1 = set(dot.list_arguments()) + with mx.AttrScope(ctx_group='stage2'): + softmax = mx.symbol.SoftmaxOutput(data = dot, name = 'softmax') + + set_stage2 = set(softmax.list_arguments()) - set_stage1 + + group2ctx = { + 'stage1' : mx.cpu(1), + 'stage2' : mx.cpu(2) + } + texec = softmax.simple_bind(mx.cpu(0), group2ctx=group2ctx, + lhs=(32,200), rhs=(200, 5)) + + for arr, name in zip(texec.arg_arrays, softmax.list_arguments()): + if name in set_stage1: + assert arr.context == group2ctx['stage1'] + else: + assert arr.context == group2ctx['stage2'] + if __name__ == '__main__': test_ctx_group() + test_ctx_group_sparse() diff --git a/tests/python/unittest/test_ndarray.py b/tests/python/unittest/test_ndarray.py index eae364eeaecf..3e0ac66c168d 100644 --- a/tests/python/unittest/test_ndarray.py +++ b/tests/python/unittest/test_ndarray.py @@ -373,6 +373,7 @@ def test_dot(): assert_almost_equal(c, C.asnumpy()) + def test_reduce(): sample_num = 200 def test_reduce_inner(numpy_reduce_func, nd_reduce_func, multi_axes): diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py index a33cb039c849..11d0ea22319a 100644 --- a/tests/python/unittest/test_operator.py +++ b/tests/python/unittest/test_operator.py @@ -855,75 +855,88 @@ def test_nearest_upsampling(): check_nearest_upsampling_with_shape(shapes, scale, root_scale) def test_batchnorm_training(): - for shape in [(2, 3), (2, 3, 2, 2)]: - data_tmp = np.random.normal(-0.1, 0.1, size=shape) - s = shape[1], - gamma = np.ones(s) - beta = np.ones(s) - gamma[1] = 3 - beta[0] = 3 + def check_batchnorm_training(stype): + for shape in [(2, 3), (2, 3, 2, 2)]: + data_tmp = np.random.normal(-0.1, 0.1, size=shape) + s = shape[1], + gamma = np.ones(s) + beta = np.ones(s) + gamma[1] = 3 + beta[0] = 3 - rolling_mean = np.random.uniform(size=s) - rolling_std = np.random.uniform(size=s) + rolling_mean = np.random.uniform(size=s) + rolling_std = np.random.uniform(size=s) - data = mx.symbol.Variable('data') + data = mx.symbol.Variable('data', stype=stype) + in_location = [mx.nd.array(data_tmp).tostype(stype), mx.nd.array(gamma).tostype(stype), + mx.nd.array(beta).tostype(stype)] + mean_std = [mx.nd.array(rolling_mean).tostype(stype), mx.nd.array(rolling_std).tostype(stype)] - test = mx.symbol.BatchNorm_v1(data, fix_gamma=True) - check_numeric_gradient(test, [data_tmp, gamma, beta], [rolling_mean, rolling_std], numeric_eps=1e-2, rtol=0.16) + test = mx.symbol.BatchNorm_v1(data, fix_gamma=True) + check_numeric_gradient(test, in_location, mean_std, numeric_eps=1e-2, rtol=0.16) - test = mx.symbol.BatchNorm(data, fix_gamma=True) - check_numeric_gradient(test, [data_tmp, gamma, beta], [rolling_mean, rolling_std], numeric_eps=1e-2, rtol=0.16) + test = mx.symbol.BatchNorm(data, fix_gamma=True) + check_numeric_gradient(test, in_location, mean_std, numeric_eps=1e-2, rtol=0.16) - test = mx.symbol.BatchNorm_v1(data, fix_gamma=True, use_global_stats=True) - check_numeric_gradient(test, [data_tmp, gamma, beta], [rolling_mean, rolling_std], numeric_eps=1e-2, rtol=0.16) + test = mx.symbol.BatchNorm_v1(data, fix_gamma=True, use_global_stats=True) + check_numeric_gradient(test, in_location, mean_std, numeric_eps=1e-2, rtol=0.16) - test = mx.symbol.BatchNorm(data, fix_gamma=True, use_global_stats=True) - check_numeric_gradient(test, [data_tmp, gamma, beta], [rolling_mean, rolling_std], numeric_eps=1e-2, rtol=0.16) + test = mx.symbol.BatchNorm(data, fix_gamma=True, use_global_stats=True) + check_numeric_gradient(test, in_location, mean_std, numeric_eps=1e-2, rtol=0.16) - test = mx.symbol.BatchNorm_v1(data, fix_gamma=False) - check_numeric_gradient(test, [data_tmp, gamma, beta], [rolling_mean, rolling_std], numeric_eps=1e-2, rtol=0.16) + test = mx.symbol.BatchNorm_v1(data, fix_gamma=False) + check_numeric_gradient(test, in_location, mean_std, numeric_eps=1e-2, rtol=0.16) - test = mx.symbol.BatchNorm(data, fix_gamma=False) - check_numeric_gradient(test, [data_tmp, gamma, beta], [rolling_mean, rolling_std], numeric_eps=1e-2, rtol=0.16) + test = mx.symbol.BatchNorm(data, fix_gamma=False) + check_numeric_gradient(test, in_location, mean_std, numeric_eps=1e-2, rtol=0.16) - test = mx.symbol.BatchNorm_v1(data, fix_gamma=False, use_global_stats=True) - check_numeric_gradient(test, [data_tmp, gamma, beta], [rolling_mean, rolling_std], numeric_eps=1e-2, rtol=0.16) + test = mx.symbol.BatchNorm_v1(data, fix_gamma=False, use_global_stats=True) + check_numeric_gradient(test, in_location, mean_std, numeric_eps=1e-2, rtol=0.16) - test = mx.symbol.BatchNorm(data, fix_gamma=False, use_global_stats=True) - check_numeric_gradient(test, [data_tmp, gamma, beta], [rolling_mean, rolling_std], numeric_eps=1e-2, rtol=0.16) + test = mx.symbol.BatchNorm(data, fix_gamma=False, use_global_stats=True) + check_numeric_gradient(test, in_location, mean_std, numeric_eps=1e-2, rtol=0.16) - # Test varying channel axis - dim = len(shape) - for chaxis in range(-dim, dim): - chaxis_true = chaxis - if chaxis < 0: - chaxis_true = dim + chaxis + # Test varying channel axis + dim = len(shape) + for chaxis in range(-dim, dim): + chaxis_true = chaxis + if chaxis < 0: + chaxis_true = dim + chaxis - shapex = shape + shapex = shape - channel_count = shapex[chaxis_true] - data_tmp = np.random.normal(-0.1, 0.1, size=shapex) + channel_count = shapex[chaxis_true] + data_tmp = np.random.normal(-0.1, 0.1, size=shapex) - gamma = np.ones(channel_count) - beta = np.ones(channel_count) - if channel_count > 1: - gamma[1] = 3 - beta[0] = 3 + gamma = np.ones(channel_count) + beta = np.ones(channel_count) + if channel_count > 1: + gamma[1] = 3 + beta[0] = 3 + + in_location = [mx.nd.array(data_tmp).tostype(stype), mx.nd.array(gamma).tostype(stype), + mx.nd.array(beta).tostype(stype)] + + xrolling_mean = np.random.uniform(size=channel_count) + xrolling_std = np.random.uniform(size=channel_count) + xmean_std = [mx.nd.array(xrolling_mean).tostype(stype), + mx.nd.array(xrolling_std).tostype(stype)] - xrolling_mean = np.random.uniform(size=channel_count) - xrolling_std = np.random.uniform(size=channel_count) + test = mx.symbol.BatchNorm(data, fix_gamma=True, axis=chaxis) + check_numeric_gradient(test, in_location, xmean_std, numeric_eps=1e-2, rtol=0.2, atol=0.01) - test = mx.symbol.BatchNorm(data, fix_gamma=True, axis=chaxis) - check_numeric_gradient(test, [data_tmp, gamma, beta], [xrolling_mean, xrolling_std], numeric_eps=1e-2, rtol=0.2, atol=0.01) + test = mx.symbol.BatchNorm(data, fix_gamma=True, use_global_stats=True, axis=chaxis) + check_numeric_gradient(test, in_location, xmean_std, numeric_eps=1e-2, rtol=0.2, atol=0.01) - test = mx.symbol.BatchNorm(data, fix_gamma=True, use_global_stats=True, axis=chaxis) - check_numeric_gradient(test, [data_tmp, gamma, beta], [xrolling_mean, xrolling_std], numeric_eps=1e-2, rtol=0.2, atol=0.01) + test = mx.symbol.BatchNorm(data, fix_gamma=False, axis=chaxis) + check_numeric_gradient(test, in_location, xmean_std, numeric_eps=1e-2, rtol=0.2, atol=0.01) - test = mx.symbol.BatchNorm(data, fix_gamma=False, axis=chaxis) - check_numeric_gradient(test, [data_tmp, gamma, beta], [xrolling_mean, xrolling_std], numeric_eps=1e-2, rtol=0.2, atol=0.01) + test = mx.symbol.BatchNorm(data, fix_gamma=False, use_global_stats=True, axis=chaxis) + check_numeric_gradient(test, in_location, xmean_std, numeric_eps=1e-2, rtol=0.2, atol=0.01) - test = mx.symbol.BatchNorm(data, fix_gamma=False, use_global_stats=True, axis=chaxis) - check_numeric_gradient(test, [data_tmp, gamma, beta], [xrolling_mean, xrolling_std], numeric_eps=1e-2, rtol=0.2, atol=0.01) + stypes = ['row_sparse', 'default'] + for stype in stypes: + check_batchnorm_training(stype) def test_convolution_grouping(): num_filter = 4 diff --git a/tests/python/unittest/test_optimizer.py b/tests/python/unittest/test_optimizer.py index 3b3b92b372d8..055f6464f0ef 100644 --- a/tests/python/unittest/test_optimizer.py +++ b/tests/python/unittest/test_optimizer.py @@ -47,26 +47,43 @@ def test_lr_wd_mult(): assert not mx.test_utils.almost_equal(args1['fc2_weight'], args2['fc2_weight'], 1e-1) -def compare_optimizer(opt1, opt2, shape, dtype): - w1 = mx.random.uniform(shape=shape, ctx=default_context(), dtype=dtype) - g1 = mx.random.uniform(shape=shape, ctx=default_context(), dtype=dtype) - - w2 = w1.copyto(default_context()) - g2 = g1.copyto(default_context()) +def compare_optimizer(opt1, opt2, shape, dtype, w_stype='default', g_stype='default'): + if w_stype == 'default': + w2 = mx.random.uniform(shape=shape, ctx=default_context(), dtype=dtype) + w1 = w2.copyto(default_context()) + elif w_stype == 'row_sparse' or w_stype == 'csr': + w2 = rand_ndarray(shape, w_stype, density=1, dtype=dtype) + w1 = w2.copyto(default_context()).tostype('default') + else: + raise Exception("type not supported yet") + if g_stype == 'default': + g2 = mx.random.uniform(shape=shape, ctx=default_context(), dtype=dtype) + g1 = g2.copyto(default_context()) + elif g_stype == 'row_sparse' or g_stype == 'csr': + g2 = rand_ndarray(shape, g_stype, dtype=dtype) + g1 = g2.copyto(default_context()).tostype('default') + else: + raise Exception("type not supported yet") state1 = opt1.create_state(0, w1) state2 = opt2.create_state(0, w2) if state1 is not None and state2 is not None: - for s1, s2, in zip(state1, state2): - if s1 is not None or s2 is not None: - assert(same(s1.asnumpy(), s2.asnumpy())) + if isinstance(state1, tuple): + for s1, s2, in zip(state1, state2): + if s1 is not None or s2 is not None: + assert(same(s1.asnumpy(), s2.asnumpy())) + else: + assert_almost_equal(state1.asnumpy(), state2.asnumpy()) opt1.update(0, w1, g1, state1) opt2.update(0, w2, g2, state2) if state1 is not None and state2 is not None: - for s1, s2, in zip(state1, state2): - if s1 is not None or s2 is not None: - assert_almost_equal(s1.asnumpy(), s2.asnumpy(), rtol=1e-4, atol=1e-5) + if isinstance(state1, tuple): + for s1, s2, in zip(state1, state2): + if s1 is not None or s2 is not None: + assert_almost_equal(s1.asnumpy(), s2.asnumpy(), rtol=1e-4, atol=1e-5) + else: + assert_almost_equal(state1.asnumpy(), state2.asnumpy()) assert_almost_equal(w1.asnumpy(), w2.asnumpy(), rtol=1e-4, atol=1e-5) # SGD @@ -186,18 +203,122 @@ def test_sgd(): not kwarg['multi_precision'])): continue compare_optimizer(opt1(**kwarg), opt2(**kwarg), shape, dtype) + # test operator fallback on cpu + if (default_context() == mx.cpu()): + compare_optimizer(opt1(**kwarg), opt2(**kwarg), shape, dtype, + g_stype='row_sparse') + if dtype != np.float16: + compare_optimizer(opt1(**kwarg), opt2(**kwarg), shape[:2], + dtype, w_stype='csr', g_stype='csr') + +class PySparseSGD(mx.optimizer.Optimizer): + """python reference implemenation of sgd""" + def __init__(self, learning_rate=0.01, momentum=0.0, **kwargs): + super(PySparseSGD, self).__init__(learning_rate=learning_rate, **kwargs) + self.momentum = momentum + + def create_state(self, index, weight): + """Create additional optimizer state: momentum + + Parameters + ---------- + weight : NDArray + The weight data + + """ + if self.momentum == 0.0: + return None + else: + return mx.nd.zeros(weight.shape, weight.context, dtype=weight.dtype) + + def update(self, index, weight, grad, state): + """Update the parameters. + + Parameters + ---------- + index : int + An unique integer key used to index the parameters + + weight : NDArray + weight ndarray + + grad : NDArray + grad ndarray + + state : NDArray or other objects returned by init_state + The auxiliary state used in optimization. + """ + lr = self._get_lr(index) + wd = self._get_wd(index) + self._update_count(index) + num_rows = weight.shape[0] + if self.momentum == 0.0: + # Update on a per row basis, skip all-zero rows + for row in range(num_rows): + grad_row = grad[row].asnumpy() + all_zeros = mx.test_utils.almost_equal(grad_row, np.zeros_like(grad_row)) + if all_zeros: + continue + if self.clip_gradient is not None: + weight[row] = ((1 - lr*wd)*weight[row] - + lr*mx.nd.clip(grad[row]*self.rescale_grad, + -self.clip_gradient, self.clip_gradient)) + else: + weight[row] = (1 - lr*wd)*weight[row] - lr*self.rescale_grad*grad[row] + else: + mom = state + for row in range(num_rows): + grad_row = grad[row].asnumpy() + all_zeros = mx.test_utils.almost_equal(grad_row, np.zeros_like(grad_row)) + if all_zeros: + continue + if self.clip_gradient is not None: + mom[row] = (self.momentum*mom[row] - lr*wd*weight[row] - + lr*mx.nd.clip(grad[row]*self.rescale_grad, -self.clip_gradient, self.clip_gradient)) + weight[row] += mom[row] + else: + mom[row] = self.momentum*mom[row] - lr*wd*weight[row] - lr*self.rescale_grad*grad[row] + weight[row] += mom[row] + +def test_sparse_sgd(): + mx.random.seed(0) + opt1 = PySparseSGD + opt2 = mx.optimizer.SGD + shape = (3, 4, 5) + mom_options = [{}, {'momentum': 0.9}] + cg_options = [{}, {'clip_gradient': 0.4}, {'clip_gradient': 0.5}] + rg_options = [{}, {'rescale_grad': 0.14}, {'rescale_grad': 0.8}] + wd_options = [{}, {'wd': 0.03}, {'wd': 0.05}, {'wd': 0.07}] + mp_options = [{}] + for dtype in [np.float32]: + for mom_option in mom_options: + for cg_option in cg_options: + for rg_option in rg_options: + for wd_option in wd_options: + for mp_option in mp_options: + kwarg = {} + kwarg.update(mom_option) + kwarg.update(cg_option) + kwarg.update(rg_option) + kwarg.update(wd_option) + kwarg.update(mp_option) + compare_optimizer(opt1(**kwarg), opt2(**kwarg), shape, dtype, + w_stype='row_sparse', g_stype='row_sparse') + compare_optimizer(opt1(**kwarg), opt2(**kwarg), shape, dtype, + w_stype='row_sparse', g_stype='default') # ADAM class PyAdam(mx.optimizer.Optimizer): """python reference implemenation of adam""" def __init__(self, learning_rate=0.001, beta1=0.9, beta2=0.999, epsilon=1e-8, - decay_factor=(1 - 1e-8), **kwargs): + decay_factor=(1 - 1e-8), sparse_update=False, **kwargs): super(PyAdam, self).__init__(learning_rate=learning_rate, **kwargs) self.beta1 = beta1 self.beta2 = beta2 self.epsilon = epsilon self.decay_factor = decay_factor + self.sparse_update = sparse_update def create_state(self, index, weight): """Create additional optimizer state: mean, variance @@ -235,21 +356,28 @@ def update(self, index, weight, grad, state): mean, variance = state wd = self._get_wd(index) - grad = grad * self.rescale_grad + wd * weight - if self.clip_gradient is not None: - mx.nd.clip(grad, -self.clip_gradient, self.clip_gradient, out=grad) - - mean *= self.beta1 - mean += grad * (1. - self.beta1) - - variance *= self.beta2 - variance += (1 - self.beta2) * mx.nd.square(grad, out=grad) - + num_rows = weight.shape[0] coef1 = 1. - self.beta1**t coef2 = 1. - self.beta2**t lr *= math.sqrt(coef2)/coef1 - - weight -= lr*mean/(mx.nd.sqrt(variance) + self.epsilon) + for row in range(num_rows): + # check row slices of all zeros + all_zeros = mx.test_utils.almost_equal(grad[row].asnumpy(), np.zeros_like(grad[row].asnumpy())) + # skip zeros during sparse update + if all_zeros and self.sparse_update: + continue + grad[row] = grad[row] * self.rescale_grad + wd * weight[row] + # clip gradients + if self.clip_gradient is not None: + mx.nd.clip(grad[row], -self.clip_gradient, self.clip_gradient, out=grad[row]) + # update mean + mean[row] *= self.beta1 + mean[row] += grad[row] * (1. - self.beta1) + # update variance + variance[row] *= self.beta2 + variance[row] += (1 - self.beta2) * mx.nd.square(grad[row], out=grad[row]) + # update weight + weight[row] -= lr*mean[row]/(mx.nd.sqrt(variance[row]) + self.epsilon) def test_adam(): @@ -266,6 +394,8 @@ def test_adam(): {'rescale_grad': 0.8, 'wd': 0.05}] for kwarg in kwargs: compare_optimizer(opt1(**kwarg), opt2(**kwarg), shape, np.float32) + compare_optimizer(opt1(sparse_update=True, **kwarg), opt2(**kwarg), shape, + np.float32, w_stype='row_sparse', g_stype='row_sparse') # RMSProp class PyRMSProp(mx.optimizer.Optimizer): @@ -406,8 +536,10 @@ def test_rms(): {'rescale_grad': 0.8, 'wd': 0.05, 'centered': True, 'clip_weights': 0.01}] for kwarg in kwargs: compare_optimizer(opt1(**kwarg), opt2(**kwarg), shape, np.float32) + compare_optimizer(opt1(**kwarg), opt2(**kwarg), shape, np.float32, g_stype='row_sparse') if __name__ == '__main__': test_adam() test_rms() test_sgd() + test_sparse_sgd() diff --git a/tests/python/unittest/test_sparse_ndarray.py b/tests/python/unittest/test_sparse_ndarray.py new file mode 100644 index 000000000000..a77343436945 --- /dev/null +++ b/tests/python/unittest/test_sparse_ndarray.py @@ -0,0 +1,524 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pickle as pkl + +from mxnet.ndarray import NDArray +from mxnet.test_utils import * +from numpy.testing import assert_allclose +import numpy.random as rnd + +from mxnet.ndarray.sparse import RowSparseNDArray, CSRNDArray + + +def assert_fcompex(f, *args, **kwargs): + prev_val = mx.test_utils.set_env_var("MXNET_EXEC_STORAGE_FALLBACK", "0", "1") + f(*args, **kwargs) + mx.test_utils.set_env_var("MXNET_EXEC_STORAGE_FALLBACK", prev_val) + + +def sparse_nd_ones(shape, stype): + return mx.nd.ones(shape).tostype(stype) + + +def check_sparse_nd_elemwise_binary(shapes, stypes, f, g): + # generate inputs + nds = [] + for i, stype in enumerate(stypes): + if stype == 'row_sparse': + nd, _ = rand_sparse_ndarray(shapes[i], stype) + elif stype == 'default': + nd = mx.nd.array(random_arrays(shapes[i]), dtype = np.float32) + else: + assert(False) + nds.append(nd) + # check result + test = f(nds[0], nds[1]) + assert_almost_equal(test.asnumpy(), g(nds[0].asnumpy(), nds[1].asnumpy())) + + +def test_sparse_nd_elemwise_add(): + num_repeats = 10 + g = lambda x,y: x + y + op = mx.nd.elemwise_add + for i in range(num_repeats): + shape = [rand_shape_2d()] * 2 + assert_fcompex(check_sparse_nd_elemwise_binary, + shape, ['default'] * 2, op, g) + assert_fcompex(check_sparse_nd_elemwise_binary, + shape, ['default', 'row_sparse'], op, g) + assert_fcompex(check_sparse_nd_elemwise_binary, + shape, ['row_sparse', 'row_sparse'], op, g) + + +def test_sparse_nd_copy(): + def check_sparse_nd_copy(from_stype, to_stype, shape): + from_nd = rand_ndarray(shape, from_stype) + # copy to ctx + to_ctx = from_nd.copyto(default_context()) + # copy to stype + to_nd = rand_ndarray(shape, to_stype) + to_nd = from_nd.copyto(to_nd) + assert np.sum(np.abs(from_nd.asnumpy() != to_ctx.asnumpy())) == 0.0 + assert np.sum(np.abs(from_nd.asnumpy() != to_nd.asnumpy())) == 0.0 + + shape = rand_shape_2d() + shape_3d = rand_shape_3d() + stypes = ['row_sparse', 'csr'] + for stype in stypes: + check_sparse_nd_copy(stype, 'default', shape) + check_sparse_nd_copy('default', stype, shape) + check_sparse_nd_copy('row_sparse', 'row_sparse', shape_3d) + check_sparse_nd_copy('row_sparse', 'default', shape_3d) + check_sparse_nd_copy('default', 'row_sparse', shape_3d) + +def test_sparse_nd_basic(): + def check_sparse_nd_basic_rsp(): + storage_type = 'row_sparse' + shape = rand_shape_2d() + nd, (v, idx) = rand_sparse_ndarray(shape, storage_type) + assert(nd._num_aux == 1) + assert(nd.indices.dtype == np.int64) + assert(nd.stype == 'row_sparse') + + check_sparse_nd_basic_rsp() + + +def test_sparse_nd_setitem(): + def check_sparse_nd_setitem(stype, shape, dst): + x = mx.nd.zeros(shape=shape, stype=stype) + x[:] = dst + dst_nd = mx.nd.array(dst) if isinstance(dst, (np.ndarray, np.generic)) else dst + assert same(x.asnumpy(), dst_nd.asnumpy()) + + shape = rand_shape_2d() + for stype in ['row_sparse', 'csr']: + # ndarray assignment + check_sparse_nd_setitem(stype, shape, rand_ndarray(shape, 'default')) + check_sparse_nd_setitem(stype, shape, rand_ndarray(shape, stype)) + # numpy assignment + check_sparse_nd_setitem(stype, shape, np.ones(shape)) + + +def test_sparse_nd_slice(): + def check_sparse_nd_csr_slice(shape): + stype = 'csr' + A, _ = rand_sparse_ndarray(shape, stype) + A2 = A.asnumpy() + start = rnd.randint(0, shape[0] - 1) + end = rnd.randint(start + 1, shape[0]) + assert same(A[start:end].asnumpy(), A2[start:end]) + assert same(A[start:].asnumpy(), A2[start:]) + assert same(A[:end].asnumpy(), A2[:end]) + + shape = (rnd.randint(2, 10), rnd.randint(1, 10)) + check_sparse_nd_csr_slice(shape) + + +def test_sparse_nd_equal(): + for stype in ['row_sparse', 'csr']: + shape = rand_shape_2d() + x = mx.nd.zeros(shape=shape, stype=stype) + y = sparse_nd_ones(shape, stype) + z = x == y + assert (z.asnumpy() == np.zeros(shape)).all() + z = 0 == x + assert (z.asnumpy() == np.ones(shape)).all() + + +def test_sparse_nd_not_equal(): + for stype in ['row_sparse', 'csr']: + shape = rand_shape_2d() + x = mx.nd.zeros(shape=shape, stype=stype) + y = sparse_nd_ones(shape, stype) + z = x != y + assert (z.asnumpy() == np.ones(shape)).all() + z = 0 != x + assert (z.asnumpy() == np.zeros(shape)).all() + + +def test_sparse_nd_greater(): + for stype in ['row_sparse', 'csr']: + shape = rand_shape_2d() + x = mx.nd.zeros(shape=shape, stype=stype) + y = sparse_nd_ones(shape, stype) + z = x > y + assert (z.asnumpy() == np.zeros(shape)).all() + z = y > 0 + assert (z.asnumpy() == np.ones(shape)).all() + z = 0 > y + assert (z.asnumpy() == np.zeros(shape)).all() + + +def test_sparse_nd_greater_equal(): + for stype in ['row_sparse', 'csr']: + shape = rand_shape_2d() + x = mx.nd.zeros(shape=shape, stype=stype) + y = sparse_nd_ones(shape, stype) + z = x >= y + assert (z.asnumpy() == np.zeros(shape)).all() + z = y >= 0 + assert (z.asnumpy() == np.ones(shape)).all() + z = 0 >= y + assert (z.asnumpy() == np.zeros(shape)).all() + z = y >= 1 + assert (z.asnumpy() == np.ones(shape)).all() + + +def test_sparse_nd_lesser(): + for stype in ['row_sparse', 'csr']: + shape = rand_shape_2d() + x = mx.nd.zeros(shape=shape, stype=stype) + y = sparse_nd_ones(shape, stype) + z = y < x + assert (z.asnumpy() == np.zeros(shape)).all() + z = 0 < y + assert (z.asnumpy() == np.ones(shape)).all() + z = y < 0 + assert (z.asnumpy() == np.zeros(shape)).all() + + +def test_sparse_nd_lesser_equal(): + for stype in ['row_sparse', 'csr']: + shape = rand_shape_2d() + x = mx.nd.zeros(shape=shape, stype=stype) + y = sparse_nd_ones(shape, stype) + z = y <= x + assert (z.asnumpy() == np.zeros(shape)).all() + z = 0 <= y + assert (z.asnumpy() == np.ones(shape)).all() + z = y <= 0 + assert (z.asnumpy() == np.zeros(shape)).all() + z = 1 <= y + assert (z.asnumpy() == np.ones(shape)).all() + + +def test_sparse_nd_binary(): + N = 10 + def check_binary(fn, stype): + for _ in range(N): + ndim = 2 + oshape = np.random.randint(1, 6, size=(ndim,)) + bdim = 2 + lshape = list(oshape) + rshape = list(oshape[ndim-bdim:]) + for i in range(bdim): + sep = np.random.uniform(0, 1) + if sep < 0.33: + lshape[ndim-i-1] = 1 + elif sep < 0.66: + rshape[bdim-i-1] = 1 + lhs = np.random.uniform(0, 1, size=lshape) + rhs = np.random.uniform(0, 1, size=rshape) + lhs_nd = mx.nd.array(lhs).tostype(stype) + rhs_nd = mx.nd.array(rhs).tostype(stype) + assert_allclose(fn(lhs, rhs), fn(lhs_nd, rhs_nd).asnumpy(), rtol=1e-4, atol=1e-4) + + stypes = ['row_sparse', 'csr'] + for stype in stypes: + check_binary(lambda x, y: x + y, stype) + check_binary(lambda x, y: x - y, stype) + check_binary(lambda x, y: x * y, stype) + check_binary(lambda x, y: x / y, stype) + check_binary(lambda x, y: x ** y, stype) + check_binary(lambda x, y: x > y, stype) + check_binary(lambda x, y: x < y, stype) + check_binary(lambda x, y: x >= y, stype) + check_binary(lambda x, y: x <= y, stype) + check_binary(lambda x, y: x == y, stype) + + +def test_sparse_nd_binary_rop(): + N = 10 + def check(fn, stype): + for _ in range(N): + ndim = 2 + shape = np.random.randint(1, 6, size=(ndim,)) + npy = np.random.normal(0, 1, size=shape) + nd = mx.nd.array(npy).tostype(stype) + assert_allclose(fn(npy), fn(nd).asnumpy(), rtol=1e-4, atol=1e-4) + + stypes = ['row_sparse', 'csr'] + for stype in stypes: + check(lambda x: 1 + x, stype) + check(lambda x: 1 - x, stype) + check(lambda x: 1 * x, stype) + check(lambda x: 1 / x, stype) + check(lambda x: 2 ** x, stype) + check(lambda x: 1 > x, stype) + check(lambda x: 0.5 > x, stype) + check(lambda x: 0.5 < x, stype) + check(lambda x: 0.5 >= x, stype) + check(lambda x: 0.5 <= x, stype) + check(lambda x: 0.5 == x, stype) + +def test_sparse_nd_binary_iop(): + N = 10 + def check_binary(fn, stype): + for _ in range(N): + ndim = 2 + oshape = np.random.randint(1, 6, size=(ndim,)) + lshape = list(oshape) + rshape = list(oshape) + lhs = np.random.uniform(0, 1, size=lshape) + rhs = np.random.uniform(0, 1, size=rshape) + lhs_nd = mx.nd.array(lhs).tostype(stype) + rhs_nd = mx.nd.array(rhs).tostype(stype) + assert_allclose(fn(lhs, rhs), + fn(lhs_nd, rhs_nd).asnumpy(), + rtol=1e-4, atol=1e-4) + + def inplace_add(x, y): + x += y + return x + def inplace_mul(x, y): + x *= y + return x + stypes = ['csr', 'row_sparse'] + fns = [inplace_add, inplace_mul] + for stype in stypes: + for fn in fns: + check_binary(fn, stype) + +def test_sparse_nd_negate(): + def check_sparse_nd_negate(shape, stype): + npy = np.random.uniform(-10, 10, rand_shape_2d()) + arr = mx.nd.array(npy).tostype(stype) + assert_almost_equal(npy, arr.asnumpy()) + assert_almost_equal(-npy, (-arr).asnumpy()) + + # a final check to make sure the negation (-) is not implemented + # as inplace operation, so the contents of arr does not change after + # we compute (-arr) + assert_almost_equal(npy, arr.asnumpy()) + + shape = rand_shape_2d() + stypes = ['csr', 'row_sparse'] + for stype in stypes: + check_sparse_nd_negate(shape, stype) + +def test_sparse_nd_broadcast(): + sample_num = 1000 + # TODO(haibin) test with more than 2 dimensions + def test_broadcast_to(stype): + for i in range(sample_num): + ndim = 2 + target_shape = np.random.randint(1, 11, size=ndim) + shape = target_shape.copy() + axis_flags = np.random.randint(0, 2, size=ndim) + axes = [] + for (axis, flag) in enumerate(axis_flags): + if flag: + shape[axis] = 1 + dat = np.random.rand(*shape) - 0.5 + numpy_ret = dat + ndarray = mx.nd.array(dat).tostype(stype) + ndarray_ret = ndarray.broadcast_to(shape=target_shape) + if type(ndarray_ret) is mx.ndarray.NDArray: + ndarray_ret = ndarray_ret.asnumpy() + assert (ndarray_ret.shape == target_shape).all() + err = np.square(ndarray_ret - numpy_ret).mean() + assert err < 1E-8 + stypes = ['csr', 'row_sparse'] + for stype in stypes: + test_broadcast_to(stype) + + +def test_sparse_nd_transpose(): + npy = np.random.uniform(-10, 10, rand_shape_2d()) + stypes = ['csr', 'row_sparse'] + for stype in stypes: + nd = mx.nd.array(npy).tostype(stype) + assert_almost_equal(npy.T, (nd.T).asnumpy()) + +def test_sparse_nd_output_fallback(): + shape = (10, 10) + out = mx.nd.zeros(shape=shape, stype='row_sparse') + mx.nd.random_normal(shape=shape, out=out) + assert(np.sum(out.asnumpy()) != 0) + +def test_sparse_nd_random(): + """ test sparse random operator on cpu """ + # gpu random operator doesn't use fixed seed + if default_context().device_type is 'gpu': + return + shape = (100, 100) + fns = [mx.nd.random_uniform, mx.nd.random_normal, mx.nd.random_gamma] + for fn in fns: + rsp_out = mx.nd.zeros(shape=shape, stype='row_sparse') + dns_out = mx.nd.zeros(shape=shape, stype='default') + mx.random.seed(0) + np.random.seed(0) + fn(shape=shape, out=dns_out) + mx.random.seed(0) + np.random.seed(0) + fn(shape=shape, out=rsp_out) + assert_almost_equal(dns_out.asnumpy(), rsp_out.asnumpy()) + + +def test_sparse_nd_astype(): + stypes = ['row_sparse', 'csr'] + for stype in stypes: + x = mx.nd.zeros(shape=rand_shape_2d(), stype=stype, dtype='float32') + y = x.astype('int32') + assert(y.dtype == np.int32), y.dtype + + +def test_sparse_nd_pickle(): + np.random.seed(0) + repeat = 10 + dim0 = 40 + dim1 = 40 + stypes = ['row_sparse', 'csr'] + densities = [0, 0.01, 0.1, 0.2, 0.5] + stype_dict = {'row_sparse': RowSparseNDArray, 'csr': CSRNDArray} + for _ in range(repeat): + shape = rand_shape_2d(dim0, dim1) + for stype in stypes: + for density in densities: + a, _ = rand_sparse_ndarray(shape, stype, density) + assert isinstance(a, stype_dict[stype]) + data = pkl.dumps(a) + b = pkl.loads(data) + assert isinstance(b, stype_dict[stype]) + assert same(a.asnumpy(), b.asnumpy()) + + +def test_sparse_nd_save_load(): + np.random.seed(0) + repeat = 1 + stypes = ['default', 'row_sparse', 'csr'] + stype_dict = {'default': NDArray, 'row_sparse': RowSparseNDArray, 'csr': CSRNDArray} + num_data = 20 + densities = [0, 0.01, 0.1, 0.2, 0.5] + fname = 'tmp_list.bin' + for _ in range(repeat): + data_list1 = [] + for i in range(num_data): + stype = stypes[np.random.randint(0, len(stypes))] + shape = rand_shape_2d(dim0=40, dim1=40) + density = densities[np.random.randint(0, len(densities))] + data_list1.append(rand_ndarray(shape, stype, density)) + assert isinstance(data_list1[-1], stype_dict[stype]) + mx.nd.save(fname, data_list1) + + data_list2 = mx.nd.load(fname) + assert len(data_list1) == len(data_list2) + for x, y in zip(data_list1, data_list2): + assert same(x.asnumpy(), y.asnumpy()) + + data_map1 = {'ndarray xx %s' % i: x for i, x in enumerate(data_list1)} + mx.nd.save(fname, data_map1) + data_map2 = mx.nd.load(fname) + assert len(data_map1) == len(data_map2) + for k, x in data_map1.items(): + y = data_map2[k] + assert same(x.asnumpy(), y.asnumpy()) + os.remove(fname) + +def test_sparse_nd_unsupported(): + nd = mx.nd.zeros((2,2), stype='row_sparse') + fn_slice = lambda x: x._slice(None, None) + fn_at = lambda x: x._at(None) + fn_reshape = lambda x: x.reshape(None) + fns = [fn_slice, fn_at, fn_reshape] + for fn in fns: + try: + fn(nd) + assert(False) + except: + pass + +def test_create_csr(): + dim0 = 50 + dim1 = 50 + densities = [0, 0.01, 0.1, 0.2, 0.5] + for density in densities: + shape = rand_shape_2d(dim0, dim1) + matrix = rand_ndarray(shape, 'csr', density) + data = matrix.data + indptr = matrix.indptr + indices = matrix.indices + csr_created = mx.nd.sparse.csr_matrix(data=data, indptr=indptr, + indices=indices, shape=shape) + assert csr_created.stype == 'csr' + assert same(csr_created.data.asnumpy(), data.asnumpy()) + assert same(csr_created.indptr.asnumpy(), indptr.asnumpy()) + assert same(csr_created.indices.asnumpy(), indices.asnumpy()) + csr_copy = mx.nd.array(csr_created) + assert(same(csr_copy.asnumpy(), csr_created.asnumpy())) + + +def test_create_row_sparse(): + dim0 = 50 + dim1 = 50 + densities = [0, 0.01, 0.1, 0.2, 0.5] + for density in densities: + shape = rand_shape_2d(dim0, dim1) + matrix = rand_ndarray(shape, 'row_sparse', density) + data = matrix.data + indices = matrix.indices + rsp_created = mx.nd.sparse.row_sparse_array(data=data, indices=indices, shape=shape) + assert rsp_created.stype == 'row_sparse' + assert same(rsp_created.data.asnumpy(), data.asnumpy()) + assert same(rsp_created.indices.asnumpy(), indices.asnumpy()) + rsp_copy = mx.nd.array(rsp_created) + assert(same(rsp_copy.asnumpy(), rsp_created.asnumpy())) + +def test_sparse_nd_empty(): + stypes = ['csr', 'row_sparse', 'default'] + for stype in stypes: + nd = mx.nd.empty((2,2), stype=stype) + assert(nd.stype == stype) + + +def test_synthetic_dataset_generator(): + def test_powerlaw_generator(csr_arr, final_row=1): + """Test power law distribution + Total Elements: 32000, Number of zeros: 3200 + Every row has 2 * non zero elements of the previous row. + Also since (2047 < 3200 < 4095) this will be true till 10th row""" + indices = csr_arr.indices.asnumpy() + indptr = csr_arr.indptr.asnumpy() + for row in range(1, final_row + 1): + nextrow = row + 1 + current_row_nnz = indices[indptr[row] - 1] + 1 + next_row_nnz = indices[indptr[nextrow] - 1] + 1 + assert next_row_nnz == 2 * current_row_nnz + + # Test if density is preserved + csr_arr_cols, _ = rand_sparse_ndarray(shape=(32, 10000), stype="csr", + density=0.01, distribution="powerlaw") + + csr_arr_small, _ = rand_sparse_ndarray(shape=(5, 5), stype="csr", + density=0.5, distribution="powerlaw") + + csr_arr_big, _ = rand_sparse_ndarray(shape=(32, 1000000), stype="csr", + density=0.4, distribution="powerlaw") + + csr_arr_square, _ = rand_sparse_ndarray(shape=(1600, 1600), stype="csr", + density=0.5, distribution="powerlaw") + assert len(csr_arr_cols.data) == 3200 + test_powerlaw_generator(csr_arr_cols, final_row=9) + test_powerlaw_generator(csr_arr_small, final_row=1) + test_powerlaw_generator(csr_arr_big, final_row=4) + test_powerlaw_generator(csr_arr_square, final_row=6) + + +if __name__ == '__main__': + import nose + nose.runmodule() diff --git a/tests/python/unittest/test_sparse_operator.py b/tests/python/unittest/test_sparse_operator.py new file mode 100644 index 000000000000..2875d7b4b645 --- /dev/null +++ b/tests/python/unittest/test_sparse_operator.py @@ -0,0 +1,373 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from mxnet.test_utils import * + + +def check_elemwise_add_ex(lhs_stype, rhs_stype, shape, lhs_grad_stype=None, rhs_grad_stype=None): + lhs = mx.symbol.Variable('lhs', stype=lhs_stype) + rhs = mx.symbol.Variable('rhs', stype=rhs_stype) + lhs_nd = rand_ndarray(shape, lhs_stype) + rhs_nd = rand_ndarray(shape, rhs_stype) + lhs_np = lhs_nd.asnumpy() + rhs_np = rhs_nd.asnumpy() + + out_np = lhs_np + rhs_np + test = mx.symbol.sparse.elemwise_add(lhs, rhs) + location = {'lhs': lhs_nd, 'rhs': rhs_nd} + check_symbolic_forward(test, location, [out_np]) + check_numeric_gradient(test, location) + grad_stypes = {} + if lhs_grad_stype is not None and lhs_grad_stype != 'default': + grad_stypes['lhs'] = lhs_grad_stype + if rhs_grad_stype is not None and rhs_grad_stype != 'default': + grad_stypes['rhs'] = rhs_grad_stype + check_symbolic_backward(test, location, [out_np], [out_np, out_np], + grad_stypes=grad_stypes) + + +def test_elemwise_add_ex(): + shapes = [rand_shape_2d(), rand_shape_3d()] + for shape in shapes: + check_elemwise_add_ex('default', 'default', shape) + check_elemwise_add_ex('default', 'row_sparse', shape) + check_elemwise_add_ex('row_sparse', 'default', shape) + check_elemwise_add_ex('row_sparse', 'row_sparse', shape, + lhs_grad_stype='row_sparse', rhs_grad_stype='row_sparse') + + +# TODO(haibin) randomize this test +def test_elemwise_add_ex_multiple_stages(): + # prep data + shape = (4, 2) + ds_np = np.array([[1, 2], [3, 4], [5, 6], [7, 8]]) + sp_np1 = np.array([[5, 10], [0, 0], [0, 0], [0, 0]]) + sp_np2 = np.array([[0, 0], [5, 10], [0, 0], [0, 0]]) + + val1 = mx.nd.array([[5, 10]]); + val2 = mx.nd.array([[5, 10]]); + idx1 = mx.nd.array([0], dtype=np.int64); + idx2 = mx.nd.array([1], dtype=np.int64); + sp_nd1 = mx.nd.sparse.row_sparse_array(val1, idx1, shape) + sp_nd2 = mx.nd.sparse.row_sparse_array(val2, idx2, shape) + ds_nd = mx.nd.array(ds_np) + + # sparse + sparse = sparse + sp_data1 = mx.symbol.Variable('sp_data1', stype='row_sparse') + sp_data2 = mx.symbol.Variable('sp_data2', stype='row_sparse') + ds_data = mx.symbol.Variable('ds_data') + plus = mx.symbol.sparse.elemwise_add(sp_data1, sp_data2, name='plus') + # sparse + dense = dense + test = mx.symbol.sparse.elemwise_add(plus, ds_data) + check_symbolic_forward(test, {'sp_data1': sp_nd1, 'sp_data2': sp_nd2, + 'ds_data': ds_nd}, [sp_np1 + sp_np2 + ds_np]) + + arr_grads = [mx.nd.zeros(shape) for i in range(3)] + exec_test = test.bind(default_context(), args={'sp_data1': sp_nd1, 'sp_data2': sp_nd2, + 'ds_data': ds_nd}, args_grad=arr_grads) + exec_test.forward(is_train=True) + assert_almost_equal(exec_test.outputs[0].asnumpy(), sp_np1 + sp_np2 + ds_np) + exec_test.backward(out_grads=exec_test.outputs) + assert_almost_equal(arr_grads[0].asnumpy(), arr_grads[1].asnumpy()) + +def test_cast_storage_ex(): + def check_cast_storage(shape, density, from_stype, to_stype, check_numeric_grad=True): + x = mx.symbol.Variable('x', stype=from_stype) + x_nd = rand_ndarray(shape, from_stype, density=density) + x_np = x_nd.asnumpy() + out_np = x_np + test = mx.symbol.cast_storage(x, stype=to_stype) + location = {'x': x_nd} + check_symbolic_forward(test, location, [out_np]) + # consider disable the numeric grad check for gpu block kernel since the input is large + if check_numeric_grad: + check_numeric_gradient(test, location) + grad_stypes = {'x': to_stype} + check_symbolic_backward(test, location, [out_np], [out_np], grad_stypes=grad_stypes) + + density = [1.00, 0.50, 0.10, 0.05, 0.01] + for d in density: + shape_2d = rand_shape_2d() + shape_3d = rand_shape_3d() + check_cast_storage(shape_2d, d, 'csr', 'default') + check_cast_storage(shape_2d, d, 'default', 'csr') + check_cast_storage(shape_2d, d, 'row_sparse', 'default') + check_cast_storage(shape_2d, d, 'default', 'row_sparse') + check_cast_storage(shape_3d, d, 'row_sparse', 'default') + check_cast_storage(shape_3d, d, 'default', 'row_sparse') + for i in range(4, 6): + shape = rand_shape_nd(i, 5) + check_cast_storage(shape, d, 'default', 'row_sparse') + check_cast_storage(shape, d, 'row_sparse', 'default') + # Test specific gpu kernels + if default_context().device_type is 'gpu': + dim0 = rnd.randint(1, 10) + # test gpu thread kernel + check_cast_storage((dim0, rnd.randint( 1, 32)), d, 'default', 'csr') + # test gpu warp kernel + check_cast_storage((dim0, rnd.randint( 32, 512)), d, 'default', 'csr') + # test gpu block kernel + check_cast_storage((dim0, rnd.randint(512, 1024)), d, 'default', 'csr', + check_numeric_grad=False) + # test gpu thread kernel + check_cast_storage((dim0, rnd.randint( 1, 32)), d, 'default', 'row_sparse') + # test gpu warp kernel + check_cast_storage((dim0, rnd.randint( 32, 512)), d, 'default', 'row_sparse') + # test gpu block kernel + check_cast_storage((dim0, rnd.randint(512, 1024)), d, 'default', 'row_sparse', + check_numeric_grad=False) + +def test_sparse_dot(): + def test_dot_csr(lhs_shape, rhs_shape, rhs_stype, trans_lhs, lhs_density, rhs_density): + lhs_nd = rand_ndarray(lhs_shape, 'csr', density=lhs_density) + lhs_dns = lhs_nd.tostype('default') + rhs_nd = rand_ndarray(rhs_shape, rhs_stype, density=rhs_density) + rhs_dns = rhs_nd if rhs_stype == 'default' else rhs_nd.tostype('default') + + out = mx.nd.dot(lhs_nd, rhs_nd, transpose_a=trans_lhs) + out_dns = mx.nd.dot(lhs_dns, rhs_dns, transpose_a=trans_lhs) + out_np = out_dns.asnumpy() + assert_almost_equal(out.asnumpy(), out_np, rtol=1e-4, atol=1e-5) + + # test symbolic forward + lhs = mx.symbol.Variable('lhs', stype='csr') + rhs = mx.symbol.Variable('rhs', stype=rhs_stype) + out = mx.symbol.sparse.dot(lhs, rhs, transpose_a=trans_lhs) + location = {'lhs': lhs_nd, 'rhs': rhs_nd} + check_symbolic_forward(out, location, [out_np], rtol=1e-3, atol=1e-4) + + # test symbolic backward + backward_trans = not trans_lhs + rhs_backward_grad = mx.nd.dot(lhs_dns, out_dns, transpose_a=backward_trans).asnumpy() + expected = {'rhs': rhs_backward_grad} + check_symbolic_backward(out, location, [out_np], expected, + grad_req={'lhs': 'null', 'rhs': 'write'}, + rtol=1e-3, atol=1e-4) + + density = [1.00, 0.50, 0.10, 0.05, 0.01] + for lhs_d in density: + lhs_shape = rand_shape_2d(50, 200) + rhs_d = 1 + test_dot_csr(lhs_shape, (lhs_shape[1], 1), 'default', False, lhs_d, rhs_d) # test gpu SpMV + test_dot_csr(lhs_shape, (lhs_shape[0], 1), 'default', True , lhs_d, rhs_d) # (vector kernel) + test_dot_csr(lhs_shape, (lhs_shape[1], rnd.randint(5, 10)), 'default', False, lhs_d, rhs_d) # test gpu SpMM + test_dot_csr(lhs_shape, (lhs_shape[0], rnd.randint(5, 10)), 'default', True , lhs_d, rhs_d) # (scalar kernel) + for rhs_d in density: + test_dot_csr(lhs_shape, (lhs_shape[1], rnd.randint(1, 10)), 'row_sparse', False, lhs_d, rhs_d) + test_dot_csr(lhs_shape, (lhs_shape[0], rnd.randint(1, 10)), 'row_sparse', True, lhs_d, rhs_d) + + +def test_sparse_slice(): + def check_csr_slice(shape, slice_input): + storage_type = 'csr' + B, _ = rand_sparse_ndarray(shape, storage_type) + np = B.asnumpy() + begin = rnd.randint(0, B.shape[0] - 1) + end = rnd.randint(begin + 1, B.shape[0]) + nd_slice = mx.nd.crop(B, begin=begin, end=end) + assert same(nd_slice.asnumpy(), np[begin:end]), (nd_slice.asnumpy(), np[begin:end]) + + shape = (rnd.randint(7, 15), rnd.randint(1, 10)) + check_csr_slice(shape, True) + check_csr_slice(shape, False) + + +def test_sparse_retain(): + def check_sparse_retain(shape, density, index_type=np.int64): + num_rows = shape[0] + rsp, _ = rand_sparse_ndarray(shape=shape, stype='row_sparse', density=density) + length = np.random.randint(1, num_rows + 1) + idx = random_sample(list(range(0, num_rows)), length) + idx.sort() + dns = rsp.asnumpy() + tensor_retained_expected = np.zeros(shape) + for i in idx: + tensor_retained_expected[i][:] = dns[i] + indices = mx.nd.array(idx, dtype=index_type) + rsp_retained = mx.nd.sparse.retain(rsp, indices=indices) + assert same(tensor_retained_expected, rsp_retained.asnumpy()) + + # check numeric gradient + data = mx.symbol.Variable('data') + idx = mx.symbol.Variable('indices') + sym = mx.sym.sparse.retain(data=data, indices=idx) + check_numeric_gradient(sym, [rsp, indices], grad_nodes=['data'], + grad_stype_dict={'data': 'row_sparse'}) + + shape = rand_shape_2d() + shape_3d = rand_shape_3d() + densities = [0.01, 0.1, 0.2, 0.5, 0.8, 1.0] + index_types = [np.float32, np.int32, np.int64] + for density in densities: + for itype in index_types: + check_sparse_retain(shape, density, itype) + check_sparse_retain(shape_3d, density, itype) + + +def test_sparse_nd_zeros(): + def check_sparse_nd_zeros(stype, shape): + zero = mx.nd.zeros(shape) + sparse_zero = mx.nd.zeros(shape=shape, stype=stype) + assert_almost_equal(sparse_zero.asnumpy(), zero.asnumpy()) + + shape = rand_shape_2d() + check_sparse_nd_zeros('row_sparse', shape) + check_sparse_nd_zeros('csr', shape) + check_sparse_nd_zeros('default', shape) + + +def test_sparse_square_sum(): + dim0 = 30 + dim1 = 30 + axes = [0, 1] + keepdims = [False, True] + densities = [0, 0.01, 0.1, 0.2, 0.5] + for density in densities: + shape = rand_shape_2d(dim0, dim1) + rsp = rand_ndarray(shape, 'row_sparse', density) + dns = rsp.tostype('default') + for axis in axes: + for keepdim in keepdims: + ret = mx.nd._internal._square_sum(rsp, axis=axis, keepdims=keepdim) + if axis == 1 and keepdim: + assert ret.stype == 'row_sparse' + else: + assert ret.stype == 'default' + ret_expected = mx.nd.sum(dns*dns, axis=axis, keepdims=keepdim) + # check forward result + assert same(ret.asnumpy(), ret_expected.asnumpy()) + + rsp_data = mx.sym.Variable('data', stype='row_sparse') + test = mx.symbol._internal._square_sum(rsp_data, axis=axis, keepdims=keepdim) + + # check symbolic backward since ograd can be a rsp + # and cannot be checked through check_numeric_gradient + # because it will add a loss layer as the output layer + # which makes ograd of the square_sum dense + if axis == 1 and keepdims: + dns_data = mx.sym.Variable('data') + baseline = mx.sym.sum(mx.sym.square(dns_data), axis=axis, keepdims=keepdim) + igrad_expected = mx.nd.empty(dns.shape) + baseline_exec = baseline.bind(default_context(), args=[dns], + args_grad=[igrad_expected]) + baseline_exec.forward(is_train=True) + baseline_exec.backward([ret_expected]) + check_symbolic_backward(test, [rsp], [ret], [igrad_expected.asnumpy()], + grad_stypes={'data': 'row_sparse'}) + + # check numeric gradient + check_numeric_gradient(test, [rsp], grad_stype_dict={'data': 'row_sparse'}, + atol=1e-2, rtol=0.1) + +def test_sparse_storage_fallback(): + """ test operators which don't implement FComputeEx or FStatefulComputeEx """ + def check_broadcast_add(shape, lhs_stype, rhs_stype): + lhs = mx.symbol.Variable('lhs', stype=lhs_stype) + rhs = mx.symbol.Variable('rhs', stype=rhs_stype) + lhs_nd = rand_ndarray(shape, lhs_stype) + rhs_nd = rand_ndarray(shape, rhs_stype) + lhs_dns = mx.nd.cast_storage(lhs_nd, stype='default') + rhs_dns = mx.nd.cast_storage(rhs_nd, stype='default') + + out_dns = (lhs_dns + rhs_dns).asnumpy() + test = mx.symbol.broadcast_add(lhs, rhs) + location = {'lhs': lhs_nd, 'rhs': rhs_nd} + check_symbolic_forward(test, location, [out_dns]) + check_numeric_gradient(test, location) + check_symbolic_backward(test, location, [out_dns], [out_dns, out_dns]) + + def np_softmax(x, axis=-1): + # fix for old numpy on Travis not supporting keepdims + # x = x - np.max(x, axis=-1, keepdims=True) + x = x - np.max(x, axis=axis, keepdims=True) + x = np.exp(x) + # x /= np.sum(x, axis=-1, keepdims=True) + x /= np.sum(x, axis=axis, keepdims=True) + return x + + def check_softmax_with_shape(lhs_stype, rhs_stype, shape, preserve_shape=False): + # bind with label + ctx = default_context() + X = mx.symbol.Variable('X', stype=lhs_stype) + L = mx.symbol.Variable('L', stype=rhs_stype) + Y = mx.symbol.SoftmaxOutput(data=X, label=L, preserve_shape=preserve_shape) + x = rand_ndarray(shape, lhs_stype) + l = rand_ndarray(shape, rhs_stype) + l[:] = np_softmax(l.asnumpy()) + grad = mx.nd.empty(shape, ctx=ctx) + exec1 = Y.bind(ctx, args = [x, l], args_grad = {'X': grad}) + exec1.forward(is_train=True) + out = exec1.outputs[0].asnumpy() + assert_almost_equal(out, np_softmax(x.asnumpy()), rtol=1e-4) + exec1.backward() + assert_almost_equal(grad.asnumpy(), np_softmax(x.asnumpy()) - l.asnumpy(), + rtol=1e-3, atol=1e-4) + + def check_concat(shape, lhs_stype, rhs_stype): + x = mx.symbol.Variable('x', stype=lhs_stype) + w = mx.symbol.Variable('w', stype=rhs_stype) + test = mx.sym.Concat(x, w) + x_nd = rand_ndarray(shape, lhs_stype) + w_nd = rand_ndarray(shape, rhs_stype) + location = {'x': x_nd, 'w': w_nd} + check_numeric_gradient(test, location) + + shape = rand_shape_2d() + stypes = ['default', 'csr', 'row_sparse'] + for lhs in stypes: + for rhs in stypes: + check_broadcast_add(shape, lhs, rhs) + check_concat(shape, lhs, rhs) + check_softmax_with_shape(lhs, rhs, shape, preserve_shape=False) + check_softmax_with_shape(rhs, rhs, shape, preserve_shape=True) + + +def test_sparse_elementwise_sum(): + def check_sparse_elementwise_sum_with_shape(stype, shape, n): + # forward + inputs = [mx.symbol.Variable('arg%d' % i) for i in range(n)] + out = mx.symbol.sparse.add_n(*inputs, name='esum') + arr = [] + arr_grad = [mx.nd.empty(shape) for _ in range(n)] + densities = [0, 0.01, 0.1, 0.2, 0.3, 0.4, 0.5] + for i in range(n): + arr.append(rand_ndarray(shape, stype, np.random.randint(0, len(densities)))) + + exec1 = out.bind(default_context(), + args=arr, + args_grad=arr_grad) + exec1.forward(is_train=True) + out1 = exec1.outputs[0].asnumpy() + out = sum(a.asnumpy() for a in arr) + assert_almost_equal(out, out1) + + out_grad = mx.nd.empty(shape) + out_grad[:] = np.random.uniform(-10, 10, shape) + # backward + exec1.backward([out_grad]) + for a in arr_grad: + assert_almost_equal(a.asnumpy(), out_grad.asnumpy()) + + maxdim = 5 + for dim in range(2, maxdim): + shape = tuple(np.random.randint(5, 10, size=dim)) + check_sparse_elementwise_sum_with_shape('row_sparse', shape, np.random.randint(1, 9)) + + +if __name__ == '__main__': + import nose + nose.runmodule() diff --git a/tests/travis/run_test.sh b/tests/travis/run_test.sh index fb1869f842b1..fd23f0e82b24 100755 --- a/tests/travis/run_test.sh +++ b/tests/travis/run_test.sh @@ -117,21 +117,21 @@ if [ ${TASK} == "python_test" ]; then mkdir -p ${PWD}/data if [ ${TRAVIS_OS_NAME} == "osx" ]; then - python -m nose tests/python/unittest || exit -1 - python3 -m nose tests/python/unittest || exit -1 + python -m nose -v tests/python/unittest || exit -1 + python3 -m nose -v tests/python/unittest || exit -1 # make cython3 # cython tests # export MXNET_ENFORCE_CYTHON=1 # python3 -m nose tests/python/unittest || exit -1 - python3 -m nose tests/python/train || exit -1 - python -m nose tests/python/doctest || exit -1 - python3 -m nose tests/python/doctest || exit -1 + python3 -m nose -v tests/python/train || exit -1 + python -m nose -v tests/python/doctest || exit -1 + python3 -m nose -v tests/python/doctest || exit -1 else - nosetests tests/python/unittest || exit -1 - nosetests3 tests/python/unittest || exit -1 - nosetests3 tests/python/train || exit -1 - nosetests tests/python/doctest || exit -1 - nosetests3 tests/python/doctest || exit -1 + nosetests -v tests/python/unittest || exit -1 + nosetests3 -v tests/python/unittest || exit -1 + nosetests3 -v tests/python/train || exit -1 + nosetests -v tests/python/doctest || exit -1 + nosetests3 -v tests/python/doctest || exit -1 fi exit 0 fi diff --git a/tests/travis/setup.sh b/tests/travis/setup.sh index 94d674f3943e..f479306a31a8 100755 --- a/tests/travis/setup.sh +++ b/tests/travis/setup.sh @@ -33,8 +33,8 @@ if [ ${TRAVIS_OS_NAME} == "osx" ]; then brew install ImageMagick brew install swig if [ ${TASK} == "python_test" ]; then - python -m pip install --user nose numpy cython - python3 -m pip install --user nose numpy cython + python -m pip install --user nose numpy cython scipy + python3 -m pip install --user nose numpy cython scipy fi fi From 54b92401d7677040bdccb46e2c5a4a95a4c6130d Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Tue, 22 Aug 2017 17:03:07 -0700 Subject: [PATCH 403/834] add flatten option to fc (#7548) * add last_axis option to fc * update per comments * clean up --- python/mxnet/gluon/nn/basic_layers.py | 30 +++++++++----- src/operator/fully_connected-inl.h | 57 +++++++++++++++++++++------ src/operator/fully_connected.cc | 13 ++++-- tests/python/unittest/test_gluon.py | 24 +++++++++-- 4 files changed, 97 insertions(+), 27 deletions(-) diff --git a/python/mxnet/gluon/nn/basic_layers.py b/python/mxnet/gluon/nn/basic_layers.py index 7901a7ae2350..2c9ff49db1e9 100644 --- a/python/mxnet/gluon/nn/basic_layers.py +++ b/python/mxnet/gluon/nn/basic_layers.py @@ -102,7 +102,7 @@ def __len__(self): class Dense(HybridBlock): - """Just your regular densely-connected NN layer. + r"""Just your regular densely-connected NN layer. `Dense` implements the operation: `output = activation(dot(input, weight) + bias)` @@ -124,6 +124,11 @@ class Dense(HybridBlock): (ie. "linear" activation: `a(x) = x`). use_bias : bool Whether the layer uses a bias vector. + flatten: bool + Whether the input tensor should be flattened. + If true, all but the first axis of input data are collapsed together. + If false, all but the last axis of input data are kept the same, and the transformation + applies on the last axis. weight_initializer : str or `Initializer` Initializer for the `kernel` weights matrix. bias_initializer: str or `Initializer` @@ -138,16 +143,27 @@ class Dense(HybridBlock): See document of `Block`. + If ``flatten`` is set to be True, then the shapes are: Input shape: - A 2D input with shape `(batch_size, in_units)`. + An N-D input with shape + `(batch_size, x1, x2, ..., xn) with x1 * x2 * ... * xn equal to in_units`. Output shape: The output would have shape `(batch_size, units)`. + + If ``flatten`` is set to be false, then the shapes are: + Input shape: + An N-D input with shape + `(x1, x2, ..., xn, in_units)`. + + Output shape: + The output would have shape `(x1, x2, ..., xn, units)`. """ - def __init__(self, units, activation=None, use_bias=True, + def __init__(self, units, activation=None, use_bias=True, flatten=True, weight_initializer=None, bias_initializer='zeros', in_units=0, **kwargs): super(Dense, self).__init__(**kwargs) + self._flatten = flatten with self.name_scope(): self._units = units self._in_units = in_units @@ -166,12 +182,8 @@ def __init__(self, units, activation=None, use_bias=True, self.act = None def hybrid_forward(self, F, x, weight, bias=None): - if bias is None: - act = F.FullyConnected(x, weight, no_bias=True, num_hidden=self._units, - name='fwd') - else: - act = F.FullyConnected(x, weight, bias, num_hidden=self._units, - name='fwd') + act = F.FullyConnected(x, weight, bias, no_bias=bias is None, num_hidden=self._units, + flatten=self._flatten, name='fwd') if self.act is not None: act = self.act(act) return act diff --git a/src/operator/fully_connected-inl.h b/src/operator/fully_connected-inl.h index cf13655d9c97..6f0cf544d633 100644 --- a/src/operator/fully_connected-inl.h +++ b/src/operator/fully_connected-inl.h @@ -48,12 +48,15 @@ enum FullyConnectedOpOutputs {kOut}; struct FullyConnectedParam : public dmlc::Parameter { int num_hidden; bool no_bias; + bool flatten; DMLC_DECLARE_PARAMETER(FullyConnectedParam) { // TODO(bing) add support for boolean DMLC_DECLARE_FIELD(num_hidden).set_lower_bound(1) .describe("Number of hidden nodes of the output."); DMLC_DECLARE_FIELD(no_bias).set_default(false) .describe("Whether to disable bias parameter."); + DMLC_DECLARE_FIELD(flatten).set_default(true) + .describe("Whether to collapse all but the first axis of the input data tensor."); } }; @@ -91,11 +94,20 @@ class FullyConnectedOp : public Operator { const TShape& ishape = in_data[fullc::kData].shape_; const TShape& oshape = out_data[fullc::kOut].shape_; - Tensor data = in_data[fullc::kData].get_with_shape( - Shape2(ishape[0], ishape.ProdShape(1, ishape.ndim())), s); Tensor wmat = in_data[fullc::kWeight].get(s); - Tensor out = out_data[fullc::kOut].get_with_shape( - Shape2(oshape[0], oshape.ProdShape(1, oshape.ndim())), s); + Tensor data, out; + if (!param_.flatten) { + data = in_data[fullc::kData].get_with_shape( + Shape2(ishape.ProdShape(0, ishape.ndim()-1), ishape[ishape.ndim()-1]), s); + out = out_data[fullc::kOut].get_with_shape( + Shape2(oshape.ProdShape(0, oshape.ndim()-1), oshape[oshape.ndim()-1]), s); + } else { + data = in_data[fullc::kData].get_with_shape( + Shape2(ishape[0], ishape.ProdShape(1, ishape.ndim())), s); + out = out_data[fullc::kOut].get_with_shape( + Shape2(oshape[0], oshape.ProdShape(1, oshape.ndim())), s); + } + // Legacy approach shown here for comparison: // out = dot(data, wmat.T()); linalg_gemm(data, wmat, out, false, true, s); @@ -124,11 +136,23 @@ class FullyConnectedOp : public Operator { const TShape& ishape = in_data[fullc::kData].shape_; const TShape& oshape = out_grad[fullc::kOut].shape_; - Tensor data = in_data[fullc::kData].get_with_shape( - Shape2(ishape[0], ishape.ProdShape(1, ishape.ndim())), s); Tensor wmat = in_data[fullc::kWeight].get(s); - Tensor grad = out_grad[fullc::kOut].get_with_shape( - Shape2(oshape[0], oshape.ProdShape(1, oshape.ndim())), s); + Tensor data, grad, gdata; + if (!param_.flatten) { + data = in_data[fullc::kData].get_with_shape( + Shape2(ishape.ProdShape(0, ishape.ndim()-1), ishape[ishape.ndim()-1]), s); + grad = out_grad[fullc::kOut].get_with_shape( + Shape2(oshape.ProdShape(0, oshape.ndim()-1), oshape[oshape.ndim()-1]), s); + gdata = in_grad[fullc::kData].get_with_shape( + Shape2(ishape.ProdShape(0, ishape.ndim()-1), ishape[ishape.ndim()-1]), s); + } else { + data = in_data[fullc::kData].get_with_shape( + Shape2(ishape[0], ishape.ProdShape(1, ishape.ndim())), s); + grad = out_grad[fullc::kOut].get_with_shape( + Shape2(oshape[0], oshape.ProdShape(1, oshape.ndim())), s); + gdata = in_grad[fullc::kData].get_with_shape( + Shape2(ishape[0], ishape.ProdShape(1, ishape.ndim())), s); + } #if defined(__CUDACC__) CHECK_EQ(s->blas_handle_ownership_, Stream::OwnHandle) @@ -147,8 +171,6 @@ class FullyConnectedOp : public Operator { Assign(gbias, req[fullc::kBias], sum_rows(grad)); } // gradient of data - Tensor gdata = in_grad[fullc::kData].get_with_shape( - Shape2(ishape[0], ishape.ProdShape(1, ishape.ndim())), s); // Legacy approach shown here for comparison: // Assign(gdata, req[fullc::kData], dot(grad, wmat)); linalg_gemm(grad, wmat, gdata, false, false, s, req[fullc::kData]); @@ -199,13 +221,24 @@ class FullyConnectedProp : public OperatorProperty { // require data to be known if (dshape.ndim() == 0) return false; - index_t num_input = dshape.ProdShape(1, dshape.ndim()); + index_t num_input; + if (!param_.flatten) { + num_input = dshape[dshape.ndim()-1]; + } else { + num_input = dshape.ProdShape(1, dshape.ndim()); + } SHAPE_ASSIGN_CHECK(*in_shape, fullc::kWeight, Shape2(param_.num_hidden, num_input)); if (!param_.no_bias) { SHAPE_ASSIGN_CHECK(*in_shape, fullc::kBias, Shape1(param_.num_hidden)); } - SHAPE_ASSIGN_CHECK(*out_shape, 0, Shape2(dshape[0], param_.num_hidden)); + if (!param_.flatten) { + TShape result_shape(dshape); + result_shape[dshape.ndim()-1] = param_.num_hidden; + SHAPE_ASSIGN_CHECK(*out_shape, 0, result_shape); + } else { + SHAPE_ASSIGN_CHECK(*out_shape, 0, Shape2(dshape[0], param_.num_hidden)); + } if (oshape.ndim() != 0) { dshape[0] = oshape[0]; SHAPE_ASSIGN_CHECK(*in_shape, fullc::kData, dshape); diff --git a/src/operator/fully_connected.cc b/src/operator/fully_connected.cc index 5dbaf8c82005..82c32a7d2546 100644 --- a/src/operator/fully_connected.cc +++ b/src/operator/fully_connected.cc @@ -76,13 +76,20 @@ DMLC_REGISTER_PARAMETER(FullyConnectedParam); MXNET_REGISTER_OP_PROPERTY(FullyConnected, FullyConnectedProp) .describe(R"code(Applies a linear transformation: :math:`Y = XW^T + b`. -Shapes: +If ``flatten`` is set to be true, then the shapes are: -- **data**: `(batch_size, input_dim)` -- **weight**: `(num_hidden, input_dim)` +- **data**: `(batch_size, x1, x2, ..., xn)` +- **weight**: `(num_hidden, x1 * x2 * ... * xn)` - **bias**: `(num_hidden,)` - **out**: `(batch_size, num_hidden)` +If ``flatten`` is set to be false, then the shapes are: + +- **data**: `(x1, x2, ..., xn, input_dim)` +- **weight**: `(num_hidden, input_dim)` +- **bias**: `(num_hidden,)` +- **out**: `(x1, x2, ..., xn, num_hidden)` + The learnable parameters include both ``weight`` and ``bias``. If ``no_bias`` is set to be true, then the ``bias`` term is ignored. diff --git a/tests/python/unittest/test_gluon.py b/tests/python/unittest/test_gluon.py index cafa08bc04ca..726213dd5455 100644 --- a/tests/python/unittest/test_gluon.py +++ b/tests/python/unittest/test_gluon.py @@ -67,9 +67,9 @@ def forward(self, x): def test_basic(): model = nn.Sequential() - model.add(nn.Dense(128, activation='tanh', in_units=10)) + model.add(nn.Dense(128, activation='tanh', in_units=10, flatten=False)) model.add(nn.Dropout(0.5)) - model.add(nn.Dense(64, activation='tanh', in_units=128)) + model.add(nn.Dense(64, activation='tanh', in_units=256)) model.add(nn.Dense(32, in_units=64)) model.add(nn.Activation('relu')) @@ -80,7 +80,7 @@ def test_basic(): # ndarray model.collect_params().initialize(mx.init.Xavier(magnitude=2.24)) - x = model(mx.nd.zeros((32, 10))) + x = model(mx.nd.zeros((32, 2, 10))) assert x.shape == (32, 32) x.wait_to_read() @@ -90,6 +90,24 @@ def test_basic(): assert list(model.collect_params().values())[0]._grad is not None +def test_dense(): + model = nn.Dense(128, activation='tanh', in_units=10, flatten=False, prefix='test_') + inputs = mx.sym.Variable('data') + outputs = model(inputs) + assert set(model.collect_params().keys()) == set(['test_weight', 'test_bias']) + assert outputs.list_outputs() == ['test_tanh_fwd_output'] + args, outs, auxs = outputs.infer_shape(data=(2, 3, 10)) + assert outs == [(2, 3, 128)] + + model = nn.Dense(128, activation='relu', in_units=30, flatten=True, prefix='test2_') + inputs = mx.sym.Variable('data') + outputs = model(inputs) + assert set(model.collect_params().keys()) == set(['test2_weight', 'test2_bias']) + assert outputs.list_outputs() == ['test2_relu_fwd_output'] + args, outs, auxs = outputs.infer_shape(data=(17, 2, 5, 3)) + assert outs == [(17, 128)] + + def test_symbol_block(): model = nn.HybridSequential() model.add(nn.Dense(128, activation='tanh')) From 68cd9c924a20ce94e31695cfd431b8ddf99d560b Mon Sep 17 00:00:00 2001 From: mbaijal <30911248+mbaijal@users.noreply.github.com> Date: Tue, 22 Aug 2017 17:04:17 -0700 Subject: [PATCH 404/834] Updating the LICENSE and NOTICE Files (#7563) --- LICENSE | 157 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ NOTICE | 40 +++++++++++++-- 2 files changed, 193 insertions(+), 4 deletions(-) diff --git a/LICENSE b/LICENSE index d64569567334..01dfcf46792d 100644 --- a/LICENSE +++ b/LICENSE @@ -200,3 +200,160 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + + ======================================================================= + Apache MXNET (incubating) Subcomponents: + + The Apache MXNET (incubating) project contains subcomponents with separate copyright + notices and license terms. Your use of the source code for the these + subcomponents is subject to the terms and conditions of the following + licenses. + + ======================================================================== + Apache-2.0 licenses + ======================================================================== + + The following components are provided under an Apache 2.0 license. + + 1. MXNet Cpp-package - For details, /cpp-package/LICENSE + 2. MXNet rcnn - For details, see, example/rcnn/LICENSE + 3. scala-package - For details, see, scala-package/LICENSE + 4. Warp-CTC - For details, see, src/operator/contrib/ctc_include/LICENSE + 5. dlpack - For details, see, dlpack/LICENSE + 6. dmlc-core - For details, see, dmlc-core/LICENSE + 7. mshadow - For details, see, mshadow/LICENSE + 8. nnvm/dmlc-core - For details, see, nnvm/dmlc-core/LICENSE + 9. nnvm - For details, see, nnvm/LICENSE + 10. nnvm-fusion - For details, see, nnvm/plugin/nnvm-fusion/LICENSE + 11. ps-lite - For details, see, ps-lite/LICENSE + + ======================================================================== + MIT licenses + ======================================================================== + + 1. Fast R-CNN - For details, see example/rcnn/LICENSE + 2. Faster R-CNN - For details, see example/rcnn/LICENSE + 3. tree_lstm - For details, see example/gluon/tree_lstm/LICENSE + + + ======================================================================== + NVIDIA Licenses + ======================================================================== + + 1. Warp-CTC + For details, see, src/operator/contrib/ctc_include/contrib/moderngpu/LICENSE + + /****************************************************************************** + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + + 2. CUB Library + For details, see, cub/LICENSE.TXT + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the NVIDIA CORPORATION nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + + ======================================================================== + Other Licenses + ======================================================================== + + 1. Caffe + For details, see, example/rcnn/LICENSE + + LICENSE + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + CONTRIBUTION AGREEMENT + + By contributing to the BVLC/caffe repository through pull-request, comment, + or otherwise, the contributor releases their content to the + license and copyright terms herein. + + + 2. MS COCO API + For details, see, example/rcnn/LICENSE + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + The views and conclusions contained in the software and documentation are those + of the authors and should not be interpreted as representing official policies, + either expressed or implied, of the FreeBSD Project. + diff --git a/NOTICE b/NOTICE index 03695607e3e9..2341ea27f67a 100644 --- a/NOTICE +++ b/NOTICE @@ -1,5 +1,37 @@ -Apache MXNET (incubating) -Copyright [2015-2017] The Apache Software Foundation + Apache MXNET (incubating) + Copyright 2015-2017 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (http://www.apache.org/). + + Warp-CTC + Copyright (c) 2013, NVIDIA CORPORATION. + + CUB Library + Copyright (c) 2010-2011, Duane Merrill. All rights reserved. + Copyright (c) 2011-2016, NVIDIA CORPORATION. All rights reserved. + + Caffe + COPYRIGHT + All contributions by the University of California: + Copyright (c) 2014, 2015, The Regents of the University of California (Regents) + All rights reserved. + All other contributions: + Copyright (c) 2014, 2015, the respective contributors + All rights reserved. + Caffe uses a shared copyright model: each contributor holds copyright over + their contributions to Caffe. The project versioning records all such + contribution and copyright details. If a contributor wants to further mark + their specific copyright on a particular contribution, they should indicate + their copyright solely in the commit message of the change when it is + committed. + + + MS COCO API + Copyright (c) 2014, Piotr Dollar and Tsung-Yi Lin + + + + + -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). From 491f81e648639c53a68155585c53c3993a33ead5 Mon Sep 17 00:00:00 2001 From: "Joshua Z. Zhang" Date: Tue, 22 Aug 2017 19:57:32 -0700 Subject: [PATCH 405/834] add resnet50_v2 pretrained (#7564) --- python/mxnet/gluon/model_zoo/model_store.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/mxnet/gluon/model_zoo/model_store.py b/python/mxnet/gluon/model_zoo/model_store.py index e524f215416d..6bc4b2805afd 100644 --- a/python/mxnet/gluon/model_zoo/model_store.py +++ b/python/mxnet/gluon/model_zoo/model_store.py @@ -38,6 +38,7 @@ ('2a903ab21260c85673a78fe65037819a843a1f43', 'resnet50_v1'), ('8aacf80ff4014c1efa2362a963ac5ec82cf92d5b', 'resnet18_v2'), ('0ed3cd06da41932c03dea1de7bc2506ef3fb97b3', 'resnet34_v2'), + ('eb7a368774aa34a12ed155126b641ae7556dad9d', 'resnet50_v2'), ('264ba4970a0cc87a4f15c96e25246a1307caf523', 'squeezenet1.0'), ('33ba0f93753c83d86e1eb397f38a667eaf2e9376', 'squeezenet1.1'), ('dd221b160977f36a53f464cb54648d227c707a05', 'vgg11'), From 393293115701b27694b8f9105f8c6360ebbbc557 Mon Sep 17 00:00:00 2001 From: Stefan Henneking Date: Wed, 23 Aug 2017 12:10:50 -0700 Subject: [PATCH 406/834] fixed minor typo (#7581) --- src/operator/tensor/dot-inl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/operator/tensor/dot-inl.h b/src/operator/tensor/dot-inl.h index aaf242e26fe1..ecaf5f9a0e95 100644 --- a/src/operator/tensor/dot-inl.h +++ b/src/operator/tensor/dot-inl.h @@ -71,7 +71,7 @@ void DotForward_(const nnvm::NodeAttrs& attrs, << "dot only supports float32 and float64"; MSHADOW_SGL_DBL_TYPE_SWITCH(outputs[0].type_flag_, DType, { if (inputs[0].ndim() == 1 && inputs[1].ndim() == 1) { - CHECK_NE(req[0], kAddTo) << "AddTo not yet suported"; + CHECK_NE(req[0], kAddTo) << "AddTo not yet supported"; Tensor out = outputs[0].get(s); VectorDot(out, inputs[0].get(s), From 6d9b6a3fd8a251dad2ce846a0ce4ade037ce7bbb Mon Sep 17 00:00:00 2001 From: qingzhouzhen <576591769@qq.com> Date: Thu, 24 Aug 2017 04:33:57 +0800 Subject: [PATCH 407/834] modify parameters counting of FC and CONV (#7568) --- python/mxnet/visualization.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/python/mxnet/visualization.py b/python/mxnet/visualization.py index 4dbf680c2e3a..aa00488d96a7 100644 --- a/python/mxnet/visualization.py +++ b/python/mxnet/visualization.py @@ -134,12 +134,20 @@ def print_layer_summary(node, out_shape): pre_filter = pre_filter + int(shape[0]) cur_param = 0 if op == 'Convolution': - cur_param = pre_filter * int(node["attr"]["num_filter"]) - for k in _str2tuple(node["attr"]["kernel"]): - cur_param *= int(k) - cur_param += int(node["attr"]["num_filter"]) + if ("no_bias" in node["attr"]) and (node["attr"]["no_bias"] == 'True'): + cur_param = pre_filter * int(node["attr"]["num_filter"]) + for k in _str2tuple(node["attr"]["kernel"]): + cur_param *= int(k) + else: + cur_param = pre_filter * int(node["attr"]["num_filter"]) + for k in _str2tuple(node["attr"]["kernel"]): + cur_param *= int(k) + cur_param += int(node["attr"]["num_filter"]) elif op == 'FullyConnected': - cur_param = pre_filter * (int(node["attr"]["num_hidden"]) + 1) + if ("no_bias" in node["attr"]) and (node["attr"]["no_bias"] == 'True'): + cur_param = pre_filter * (int(node["attr"]["num_hidden"])) + else: + cur_param = (pre_filter+1) * (int(node["attr"]["num_hidden"])) elif op == 'BatchNorm': key = node["name"] + "_output" if show_shape: From f68cb40df7fb0fe5fade47b65eefac07dd35a9b5 Mon Sep 17 00:00:00 2001 From: Dick Carter Date: Wed, 23 Aug 2017 14:15:55 -0700 Subject: [PATCH 408/834] FP16-I/O conv/deconv to use pseudo-fp16, ignoring MSHADOW_USE_PASCAL. (#7527) * FP16-I/O conv and deconv will use pseudo-fp16, ignoring MSHADOW_USE_PASCAL. * Fixing cpplint error. * Empty commit to trigger CI. --- src/operator/convolution.cu | 55 ++++--------------------------- src/operator/deconvolution.cu | 61 +++++------------------------------ 2 files changed, 14 insertions(+), 102 deletions(-) diff --git a/src/operator/convolution.cu b/src/operator/convolution.cu index f5777c1714a4..b327f3cff424 100644 --- a/src/operator/convolution.cu +++ b/src/operator/convolution.cu @@ -60,61 +60,18 @@ Operator* CreateOp(ConvolutionParam param, int dtype, } #if MXNET_USE_CUDNN == 1 - // The NVIDIA Pascal architecture was the first to include 16-bit ALUs. - // Thus, when the framework is compiled with MSHADOW_USE_PASCAL == 1, we - // perform the convolution calculation in 16-bit when the tensor type is - // also 16-bit. For NVIDIA architectures earlier than Pascal (so Maxwell - // and Kepler), the computation precision is always at least 32-bits. -#if MSHADOW_USE_PASCAL == 1 - // true fp16 - int desired_forward_compute_type = dtype; - int desired_backward_compute_type = dtype; -#else - // pseudo fp16 - int desired_forward_compute_type = - (dtype == mshadow::kFloat16) ? mshadow::kFloat32 : dtype; - int desired_backward_compute_type = - (dtype == mshadow::kFloat16) ? mshadow::kFloat32 : dtype; -#endif // MSHADOW_USE_PASCAL == 1 + // On fp16-I/O instances, use fp32 compute (i.e. pseudo-fp16). + int compute_type = (dtype == mshadow::kFloat16) ? mshadow::kFloat32 : dtype; MSHADOW_REAL_TYPE_SWITCH(dtype, DType, { if (param.cudnn_off) { op = new ConvolutionOp(param); + } else if (!CuDNNConvolutionOp::Supports(param, compute_type, compute_type, ctx)) { + LOG(WARNING) << "This convolution is not supported by cudnn, MXNET convolution is applied."; + op = new ConvolutionOp(param); } else { - int forward_compute_type = desired_forward_compute_type; - int backward_compute_type = desired_backward_compute_type; - bool convolutionIsSupported = CuDNNConvolutionOp::Supports(param, - forward_compute_type, - backward_compute_type, ctx); - - // If cuDNN can't handle this case with fp16 backprop kernels, try fp32 backprop. - if (!convolutionIsSupported && backward_compute_type == mshadow::kFloat16) { - backward_compute_type = mshadow::kFloat32; - convolutionIsSupported = CuDNNConvolutionOp::Supports(param, - forward_compute_type, - backward_compute_type, ctx); - } - - // If cuDNN can't handle this case with fp16 forward kernels, try fp32 - if (!convolutionIsSupported && forward_compute_type == mshadow::kFloat16) { - forward_compute_type = mshadow::kFloat32; - convolutionIsSupported = CuDNNConvolutionOp::Supports(param, - forward_compute_type, - backward_compute_type, ctx); - } - if (!convolutionIsSupported) { - LOG(WARNING) << "This convolution is not supported by cudnn, MXNET convolution is applied."; - op = new ConvolutionOp(param); - } else { - if (forward_compute_type != desired_forward_compute_type) - LOG(WARNING) << "Requested forward compute precision not supported, using fp32."; - if (backward_compute_type != desired_backward_compute_type) - LOG(WARNING) << "Requested backward compute precision not supported, using fp32."; - op = new CuDNNConvolutionOp(param, - forward_compute_type, - backward_compute_type, + op = new CuDNNConvolutionOp(param, compute_type, compute_type, *in_shape, *out_shape, ctx); - } } }) #else diff --git a/src/operator/deconvolution.cu b/src/operator/deconvolution.cu index e9b5cb8e3c7f..de7dff5569ed 100644 --- a/src/operator/deconvolution.cu +++ b/src/operator/deconvolution.cu @@ -45,64 +45,19 @@ Operator* CreateOp(DeconvolutionParam param, int dtype, return op; } #if MXNET_USE_CUDNN == 1 - // The NVIDIA Pascal architecture was the first to include 16-bit ALUs. - // Thus, when the framework is compiled with MSHADOW_USE_PASCAL == 1, we - // perform the deconvolution calculation in 16-bit when the tensor type is - // also 16-bit. For NVIDIA architectures earlier than Pascal (so Maxwell - // and Kepler), the computation precision is always at least 32-bits. -#if MSHADOW_USE_PASCAL == 1 - // true fp16 - int desired_forward_compute_type = dtype; - int desired_backward_compute_type = dtype; -#else - // pseudo fp16 - int desired_forward_compute_type = - (dtype == mshadow::kFloat16) ? mshadow::kFloat32 : dtype; - int desired_backward_compute_type = - (dtype == mshadow::kFloat16) ? mshadow::kFloat32 : dtype; -#endif // MSHADOW_USE_PASCAL == 1 + // On fp16-I/O instances, use fp32 compute (i.e. pseudo-fp16). + int compute_type = (dtype == mshadow::kFloat16) ? mshadow::kFloat32 : dtype; MSHADOW_REAL_TYPE_SWITCH(dtype, DType, { if (param.cudnn_off) { op = new DeconvolutionOp(param); + } else if (!CuDNNDeconvolutionOp::Supports(param, compute_type, compute_type, ctx)) { + LOG(WARNING) << + "This deconvolution is not supported by cudnn, MXNET deconvolution is applied."; + op = new DeconvolutionOp(param); } else { - int forward_compute_type = desired_forward_compute_type; - int backward_compute_type = desired_backward_compute_type; - bool deconvolutionIsSupported = CuDNNDeconvolutionOp::Supports(param, - forward_compute_type, - backward_compute_type, ctx); - - // If cuDNN can't handle this case with fp16 backprop kernels, try fp32 backprop. - if (!deconvolutionIsSupported && backward_compute_type == mshadow::kFloat16) { - backward_compute_type = mshadow::kFloat32; - deconvolutionIsSupported = CuDNNDeconvolutionOp::Supports(param, - forward_compute_type, - backward_compute_type, ctx); - } - - // If cuDNN can't handle this case with fp16 forward kernels, try fp32 - if (!deconvolutionIsSupported && forward_compute_type == mshadow::kFloat16) { - forward_compute_type = mshadow::kFloat32; - deconvolutionIsSupported = CuDNNDeconvolutionOp::Supports(param, - forward_compute_type, - backward_compute_type, ctx); - } - if (!deconvolutionIsSupported) { - LOG(WARNING) << - "This deconvolution is not supported by cudnn, MXNET deconvolution is applied."; - op = new DeconvolutionOp(param); - } else { - if ((forward_compute_type != desired_forward_compute_type) || - (backward_compute_type != desired_backward_compute_type)) { - LOG(WARNING) << - "True fp16 deconvolution by cudnn not supported in this configuration. " << - "Falling back to pseudo fp16."; - } - op = new CuDNNDeconvolutionOp(param, - forward_compute_type, - backward_compute_type, - *in_shape, *out_shape, ctx); - } + op = new CuDNNDeconvolutionOp(param, compute_type, compute_type, + *in_shape, *out_shape, ctx); } }) #else From 050d85e338beed7880577b76aaf73c2948fc8a35 Mon Sep 17 00:00:00 2001 From: Dick Carter Date: Wed, 23 Aug 2017 14:41:17 -0700 Subject: [PATCH 409/834] Set dev_id in streams, also update mshadow. (#7526) * Set dev_id in streams, also update mshadow. * Fix cpplint error. * Empty commit to trigger CI. * Further update of mshadow to match current hash. --- src/common/cuda_utils.h | 19 ++++++++++++------- src/engine/naive_engine.cc | 2 +- src/engine/stream_manager.h | 4 ++-- src/engine/threaded_engine_perdevice.cc | 4 ++-- tests/cpp/include/test_op.h | 3 ++- 5 files changed, 19 insertions(+), 13 deletions(-) diff --git a/src/common/cuda_utils.h b/src/common/cuda_utils.h index 0213c73177b3..0f63895d3284 100644 --- a/src/common/cuda_utils.h +++ b/src/common/cuda_utils.h @@ -274,26 +274,31 @@ inline int SMArch(int device_id) { /*! * \brief Determine whether a cuda-capable gpu's architecture supports float16 math. + * Assume not if device_id is negative. * \param device_id The device index of the cuda-capable gpu of interest. * \return whether the gpu's architecture supports float16 math. */ inline bool SupportsFloat16Compute(int device_id) { - // Kepler and most Maxwell GPUs do not support fp16 compute - int computeCapabilityMajor = ComputeCapabilityMajor(device_id); - int computeCapabilityMinor = ComputeCapabilityMinor(device_id); - return (computeCapabilityMajor > 5) || - (computeCapabilityMajor == 5 && computeCapabilityMinor >= 3); + if (device_id < 0) { + return false; + } else { + // Kepler and most Maxwell GPUs do not support fp16 compute + int computeCapabilityMajor = ComputeCapabilityMajor(device_id); + return (computeCapabilityMajor > 5) || + (computeCapabilityMajor == 5 && ComputeCapabilityMinor(device_id) >= 3); + } } /*! * \brief Determine whether a cuda-capable gpu's architecture supports Tensor Core math. + * Assume not if device_id is negative. * \param device_id The device index of the cuda-capable gpu of interest. * \return whether the gpu's architecture supports Tensor Core math. */ inline bool SupportsTensorCore(int device_id) { // Volta (sm_70) supports TensorCore algos - int computeCapabilityMajor = ComputeCapabilityMajor(device_id); - return (computeCapabilityMajor >= 7); + return device_id >= 0 && + ComputeCapabilityMajor(device_id) >=7; } // The policy if the user hasn't set the environment variable MXNET_CUDA_ALLOW_TENSOR_CORE diff --git a/src/engine/naive_engine.cc b/src/engine/naive_engine.cc index 85ec3ae672e2..b354418288aa 100644 --- a/src/engine/naive_engine.cc +++ b/src/engine/naive_engine.cc @@ -154,7 +154,7 @@ class NaiveEngine final : public Engine { streams_.resize(dev_id + 1, nullptr); } if (streams_[dev_id] == nullptr) { - streams_[dev_id] = mshadow::NewStream(true, MXNET_USE_CUDNN != 0); + streams_[dev_id] = mshadow::NewStream(true, MXNET_USE_CUDNN != 0, dev_id); } exec_fun(RunContext{exec_ctx, streams_[dev_id]}, callback); #else diff --git a/src/engine/stream_manager.h b/src/engine/stream_manager.h index 1a66277bb4ec..cd6db53f14c6 100644 --- a/src/engine/stream_manager.h +++ b/src/engine/stream_manager.h @@ -77,7 +77,7 @@ RunContext StreamManager::GetRunContext( auto&& counter = gpu_cnt_.at(ctx.dev_id); if (counter == -1) { for (auto&& i : gpu_streams_.at(ctx.dev_id)) { - i = mshadow::NewStream(true, MXNET_USE_CUDNN != 0); + i = mshadow::NewStream(true, MXNET_USE_CUDNN != 0, ctx.dev_id); } counter = 0; } @@ -108,7 +108,7 @@ RunContext StreamManager::GetIORunContext( { std::lock_guard lock{m_}; if (gpu_io_streams_.at(ctx.dev_id) == nullptr) { - gpu_io_streams_.at(ctx.dev_id) = mshadow::NewStream(false, false); + gpu_io_streams_.at(ctx.dev_id) = mshadow::NewStream(false, false, ctx.dev_id); } } ret = RunContext{ctx, gpu_io_streams_.at(ctx.dev_id)}; diff --git a/src/engine/threaded_engine_perdevice.cc b/src/engine/threaded_engine_perdevice.cc index 66cfc9de1468..5cd8ca049dd3 100644 --- a/src/engine/threaded_engine_perdevice.cc +++ b/src/engine/threaded_engine_perdevice.cc @@ -183,9 +183,9 @@ class ThreadedEnginePerDevice : public ThreadedEngine { // allocate stream mshadow::SetDevice(ctx.dev_id); if (is_copy_worker) { - stream = mshadow::NewStream(false, false); + stream = mshadow::NewStream(false, false, ctx.dev_id); } else { - stream = mshadow::NewStream(true, MXNET_USE_CUDNN != 0); + stream = mshadow::NewStream(true, MXNET_USE_CUDNN != 0, ctx.dev_id); } } while (false); // execute task diff --git a/tests/cpp/include/test_op.h b/tests/cpp/include/test_op.h index d8f90df8447e..951affa208f8 100644 --- a/tests/cpp/include/test_op.h +++ b/tests/cpp/include/test_op.h @@ -75,7 +75,8 @@ class BasicOperatorData { : opContext_(*opContext) { CHECK_EQ(opContext_.run_ctx.stream == nullptr, true) << "Invalid runtime context stream state"; - opContext_.run_ctx.stream = mshadow::NewStream(true, true); + auto device_id = opContext->run_ctx.get_ctx().dev_id; + opContext_.run_ctx.stream = mshadow::NewStream(true, true, device_id); CHECK_EQ(opContext_.run_ctx.stream != nullptr, true) << "Unable to allocate a GPU stream"; } From d839abc09ce656205028e9f3df06067d87a2fc6c Mon Sep 17 00:00:00 2001 From: moin Date: Thu, 24 Aug 2017 19:06:01 +0200 Subject: [PATCH 410/834] fix for amalgamation build with MIN=1 (#7597) --- amalgamation/amalgamation.py | 4 ++++ src/operator/fully_connected-inl.h | 23 +++++++++++++++++------ 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/amalgamation/amalgamation.py b/amalgamation/amalgamation.py index 22b421d79fba..2aba8f4bdc77 100644 --- a/amalgamation/amalgamation.py +++ b/amalgamation/amalgamation.py @@ -32,6 +32,10 @@ minimum = int(sys.argv[6]) if len(sys.argv) > 5 else 0 android = int(sys.argv[7]) if len(sys.argv) > 6 else 0 +# blacklist linear algebra headers when building without blas. +if minimum != 0: + blacklist.append('linalg.h') + def pprint(lst): for item in lst: print item diff --git a/src/operator/fully_connected-inl.h b/src/operator/fully_connected-inl.h index 6f0cf544d633..7120b5672f60 100644 --- a/src/operator/fully_connected-inl.h +++ b/src/operator/fully_connected-inl.h @@ -33,7 +33,9 @@ #include #include "./operator_common.h" #include "./elemwise_op_common.h" +#if (MSHADOW_USE_CBLAS != 0) #include "linalg.h" +#endif namespace mxnet { namespace op { @@ -108,9 +110,12 @@ class FullyConnectedOp : public Operator { Shape2(oshape[0], oshape.ProdShape(1, oshape.ndim())), s); } - // Legacy approach shown here for comparison: - // out = dot(data, wmat.T()); +#if (MSHADOW_USE_CBLAS == 0) + // Legacy approach for amalgamation build w/out cblas + out = dot(data, wmat.T()); +#else linalg_gemm(data, wmat, out, false, true, s); +#endif if (!param_.no_bias) { Tensor bias = in_data[fullc::kBias].get(s); out += repmat(bias, data.size(0)); @@ -162,18 +167,24 @@ class FullyConnectedOp : public Operator { CHECK_NE(req[fullc::kWeight], kWriteInplace) << "cannot write weight inplace"; // gradient of weight Tensor gwmat = in_grad[fullc::kWeight].get(s); - // Legacy approach shown here for comparison: - // Assign(gwmat, req[fullc::kWeight], dot(grad.T(), data)); +#if (MSHADOW_USE_CBLAS == 0) + // Legacy approach for amalgamation build w/out cblas + Assign(gwmat, req[fullc::kWeight], dot(grad.T(), data)); +#else linalg_gemm(grad, data, gwmat, true, false, s, req[fullc::kWeight]); +#endif // gradient of bias if (!param_.no_bias) { Tensor gbias = in_grad[fullc::kBias].get(s); Assign(gbias, req[fullc::kBias], sum_rows(grad)); } // gradient of data - // Legacy approach shown here for comparison: - // Assign(gdata, req[fullc::kData], dot(grad, wmat)); +#if (MSHADOW_USE_CBLAS == 0) + // Legacy approach for amalgamation build w/out cblas + Assign(gdata, req[fullc::kData], dot(grad, wmat)); +#else linalg_gemm(grad, wmat, gdata, false, false, s, req[fullc::kData]); +#endif } private: From 9296907c8343ace2f7ed5cfef757849e63176382 Mon Sep 17 00:00:00 2001 From: reminisce Date: Thu, 24 Aug 2017 10:29:48 -0700 Subject: [PATCH 411/834] Fix import error of broadcast max, min, mod in ndarray.py and add unit tests (#7572) --- python/mxnet/ndarray/ndarray.py | 4 +- tests/python/unittest/test_operator.py | 132 +++++++++++++++++++++---- 2 files changed, 117 insertions(+), 19 deletions(-) diff --git a/python/mxnet/ndarray/ndarray.py b/python/mxnet/ndarray/ndarray.py index 20ca2262f0cd..7322325722d6 100644 --- a/python/mxnet/ndarray/ndarray.py +++ b/python/mxnet/ndarray/ndarray.py @@ -41,11 +41,11 @@ from . import broadcast_add, broadcast_mul, transpose, broadcast_not_equal, broadcast_power from . import broadcast_sub, broadcast_div, broadcast_to, broadcast_equal, cast_storage from . import broadcast_greater, broadcast_greater_equal, broadcast_lesser, broadcast_lesser_equal -from . import zeros_like, slice +from . import zeros_like, slice, broadcast_minimum, broadcast_maximum, broadcast_mod __all__ = ["NDArray", "concatenate", "_DTYPE_NP_TO_MX", "_DTYPE_MX_TO_NP", "_GRAD_REQ_MAP", "ones", "add", "arange", "divide", "equal", "full", "greater", "greater_equal", - "imdecode", "lesser", "lesser_equal", "maximum", "minimum", "moveaxis", + "imdecode", "lesser", "lesser_equal", "maximum", "minimum", "moveaxis", "modulo", "multiply", "negative", "not_equal", "onehot_encode", "power", "subtract", "true_divide", "waitall", "_new_empty_handle"] diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py index 11d0ea22319a..ceb11ed07c02 100644 --- a/tests/python/unittest/test_operator.py +++ b/tests/python/unittest/test_operator.py @@ -24,6 +24,7 @@ from numpy.testing import assert_allclose, assert_array_equal from mxnet.test_utils import * + def np_softmax(x, axis=-1): # fix for old numpy on Travis not supporting keepdims # x = x - np.max(x, axis=-1, keepdims=True) @@ -58,6 +59,7 @@ def check_elementwise_sum_with_shape(shape, n): for a in arr_grad: assert_almost_equal(a.asnumpy(), out_grad.asnumpy()) + def test_elementwise_sum(): np.random.seed(0) nrepeat = 2 @@ -112,6 +114,7 @@ def check_concat_with_shape(shapes, dimension, skip_second): np_grad = arr_np[i] assert_almost_equal(grad.asnumpy(), np_grad + 1) + def test_concat(): for dimension in range(4): n = 2 @@ -158,6 +161,7 @@ def test_concat(): check_concat_with_shape(shapes,dimension,True) check_concat_with_shape(shapes,dimension,False) + def test_slice_channel(): def check_slice_channel(data_ndim, axis, num_outputs, squeeze_axis): ins = [] @@ -221,6 +225,7 @@ def check_regression(symbol, forward, backward): npout = backward(npout, arr_label.asnumpy().reshape(npout.shape)) assert_almost_equal(npout, arr_grad.asnumpy()) + def test_regression(): check_regression(mx.symbol.LogisticRegressionOutput, lambda x: 1.0 / (1.0 + np.exp(-x)), @@ -229,6 +234,7 @@ def test_regression(): lambda x: x, lambda x, y : x - y) + def check_softmax_with_ignore_label(xpu): X = mx.symbol.Variable('X') L = mx.symbol.Variable('L') @@ -261,6 +267,7 @@ def check_softmax_with_ignore_label(xpu): assert abs(np.sum(grad1[:int(shape[0]/2)])) < 1e-5 assert_almost_equal(grad0[int(shape[0]/2):], grad1[int(shape[0]/2):]) + def check_softmax_with_shape(shape, xpu, preserve_shape=False): # bind with label X = mx.symbol.Variable('X') @@ -277,11 +284,13 @@ def check_softmax_with_shape(shape, xpu, preserve_shape=False): exec1.backward() assert_almost_equal(grad.asnumpy(), np_softmax(x.asnumpy()) - l.asnumpy(), rtol=1e-4) + def test_softmax(): check_softmax_with_shape((3, 4), default_context(), preserve_shape=False) check_softmax_with_shape((3, 4), default_context(), preserve_shape=True) check_softmax_with_shape((3, 4, 2), default_context(), preserve_shape=True) + def test_python_op(): X = mx.symbol.Variable('X') op = mx.operator.NumpyOp() @@ -296,6 +305,7 @@ def test_python_op(): exec1.backward(dy) assert_almost_equal(dy.asnumpy(), dx.asnumpy()) + def test_swapaxes(): data = mx.symbol.Variable('data') shape = (2, 3, 4) @@ -314,6 +324,7 @@ def test_swapaxes(): assert_almost_equal(out, swap_) + def test_scalarop(): data = mx.symbol.Variable('data') shape = (3, 4) @@ -344,6 +355,7 @@ def test_scalar_pow(): check_symbolic_forward(test, [data_tmp], [data_tmp ** 2]) check_symbolic_backward(test, [data_tmp], [np.ones(shape)], [2 * data_tmp]) + def test_symbol_pow(): shape = (1, 1) @@ -362,6 +374,7 @@ def test_symbol_pow(): exp_dir = data_tmp**(exp_tmp) * np.log(data_tmp) check_symbolic_backward(test, [data_tmp, exp_tmp], [np.ones(shape)], [data_dir, exp_dir]) + def test_pow_fn(): shape = (3, 4) exp = mx.symbol.Variable("exp") @@ -371,6 +384,7 @@ def test_pow_fn(): check_symbolic_forward(y, [x], [2**x]) check_symbolic_backward(y, [x], [np.ones(shape)], [np.log(2) * 2**x]) + def test_relu(): def frelu(x): return np.maximum(x, 0.0) @@ -386,6 +400,7 @@ def frelu_grad(x): check_symbolic_forward(y, [xa], [ya]) check_symbolic_backward(y, [xa], [np.ones(shape)], [ga]) + def test_sigmoid(): def fsigmoid(a): return np.divide(1.0, (1.0 + np.exp(-a))) @@ -398,6 +413,7 @@ def fsigmoid(a): check_symbolic_forward(y, [xa], [ya]) check_symbolic_backward(y, [xa], [np.ones(shape)], [ya * (1 - ya)]) + def test_binary_logic(): def _inner_test(forward_gt, logic_sym, x_shape, y_shape, test_scalar=True): x = mx.symbol.Variable("x") @@ -453,6 +469,7 @@ def _inner_test(forward_gt, logic_sym, x_shape, y_shape, test_scalar=True): logic_sym=lambda x, y: mx.sym.broadcast_not_equal(x, y), x_shape=(1, 10), y_shape=(10, 1), test_scalar=False) + def test_embedding(): in_dim = 10 out_dim = 4 @@ -479,6 +496,7 @@ def test_embedding(): exe_test.backward([grad]) assert_almost_equal(grad_map["embed_weight"].asnumpy(), np.dot(np_onehot.T, np_grad)) + # check ops handle duplicate input correctly. def test_binary_op_duplicate_input(): data = mx.symbol.Variable('data') @@ -497,6 +515,7 @@ def test_binary_op_duplicate_input(): exe_square.backward(out_grad) assert_almost_equal(arr_grad.asnumpy(), 2.0 * data_tmp) + def test_sign(): data = mx.symbol.Variable('data') shape = (3, 4) @@ -520,6 +539,7 @@ def test_sign(): exe_test.backward(out_grad) assert_almost_equal(arr_grad.asnumpy(), npout_grad) + def test_round_ceil_floor(): data = mx.symbol.Variable('data') shape = (3, 4) @@ -536,6 +556,7 @@ def test_round_ceil_floor(): npout = np.round(data_tmp) + np.ceil(data_tmp) + np.floor(data_tmp) assert_almost_equal(out, npout) + def test_trunc(): data_tmp = np.random.rand(3, 4) * 10 - 5 arr_data = mx.nd.array(data_tmp) @@ -549,6 +570,7 @@ def test_trunc(): assert_almost_equal(out, npout) + def test_rsqrt_cos_sin(): data = mx.symbol.Variable('data') shape = (3, 4) @@ -572,6 +594,7 @@ def test_rsqrt_cos_sin(): exe_test.backward(out_grad) assert_almost_equal(arr_grad.asnumpy(), npout_grad) + def test_maximum_minimum(): data1 = mx.symbol.Variable('data') data2 = mx.symbol.Variable('data') @@ -584,11 +607,9 @@ def test_maximum_minimum(): arr_data1 = mx.nd.array(data_tmp1) arr_data2 = mx.nd.array(data_tmp2) - arr_grad1 = mx.nd.empty(shape) arr_grad2 = mx.nd.empty(shape) - test = mx.sym.maximum(data1,data2) + mx.sym.minimum(data1,data2); exe_test = test.bind(default_context(), args=[arr_data1,arr_data2], args_grad=[arr_grad1,arr_grad2]) exe_test.forward(is_train=True) @@ -610,6 +631,7 @@ def test_maximum_minimum(): assert_almost_equal(arr_grad1.asnumpy(), npout_grad1) assert_almost_equal(arr_grad2.asnumpy(), npout_grad2) + def test_maximum_minimum_scalar(): data1 = mx.symbol.Variable('data') shape = (3, 4) @@ -640,6 +662,7 @@ def test_maximum_minimum_scalar(): assert_almost_equal(arr_grad1.asnumpy(), npout_grad1) + def test_abs(): data = mx.symbol.Variable('data') shape = (3, 4) @@ -663,6 +686,7 @@ def test_abs(): exe_test.backward(out_grad) assert_almost_equal(arr_grad.asnumpy(), npout_grad) + def check_deconvolution_forward_backward(input_shape, num_filter, kernel, stride, pad): """configure A: input --> conv --> deconv --> output. the convolution and deconvoluiton has similar parameter which ensure @@ -761,6 +785,7 @@ def check_deconvolution_gradient(input_shape, num_filter, pad): assert_almost_equal(conv_args_grad[1].asnumpy() + deconv_addto_args_grad_npy[1], deconv_addto_args_grad[1].asnumpy(), rtol=1e-3, atol=1e-2) + def check_deconvolution_target_shape(input_shape, kernel, stride, pad, adj, target_shape=None): data = mx.sym.Variable(name="data") if target_shape: @@ -774,6 +799,7 @@ def check_deconvolution_target_shape(input_shape, kernel, stride, pad, adj, targ arg_shapes, out_shapes, _ = deconv.infer_shape(data=input_shape) assert out_shapes[0] == (input_shape[0], 5, 8, 8) + def test_deconvolution(): check_deconvolution_target_shape( input_shape = (2,3,4,4), @@ -822,6 +848,7 @@ def test_deconvolution(): pad = (3,3) ) + def check_nearest_upsampling_with_shape(shapes, scale, root_scale): arr = {'arg_%d'%i: mx.random.uniform(-10.0, 10.0, shape, ctx=mx.cpu()).copyto(default_context()) for i, shape in zip(range(len(shapes)), shapes)} arr_grad = {'arg_%d'%i: mx.nd.zeros(shape) for i, shape in zip(range(len(shapes)), shapes)} @@ -834,6 +861,7 @@ def check_nearest_upsampling_with_shape(shapes, scale, root_scale): name = 'arg_%d'%k assert_allclose(arr[name].asnumpy()*root_scale**2*scale**(2*k), arr_grad[name].asnumpy(), rtol=1e-4) + def check_bilinear_upsampling_with_shape(shapes, scale, root_scale): arr = {'arg_%d'%i: mx.random.uniform(-10.0, 10.0, shape, ctx=mx.cpu()).copyto(default_context()) for i, shape in zip(range(len(shapes)), shapes)} arr_grad = {'arg_%d'%i: mx.nd.zeros(shape) for i, shape in zip(range(len(shapes)), shapes)} @@ -846,6 +874,7 @@ def check_bilinear_upsampling_with_shape(shapes, scale, root_scale): name = 'arg_%d'%k assert_allclose(arr[name].asnumpy()*root_scale**2*scale**(2*k), arr_grad[name].asnumpy(), rtol=1e-4) + def test_nearest_upsampling(): for root_scale in [1,2,3]: for scale in [1,2,3]: @@ -854,6 +883,7 @@ def test_nearest_upsampling(): shapes = [(1,3,base*root_scale*scale**(num_shape-1-i),base*root_scale*scale**(num_shape-1-i)) for i in range(num_shape)] check_nearest_upsampling_with_shape(shapes, scale, root_scale) + def test_batchnorm_training(): def check_batchnorm_training(stype): for shape in [(2, 3), (2, 3, 2, 2)]: @@ -938,6 +968,7 @@ def check_batchnorm_training(stype): for stype in stypes: check_batchnorm_training(stype) + def test_convolution_grouping(): num_filter = 4 num_group = 2 @@ -1006,6 +1037,7 @@ def test_depthwise_convolution(): for arr1, arr2 in zip(exe1.outputs + exe1.grad_arrays, exe2.outputs + exe2.grad_arrays): np.testing.assert_allclose(arr1.asnumpy(), arr2.asnumpy(), rtol=1e-3, atol=1e-4) + def gen_broadcast_data(idx): # Manually set test cases binary_op_data_shape = np.array( @@ -1061,27 +1093,35 @@ def gen_broadcast_data(idx): r_shape[np.where(r_axis_flags == 0)] = 1 return [np.random.random(l_shape), np.random.random(r_shape)] + def gen_broadcast_data_int(idx): d = gen_broadcast_data(idx); return [np.round(d[0]*100).astype(int), np.round(d[1]*100).astype(int)] + def gen_binary_data(dummy): ndim = np.random.randint(1, 6) shape = np.random.randint(1, 6, size=(ndim,)) return [np.random.random(shape), np.random.random(shape)] + def gen_binary_data_int(dummy): d = gen_binary_data(dummy); return [np.round(d[0]*100).astype(int), np.round(d[1]*100).astype(int)] -def check_binary_op_forward(symbol, baseline, gen_data, rtol=1e-3, atol=1e-5): + +def check_binary_op_forward(symbol, baseline, gen_data, rtol=1e-3, atol=1e-5, mx_nd_func=None): sample_num = 200 for i in range(sample_num): d = gen_data(i) x = baseline(d[0], d[1]) - y = symbol.bind(default_context(), args={'a': mx.nd.array(d[0]), 'b' : mx.nd.array(d[1])}) + y = symbol.bind(default_context(), args={'a': mx.nd.array(d[0]), 'b': mx.nd.array(d[1])}) y.forward(is_train=True) y = y.outputs[0].asnumpy() + if mx_nd_func is not None: + d0 = mx.nd.array(d[0], dtype=d[0].dtype) + d1 = mx.nd.array(d[1], dtype=d[1].dtype) + assert_almost_equal(y, mx_nd_func(d0, d1).asnumpy(), rtol=rtol, atol=atol) idx = np.abs(x-y) > atol+rtol*np.abs(x) if idx.any(): print('found precision problem') @@ -1097,11 +1137,13 @@ def check_binary_op_forward(symbol, baseline, gen_data, rtol=1e-3, atol=1e-5): print('diff: {}'.format(np.abs(x-y)[idx] - atol-rtol*np.abs(x)[idx])) assert_allclose(y, x, rtol=rtol, atol=atol) + def check_binary_op_backward(symbol, baseline, gen_data, rtol=1e-3, atol=1e-5): sample_num = 200 for i in range(sample_num): d = gen_data(i) out = np.random.random((d[0] + d[1]).shape) + def reduce_op(shape, x): if shape == x.shape: return x @@ -1111,18 +1153,20 @@ def reduce_op(shape, x): keepdims_shape[i] = 1 x = np.sum(x, axis=i).reshape(keepdims_shape) return x + baseline_grad1, baseline_grad2 = baseline(out, d[0], d[1]) x_1 = reduce_op(d[0].shape, baseline_grad1) x_2 = reduce_op(d[1].shape, baseline_grad2) y_1 = mx.nd.empty(d[0].shape) y_2 = mx.nd.empty(d[1].shape) - y = symbol.bind(default_context(), args={'a': mx.nd.array(d[0]), 'b' : mx.nd.array(d[1])}, + y = symbol.bind(default_context(), args={'a': mx.nd.array(d[0]), 'b': mx.nd.array(d[1])}, args_grad=[y_1, y_2]) y.forward(is_train=True) y.backward([mx.nd.array(out)]) assert_allclose(y_1.asnumpy(), x_1, rtol=rtol, atol=atol) assert_allclose(y_2.asnumpy(), x_2, rtol=rtol, atol=atol) + def test_binary_op(): a = mx.sym.Variable('a') b = mx.sym.Variable('b') @@ -1177,51 +1221,65 @@ def test_bneq(a, b): test_bpow(a, b) test_bneq(a, b) + def test_broadcast_binary_op(): a = mx.sym.Variable('a') b = mx.sym.Variable('b') def test_bplus(a, b): c = mx.sym.broadcast_plus(a, b) - check_binary_op_forward(c, lambda a, b: a + b, gen_broadcast_data) + check_binary_op_forward(c, lambda a, b: a + b, gen_broadcast_data, mx_nd_func=mx.nd.add) check_binary_op_backward(c, lambda g_out, a, b: (g_out, g_out), gen_broadcast_data) def test_bminus(a, b): c = mx.sym.broadcast_minus(a, b) - check_binary_op_forward(c, lambda a, b: a - b, gen_broadcast_data) + check_binary_op_forward(c, lambda a, b: a - b, gen_broadcast_data, mx_nd_func=mx.nd.subtract) check_binary_op_backward(c, lambda g_out, a, b: (g_out, - g_out), gen_broadcast_data) def test_bmul(a, b): c = mx.sym.broadcast_mul(a, b) - check_binary_op_forward(c, lambda a, b: a * b, gen_broadcast_data) + check_binary_op_forward(c, lambda a, b: a * b, gen_broadcast_data, mx_nd_func=mx.nd.multiply) check_binary_op_backward(c, lambda g_out, a, b: (g_out * b, g_out * a), gen_broadcast_data) def test_bdiv(a, b): c = mx.sym.broadcast_div(a, b) - check_binary_op_forward(c, lambda a, b: a / b, gen_broadcast_data) + check_binary_op_forward(c, lambda a, b: a / b, gen_broadcast_data, mx_nd_func=mx.nd.divide) check_binary_op_backward(c, lambda g_out, a, b: (g_out / b, - g_out * a / (b * b)), gen_broadcast_data) def test_bmod(a, b): c = mx.sym.broadcast_mod(a, b) - check_binary_op_forward(c, lambda a, b: a % b, gen_broadcast_data, atol=1) + check_binary_op_forward(c, lambda a, b: a % b, gen_broadcast_data, atol=1, mx_nd_func=mx.nd.modulo) check_binary_op_backward(c, lambda g_out, a, b: (g_out, - g_out * (a // b)), gen_broadcast_data, atol=1) def test_bmod_int(a, b): c = mx.sym.broadcast_mod(mx.sym.cast(a, dtype='int32'), mx.sym.cast(b, dtype='int32')) - check_binary_op_forward(c, lambda a, b: a % b, gen_broadcast_data_int) + check_binary_op_forward(c, lambda a, b: a % b, gen_broadcast_data_int, mx_nd_func=mx.nd.modulo) check_binary_op_backward(c, lambda g_out, a, b: (np.zeros_like(a), np.zeros_like(b)), gen_broadcast_data_int) def test_bpow(a, b): c = mx.sym.broadcast_power(a, b) - check_binary_op_forward(c, lambda a, b: a ** b, gen_broadcast_data) + check_binary_op_forward(c, lambda a, b: a ** b, gen_broadcast_data, mx_nd_func=mx.nd.power) check_binary_op_backward(c, lambda g_out, a, b: (g_out * a **(b - 1) * b, g_out * a ** b * np.log(a)), gen_broadcast_data) def test_bequal(a, b): c = mx.sym.broadcast_equal(a, b) - check_binary_op_forward(c, lambda a, b: (a == b).astype(a.dtype), gen_broadcast_data_int) + check_binary_op_forward(c, lambda a, b: (a == b).astype(a.dtype), gen_broadcast_data_int, + mx_nd_func=mx.nd.equal) check_binary_op_backward(c, lambda g_out, a, b: (np.zeros_like(a), np.zeros_like(b)), gen_broadcast_data_int) + def test_bmax(a, b): + c = mx.sym.broadcast_maximum(a, b) + check_binary_op_forward(c, lambda x, y: np.maximum(x, y), gen_broadcast_data, mx_nd_func=mx.nd.maximum) + # pass idx=200 to gen_broadcast_data so that generated ndarrays' sizes are not too big + check_numeric_gradient(c, gen_broadcast_data(idx=200), rtol=1e-2, atol=1e-3) + + def test_bmin(a, b): + c = mx.sym.broadcast_minimum(a, b) + check_binary_op_forward(c, lambda x, y: np.minimum(x, y), gen_broadcast_data, mx_nd_func=mx.nd.minimum) + # pass idx=200 to gen_broadcast_data so that generated ndarrays' sizes are not too big + check_numeric_gradient(c, gen_broadcast_data(idx=200), rtol=1e-2, atol=1e-3) + test_bplus(a, b) test_bminus(a, b) test_bmul(a, b) @@ -1230,6 +1288,9 @@ def test_bequal(a, b): test_bmod_int(a, b) test_bpow(a, b) test_bequal(a, b) + test_bmax(a, b) + test_bmin(a, b) + def test_run_convolution_dilated_impulse_response(dil=(1,1), kernel_shape=(3,3), verbose=False): # Input for spike response @@ -1238,7 +1299,6 @@ def test_run_convolution_dilated_impulse_response(dil=(1,1), kernel_shape=(3,3), spike_img = mx.nd.array(spike_imgs) spike_img2 = mx.nd.array(spike_imgs) - kernel_weights = mx.nd.ones(shape=tuple([1,1]+list(kernel_shape)), dtype=np.float32) kernel_weights2 = mx.nd.ones(shape=tuple([1,1]+list(kernel_shape)), dtype=np.float32) @@ -1374,6 +1434,7 @@ def test_reshape_new(src_shape, shape_args, reverse, dst_shape): exe.backward(out_grads=[mx.nd.array(out_grad_npy, ctx=default_context())]) assert_allclose(exe.grad_arrays[0].asnumpy(), out_grad_npy.reshape((5, 4, 3, 7))) + def test_reduce(): sample_num = 500 def test_reduce_inner(numpy_reduce_func, numpy_reduce_grad_func, mx_reduce_sym, nan_prob = 0): @@ -1490,6 +1551,7 @@ def test_broadcasting_ele(sym_bcast): test_broadcasting_ele(sym_bcast_axis) test_broadcasting_ele(sym_bcast_to) + def test_transpose(): for ndim in range(1, 7): for t in range(5): @@ -1589,6 +1651,7 @@ def test_slice_axis(): xx[idx] = x.asnumpy()[idx] assert_allclose(xx + x_grad_npy, xgrad.asnumpy(), atol=1E-5) + def test_flip(): for ndim in range(1, 6): for t in range(5): @@ -1677,18 +1740,22 @@ def dot_sym(data_type): x = mx.sym.Variable('x', dtype=data_type) y = mx.sym.Variable('y', dtype=data_type) return mx.sym.dot(x, y) + def dot_sym_xT(data_type): x = mx.sym.Variable('x', dtype=data_type) y = mx.sym.Variable('y', dtype=data_type) return mx.sym.dot(x, y, transpose_a=True) + def dot_sym_yT(data_type): x = mx.sym.Variable('x', dtype=data_type) y = mx.sym.Variable('y', dtype=data_type) return mx.sym.dot(x, y, transpose_b=True) + def dot_sym_xT_yT(data_type): x = mx.sym.Variable('x', dtype=data_type) y = mx.sym.Variable('y', dtype=data_type) return mx.sym.dot(x, y, transpose_a=True, transpose_b=True) + for data_type in dtypes: for ashape, bshape in [((3, 4), (4, 5)), ((2, 3, 4), (4, 5, 6))]: m1_npy = np.random.uniform(-1, 1, ashape) @@ -1700,6 +1767,7 @@ def dot_sym_xT_yT(data_type): check_numeric_gradient(dot_sym_yT(data_type), [m1_npy, m2_npy.T], numeric_eps=1e-1, rtol=2e-2, atol=1e-3) check_numeric_gradient(dot_sym_xT_yT(data_type), [m1_npy.T, m2_npy.T], numeric_eps=1e-1, rtol=2e-2, atol=1e-3) + def test_batch_dot(): dtypes = ['float32', 'float64'] @@ -1756,6 +1824,7 @@ def test_batch_dot(): assert_almost_equal(exe_add.grad_dict['b'].asnumpy(), bgrad_npy + b_init_grad_npy, rtol=1e-3, atol=1e-4) + def get_correlation(data1,data2,kernel_size,max_displacement,stride1,stride2,pad_size,is_multiply): img1 = mx.sym.Variable('img1') @@ -1763,6 +1832,7 @@ def get_correlation(data1,data2,kernel_size,max_displacement,stride1,stride2,pad return mx.sym.Correlation(data1=img1,data2=img2,kernel_size =kernel_size,max_displacement = max_displacement, stride1 = stride1,stride2 = stride2,pad_size= pad_size,is_multiply = is_multiply) + def correlation_forward(data1,data2,pad_size,kernel_size,stride1,stride2,max_displacement,is_multiply): # compute output's dimension @@ -1810,6 +1880,7 @@ def correlation_forward(data1,data2,pad_size,kernel_size,stride1,stride2,max_dis out /= float(kernel_size**2*data1.shape[1]) return out,tmp1,tmp2 + def correlation_backward(out_grad,tmp1,tmp2,data1,data2,pad_size,kernel_size,stride1,stride2,max_displacement,is_multiply): # compute output's dimension @@ -1859,6 +1930,7 @@ def correlation_backward(out_grad,tmp1,tmp2,data1,data2,pad_size,kernel_size,str tmp2_grad = tmp2_grad / float(kernel_size**2*data1.shape[1]) return tmp1_grad[:,:,pad_size:pad_size+data1.shape[2],pad_size:pad_size+data1.shape[3]],tmp2_grad[:,:,pad_size:pad_size+data1.shape[2],pad_size:pad_size+data1.shape[3]], + def unittest_correlation(data_shape,kernel_size,max_displacement,stride1,stride2,pad_size,is_multiply): img1 = np.random.random(data_shape) @@ -1891,8 +1963,8 @@ def unittest_correlation(data_shape,kernel_size,max_displacement,stride1,stride2 assert_almost_equal(exe1.grad_dict['img1'].asnumpy(), grad1, rtol=1e-3, atol=1e-4) assert_almost_equal(exe1.grad_dict['img2'].asnumpy(), grad2, rtol=1e-3, atol=1e-4) -def test_correlation(): +def test_correlation(): unittest_correlation((1,3,10,10), kernel_size = 1,max_displacement = 4,stride1 = 1,stride2 = 1,pad_size = 4,is_multiply = False) unittest_correlation((5,1,15,15), kernel_size = 1,max_displacement = 5,stride1 = 1,stride2 = 1,pad_size = 5,is_multiply = False) unittest_correlation((5,1,15,15), kernel_size = 1,max_displacement = 5,stride1 = 1,stride2 = 1,pad_size = 5,is_multiply = True) @@ -1932,6 +2004,7 @@ def test_support_vector_machine_l1_svm(): assert_almost_equal(grad_np, grad.asnumpy()) + def test_support_vector_machine_l2_svm(): xpu = default_context() shape = (20, 10) @@ -1979,6 +2052,7 @@ def test_roipooling(): grad_nodes={'data':'add', 'rois':'null'}, numeric_eps=1e-4, rtol=1e-1, atol=1E-4) + def check_pad_with_shape(shape, xpu, pad_width, mode): # bind with label X = mx.symbol.Variable('X') @@ -1997,6 +2071,7 @@ def check_pad_with_shape(shape, xpu, pad_width, mode): # grad check check_numeric_gradient(Y, [x.asnumpy()], numeric_eps=1e-2, rtol=1e-2) + def test_pad(): shape1 = (2, 3, 3, 5) pad1 = (0, 0, 0, 0, 1, 2, 3, 4) @@ -2009,6 +2084,7 @@ def test_pad(): check_pad_with_shape(shape1, default_context(), pad1, 'reflect') check_pad_with_shape(shape2, default_context(), pad2, 'reflect') + def np_instance_norm(data, weight, bias, eps): spatial_dims = data.shape[2::] num_spatial_vals = np.prod(np.array(spatial_dims)) @@ -2025,6 +2101,7 @@ def np_instance_norm(data, weight, bias, eps): biasBatch = np.reshape(np.repeat(biasBatch, num_spatial_vals), data.shape) return weightBatch * (data - mean)/np.sqrt(var + eps) + biasBatch + def check_instance_norm_with_shape(shape, xpu): # bind with label eps = 0.001 @@ -2045,12 +2122,14 @@ def check_instance_norm_with_shape(shape, xpu): check_numeric_gradient(Y, {'X':x.asnumpy(), 'G':gamma.asnumpy(), 'B':beta.asnumpy()}, numeric_eps=1e-2, rtol=1e-2, atol=1e-2) + def test_instance_normalization(): check_instance_norm_with_shape((1, 1, 1), default_context()) check_instance_norm_with_shape((2, 1, 2), default_context()) check_instance_norm_with_shape((2,4,5,6), default_context()) check_instance_norm_with_shape((3,3,2,3,2,1,1), default_context()) + def check_l2_normalization(in_shape, mode, ctx=default_context(), norm_eps=1e-10): data = mx.symbol.Variable('data') out = mx.symbol.L2Normalization(data=data, mode=mode, eps=norm_eps) @@ -2084,6 +2163,7 @@ def check_l2_normalization(in_shape, mode, ctx=default_context(), norm_eps=1e-10 # check gradient check_numeric_gradient(out, [in_data], numeric_eps=1e-3, rtol=1e-2, atol=1e-3) + def test_l2_normalization(): for mode in ['channel', 'spatial', 'instance']: for nbatch in [1, 4]: @@ -2093,6 +2173,7 @@ def test_l2_normalization(): for width in [5, 7]: check_l2_normalization((nbatch, nchannel, height, width), mode) + def sequence_mask_numpy(array, lengths, value): arrayMask = array.copy() shape = array.shape @@ -2101,6 +2182,7 @@ def sequence_mask_numpy(array, lengths, value): arrayMask[int(lengths[i]):, i] = value return arrayMask + def check_sequence_mask(shape, xpu, mask_value): # bind with label X = mx.symbol.Variable('X') @@ -2123,12 +2205,14 @@ def check_sequence_mask(shape, xpu, mask_value): check_numeric_gradient(Y, [x.asnumpy(), l.asnumpy()], grad_nodes={'X':'write'}, numeric_eps=1e-3, rtol=1e-2) + def test_sequence_mask(): shape1 = (4, 2, 2, 3) shape2 = (1, 2, 2, 3, 1, 1) check_sequence_mask(shape1, default_context(), 2.1) check_sequence_mask(shape2, default_context(), 0.1) + def check_sequence_reverse(xpu): # sample data @@ -2192,6 +2276,7 @@ def test_wrapper(arr, xpu, sequence_length=None, use_sequence_length=False): def test_sequence_reverse(): check_sequence_reverse(mx.cpu()) + def mathematical_core_binary(name, forward_mxnet_call, forward_numpy_call, @@ -2236,6 +2321,7 @@ def mathematical_core_binary(name, assert_almost_equal(arr_grad1, npout_grad1) assert_almost_equal(arr_grad2, npout_grad2) + def mathematical_core(name, forward_mxnet_call, forward_numpy_call, backward_numpy_call, data_init=5., grad_init=2.): data = mx.symbol.Variable('data') shape = (3, 4) @@ -2264,6 +2350,7 @@ def mathematical_core(name, forward_mxnet_call, forward_numpy_call, backward_num # print(npout_grad) assert_almost_equal(arr_grad, npout_grad) + def test_special_functions_using_scipy(): try: from scipy import special as scipy_special @@ -2294,6 +2381,7 @@ def rounding(name, forward_mxnet_call, forward_numpy_call, data_init=5., grad_in npout = forward_numpy_call(data_tmp) assert_almost_equal(out, npout) + def test_mathematical(): # rsqrt mathematical_core("rsqrt", @@ -2380,6 +2468,7 @@ def test_mathematical(): # fix rounding("fix", lambda x: mx.sym.fix(x), lambda x: np.fix(x)) + def test_special_functions_using_scipy(): try: from scipy import special as scipy_special @@ -2395,6 +2484,7 @@ def test_special_functions_using_scipy(): mathematical_core("gammaln", lambda x: mx.sym.gammaln(x), lambda x: scipy_special.gammaln(x), lambda x: scipy_special.psi(x), 0.5, 0.5) + def test_clip(): data = mx.symbol.Variable('data') shape = (30, 30) @@ -2404,6 +2494,7 @@ def test_clip(): check_symbolic_backward(test, [data_tmp], [np.ones(shape)], [np.where(data_tmp < 0.6, [1], [0]) * np.where(data_tmp > -0.6, [1], [0])]) + def test_init(): def test_basic_val_init(sym_func, np_func, shape, dtype): x = sym_func(shape=shape, dtype=dtype) @@ -2411,6 +2502,7 @@ def test_basic_val_init(sym_func, np_func, shape, dtype): exe.forward(is_train=True) assert_almost_equal(exe.outputs[0].asnumpy(), np_func(shape=shape, dtype=dtype)) assert exe.outputs[0].asnumpy().dtype == dtype + def test_arange(): for i in range(5): start = np.random.rand() * 10 @@ -2432,6 +2524,7 @@ def test_arange(): def test_order(): ctx = default_context() + def gt_topk(dat, axis, ret_typ, k, is_ascend): if ret_typ == "indices": if is_ascend: @@ -2538,6 +2631,7 @@ def test_blockgrad(): assert_almost_equal(exe.outputs[0].asnumpy(), a_npy) exe.backward() # No error if BlockGrad works + def test_take(): def check_output_n_grad(data_shape, idx_shape): exe = result.simple_bind(default_context(), a=data_shape, @@ -2691,7 +2785,6 @@ def bilinear_forward_numpy(data, grid): +(1-xWeightTopLeft) * (1-yWeightTopLeft) * inBottomRight return out - def bilinear_backward_numpy(out_grad, data, grid): data_grad = np.zeros(data.shape, dtype=np.float32) @@ -2802,6 +2895,7 @@ def bilinear_backward_numpy(out_grad, data, grid): assert_almost_equal(exe_addto.grad_dict['data'].asnumpy(), data_grad + data_initial_grid, rtol=1e-3,atol=1e-5) assert_almost_equal(exe_addto.grad_dict['grid'].asnumpy(), grid_grad + grid_initial_grid, rtol=1e-3,atol=1e-5) + def test_index2d(): for _ in range(30): n = np.random.randint(1, 100) @@ -2811,6 +2905,7 @@ def test_index2d(): r = mx.nd.batch_take(data, x) assert_almost_equal(r.asnumpy(), data.asnumpy()[np.arange(n), x.asnumpy()]) + def test_cast(): for srctype in [np.int32, np.float32, np.float16]: for dsttype in [np.float32, np.int32, np.float16]: @@ -3277,6 +3372,7 @@ def check_ctc_loss(acts, labels, loss_truth): # test grad check_numeric_gradient(ctc, [acts, labels], grad_nodes=['input'], rtol=0.05, atol=1e-3) + def test_ctc_loss(): # Test 1: check that batches are same + check against Torch WarpCTC acts = np.array([ @@ -3310,6 +3406,7 @@ def test_quantization_op(): assert same(qa.asnumpy(), qa_real.asnumpy()) assert same(a_.asnumpy(), a_real.asnumpy()) + def test_reciprocal_op(): data_tmp = np.random.rand(3, 4) * 10 - 5 # Avoid possible division by 0 errors @@ -3320,6 +3417,7 @@ def test_reciprocal_op(): check_numeric_gradient(test, [data_tmp]) check_symbolic_forward(test, [data_tmp], [np.reciprocal(data_tmp)]) + def test_custom_op(): class Sqr(mx.operator.CustomOp): def forward(self, is_train, req, in_data, out_data, aux): @@ -3391,6 +3489,7 @@ def test_psroipooling(): check_numeric_gradient(op, [im_data, rois_data], rtol=rtol, atol=atol, grad_nodes=grad_nodes, ctx=mx.gpu(0)) + def test_deformable_convolution(): for num_batch in [1, 2]: for num_channel_data, num_deformable_group in itertools.product([4, 8], [1, 2]): @@ -3461,7 +3560,6 @@ def test_deformable_psroipooling(): grad_nodes=grad_nodes, ctx=mx.gpu(0)) - def test_laop(): # enable numerical checking of gradients From b0b46641a3d3b58f7b38d8bf84b9bf0c392f3873 Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Thu, 24 Aug 2017 10:43:52 -0700 Subject: [PATCH 412/834] add ctx to begin_state in rnn_layer (#7580) * add ctx to begin_state * fix image classification --- example/gluon/image_classification.py | 8 ++++++-- python/mxnet/gluon/rnn/rnn_layer.py | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/example/gluon/image_classification.py b/example/gluon/image_classification.py index 3f84ff8602ed..b8d018d3098a 100644 --- a/example/gluon/image_classification.py +++ b/example/gluon/image_classification.py @@ -57,6 +57,8 @@ help='enable batch normalization or not in vgg. default is false.') parser.add_argument('--use-pretrained', action='store_true', help='enable using pretrained model from gluon.') +parser.add_argument('--kvstore', type=str, default='device', + help='kvstore to use for trainer/module.') parser.add_argument('--log-interval', type=int, default=50, help='Number of batches to wait before logging.') opt = parser.parse_args() @@ -116,7 +118,8 @@ def train(epochs, ctx): if isinstance(ctx, mx.Context): ctx = [ctx] net.initialize(mx.init.Xavier(magnitude=2.24), ctx=ctx) - trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': opt.lr, 'wd': opt.wd}) + trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': opt.lr, 'wd': opt.wd}, + kvstore = opt.kvstore) metric = mx.metric.Accuracy() loss = gluon.loss.SoftmaxCrossEntropyLoss() @@ -162,7 +165,8 @@ def train(epochs, ctx): out = net(data) softmax = mx.sym.SoftmaxOutput(out, name='softmax') mod = mx.mod.Module(softmax, context=[mx.gpu(i) for i in range(gpus)] if gpus > 0 else [mx.cpu()]) - mod.fit(train_data, num_epoch=opt.epochs, batch_end_callback = mx.callback.Speedometer(batch_size, 1)) + mod.fit(train_data, num_epoch=opt.epochs, kvstore=opt.kvstore, + batch_end_callback = mx.callback.Speedometer(batch_size, 1)) else: if opt.mode == 'hybrid': net.hybridize() diff --git a/python/mxnet/gluon/rnn/rnn_layer.py b/python/mxnet/gluon/rnn/rnn_layer.py index 86b7c618e503..063d56654f9f 100644 --- a/python/mxnet/gluon/rnn/rnn_layer.py +++ b/python/mxnet/gluon/rnn/rnn_layer.py @@ -141,7 +141,7 @@ def begin_state(self, batch_size=0, func=ndarray.zeros, **kwargs): batch_size: int Only required for `NDArray` API. Size of the batch ('N' in layout). Dimension of the input. - func : callable, default `symbol.zeros` + func : callable, default `ndarray.zeros` Function for creating initial state. For Symbol API, func can be `symbol.zeros`, `symbol.uniform`, @@ -172,7 +172,7 @@ def forward(self, inputs, states=None): batch_size = inputs.shape[self._layout.find('N')] skip_states = states is None if skip_states: - states = self.begin_state(batch_size) + states = self.begin_state(batch_size, ctx=inputs.context) if isinstance(states, ndarray.NDArray): states = [states] for state, info in zip(states, self.state_info(batch_size)): From f489810e0243aec05bc5107e94a9742cf55e1a1c Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Thu, 24 Aug 2017 12:02:41 -0700 Subject: [PATCH 413/834] contrib ctc interface changes, cudnn7 CTC, and gluon CTC (#7442) * contrib ctc interface changes for compatibility * cudnn ctc * update per comments --- python/mxnet/gluon/loss.py | 90 +++++++ src/operator/contrib/ctc_loss-inl.h | 331 ++++++++++++++++++++++---- src/operator/contrib/ctc_loss.cc | 12 +- src/operator/sequence_op_common.h | 18 +- tests/python/gpu/test_operator_gpu.py | 1 + tests/python/unittest/test_loss.py | 30 +++ 6 files changed, 430 insertions(+), 52 deletions(-) diff --git a/python/mxnet/gluon/loss.py b/python/mxnet/gluon/loss.py index 583910590868..bb45e8926e95 100644 --- a/python/mxnet/gluon/loss.py +++ b/python/mxnet/gluon/loss.py @@ -21,6 +21,8 @@ from __future__ import absolute_import from .. import ndarray +from ..contrib import symbol as symbol_contrib +from ..contrib import ndarray as ndarray_contrib from ..base import numeric_types from .block import HybridBlock @@ -295,3 +297,91 @@ def hybrid_forward(self, F, output, label, sample_weight=None): loss = label * (F.log(label+1e-8) - output) loss = _apply_weighting(F, loss, self._weight, sample_weight) return F.mean(loss, axis=self._batch_axis, exclude=True) + + +class CTCLoss(Loss): + r"""Connectionist Temporal Classification Loss. + + See `"Connectionist Temporal Classification: Labelling Unsegmented + Sequence Data with Recurrent Neural Networks" + `_ paper for more information. + + Parameters + ---------- + layout : str, default 'NTC' + Layout of the output sequence activation vector. + label_layout : str, default 'NT' + Layout of the labels. + padding_mask : int or None, default -1 + This is the label value to be considered padding, which is used to derive the actual + lengths of labels. Only required when `label_lengths` is None. + weight : float or None + Global scalar weight for loss. + sample_weight : Symbol or None + Per sample weighting. Must be broadcastable to + the same shape as loss. For example, if loss has + shape (64, 10) and you want to weight each sample + in the batch, `sample_weight` should have shape (64, 1). + This should be used as the fifth argument when calling this loss. + + Input shapes: + `data` is an activation tensor without softmax. + Its shape depends on `layout`. For `layout='TNC'`, this + input has shape `(sequence_length, batch_size, alphabet_size)` + + `label` is the label index matrix. + Its shape depends on `label_layout`. For `label_layout='TN'`, this + input has shape `(label_sequence_length, batch_size)` + When `label_lengths` is not specified, the first occurrence of `padding_mask` + in each sample marks the end of the label sequence of that sample. + For example, suppose there are two samples, with *label_sequence_length* = 4. + The two sequences of labels are [2, 1] and [3, 2, 2], and their actual lengths + are smaller than 4. Thus, given *padding_mask* = 0, the resulting ```label``` + tensor should be padded to be:: + + [[2, 1, 0, 0], [3, 2, 2, 0]] + + `data_lengths` is optional and defaults to None. + When specified, it represents the actual lengths of data. + The shape should be (batch_size,). + If None, the data lengths are treated as being equal to the max sequence length. + This should be used as the third argument when calling this loss. + + `label_lengths` is optional and defaults to None. + When specified, it represents the actual lengths of labels. + The shape should be (batch_size,). + If None, the label lengths are derived from the first occurrence of + the value specified by `padding_mask`. + This should be used as the fourth argument when calling this loss. + + Output shape: + The CTC loss output has the shape (batch_size,). + """ + def __init__(self, layout='NTC', label_layout='NT', padding_mask=-1, + weight=None, **kwargs): + assert layout in ['NTC', 'TNC'],\ + "Only 'NTC' and 'TNC' layouts for output are supported. Got: %s"%layout + assert label_layout in ['NT', 'TN'],\ + "Only 'NT' and 'TN' layouts for label are supported. Got: %s"%label_layout + self._layout = layout + self._label_layout = label_layout + self._padding_mask = padding_mask + batch_axis = label_layout.find('N') + super(CTCLoss, self).__init__(weight, batch_axis, **kwargs) + + def hybrid_forward(self, F, data, label, + data_lengths=None, label_lengths=None, sample_weight=None): + if self._layout == 'NTC': + data = F.swapaxes(data, 0, 1) + if self._batch_axis == 1: + label = F.swapaxes(label, 0, 1) + if F is ndarray: + F_contrib = ndarray_contrib + else: + F_contrib = symbol_contrib + loss = F_contrib.CTCLoss(data, label, + use_data_lengths=data_lengths is not None, + use_label_lengths=label_lengths is not None, + data_lengths=data_lengths, label_lengths=label_lengths, + padding_mask=self._padding_mask) + return _apply_weighting(F, loss, self._weight, sample_weight) diff --git a/src/operator/contrib/ctc_loss-inl.h b/src/operator/contrib/ctc_loss-inl.h index 0d0c0bf4cd09..13ce1f240afd 100644 --- a/src/operator/contrib/ctc_loss-inl.h +++ b/src/operator/contrib/ctc_loss-inl.h @@ -41,6 +41,11 @@ #include "../sequence_op_common.h" #include "../mshadow_op.h" +#if defined(__CUDACC__) && MXNET_USE_CUDNN == 1 && CUDNN_MAJOR >= 7 +#define CUDNN_LABEL_LENGTH_LIMIT 256 +#include "../nn/softmax-inl.h" +#endif + namespace mxnet { namespace op { @@ -52,14 +57,14 @@ enum CTCLossOpForwardResource { kTempSpace }; template inline void get_workspace_size(std::vector *label_lengths, - std::vector *input_lengths, + std::vector *data_lengths, int alphabet_size, int minibatch, bool gpu, size_t *size_bytes) { // This is the max of all S and T for all examples in the minibatch. int maxL = *std::max_element(label_lengths->data(), label_lengths->data() + minibatch); - int maxT = *std::max_element(input_lengths->data(), - input_lengths->data() + minibatch); + int maxT = *std::max_element(data_lengths->data(), + data_lengths->data() + minibatch); const int S = 2 * maxL + 1; @@ -125,34 +130,109 @@ inline void get_workspace_size(std::vector *label_lengths, } // Takes a tensor of labels, and interprets 0-elements at the end of the vector -// as padding. The tensor is packed into a std::vector without padding -// characters. The sequence lengths are also inferred from the padding chars +// as padding. The tensor is packed into an std::vector without padding +// characters. The label sequence lengths are also inferred from the padding chars. +// When cudnn is enabled, the return value signifies whether the cudnn length limit is exceeded. template -inline void LabelTensorToPackedVector(mshadow::Tensor labels, +inline bool LabelTensorToPackedVector(mshadow::Tensor labels, + int padding_mask, std::vector *packed_labels, std::vector *label_lengths) { int batch = labels.size(0); int max_num_labels = labels.size(1); - std::vector cpu_labels(max_num_labels); + bool exceed_limit = false; + + std::vector cpu_labels(max_num_labels*batch); + mshadow::Tensor flat_labels = labels.FlatTo1D(); + IndexTensorToVector(flat_labels, &cpu_labels); + + for (int b = 0; b < batch; ++b) { + auto start = cpu_labels.data()+b*max_num_labels; + auto res = std::find(start, start+max_num_labels, padding_mask); + int len = std::distance(start, res); +#if defined(__CUDACC__) && MXNET_USE_CUDNN == 1 && CUDNN_MAJOR >= 7 + exceed_limit = exceed_limit || len > CUDNN_LABEL_LENGTH_LIMIT; +#endif + std::copy(start, start + len, + std::back_inserter(*packed_labels)); + label_lengths->at(b) = len; + } + return exceed_limit; +} + +// Takes a tensor of labels, and a vector which specifies the actual length of each label +// The tensor is packed into an std::vector without padding characters. +// The label length vector is copied into an std::vector. +// When cudnn is enabled, the return value signifies whether the cudnn length limit is exceeded. +template +inline bool PackLabelByLength(mshadow::Tensor labels, + mshadow::Tensor in_label_lengths, + std::vector *packed_labels, + std::vector *label_lengths) { + int batch = labels.size(0); + int max_num_labels = labels.size(1); + bool exceed_limit = false; + + IndexTensorToVector(in_label_lengths, label_lengths); + + std::vector cpu_labels(max_num_labels*batch); + mshadow::Tensor flat_labels = labels.FlatTo1D(); + IndexTensorToVector(flat_labels, &cpu_labels); for (int b = 0; b < batch; ++b) { - IndexTensorToVector(labels[b], &cpu_labels); - auto res = std::find(cpu_labels.begin(), cpu_labels.end(), 0); - int len = std::distance(cpu_labels.begin(), res); - std::copy(cpu_labels.begin(), cpu_labels.begin() + len, + auto start = cpu_labels.data()+b*max_num_labels; + int len = label_lengths->at(b); +#if defined(__CUDACC__) && MXNET_USE_CUDNN == 1 && CUDNN_MAJOR >= 7 + exceed_limit = exceed_limit || len > CUDNN_LABEL_LENGTH_LIMIT; +#endif + std::copy(start, start + len, std::back_inserter(*packed_labels)); - label_lengths->emplace_back(len); } + return exceed_limit; } struct CTCLossParam : public dmlc::Parameter { - DMLC_DECLARE_PARAMETER(CTCLossParam) {} + bool use_data_lengths; + bool use_label_lengths; + dmlc::optional padding_mask; + DMLC_DECLARE_PARAMETER(CTCLossParam) { + DMLC_DECLARE_FIELD(use_data_lengths).set_default(false) + .describe("Whether the data lenghts are decided by `data_lengths`. " + "If false, the lengths are equal to the max sequence length."); + DMLC_DECLARE_FIELD(use_label_lengths).set_default(false) + .describe("Whether the label lenghts are decided by " + "`label_lengths`, or derived from `padding_mask`. " + "If false, the lengths are derived from the " + "first occurrence of the value of `padding_mask`."); + DMLC_DECLARE_FIELD(padding_mask).set_default(dmlc::optional(0)) + .describe("int or None. This is the label value to be considered padding. " + "Only required when `use_label_lengths` is false. " + "Labels before the first occurrence of `padding_mask` are included " + "in calculation."); + } }; template class CTCLossOp : public Operator { public: - explicit CTCLossOp(CTCLossParam p) { this->param_ = p; } + explicit CTCLossOp(CTCLossParam p) { + this->param_ = p; + exceed_cudnn_limit = false; +#if defined(__CUDACC__) && MXNET_USE_CUDNN == 1 && CUDNN_MAJOR >= 7 + CUDNN_CALL(cudnnCreateCTCLossDescriptor(&ctc_desc_)); + CUDNN_CALL(cudnnSetCTCLossDescriptor(ctc_desc_, CUDNN_DATA_FLOAT)); + CUDNN_CALL(cudnnCreateTensorDescriptor(&prob_desc_)); + CUDNN_CALL(cudnnCreateTensorDescriptor(&grad_desc_)); +#endif + } + + ~CTCLossOp() { +#if defined(__CUDACC__) && MXNET_USE_CUDNN == 1 && CUDNN_MAJOR >= 7 + CUDNN_CALL(cudnnDestroyCTCLossDescriptor(ctc_desc_)); + CUDNN_CALL(cudnnDestroyTensorDescriptor(prob_desc_)); + CUDNN_CALL(cudnnDestroyTensorDescriptor(grad_desc_)); +#endif + } virtual void Forward(const OpContext &ctx, const std::vector &in_data, const std::vector &req, @@ -160,8 +240,9 @@ class CTCLossOp : public Operator { const std::vector &aux_args) { using namespace mshadow; using namespace mshadow::expr; - CHECK_EQ(in_data.size(), 2U); + CHECK_EQ(in_data.size(), 2U+param_.use_data_lengths+param_.use_label_lengths); CHECK_EQ(out_data.size(), 2U); + exceed_cudnn_limit = false; Stream *s = ctx.get_stream(); Tensor data = @@ -178,27 +259,41 @@ class CTCLossOp : public Operator { int batch_size = data.size(1); int alphabet_size = data.size(2); + // data_lengths + std::vector data_lengths(batch_size, max_seq_len); + if (param_.use_data_lengths) { + int kInputLength = 2; + IndexTensorToVector(in_data[kInputLength].get(s), &data_lengths); + } + // label_lengths std::vector packed_labels; - std::vector label_lengths; - LabelTensorToPackedVector(labels, &packed_labels, &label_lengths); - - // allocate temporary workspace - std::vector input_lengths(batch_size, max_seq_len); - size_t size_bytes; - bool gpu = data.kDevCPU ? false : true; - get_workspace_size(&label_lengths, &input_lengths, alphabet_size, - batch_size, gpu, &size_bytes); - - // round-up so there are enough elems in memory - int num_tmp_elems = (size_bytes + sizeof(real_t) - 1) / sizeof(real_t); - Tensor workspace = - ctx.requested[ctc_loss::kTempSpace].get_space_typed( - Shape1(num_tmp_elems), s); - - compute_ctc_cost(data, costs.dptr_, grad.dptr_, packed_labels.data(), - label_lengths.data(), input_lengths.data(), - workspace.dptr_, ctx.is_train); + std::vector label_lengths(batch_size); + + if (param_.use_label_lengths) { + int kLabelLength = 2+param_.use_data_lengths; + exceed_cudnn_limit = PackLabelByLength(labels, in_data[kLabelLength].get(s), + &packed_labels, &label_lengths); + } else { + exceed_cudnn_limit = LabelTensorToPackedVector(labels, param_.padding_mask.value(), + &packed_labels, &label_lengths); + } + +#if defined(__CUDACC__) && MXNET_USE_CUDNN == 1 && CUDNN_MAJOR >= 7 + if (!param_.use_data_lengths && !exceed_cudnn_limit) { + cudnn_forward(ctx, s, data, costs, grad, + &data_lengths, &label_lengths, &packed_labels, + max_seq_len, batch_size, alphabet_size); + } else { + baidu_forward(ctx, s, data, costs, grad, + &data_lengths, &label_lengths, &packed_labels, + batch_size, alphabet_size); + } +#else + baidu_forward(ctx, s, data, costs, grad, + &data_lengths, &label_lengths, &packed_labels, + batch_size, alphabet_size); +#endif // __CUDACC__ && CUDNN } virtual void Backward(const OpContext &ctx, @@ -221,12 +316,143 @@ class CTCLossOp : public Operator { Tensor data_grad_computed = out_data[ctc_loss::kGrad].get(s); - Assign(data_grad, req[ctc_loss::kData], - broadcast<1>(output_grad, data_grad.shape_) * data_grad_computed); +#if defined(__CUDACC__) && MXNET_USE_CUDNN == 1 && CUDNN_MAJOR >= 7 + if (!param_.use_data_lengths && !exceed_cudnn_limit) { + cudnn_backward_extra(s, data_grad, output_grad, data_grad_computed); + } else { + baidu_backward_extra(req, data_grad, output_grad, data_grad_computed); + } +#else + baidu_backward_extra(req, data_grad, output_grad, data_grad_computed); +#endif } private: CTCLossParam param_; + bool exceed_cudnn_limit; + +#if defined(__CUDACC__) && MXNET_USE_CUDNN == 1 && CUDNN_MAJOR >= 7 + cudnnDataType_t dtype_; + cudnnCTCLossDescriptor_t ctc_desc_; + cudnnTensorDescriptor_t prob_desc_, grad_desc_; + + inline virtual void cudnn_forward(const OpContext &ctx, + mshadow::Stream* s, + mshadow::Tensor data, + mshadow::Tensor costs, + mshadow::Tensor grad, + std::vector* data_lengths, + std::vector* label_lengths, + std::vector* packed_labels, + int max_seq_len, + int batch_size, + int alphabet_size) { + using namespace mshadow; + + // call cudnn to calculate ctc loss + dtype_ = CUDNN_DATA_FLOAT; + int dims[3], strides[3]; + size_t workspace_bytes; + int workspace_size; + dims[0] = max_seq_len; + dims[1] = batch_size; + dims[2] = alphabet_size; + strides[0] = batch_size*alphabet_size; + strides[1] = alphabet_size; + strides[2] = 1; + cudnnCTCLossAlgo_t ctc_algo = CUDNN_CTC_LOSS_ALGO_DETERMINISTIC; + CUDNN_CALL(cudnnSetTensorNdDescriptor(prob_desc_, + dtype_, + 3, + dims, + strides)); + CUDNN_CALL(cudnnSetTensorNdDescriptor(grad_desc_, + dtype_, + 3, + dims, + strides)); + CUDNN_CALL(cudnnGetCTCLossWorkspaceSize(s->dnn_handle_, + prob_desc_, + grad_desc_, + packed_labels->data(), + label_lengths->data(), + data_lengths->data(), + ctc_algo, + ctc_desc_, + &workspace_bytes)); + workspace_size = workspace_bytes/sizeof(real_t); + + Tensor temp_space = + ctx.requested[ctc_loss::kTempSpace].get_space_typed( + mshadow::Shape1(workspace_size+data.shape_.FlatTo1D()[0]), s); + + Tensor work_space(temp_space.dptr_, + mshadow::Shape1(workspace_size), s); + Tensor prob(temp_space.dptr_+workspace_size, + data.shape_, s); + + // since the input is activation before softmax and cudnn ctc takes softmax + // apply softmax to inputs first. + mxnet_op::Softmax(s, data.dptr_, prob.dptr_, data.shape_, 2); + + CUDNN_CALL(cudnnCTCLoss(s->dnn_handle_, + prob_desc_, + prob.dptr_, + packed_labels->data(), + label_lengths->data(), + data_lengths->data(), + costs.dptr_, + grad_desc_, + grad.dptr_, + ctc_algo, + ctc_desc_, + work_space.dptr_, + workspace_bytes)); + } + inline virtual void cudnn_backward_extra(mshadow::Stream* s, + mshadow::Tensor data_grad, + mshadow::Tensor output_grad, + mshadow::Tensor data_grad_computed) { + mxnet_op::SoftmaxGrad(s, + output_grad.dptr_, data_grad_computed.dptr_, data_grad.dptr_, data_grad.shape_, 2); + } +#endif // __CUDACC__ && CUDNN + + inline virtual void baidu_forward(const OpContext &ctx, + mshadow::Stream* s, + mshadow::Tensor data, + mshadow::Tensor costs, + mshadow::Tensor grad, + std::vector* data_lengths, + std::vector* label_lengths, + std::vector* packed_labels, + int batch_size, + int alphabet_size) { + using namespace mshadow; + // allocate temporary workspace + size_t size_bytes; + bool gpu = data.kDevCPU ? false : true; + get_workspace_size(label_lengths, data_lengths, alphabet_size, + batch_size, gpu, &size_bytes); + + // round-up so there are enough elems in memory + int num_tmp_elems = (size_bytes + sizeof(real_t) - 1) / sizeof(real_t); + Tensor workspace = + ctx.requested[ctc_loss::kTempSpace].get_space_typed( + Shape1(num_tmp_elems), s); + + compute_ctc_cost(data, costs.dptr_, grad.dptr_, packed_labels->data(), + label_lengths->data(), data_lengths->data(), + workspace.dptr_, ctx.is_train); + } + + inline virtual void baidu_backward_extra(const std::vector &req, + mshadow::Tensor data_grad, + mshadow::Tensor output_grad, + mshadow::Tensor data_grad_computed) { + Assign(data_grad, req[ctc_loss::kData], + mshadow::expr::broadcast<1>(output_grad, data_grad.shape_) * data_grad_computed); + } }; // class CTCLossOp template @@ -240,15 +466,22 @@ class CTCLossProp : public OperatorProperty { int NumOutputs() const override { return 2; } std::vector ListArguments() const override { - return {"data", "label"}; + if (param_.use_data_lengths && param_.use_label_lengths) { + return {"data", "label", "data_lengths", "label_lengths"}; + } else if (param_.use_data_lengths) { + return {"data", "label", "data_lengths"}; + } else if (param_.use_label_lengths) { + return {"data", "label", "label_lengths"}; + } else { + return {"data", "label"}; + } } std::vector ListOutputs() const override { return {"output", "grad"}; } - void Init( - const std::vector> &kwargs) override { + void Init(const std::vector> &kwargs) override { param_.Init(kwargs); } @@ -259,7 +492,9 @@ class CTCLossProp : public OperatorProperty { bool InferShape(std::vector *in_shape, std::vector *out_shape, std::vector *aux_shape) const override { using namespace mshadow; - CHECK_EQ(in_shape->size(), 2U) << "Expect two inputs to the symbol."; + index_t expected_inputs = 2+param_.use_data_lengths+param_.use_label_lengths; + CHECK_EQ(in_shape->size(), expected_inputs) + << "Expect " << expected_inputs << " inputs to the symbol."; const TShape &dshape = (*in_shape)[ctc_loss::kData]; const TShape &lshape = (*in_shape)[ctc_loss::kLabel]; @@ -267,10 +502,24 @@ class CTCLossProp : public OperatorProperty { CHECK_EQ(lshape.ndim(), 2U) << "The labels array must be of rank 2."; CHECK_EQ(dshape[1], lshape[0]) << "The batch size for the labels and data arrays must be the same."; + if (param_.use_data_lengths) { + int kInputLength = 2; + const TShape &dlshape = (*in_shape)[kInputLength]; + CHECK_EQ(dlshape.ndim(), 1U) << "Data length array must be a vector."; + CHECK_EQ(dlshape[0], dshape[1]) + << "The batch size for the data and data lengths must be the same."; + } + if (param_.use_label_lengths) { + int kLabelLength = 2+param_.use_data_lengths; + const TShape &llshape = (*in_shape)[kLabelLength]; + CHECK_EQ(llshape.ndim(), 1U) << "Label length array must be a vector."; + CHECK_EQ(llshape[0], lshape[0]) + << "The batch size for the labels and label lengths must be the same."; + } CHECK_GE(dshape[0], lshape[1]) << "The max number of labels cannot exceed " "the maximum sequence length of the " - "input."; + "data."; TShape oshape(1); oshape[0] = dshape[1]; // batch size diff --git a/src/operator/contrib/ctc_loss.cc b/src/operator/contrib/ctc_loss.cc index 3727cee10b1c..d544a1fdec04 100644 --- a/src/operator/contrib/ctc_loss.cc +++ b/src/operator/contrib/ctc_loss.cc @@ -31,7 +31,7 @@ namespace mshadow { template ctcStatus_t compute_ctc_cost(const Tensor activations, DType *costs, DType *grads, int *labels, - int *label_lengths, int *input_lengths, + int *label_lengths, int *data_lengths, void *workspace, int train) { int minibatch = static_cast(activations.size(1)); int alphabet_size = static_cast(activations.size(2)); @@ -39,10 +39,10 @@ ctcStatus_t compute_ctc_cost(const Tensor activations, mxnet_warpctc::CpuCTC ctc(alphabet_size, minibatch, workspace, blank_label); if (train) return ctc.cost_and_grad(activations.dptr_, grads, costs, labels, - label_lengths, input_lengths); + label_lengths, data_lengths); else return ctc.score_forward(activations.dptr_, costs, labels, label_lengths, - input_lengths); + data_lengths); } } // namespace mshadow @@ -100,6 +100,12 @@ information. .add_argument("data", "NDArray-or-Symbol", "Input data to the ctc_loss op.") .add_argument("label", "NDArray-or-Symbol", "Ground-truth labels for the loss.") + .add_argument("data_lengths", "NDArray-or-Symbol", + "Lengths of data for each of the samples. Only required " + "when use_data_lengths is true.") + .add_argument("label_lengths", "NDArray-or-Symbol", + "Lengths of labels for each of the samples. Only required " + "when use_label_lengths is true.") .add_arguments(CTCLossParam::__FIELDS__()); NNVM_REGISTER_OP(_contrib_CTCLoss).add_alias("_contrib_ctc_loss"); diff --git a/src/operator/sequence_op_common.h b/src/operator/sequence_op_common.h index 9e5843161087..724e0e0da121 100644 --- a/src/operator/sequence_op_common.h +++ b/src/operator/sequence_op_common.h @@ -32,9 +32,10 @@ namespace mxnet { namespace op { -template -void IndexTensorToVector(mshadow::Tensor data, - std::vector *index_vec) { +template +typename std::enable_if::value>::type +IndexTensorToVector(mshadow::Tensor data, + std::vector *index_vec) { int max_seq_len = data.shape_.Size(); #if MXNET_USE_CUDA DType *temp_index = @@ -44,18 +45,19 @@ void IndexTensorToVector(mshadow::Tensor data, cudaMemcpyDeviceToHost, data.stream_->stream_); CHECK_EQ(cuda_status, cudaSuccess) << "cuda memcpy label error"; for (int i = 0; i < max_seq_len; ++i) { - (*index_vec)[i] = static_cast(temp_index[i]); + (*index_vec)[i] = static_cast(temp_index[i]); } free(temp_index); #endif } -template -void IndexTensorToVector(mshadow::Tensor data, - std::vector *index_vec) { +template +typename std::enable_if::value>::type +IndexTensorToVector(mshadow::Tensor data, + std::vector *index_vec) { int max_seq_len = data.shape_.Size(); DType *index_array = static_cast(data.dptr_); for (int i = 0; i < max_seq_len; ++i) - (*index_vec)[i] = static_cast(index_array[i]); + (*index_vec)[i] = static_cast(index_array[i]); } } // namespace op diff --git a/tests/python/gpu/test_operator_gpu.py b/tests/python/gpu/test_operator_gpu.py index 35a20f935573..11d146cae840 100644 --- a/tests/python/gpu/test_operator_gpu.py +++ b/tests/python/gpu/test_operator_gpu.py @@ -29,6 +29,7 @@ from test_optimizer import * from test_random import * from test_gluon import * +from test_loss import * #from test_rnn import * from test_gluon_rnn import * from test_sparse_operator import test_cast_storage_ex, test_sparse_dot diff --git a/tests/python/unittest/test_loss.py b/tests/python/unittest/test_loss.py index 714ea7562fdb..b864215ca1d1 100644 --- a/tests/python/unittest/test_loss.py +++ b/tests/python/unittest/test_loss.py @@ -165,6 +165,36 @@ def test_l1_loss(): assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.1 +def test_ctc_loss(): + loss = gluon.loss.CTCLoss(padding_mask=0) + l = loss(mx.nd.ones((2,20,4)), mx.nd.array([[2,1,0,0],[3,2,2,0]])) + mx.test_utils.assert_almost_equal(l.asnumpy(), np.array([18.82820702, 16.50581741])) + + loss = gluon.loss.CTCLoss(layout='TNC', padding_mask=0) + l = loss(mx.nd.ones((20,2,4)), mx.nd.array([[2,1,0,0],[3,2,2,0]])) + mx.test_utils.assert_almost_equal(l.asnumpy(), np.array([18.82820702, 16.50581741])) + + loss = gluon.loss.CTCLoss(layout='TNC', label_layout='TN', padding_mask=0) + l = loss(mx.nd.ones((20,2,4)), mx.nd.array([[2,1,0,0],[3,2,2,0]]).T) + mx.test_utils.assert_almost_equal(l.asnumpy(), np.array([18.82820702, 16.50581741])) + + loss = gluon.loss.CTCLoss(padding_mask=-1) + l = loss(mx.nd.ones((2,20,4)), mx.nd.array([[2,1,-1,-1],[3,2,2,-1]])) + mx.test_utils.assert_almost_equal(l.asnumpy(), np.array([18.82820702, 16.50581741])) + + loss = gluon.loss.CTCLoss() + l = loss(mx.nd.ones((2,20,4)), mx.nd.array([[2,1,2,2],[3,2,2,2]]), None, mx.nd.array([2,3])) + mx.test_utils.assert_almost_equal(l.asnumpy(), np.array([18.82820702, 16.50581741])) + + loss = gluon.loss.CTCLoss() + l = loss(mx.nd.ones((2,25,4)), mx.nd.array([[2,1,-1,-1],[3,2,2,-1]]), mx.nd.array([20,20])) + mx.test_utils.assert_almost_equal(l.asnumpy(), np.array([18.82820702, 16.50581741])) + + loss = gluon.loss.CTCLoss() + l = loss(mx.nd.ones((2,25,4)), mx.nd.array([[2,1,3,3],[3,2,2,3]]), mx.nd.array([20,20]), mx.nd.array([2,3])) + mx.test_utils.assert_almost_equal(l.asnumpy(), np.array([18.82820702, 16.50581741])) + + def test_sample_weight_loss(): mx.random.seed(1234) np.random.seed(1234) From 0a3ee080d9706702c6c279a7051aef1fa806fd34 Mon Sep 17 00:00:00 2001 From: Xin Li Date: Fri, 25 Aug 2017 03:03:24 +0800 Subject: [PATCH 414/834] Fix symbol load json (#6420) --- cpp-package/include/mxnet-cpp/symbol.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp-package/include/mxnet-cpp/symbol.hpp b/cpp-package/include/mxnet-cpp/symbol.hpp index ee1a11e26a40..11590fad6041 100644 --- a/cpp-package/include/mxnet-cpp/symbol.hpp +++ b/cpp-package/include/mxnet-cpp/symbol.hpp @@ -103,6 +103,7 @@ inline Symbol Symbol::Load(const std::string &file_name) { return Symbol(handle); } inline Symbol Symbol::LoadJSON(const std::string &json_str) { + op_map(); SymbolHandle handle; CHECK_EQ(MXSymbolCreateFromJSON(json_str.c_str(), &(handle)), 0); return Symbol(handle); From 7f90a39c6d3e42cfb91946120a4f90797b3a06a0 Mon Sep 17 00:00:00 2001 From: gurumurthys Date: Thu, 24 Aug 2017 12:11:18 -0700 Subject: [PATCH 415/834] Added gen_data.py and modified README.md for bi-lstm-sort example (#6549) --- example/bi-lstm-sort/README.md | 6 +++++- example/bi-lstm-sort/gen_data.py | 20 ++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 example/bi-lstm-sort/gen_data.py diff --git a/example/bi-lstm-sort/README.md b/example/bi-lstm-sort/README.md index b56b671c428e..a590a18bfbc0 100644 --- a/example/bi-lstm-sort/README.md +++ b/example/bi-lstm-sort/README.md @@ -2,9 +2,13 @@ This is an example of using bidirection lstm to sort an array. Firstly, generate data by: - cd data python gen_data.py +Move generated txt files to data directory + + mkdir data + mv *.txt data + Then, train the model by: python lstm_sort.py diff --git a/example/bi-lstm-sort/gen_data.py b/example/bi-lstm-sort/gen_data.py new file mode 100644 index 000000000000..55ce1cfba2fb --- /dev/null +++ b/example/bi-lstm-sort/gen_data.py @@ -0,0 +1,20 @@ +import random + +vocab = [str(x) for x in range(100, 1000)] +sw_train = open("sort.train.txt", "w") +sw_test = open("sort.test.txt", "w") +sw_valid = open("sort.valid.txt", "w") + +for i in range(1000000): + seq = " ".join([vocab[random.randint(0, len(vocab) - 1)] for j in range(5)]) + k = i % 50 + if k == 0: + sw_test.write(seq + "\n") + elif k == 1: + sw_valid.write(seq + "\n") + else: + sw_train.write(seq + "\n") + +sw_train.close() +sw_test.close() +sw_valid.close() From c584b516b15eedb036a5cb598674031ffb751fb4 Mon Sep 17 00:00:00 2001 From: Zehao Shi Date: Fri, 25 Aug 2017 03:14:26 +0800 Subject: [PATCH 416/834] Fix RCNN multi-gpu bucketing warning which may cause OOM error. (#6965) * Fix a spelling mistake. * FIX pad example * fix smooth l1 comment * Fix rcnn multi-gpu bucketing warning --- example/rcnn/rcnn/core/loader.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/example/rcnn/rcnn/core/loader.py b/example/rcnn/rcnn/core/loader.py index 826ee20f080c..fdd6e5c386f1 100644 --- a/example/rcnn/rcnn/core/loader.py +++ b/example/rcnn/rcnn/core/loader.py @@ -165,11 +165,16 @@ def reset(self): vert = np.logical_not(horz) horz_inds = np.where(horz)[0] vert_inds = np.where(vert)[0] + # Avoid putting different aspect ratio image into the same bucket, + # which may cause bucketing warning. + pad_horz = self.batch_size - len(horz_inds) % self.batch_size + pad_vert = self.batch_size - len(vert_inds) % self.batch_size + horz_inds = np.hstack([horz_inds, horz_inds[:pad_horz]]) + vert_inds = np.hstack([vert_inds, vert_inds[:pad_vert]]) inds = np.hstack((np.random.permutation(horz_inds), np.random.permutation(vert_inds))) - extra = inds.shape[0] % self.batch_size - inds_ = np.reshape(inds[:-extra], (-1, self.batch_size)) - row_perm = np.random.permutation(np.arange(inds_.shape[0])) - inds[:-extra] = np.reshape(inds_[row_perm, :], (-1,)) + inds = np.reshape(inds[:], (-1, self.batch_size)) + row_perm = np.random.permutation(np.arange(inds.shape[0])) + inds = np.reshape(inds[row_perm, :], (-1,)) self.index = inds else: np.random.shuffle(self.index) From 3730f549fa9e9cb30b99e98e1f25a5372f7c9421 Mon Sep 17 00:00:00 2001 From: Adam Russell Date: Thu, 24 Aug 2017 16:21:09 -0400 Subject: [PATCH 417/834] Changed next() method to use the seq_size attribute and not the global variable $seq_size. (#7521) --- perl-package/AI-MXNet/examples/char_lstm.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perl-package/AI-MXNet/examples/char_lstm.pl b/perl-package/AI-MXNet/examples/char_lstm.pl index 54a9e3672f63..9a80ddadf618 100755 --- a/perl-package/AI-MXNet/examples/char_lstm.pl +++ b/perl-package/AI-MXNet/examples/char_lstm.pl @@ -133,7 +133,7 @@ sub BUILD [$offset + 1 , $offset + $self->batch_size*$self->seq_size] )->reshape([$self->batch_size, $self->seq_size]); $self->seq_counter($self->seq_counter + 1); - if($self->seq_counter == $seq_size - 1) + if($self->seq_counter == $self->seq_size - 1) { $self->counter($self->counter + 1); $self->seq_counter(0); From 125a12631c6c051a8b73cb6d2fdaad17ea71b31d Mon Sep 17 00:00:00 2001 From: Przemyslaw Tredak Date: Thu, 24 Aug 2017 13:44:19 -0700 Subject: [PATCH 418/834] Relaxing condition in slice (#7487) * Relaxing condition in slice * Update ndarray.cc --- src/ndarray/ndarray.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ndarray/ndarray.cc b/src/ndarray/ndarray.cc index 0d2968626d79..139d97670bec 100644 --- a/src/ndarray/ndarray.cc +++ b/src/ndarray/ndarray.cc @@ -91,7 +91,8 @@ NDArray NDArray::Slice(index_t begin, index_t end) const { using namespace autograd; using namespace mshadow; CHECK(!is_none()) << "NDArray is not initialized"; - CHECK_LT(begin, end) << "Invalid slicing range [" << begin << ", " << end << ")"; + CHECK_LE(begin, end) + << "Invalid slicing range [" << begin << ", " << end << ")"; CHECK_GE(shape_[0], end) << "Slice end index out of range"; CHECK_EQ(storage_type(), kDefaultStorage); NDArray ret = *this; From d956d1962ec69df12fa31e7800ba1a89c466f36c Mon Sep 17 00:00:00 2001 From: mrkumar83 Date: Thu, 24 Aug 2017 13:51:24 -0700 Subject: [PATCH 419/834] Fixing loss function code in tutorial (#7583) * Fixing loss function code in tutorial * Updating pull request with feedback --- docs/tutorials/gluon/gluon.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/tutorials/gluon/gluon.md b/docs/tutorials/gluon/gluon.md index ac1aa3f60f5e..a1688ea121dd 100644 --- a/docs/tutorials/gluon/gluon.md +++ b/docs/tutorials/gluon/gluon.md @@ -102,7 +102,8 @@ To compute loss and backprop for one iteration, we do: label = mx.nd.arange(10) # dummy label with autograd.record(): output = net(data) - loss = gluon.loss.softmax_cross_entropy_loss(output, label) + L = gluon.loss.SoftmaxCrossEntropyLoss() + loss = L(output, label) loss.backward() print('loss:', loss) print('grad:', net.fc1.weight.grad()) @@ -127,9 +128,10 @@ this is a commonly used functionality, gluon provide a `Trainer` class for it: ```python trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': 0.01}) -with record(): +with autograd.record(): output = net(data) - loss = gluon.loss.softmax_cross_entropy_loss(output, label) + L = gluon.loss.SoftmaxCrossEntropyLoss() + loss = L(output, label) loss.backward() # do the update. Trainer needs to know the batch size of data to normalize From 7f65a3438c95bea91f99a4cba9f645b177029271 Mon Sep 17 00:00:00 2001 From: Yao Wang Date: Thu, 24 Aug 2017 15:00:57 -0700 Subject: [PATCH 420/834] Add MXNet MKL pip install (#7598) --- docs/get_started/install.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/get_started/install.md b/docs/get_started/install.md index 2ab771d4cfef..65126a53254e 100644 --- a/docs/get_started/install.md +++ b/docs/get_started/install.md @@ -150,6 +150,11 @@ pip install graphviz **Step 4** Validate the installation by running simple MXNet code described [here](#validate-mxnet-installation). +**Experimental Choice** If You would like to install mxnet with Intel MKL, try the experimental pip package with MKL: +```bash +$ pip install mxnet-mkl +``` +
    @@ -313,6 +318,11 @@ pip install graphviz **Step 4** Validate the installation by running simple MXNet code described [here](#validate-mxnet-installation). +**Experimental Choice** If You would like to install mxnet with Intel MKL, try the experimental pip package with MKL: +```bash +$ pip install mxnet-cu80mkl +``` +
    From b34580ea8f215cf57137a625143506545cbb587d Mon Sep 17 00:00:00 2001 From: Rahul Huilgol Date: Thu, 24 Aug 2017 15:08:40 -0700 Subject: [PATCH 421/834] add license to new file (#7599) --- example/bi-lstm-sort/gen_data.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/example/bi-lstm-sort/gen_data.py b/example/bi-lstm-sort/gen_data.py index 55ce1cfba2fb..55af1b45554a 100644 --- a/example/bi-lstm-sort/gen_data.py +++ b/example/bi-lstm-sort/gen_data.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import random vocab = [str(x) for x in range(100, 1000)] From b7106369c99b7d0b7413617c9331d7be5acd91f9 Mon Sep 17 00:00:00 2001 From: Chris Olivier Date: Thu, 24 Aug 2017 15:43:22 -0700 Subject: [PATCH 422/834] nightly build test mnist training and optimizer (#7559) (#7562) * nightly build stochastically choose optimizer (#7559) * Only call MKL script once * Fix 'momentum' and 'multi_precision' optimizer args * fix cmake build for active kvstore * stochastic choice of optimizer for mnist training * Run all three optimizers * Add just lenet test * Trigger CI --- CMakeLists.txt | 39 +++++---- tests/nightly/test_all.sh | 22 ++++- tests/nightly/test_image_classification.sh | 93 ++++++++++++++++++++++ 3 files changed, 135 insertions(+), 19 deletions(-) create mode 100755 tests/nightly/test_image_classification.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index dc9ca5f7bb0c..5e32f6baefe3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -354,7 +354,7 @@ if(USE_CUDA) FIND_LIBRARY(CUDA_cufft_LIBRARY nvrtc "${CUDA_TOOLKIT_ROOT_DIR}/lib/x64" "${CUDA_TOOLKIT_ROOT_DIR}/lib/win32") list(APPEND mxnet_LINKER_LIBS "${CUDA_cufft_LIBRARY}/../cufft.lib") # For fft operator FIND_LIBRARY(CUDA_cusolver_LIBRARY nvrtc "${CUDA_TOOLKIT_ROOT_DIR}/lib/x64" "${CUDA_TOOLKIT_ROOT_DIR}/lib/win32") - list(APPEND mxnet_LINKER_LIBS "${CUDA_cusolver_LIBRARY}/../cusolver.lib") # For cusolver + list(APPEND mxnet_LINKER_LIBS "${CUDA_cusolver_LIBRARY}/../cusolver.lib") # For cusolver else(MSVC) list(APPEND mxnet_LINKER_LIBS nvrtc cuda cufft cusolver) link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") @@ -419,6 +419,29 @@ else() add_library(mxnet SHARED ${SOURCE}) endif() endif() + +if(USE_DIST_KVSTORE) + if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/ps-lite/CMakeLists.txt) + add_subdirectory("ps-lite") + list(APPEND pslite_LINKER_LIBS pslite protobuf) + target_link_libraries(mxnet debug ${pslite_LINKER_LIBS_DEBUG}) + target_link_libraries(mxnet optimized ${pslite_LINKER_LIBS_RELEASE}) + if(CMAKE_BUILD_TYPE STREQUAL "Debug") + list(APPEND mxnet_LINKER_LIBS ${pslite_LINKER_LIBS_DEBUG}) + else() + list(APPEND mxnet_LINKER_LIBS ${pslite_LINKER_LIBS_RELEASE}) + endif() + target_link_libraries(mxnet debug ${pslite_LINKER_LIBS_DEBUG}) + target_link_libraries(mxnet optimized ${pslite_LINKER_LIBS_RELEASE}) + + else() + set(pslite_LINKER_LIBS protobuf zmq-static) + endif() + add_definitions(-DMXNET_USE_DIST_KVSTORE) + include_directories(SYSTEM ${pslite_INCLUDE_DIR}) + list(APPEND mxnet_LINKER_LIBS ${pslite_LINKER_LIBS}) +endif() + target_link_libraries(mxnet ${mxnet_LINKER_LIBS}) if(USE_PLUGINS_WARPCTC) @@ -433,20 +456,6 @@ if(MSVC AND USE_MXNET_LIB_NAMING) endif() -if(USE_DIST_KVSTORE) - if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/ps-lite/CMakeLists.txt) - add_subdirectory("ps-lite") - list(APPEND pslite_LINKER_LIBS pslite) - target_link_libraries(mxnet debug ${pslite_LINKER_LIBS_DEBUG}) - target_link_libraries(mxnet optimized ${pslite_LINKER_LIBS_RELEASE}) - else() - set(pslite_LINKER_LIBS protobuf zmq-static ) - endif() - add_definitions(-DMXNET_USE_DIST_KVSTORE) - target_link_libraries(mxnet ${pslite_LINKER_LIBS}) - include_directories(SYSTEM ${pslite_INCLUDE_DIR}) -endif() - if(USE_PROFILER) add_definitions(-DMXNET_USE_PROFILER) endif() diff --git a/tests/nightly/test_all.sh b/tests/nightly/test_all.sh index 32913c9f5f5b..04d895fecf21 100755 --- a/tests/nightly/test_all.sh +++ b/tests/nightly/test_all.sh @@ -72,10 +72,24 @@ check_val() { example_dir=../../example/image-classification # python: lenet + mnist test_lenet() { - python $example_dir/train_mnist.py \ - --data-dir `pwd`/data/mnist/ --network lenet --gpus $gpus --num-epochs 10 \ - 2>&1 | tee log - check_val 0.99 + optimizers="adam sgd adagrad" + for optimizer in ${optimizers}; do + echo "OPTIMIZER: $optimizer" + if [ "$optimizer" == "adam" ]; then + learning_rate=0.0005 + desired_accuracy=0.98 + else + learning_rate=0.01 + desired_accuracy=0.99 + fi + python $example_dir/train_mnist.py --lr $learning_rate \ + --network lenet --optimizer $optimizer --gpus $gpus \ + --num-epochs 10 2>&1 | tee log + if [ $? -ne 0 ]; then + return $? + fi + check_val $desired_accuracy + done } juLog -name=Python.Lenet.Mnist -error=Fail test_lenet diff --git a/tests/nightly/test_image_classification.sh b/tests/nightly/test_image_classification.sh new file mode 100755 index 000000000000..93e403a2affc --- /dev/null +++ b/tests/nightly/test_image_classification.sh @@ -0,0 +1,93 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +# setup +export LD_LIBRARY_PATH=`pwd`/`dirname $0`/lib:/usr/local/cuda/lib64:$LD_LIBRARY_PATH +export PYTHONPATH=`pwd`/`dirname $0`/python +cd `pwd`/`dirname $0` +. sh2ju.sh + +## clean last build log +juLogClean + +if [ -f $(which nvidia-smi) ]; then + if [ $# -eq 1 ]; then + num_gpus=$1 + else + num_gpus=$(nvidia-smi -L | grep "GPU" | wc -l) + fi + gpus=`seq 0 $((num_gpus-1)) | paste -sd ","` + device_arg="--gpus $gpus" +else + device_arg="" +fi + +# build +build() { + make -C ../.. clean + make -C ../.. -j8 + return $? +} + +cp ../../make/config.mk ../.. +cat >>../../config.mk < $expected) print \"$pass\"; else print \"$fail\"}" + rm -f log +} + +example_dir=../../example/image-classification +# python: lenet + mnist +test_lenet() { + optimizers="adam sgd adagrad" + for optimizer in ${optimizers}; do + echo "OPTIMIZER: $optimizer" + if [ "$optimizer" == "adam" ]; then + learning_rate=0.0005 + desired_accuracy=0.98 + else + learning_rate=0.01 + desired_accuracy=0.99 + fi + python $example_dir/train_mnist.py --lr $learning_rate \ + --network lenet --optimizer $optimizer --gpus $gpus \ + --num-epochs 10 2>&1 | tee log + if [ $? -ne 0 ]; then + return $? + fi + check_val $desired_accuracy + done +} +juLog -name=Python.Lenet.Mnist -error=Fail test_lenet + +exit $errors From 4b94360e6e970ea71252f1e79b841e7bc4105de5 Mon Sep 17 00:00:00 2001 From: Saswata Date: Thu, 24 Aug 2017 19:22:25 -0400 Subject: [PATCH 423/834] make MXDataIter work without indices (#7456) indices are optional, custom cpp iterators providing data batches without indices should work while using MXDataIter. --- python/mxnet/io.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/python/mxnet/io.py b/python/mxnet/io.py index 4e69a8a801cb..314a2b28dfa8 100644 --- a/python/mxnet/io.py +++ b/python/mxnet/io.py @@ -815,10 +815,13 @@ def getindex(self): check_call(_LIB.MXDataIterGetIndex(self.handle, ctypes.byref(index_data), ctypes.byref(index_size))) - address = ctypes.addressof(index_data.contents) - dbuffer = (ctypes.c_uint64* index_size.value).from_address(address) - np_index = np.frombuffer(dbuffer, dtype=np.uint64) - return np_index.copy() + if index_size.value: + address = ctypes.addressof(index_data.contents) + dbuffer = (ctypes.c_uint64* index_size.value).from_address(address) + np_index = np.frombuffer(dbuffer, dtype=np.uint64) + return np_index.copy() + else: + return None def getpad(self): pad = ctypes.c_int(0) From 97a15c2ac602199df45879e1a4e4daac95a1c445 Mon Sep 17 00:00:00 2001 From: Kenji Doi Date: Fri, 25 Aug 2017 13:10:42 +0900 Subject: [PATCH 424/834] NAG also has 'momentum' optimizer args (#7602) --- example/image-classification/common/fit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/image-classification/common/fit.py b/example/image-classification/common/fit.py index dfec2a886b80..aeead0f82a3b 100755 --- a/example/image-classification/common/fit.py +++ b/example/image-classification/common/fit.py @@ -171,7 +171,7 @@ def fit(args, network, data_loader, **kwargs): optimizer_params['multi_precision'] = True # Only a limited number of optimizers have 'momentum' property - has_momentum = {'sgd', 'dcasgd'} + has_momentum = {'sgd', 'dcasgd', 'nag'} if args.optimizer in has_momentum: optimizer_params['momentum'] = args.mom From bc468b07421bb6bad2d3039b7a5b4f5d2aa256cc Mon Sep 17 00:00:00 2001 From: Dick Carter Date: Thu, 24 Aug 2017 21:14:01 -0700 Subject: [PATCH 425/834] Convert dot to linalg gemm (#7603) * Expands linalg_gemm use. Legacy mshadow::dot use only if no cblas. * Fix cpplint. --- Jenkinsfile | 2 +- .../contrib/deformable_convolution-inl.h | 19 ++++--- src/operator/convolution-inl.h | 19 ++++--- src/operator/convolution_v1-inl.h | 17 ++++-- src/operator/deconvolution-inl.h | 22 ++++++-- src/operator/fully_connected-inl.h | 23 ++------ src/operator/grid_generator-inl.h | 9 ++- src/operator/linalg_impl.h | 56 +++++++++++++++++-- src/operator/spatial_transformer-inl.h | 9 ++- 9 files changed, 124 insertions(+), 52 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index bf237a589c99..2d4cc017c865 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -155,7 +155,7 @@ try { ws('workspace/amalgamation') { init_git() make('cpu', '-C amalgamation/ clean') - make('cpu', '-C amalgamation/ USE_BLAS=openblas') + make('cpu', '-C amalgamation/ USE_BLAS=openblas MIN=1') } } }, diff --git a/src/operator/contrib/deformable_convolution-inl.h b/src/operator/contrib/deformable_convolution-inl.h index a8dc6b8f09ed..18c1fa367e67 100644 --- a/src/operator/contrib/deformable_convolution-inl.h +++ b/src/operator/contrib/deformable_convolution-inl.h @@ -44,6 +44,7 @@ #include "../operator_common.h" #include "../nn/im2col.h" #include "./nn/deformable_im2col.h" +#include "../linalg.h" namespace mxnet { @@ -152,7 +153,9 @@ class DeformableConvolutionOp : public Operator { param_.num_deformable_group, col_buffer.dptr()); Tensor output_3d = output_4d[n]; for (index_t g = 0; g < group_; ++g) { - ASSIGN_DISPATCH(output_3d[g], req[conv::kOut], dot(weight_3d[g], col_buffer_3d[g])); + // Legacy approach shown here for comparison: + // Assign(output_3d[g], req[conv::kOut], dot(weight_3d[g], col_buffer_3d[g])); + linalg_gemm(weight_3d[g], col_buffer_3d[g], output_3d[g], false, false, s, req[conv::kOut]); } } if (bias_term_) { @@ -216,7 +219,9 @@ class DeformableConvolutionOp : public Operator { for (index_t n = 0; n < num_; ++n) { Tensor out_grad_3d = out_grad_4d[n]; for (index_t g = 0; g < group_; ++g) { - col_buffer_3d[g] = dot(weight_3d[g].T(), out_grad_3d[g]); + // Legacy approach shown here for comparison: + // col_buffer_3d[g] = dot(weight_3d[g].T(), out_grad_3d[g]); + linalg_gemm(weight_3d[g], out_grad_3d[g], col_buffer_3d[g], true, false, s); } // gradient w.r.t. input coordinate data @@ -243,12 +248,10 @@ class DeformableConvolutionOp : public Operator { param_.num_deformable_group, col_buffer.dptr()); for (index_t g = 0; g < group_; ++g) { - if (0 == n) { - ASSIGN_DISPATCH(dweight_3d[g], req[conv::kWeight], - dot(out_grad_3d[g], col_buffer_3d[g].T())); - } else { - dweight_3d[g] += dot(out_grad_3d[g], col_buffer_3d[g].T()); - } + auto request = (n == 0) ? req[conv::kWeight] : kAddTo; + // Legacy approach shown here for comparison: + // Assign(dweight_3d[g], request, dot(out_grad_3d[g], col_buffer_3d[g].T())); + linalg_gemm(out_grad_3d[g], col_buffer_3d[g], dweight_3d[g], false, true, s, request); } } diff --git a/src/operator/convolution-inl.h b/src/operator/convolution-inl.h index 0a2522cccb65..0edaee1dae32 100644 --- a/src/operator/convolution-inl.h +++ b/src/operator/convolution-inl.h @@ -40,6 +40,7 @@ #include #include "./operator_common.h" #include "./nn/im2col.h" +#include "./linalg.h" namespace mxnet { @@ -160,7 +161,9 @@ class ConvolutionOp : public Operator { col_buffer.dptr()); Tensor output_3d = output_4d[n]; for (index_t g = 0; g < group_; ++g) { - ASSIGN_DISPATCH(output_3d[g], req[conv::kOut], dot(weight_3d[g], col_buffer_3d[g])); + // Legacy approach shown here for comparison: + // Assign(output_3d[g], req[conv::kOut], dot(weight_3d[g], col_buffer_3d[g])); + linalg_gemm(weight_3d[g], col_buffer_3d[g], output_3d[g], false, false, s, req[conv::kOut]); } } if (bias_term_) { @@ -219,7 +222,9 @@ class ConvolutionOp : public Operator { Tensor out_grad_3d = out_grad_4d[n]; // gradient w.r.t. input data for (index_t g = 0; g < group_; ++g) { - col_buffer_3d[g] = dot(weight_3d[g].T(), out_grad_3d[g]); + // Legacy approach shown here for comparison: + // col_buffer_3d[g] = dot(weight_3d[g].T(), out_grad_3d[g]); + linalg_gemm(weight_3d[g], out_grad_3d[g], col_buffer_3d[g], true, false, s); } col2im(s, col_buffer.dptr(), in_grad[conv::kData].shape_, col_buffer.shape_, param_.kernel, param_.pad, param_.stride, param_.dilate, @@ -230,12 +235,10 @@ class ConvolutionOp : public Operator { col_buffer.shape_, param_.kernel, param_.pad, param_.stride, param_.dilate, col_buffer.dptr()); for (index_t g = 0; g < group_; ++g) { - if (0 == n) { - ASSIGN_DISPATCH(dweight_3d[g], req[conv::kWeight], - dot(out_grad_3d[g], col_buffer_3d[g].T())); - } else { - dweight_3d[g] += dot(out_grad_3d[g], col_buffer_3d[g].T()); - } + auto request = (n == 0) ? req[conv::kWeight] : kAddTo; + // Legacy approach shown here for comparison: + // Assign(dweight_3d[g], request, dot(out_grad_3d[g], col_buffer_3d[g].T())); + linalg_gemm(out_grad_3d[g], col_buffer_3d[g], dweight_3d[g], false, true, s, request); } } diff --git a/src/operator/convolution_v1-inl.h b/src/operator/convolution_v1-inl.h index f39d8e0804bc..0ac940c24b19 100644 --- a/src/operator/convolution_v1-inl.h +++ b/src/operator/convolution_v1-inl.h @@ -37,6 +37,7 @@ #include #include #include "./operator_common.h" +#include "./linalg.h" namespace mxnet { namespace op { @@ -180,7 +181,9 @@ class ConvolutionV1Op : public Operator { for (uint32_t gid = 0; gid < param_.num_group; ++gid) { mshadow::Tensor tmpc = temp_col.Slice(gstride * gid, gstride * (gid + 1)); - temp_dst[gid] = dot(wmat[gid], tmpc); + // Legacy approach shown here for comparison: + // temp_dst[gid] = dot(wmat[gid], tmpc); + linalg_gemm(wmat[gid], tmpc, temp_dst[gid], false, false, s); } out.Slice(i, i + step) = swapaxis<1, 0>(reshape(temp_dst, mshadow::Shape4(param_.num_filter, @@ -267,15 +270,21 @@ class ConvolutionV1Op : public Operator { Tensor tmpc = temp_col.Slice(gstride * gid, gstride * (gid + 1)); if (i == 0) { Tensor tmp_gwmat = gwmat[gid]; - Assign(tmp_gwmat, req[conv_v1::kWeight], dot(temp_dst[gid], tmpc.T())); + // Legacy approach shown here for comparison: + // Assign(tmp_gwmat, req[conv_v1::kWeight], dot(temp_dst[gid], tmpc.T())); + linalg_gemm(temp_dst[gid], tmpc, tmp_gwmat, false, true, s, req[conv_v1::kWeight]); } else { - gwmat[gid] += dot(temp_dst[gid], tmpc.T()); + // Legacy approach shown here for comparison: + // gwmat[gid] += dot(temp_dst[gid], tmpc.T()); + linalg_gemm(temp_dst[gid], tmpc, gwmat[gid], false, true, s, kAddTo); } } for (uint32_t gid = 0; gid < param_.num_group; ++gid) { Tensor tmpc = temp_col.Slice(gstride * gid, gstride * (gid + 1)); - tmpc = dot(wmat[gid].T(), temp_dst[gid]); + // Legacy approach shown here for comparison: + // tmpc = dot(wmat[gid].T(), temp_dst[gid]); + linalg_gemm(wmat[gid], temp_dst[gid], tmpc, true, false, s); } if (param_.pad[0] == 0 && param_.pad[1] == 0) { Assign(gdata.Slice(i, i + step), req[conv_v1::kData], diff --git a/src/operator/deconvolution-inl.h b/src/operator/deconvolution-inl.h index 9db94a8c5986..dd77c150c970 100644 --- a/src/operator/deconvolution-inl.h +++ b/src/operator/deconvolution-inl.h @@ -34,6 +34,7 @@ #include #include #include "./operator_common.h" +#include "./linalg.h" namespace mxnet { @@ -227,7 +228,9 @@ class DeconvolutionOp : public Operator { for (uint32_t gid = 0; gid < param_.num_group; ++gid) { mshadow::Tensor tmpc = temp_col.Slice(gstride * gid, gstride * (gid + 1)); - tmpc = dot(wmat[gid].T(), temp_dst[gid]); + // Legacy approach shown here for comparison: + // tmpc = dot(wmat[gid].T(), temp_dst[gid]); + linalg_gemm(wmat[gid], temp_dst[gid], tmpc, true, false, s); } if (o_pad[0] == 0 && o_pad[1] == 0) { out.Slice(i, i + step) = pack_col2patch(temp_col, @@ -335,16 +338,23 @@ class DeconvolutionOp : public Operator { Tensor tmpc = temp_col.Slice(gstride * gid, gstride * (gid + 1)); if (i == 0) { Tensor tmp_gwmat = gwmat[gid]; - Assign(tmp_gwmat, req[deconv::kWeight], dot(temp_dst[gid], tmpc.T())); + // Legacy approach shown here for comparison: + // Assign(tmp_gwmat, req[deconv::kWeight], dot(temp_dst[gid], tmpc.T())); + linalg_gemm(temp_dst[gid], tmpc, tmp_gwmat, false, true, s, req[deconv::kWeight]); } else { - gwmat[gid] += dot(temp_dst[gid], tmpc.T()); + // Legacy approach shown here for comparison: + // gwmat[gid] += dot(temp_dst[gid], tmpc.T()); + linalg_gemm(temp_dst[gid], tmpc, gwmat[gid], false, true, s, kAddTo); } } - if (req[deconv::kData] == kWriteTo || req[deconv::kData] == kWriteInplace - || req[deconv::kData] == kAddTo) { + if (req[deconv::kData] == kWriteTo || + req[deconv::kData] == kWriteInplace || + req[deconv::kData] == kAddTo) { for (uint32_t gid = 0; gid < param_.num_group; ++gid) { Tensor tmpc = temp_col.Slice(gstride * gid, gstride * (gid + 1)); - temp_dst[gid] = dot(wmat[gid], tmpc); + // Legacy approach shown here for comparison: + // temp_dst[gid] = dot(wmat[gid], tmpc); + linalg_gemm(wmat[gid], tmpc, temp_dst[gid], false, false, s); } Assign(gdata.Slice(i, i + step), req[deconv::kData], diff --git a/src/operator/fully_connected-inl.h b/src/operator/fully_connected-inl.h index 7120b5672f60..c507e4251f3e 100644 --- a/src/operator/fully_connected-inl.h +++ b/src/operator/fully_connected-inl.h @@ -33,9 +33,7 @@ #include #include "./operator_common.h" #include "./elemwise_op_common.h" -#if (MSHADOW_USE_CBLAS != 0) #include "linalg.h" -#endif namespace mxnet { namespace op { @@ -110,12 +108,9 @@ class FullyConnectedOp : public Operator { Shape2(oshape[0], oshape.ProdShape(1, oshape.ndim())), s); } -#if (MSHADOW_USE_CBLAS == 0) - // Legacy approach for amalgamation build w/out cblas - out = dot(data, wmat.T()); -#else + // Legacy approach shown here for comparison: + // out = dot(data, wmat.T()); linalg_gemm(data, wmat, out, false, true, s); -#endif if (!param_.no_bias) { Tensor bias = in_data[fullc::kBias].get(s); out += repmat(bias, data.size(0)); @@ -167,24 +162,18 @@ class FullyConnectedOp : public Operator { CHECK_NE(req[fullc::kWeight], kWriteInplace) << "cannot write weight inplace"; // gradient of weight Tensor gwmat = in_grad[fullc::kWeight].get(s); -#if (MSHADOW_USE_CBLAS == 0) - // Legacy approach for amalgamation build w/out cblas - Assign(gwmat, req[fullc::kWeight], dot(grad.T(), data)); -#else + // Legacy approach shown here for comparison: + // out = Assign(gwmat, req[fullc::kWeight], dot(grad.T(), data)); linalg_gemm(grad, data, gwmat, true, false, s, req[fullc::kWeight]); -#endif // gradient of bias if (!param_.no_bias) { Tensor gbias = in_grad[fullc::kBias].get(s); Assign(gbias, req[fullc::kBias], sum_rows(grad)); } // gradient of data -#if (MSHADOW_USE_CBLAS == 0) - // Legacy approach for amalgamation build w/out cblas - Assign(gdata, req[fullc::kData], dot(grad, wmat)); -#else + // Legacy approach shown here for comparison: + // Assign(gdata, req[fullc::kData], dot(grad, wmat)); linalg_gemm(grad, wmat, gdata, false, false, s, req[fullc::kData]); -#endif } private: diff --git a/src/operator/grid_generator-inl.h b/src/operator/grid_generator-inl.h index 65fb8ccf2e07..0be6e7806bce 100644 --- a/src/operator/grid_generator-inl.h +++ b/src/operator/grid_generator-inl.h @@ -35,6 +35,7 @@ #include #include "./mshadow_op.h" #include "./operator_common.h" +#include "./linalg.h" namespace mxnet { namespace op { @@ -101,7 +102,9 @@ class GridGeneratorOp : public Operator { grid_dst[1] = scalar(-1.0) + tcast(tcast(grid_dst[1] / scalar(param_.target_shape[1]))) * scalar(2.0/(param_.target_shape[0] - 1)); grid_dst[2] = scalar(1.0); - Assign(out, req[grid::kOut], dot(data, grid_dst)); + // Legacy approach shown here for comparison: + // Assign(out, req[grid::kOut], dot(data, grid_dst)); + linalg_gemm(data, grid_dst, out, false, false, s, req[grid::kOut]); break; } // Warping transformation @@ -150,8 +153,10 @@ class GridGeneratorOp : public Operator { param_.target_shape[0] * param_.target_shape[1]); Tensor grad = out_grad[grid::kOut] .get_with_shape(grad_shape, s); + // Legacy approach shown here for comparison: + // Assign(gdata, req[grid::kData], dot(grad, grid_dst.T())); // grad : (batch * 2, H * W) grid_dst.T : (H * W, 3) - Assign(gdata, req[grid::kData] , dot(grad, grid_dst.T())); + linalg_gemm(grad, grid_dst, gdata, false, true, s, req[grid::kData]); break; } case grid::kWarp: { diff --git a/src/operator/linalg_impl.h b/src/operator/linalg_impl.h index 1e3b0e66e641..c1e813614c72 100644 --- a/src/operator/linalg_impl.h +++ b/src/operator/linalg_impl.h @@ -56,6 +56,8 @@ inline void check_gemm(const Tensor& A, const Tensor inline \ void linalg_gemm(const Tensor& A, const Tensor& B, \ @@ -69,6 +71,17 @@ void linalg_gemm(const Tensor& A, const Tensor for DType=mshadow::half::half_t. +template<> inline +void linalg_gemm(const Tensor& A, + const Tensor& B, + const Tensor& C, + mshadow::half::half_t alpha, + mshadow::half::half_t beta, + bool tA, bool tB, Stream *s) { + LOG(FATAL) << "FP16 gemm on cpu not implemented!"; +} + #define LINALG_CPU_BATCH_GEMM(DType) \ template<> inline \ void linalg_batch_gemm(const Tensor& A, const Tensor& B, \ @@ -82,6 +95,8 @@ void linalg_batch_gemm(const Tensor& A, const Tensor< LINALG_CPU_BATCH_GEMM(float) LINALG_CPU_BATCH_GEMM(double) +#endif // (MSHADOW_USE_CBLAS != 0) + #ifdef __CUDACC__ template @@ -198,7 +213,7 @@ void linalg_batch_gemm(const Tensor& A, const Tensor< LINALG_GPU_BATCH_GEMM(SgemmBatched, float) LINALG_GPU_BATCH_GEMM(DgemmBatched, double) -#endif +#endif // __CUDACC__ //////////////////////////////// TRSM //////////////////////////////////////////// @@ -218,6 +233,8 @@ inline void check_trsm(const Tensor& A, const Tensor inline \ void linalg_trsm(const Tensor& A, const Tensor& B, \ @@ -243,6 +260,8 @@ void linalg_batch_trsm(const Tensor& A, const Tensor< LINALG_CPU_BATCH_TRSM(float) LINALG_CPU_BATCH_TRSM(double) +#endif // (MSHADOW_USE_CBLAS != 0) + #ifdef __CUDACC__ // cublas col-major processing accounted for by switching sides and fill mode @@ -297,7 +316,7 @@ void linalg_batch_trsm(const Tensor& A, const Tensor< LINALG_GPU_BATCH_TRSM(StrsmBatched, float) LINALG_GPU_BATCH_TRSM(DtrsmBatched, double) -#endif +#endif // __CUDACC__ /*! * \brief Performs gemm, setting alpha and beta as appropriate for `req`. @@ -332,6 +351,31 @@ inline void linalg_gemm(const Tensor& A, } } +// A cpu specialization for linalg_gemm that uses mshadow::dot(), if no cblas. +#if (MSHADOW_USE_CBLAS == 0) +template +inline void linalg_gemm(const Tensor& A, + const Tensor& B, + const Tensor& C, + bool tA, bool tB, Stream *s, + mxnet::OpReqType req) { + using namespace mxnet; + switch (req) { + case kNullOp: + break; + case kWriteTo: + case kWriteInplace: + C = dot(tA ? A.T() : A, tB ? B.T() : B); + break; + case kAddTo: + C += dot(tA ? A.T() : A, tB ? B.T() : B); + break; + default: + LOG(FATAL) << "not reached"; + } +} +#endif + //////////////////////////////// TRMM //////////////////////////////////////////// // CPU/GPU-versions of BLAS3 function "trmm". Please refer to the BLAS3-documentation @@ -350,6 +394,8 @@ inline void check_trmm(const Tensor& A, const Tensor inline \ void linalg_trmm(const Tensor& A, const Tensor& B, \ @@ -375,6 +421,8 @@ void linalg_batch_trmm(const Tensor& A, const Tensor< LINALG_XPU_BATCH_TRMM(cpu, float) LINALG_XPU_BATCH_TRMM(cpu, double) +#endif // (MSHADOW_USE_CBLAS != 0) + #ifdef __CUDACC__ // cublas col-major processing accounted for by switching sides and fill mode @@ -401,7 +449,7 @@ LINALG_GPU_TRMM(Dtrmm, double) LINALG_XPU_BATCH_TRMM(gpu, float) LINALG_XPU_BATCH_TRMM(gpu, double) -#endif +#endif // __CUDACC__ //////////////////////////////// POTRF //////////////////////////////////////////// @@ -437,7 +485,7 @@ void linalg_batch_potrf(const Tensor& A, bool lower, LINALG_CPU_BATCH_POTRF(float) LINALG_CPU_BATCH_POTRF(double) -#if MXNET_USE_CUSOLVER == 1 +#if defined(__CUDACC__) && MXNET_USE_CUSOLVER == 1 #define LINALG_GPU_BUFFSIZE_POTRF(fname, DType) \ inline int linalg_potrf_buffsize(const Tensor& A, bool lower, Stream *s) { \ diff --git a/src/operator/spatial_transformer-inl.h b/src/operator/spatial_transformer-inl.h index 77967579340f..e29ad49c4aa6 100644 --- a/src/operator/spatial_transformer-inl.h +++ b/src/operator/spatial_transformer-inl.h @@ -35,6 +35,7 @@ #include #include #include "./operator_common.h" +#include "./linalg.h" namespace mxnet { @@ -100,7 +101,9 @@ class SpatialTransformerOp : public Operator { Copy(grid_dst, workspace, grid_dst.stream_); for (index_t batch = 0; batch < data.size(0); batch++) { if (param_.transform_type == st::kAffine) { - grid_src[batch] = dot(loc[batch], grid_dst); + // Legacy approach shown here for comparison: + // grid_src[batch] = dot(loc[batch], grid_dst); + linalg_gemm(loc[batch], grid_dst, grid_src[batch], false, false, s); } } if (param_.sampler_type == st::kBilinear) { @@ -133,7 +136,9 @@ class SpatialTransformerOp : public Operator { } for (index_t batch = 0; batch < data.size(0); batch++) { if (param_.transform_type == st::kAffine) { - gloc[batch] = dot(grid_src[batch], grid_dst.T()); + // Legacy approach shown here for comparison: + // gloc[batch] = dot(grid_src[batch], grid_dst.T()); + linalg_gemm(grid_src[batch], grid_dst, gloc[batch], false, true, s); } } } From 942e88865f0938db1c6284264d92a109f3f5830e Mon Sep 17 00:00:00 2001 From: Hu Shiwen Date: Sat, 26 Aug 2017 00:59:05 +0800 Subject: [PATCH 426/834] fix linalg_impl (#7611) * fix linalg_impl * fix * fix * fix --- Jenkinsfile | 11 ++++++++++- src/operator/linalg_impl.h | 38 +++++++++++++++++++++++++++++++------- 2 files changed, 41 insertions(+), 8 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 2d4cc017c865..2dfc57c9a265 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -150,7 +150,7 @@ try { } } }, - 'Amalgamation': { + 'Amalgamation MIN': { node('mxnetlinux') { ws('workspace/amalgamation') { init_git() @@ -159,6 +159,15 @@ try { } } }, + 'Amalgamation': { + node('mxnetlinux') { + ws('workspace/amalgamation') { + init_git() + make('cpu', '-C amalgamation/ clean') + make('cpu', '-C amalgamation/ USE_BLAS=openblas') + } + } + }, 'GPU: MKLML': { node('mxnetlinux') { ws('workspace/build-mklml') { diff --git a/src/operator/linalg_impl.h b/src/operator/linalg_impl.h index c1e813614c72..e95eff0cc407 100644 --- a/src/operator/linalg_impl.h +++ b/src/operator/linalg_impl.h @@ -324,9 +324,9 @@ LINALG_GPU_BATCH_TRSM(DtrsmBatched, double) * \param A the first operand of the gemm * \param B the second operand of the gemm * \param C the data to be assigned - * \tparam tA whether the `A` operand should be transposed first. - * \tparam tB whether the `B` operand should be transposed first. - * \tparam s the stream to perform the operation + * \param tA whether the `A` operand should be transposed first. + * \param tB whether the `B` operand should be transposed first. + * \param s the stream to perform the operation * \param req the assignment request */ template @@ -353,8 +353,8 @@ inline void linalg_gemm(const Tensor& A, // A cpu specialization for linalg_gemm that uses mshadow::dot(), if no cblas. #if (MSHADOW_USE_CBLAS == 0) -template -inline void linalg_gemm(const Tensor& A, +template +inline void linalg_gemm(const Tensor& A, const Tensor& B, const Tensor& C, bool tA, bool tB, Stream *s, @@ -365,10 +365,34 @@ inline void linalg_gemm(const Tensor& A, break; case kWriteTo: case kWriteInplace: - C = dot(tA ? A.T() : A, tB ? B.T() : B); + if (tA) { + if (tB) { + const_cast&>(C) = dot(A.T(), B.T()); + } else { + const_cast&>(C) = dot(A.T(), B); + } + } else { + if (tB) { + const_cast&>(C) = dot(A, B.T()); + } else { + const_cast&>(C) = dot(A, B); + } + } break; case kAddTo: - C += dot(tA ? A.T() : A, tB ? B.T() : B); + if (tA) { + if (tB) { + const_cast&>(C) += dot(A.T(), B.T()); + } else { + const_cast&>(C) += dot(A.T(), B); + } + } else { + if (tB) { + const_cast&>(C) += dot(A, B.T()); + } else { + const_cast&>(C) += dot(A, B); + } + } break; default: LOG(FATAL) << "not reached"; From 3b3d824320acc41b8a2242683f9155d0fe67a8ca Mon Sep 17 00:00:00 2001 From: Rahul Huilgol Date: Fri, 25 Aug 2017 20:20:55 -0700 Subject: [PATCH 427/834] set build status to success only after job ends (#7628) Earlier code marks status as success initially. So any new PR shows jenkins status as success if we see the check mark on github. On opening the full build status, we see that builds haven't even started or are running. If something fails, variable changes to failure then. So even without this merge, a red mark on github indicates that build has failed correctly. That behavior is unchanged. --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 2dfc57c9a265..ac34e71a53f1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -10,8 +10,6 @@ docker_run = 'tests/ci_build/ci_build.sh' max_time = 60 // assign any caught errors here err = null -// set build status to success by default -currentBuild.result = "SUCCESS" // initialize source codes def init_git() { @@ -438,6 +436,8 @@ try { } } } + // set build status to success at the end + currentBuild.result = "SUCCESS" } catch (caughtError) { node("mxnetlinux") { sh "echo caught error" From c42453dfb4cd4baf6a0fe12ccc34cda62ea40df9 Mon Sep 17 00:00:00 2001 From: Rahul Huilgol Date: Fri, 25 Aug 2017 20:21:51 -0700 Subject: [PATCH 428/834] Fix build status of a test (#7629) installs bc required by sh2ju.sh and changes the regex match to capital alphabet as it clashes with a warning thrown by opencv driver --- tests/nightly/test_image_classification.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/nightly/test_image_classification.sh b/tests/nightly/test_image_classification.sh index 93e403a2affc..7ab443dc044c 100755 --- a/tests/nightly/test_image_classification.sh +++ b/tests/nightly/test_image_classification.sh @@ -21,6 +21,8 @@ # setup export LD_LIBRARY_PATH=`pwd`/`dirname $0`/lib:/usr/local/cuda/lib64:$LD_LIBRARY_PATH export PYTHONPATH=`pwd`/`dirname $0`/python +# bc is required by sh2ju.sh +apt-get install bc cd `pwd`/`dirname $0` . sh2ju.sh @@ -59,8 +61,8 @@ juLog -name=Build -error=Error build # check if the final evaluation accuracy exceed the threshold check_val() { expected=$1 - pass="Final validation >= $expected, Pass" - fail="Final validation < $expected, Fail" + pass="Final validation >= $expected, PASS" + fail="Final validation < $expected, FAIL" python ../../tools/parse_log.py log --format none | tail -n1 | \ awk "{ if (\$3~/^[.0-9]+$/ && \$3 > $expected) print \"$pass\"; else print \"$fail\"}" rm -f log @@ -88,6 +90,6 @@ test_lenet() { check_val $desired_accuracy done } -juLog -name=Python.Lenet.Mnist -error=Fail test_lenet +juLog -name=Python.Lenet.Mnist -error=FAIL test_lenet exit $errors From 50342a432327e4de8cf65d5935c4c80073b35a6a Mon Sep 17 00:00:00 2001 From: Dick Carter Date: Fri, 25 Aug 2017 21:27:34 -0700 Subject: [PATCH 429/834] entire codebase build with mshadow_use_clas=0 (#7625) --- src/operator/linalg_impl.h | 199 +++++++++++++++++++++++-------------- 1 file changed, 127 insertions(+), 72 deletions(-) diff --git a/src/operator/linalg_impl.h b/src/operator/linalg_impl.h index e95eff0cc407..27378db201bd 100644 --- a/src/operator/linalg_impl.h +++ b/src/operator/linalg_impl.h @@ -56,7 +56,7 @@ inline void check_gemm(const Tensor& A, const Tensor inline \ @@ -68,19 +68,6 @@ void linalg_gemm(const Tensor& A, const Tensor for DType=mshadow::half::half_t. -template<> inline -void linalg_gemm(const Tensor& A, - const Tensor& B, - const Tensor& C, - mshadow::half::half_t alpha, - mshadow::half::half_t beta, - bool tA, bool tB, Stream *s) { - LOG(FATAL) << "FP16 gemm on cpu not implemented!"; -} #define LINALG_CPU_BATCH_GEMM(DType) \ template<> inline \ @@ -92,10 +79,43 @@ void linalg_batch_gemm(const Tensor& A, const Tensor< linalg_gemm(A[i], B[i], C[i], alpha, beta, tA, tB); \ } \ } + +#else + +#define LINALG_CPU_GEMM(fname, DType) \ +template<> inline \ +void linalg_gemm(const Tensor& A, const Tensor& B, \ + const Tensor& C, DType alpha, DType beta, \ + bool tA, bool tB, Stream *s) { \ + LOG(FATAL) << "linalg_gemm (without req arg) not implemented by mxnet for cpu, needs cblas!"; \ +} + +#define LINALG_CPU_BATCH_GEMM(DType) \ +template<> inline \ +void linalg_batch_gemm(const Tensor& A, const Tensor& B, \ + const Tensor& C, DType alpha, DType beta, \ + bool tA, bool tB, Stream *s) { \ + LOG(FATAL) << "linalg_batch_gemm not implemented by mxnet for cpu, needs cblas!"; \ +} + +#endif // MSHADOW_USE_CBLAS == 1 + +LINALG_CPU_GEMM(sgemm, float) +LINALG_CPU_GEMM(dgemm, double) + LINALG_CPU_BATCH_GEMM(float) LINALG_CPU_BATCH_GEMM(double) -#endif // (MSHADOW_USE_CBLAS != 0) +// Specialization of linalg_gemm for DType=mshadow::half::half_t. +template<> inline +void linalg_gemm(const Tensor& A, + const Tensor& B, + const Tensor& C, + mshadow::half::half_t alpha, + mshadow::half::half_t beta, + bool tA, bool tB, Stream *s) { + LOG(FATAL) << "FP16 gemm on cpu not implemented!"; +} #ifdef __CUDACC__ @@ -233,7 +253,7 @@ inline void check_trsm(const Tensor& A, const Tensor inline \ @@ -245,8 +265,6 @@ void linalg_trsm(const Tensor& A, const Tensor inline \ @@ -257,11 +275,31 @@ void linalg_batch_trsm(const Tensor& A, const Tensor< linalg_trsm(A[i], B[i], alpha, rightside, lower, transpose); \ } \ } + +#else + +#define LINALG_CPU_TRSM(fname, DType) \ +template<> inline \ +void linalg_trsm(const Tensor& A, const Tensor& B, \ + DType alpha, bool rightside, bool lower, bool transpose, Stream *s) { \ + LOG(FATAL) << "linalg_trsm not implemented, needs cblas!"; \ +} + +#define LINALG_CPU_BATCH_TRSM(DType) \ +template<> inline \ +void linalg_batch_trsm(const Tensor& A, const Tensor& B, \ + DType alpha, bool rightside, bool lower, bool transpose, Stream *s) { \ + LOG(FATAL) << "linalg_batch_trsm not implemented, needs cblas!"; \ +} + +#endif // MSHADOW_USE_CBLAS == 1 + +LINALG_CPU_TRSM(strsm, float) +LINALG_CPU_TRSM(dtrsm, double) + LINALG_CPU_BATCH_TRSM(float) LINALG_CPU_BATCH_TRSM(double) -#endif // (MSHADOW_USE_CBLAS != 0) - #ifdef __CUDACC__ // cublas col-major processing accounted for by switching sides and fill mode @@ -351,54 +389,60 @@ inline void linalg_gemm(const Tensor& A, } } -// A cpu specialization for linalg_gemm that uses mshadow::dot(), if no cblas. -#if (MSHADOW_USE_CBLAS == 0) -template -inline void linalg_gemm(const Tensor& A, - const Tensor& B, - const Tensor& C, - bool tA, bool tB, Stream *s, - mxnet::OpReqType req) { - using namespace mxnet; - switch (req) { - case kNullOp: - break; - case kWriteTo: - case kWriteInplace: - if (tA) { - if (tB) { - const_cast&>(C) = dot(A.T(), B.T()); - } else { - const_cast&>(C) = dot(A.T(), B); - } - } else { - if (tB) { - const_cast&>(C) = dot(A, B.T()); - } else { - const_cast&>(C) = dot(A, B); - } - } - break; - case kAddTo: - if (tA) { - if (tB) { - const_cast&>(C) += dot(A.T(), B.T()); - } else { - const_cast&>(C) += dot(A.T(), B); - } - } else { - if (tB) { - const_cast&>(C) += dot(A, B.T()); - } else { - const_cast&>(C) += dot(A, B); - } - } - break; - default: - LOG(FATAL) << "not reached"; - } +#if MSHADOW_USE_CBLAS == 0 + +// A template for a cpu linalg_gemm implementation using mshadow::dot() +#define LINALG_CPU_GEMM_NO_CBLAS(DType) \ +template<> inline \ +void linalg_gemm(const Tensor& A, \ + const Tensor& B, \ + const Tensor& C, \ + bool tA, bool tB, Stream *s, \ + mxnet::OpReqType req) { \ + using namespace mxnet; \ + switch (req) { \ + case kNullOp: \ + break; \ + case kWriteTo: \ + case kWriteInplace: \ + if (tA) { \ + if (tB) { \ + const_cast&>(C) = dot(A.T(), B.T()); \ + } else { \ + const_cast&>(C) = dot(A.T(), B); \ + } \ + } else { \ + if (tB) { \ + const_cast&>(C) = dot(A, B.T()); \ + } else { \ + const_cast&>(C) = dot(A, B); \ + } \ + } \ + break; \ + case kAddTo: \ + if (tA) { \ + if (tB) { \ + const_cast&>(C) += dot(A.T(), B.T()); \ + } else { \ + const_cast&>(C) += dot(A.T(), B); \ + } \ + } else { \ + if (tB) { \ + const_cast&>(C) += dot(A, B.T()); \ + } else { \ + const_cast&>(C) += dot(A, B); \ + } \ + } \ + break; \ + default: \ + LOG(FATAL) << "not reached"; \ + } \ } -#endif + +LINALG_CPU_GEMM_NO_CBLAS(float) +LINALG_CPU_GEMM_NO_CBLAS(double) + +#endif // (MSHADOW_USE_CBLAS == 0) //////////////////////////////// TRMM //////////////////////////////////////////// @@ -418,7 +462,7 @@ inline void check_trmm(const Tensor& A, const Tensor inline \ @@ -430,8 +474,17 @@ void linalg_trmm(const Tensor& A, const Tensor inline \ +void linalg_trmm(const Tensor& A, const Tensor& B, \ + DType alpha, bool rightside, bool lower, bool transpose, Stream *s) { \ + LOG(FATAL) << "linalg_trmm not implemented, needs cblas!"; \ +} + +#endif // MSHADOW_USE_CBLAS == 1 #define LINALG_XPU_BATCH_TRMM(xpu, DType) \ template<> inline \ @@ -442,11 +495,13 @@ void linalg_batch_trmm(const Tensor& A, const Tensor< linalg_trmm(A[i], B[i], alpha, rightside, lower, transpose, s); \ } \ } + +LINALG_CPU_TRMM(strmm, float) +LINALG_CPU_TRMM(dtrmm, double) + LINALG_XPU_BATCH_TRMM(cpu, float) LINALG_XPU_BATCH_TRMM(cpu, double) -#endif // (MSHADOW_USE_CBLAS != 0) - #ifdef __CUDACC__ // cublas col-major processing accounted for by switching sides and fill mode From b7fcd090e940b923c62d24fee571b0fd4c5418cd Mon Sep 17 00:00:00 2001 From: Kai Li <1196594711@qq.com> Date: Sat, 26 Aug 2017 15:22:09 +0800 Subject: [PATCH 430/834] Update README.md (#7630) --- example/image-classification/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/image-classification/README.md b/example/image-classification/README.md index 3f514e2a391f..1c72a1d78d9f 100644 --- a/example/image-classification/README.md +++ b/example/image-classification/README.md @@ -128,7 +128,7 @@ to calculate the accuracy. | `imagenet1k-resnet-152` | 0.7653 | 0.9312 | | `imagenet1k-resnext-50` | 0.7689 | 0.9332 | | `imagenet1k-resnext-101` | 0.7828 | 0.9408 | -| `imagenet1k-rexnext-101-64x4d` | 0.7911 | 0.9430 | +| `imagenet1k-resnext-101-64x4d` | 0.7911 | 0.9430 | Note: - our Resnet does not need to specify the RGB mean due the data batch From 2e6ef8c88006a8c80dd5f7be3cc4c6ed56cbcaae Mon Sep 17 00:00:00 2001 From: Haibin Lin Date: Sat, 26 Aug 2017 11:37:10 -0700 Subject: [PATCH 431/834] unit test for csv iter, doc update for libsvmiter (#7623) * add unit test for csv iter * fix lint * add libsvm to mxnet.io doc * update libsvm doc --- docs/api/python/io.md | 1 + python/mxnet/io.py | 3 +- src/io/inst_vector.h | 2 +- src/io/iter_libsvm.cc | 90 +++++++++++++++++--------------- tests/python/unittest/test_io.py | 21 ++++++++ 5 files changed, 73 insertions(+), 44 deletions(-) diff --git a/docs/api/python/io.md b/docs/api/python/io.md index 15f8aa3ce354..ce8245b73fe8 100644 --- a/docs/api/python/io.md +++ b/docs/api/python/io.md @@ -56,6 +56,7 @@ A detailed tutorial is available at io.NDArrayIter io.CSVIter + io.LibSVMIter io.ImageRecordIter io.ImageRecordUInt8Iter io.MNISTIter diff --git a/python/mxnet/io.py b/python/mxnet/io.py index 314a2b28dfa8..b1696815274a 100644 --- a/python/mxnet/io.py +++ b/python/mxnet/io.py @@ -189,6 +189,7 @@ class DataIter(object): -------- NDArrayIter : Data-iterator for MXNet NDArray or numpy-ndarray objects. CSVIter : Data-iterator for csv data. + LibSVMIter : Data-iterator for libsvm data. ImageIter : Data-iterator for images. """ def __init__(self, batch_size=0): @@ -721,7 +722,7 @@ class MXDataIter(DataIter): """A python wrapper a C++ data iterator. This iterator is the Python wrapper to all native C++ data iterators, such - as `CSVIter, `ImageRecordIter`, `MNISTIter`, etc. When initializing + as `CSVIter`, `ImageRecordIter`, `MNISTIter`, etc. When initializing `CSVIter` for example, you will get an `MXDataIter` instance to use in your Python code. Calls to `next`, `reset`, etc will be delegated to the underlying C++ data iterators. diff --git a/src/io/inst_vector.h b/src/io/inst_vector.h index 6dc7bdfd730a..afa19e277653 100644 --- a/src/io/inst_vector.h +++ b/src/io/inst_vector.h @@ -169,7 +169,7 @@ struct TBlobBatch { } /*! \brief destructor */ ~TBlobBatch() { - delete inst_index; + delete[] inst_index; } }; // struct TBlobBatch diff --git a/src/io/iter_libsvm.cc b/src/io/iter_libsvm.cc index 803d19e74481..8e53e6f28712 100644 --- a/src/io/iter_libsvm.cc +++ b/src/io/iter_libsvm.cc @@ -198,19 +198,21 @@ class LibSVMIter: public SparseIIterator { DMLC_REGISTER_PARAMETER(LibSVMIterParam); MXNET_REGISTER_IO_ITER(LibSVMIter) -.describe(R"code(Returns the LibSVM file iterator. This iterator is experimental and -should be used with care. +.describe(R"code(Returns the libsvm file iterator which returns sparse data with `csr` +storage type. This iterator is experimental and should be used with care. -The input data is similar to libsvm file format, except that the indices are expected to be -zero-based instead of one-based. Details of the libsvm format are available at -`https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/` +The input data is stored in a format similar to libsvm file format, except that the indices +are expected to be zero-based instead of one-based. Details of the libsvm format are available +at `https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/` In this function, the `data_shape` parameter is used to set the shape of each line of the data. The dimension of both `data_shape` and `label_shape` are expected to be 1. When `label_libsvm` is set to ``NULL``, both data and label are read from the same file specified -by `data_libsvm`. Otherwise, data is read from `data_libsvm` and label from `label_libsvm`, -in this case, if `data_libsvm` contains label, it will ignored. +by `data_libsvm`. In this case, the data is stored in `csr` storage type, while the label is a 1D +dense array. Otherwise, data is read from `data_libsvm` and label from `label_libsvm`, +in this case, both data and label are stored in csr storage type. If `data_libsvm` contains label, +it will ignored. The `LibSVMIter` only support `round_batch` parameter set to ``True`` for now. So, if `batch_size` is 3 and there are 4 total rows in libsvm file, 2 more examples @@ -221,58 +223,62 @@ If ``data_libsvm = 'data/'`` is set, then all the files in this directory will b Examples:: - // Contents of libsvm file ``data.t``. + # Contents of libsvm file ``data.t``. 1.0 0:0.5 2:1.2 -2.0 -3.0 0:0.6 1:2.4 2:1.2 4 2:-1.2 - // Creates a `LibSVMIter` with `batch_size`=3. - LibSVMIter = mx.io.LibSVMIter(data_libsvm = 'data.t', data_shape = (3,), - batch_size = 3) - - // The first batch (data and label) - [[ 0.5 0. 1.2 ] - [ 0. 0. 0. ] - [ 0.6 2.4 1.2 ]] - + # Creates a `LibSVMIter` with `batch_size`=3. + >>> data_iter = mx.io.LibSVMIter(data_libsvm = 'data.t', data_shape = (3,), batch_size = 3) + # The data of the first batch is stored in csr storage type + >>> batch = data_iter.next() + >>> csr = batch.data[0] + + >>> csr.asnumpy() + [[ 0.5 0. 1.2 ] + [ 0. 0. 0. ] + [ 0.6 2.4 1.2]] + # The label of first batch + >>> label = batch.label[0] + >>> label [ 1. -2. -3.] + - // The second batch (data and label) + >>> second_batch = data_iter.next() + # The data of the second batch + >>> second_batch.data[0].asnumpy() [[ 0. 0. -1.2 ] [ 0.5 0. 1.2 ] [ 0. 0. 0. ]] - + # The label of the second batch + >>> second_batch.label[0].asnumpy() [ 4. 1. -2.] - // Contents of libsvm file ``label.t`` + # Contents of libsvm file ``label.t`` 1.0 -2.0 0:0.125 -3.0 2:1.2 4 1:1.0 2:-1.2 - // Creates a `LibSVMIter` with specified label file - LibSVMIter = mx.io.LibSVMIter(data_libsvm = 'data.t', data_shape = (3,), - label_libsvm = 'label.t', label_shape = (3,), batch_size = 3) - - // Two batches of data read from the above iterator are as follows(data and label): - // The first batch - [[ 0.5 0. 1.2 ] - [ 0. 0. 0. ] - [ 0.6 2.4 1.2 ]] - - [[ 0. 0. 0. ] - [ 0.125 0. 0. ] - [ 0. 0. 1.2 ]] - - // The second batch - [[ 0. 0. -1.2 ] - [ 0.5 0. 1.2 ] - [ 0. 0. 0. ]] - - [[ 0. 1. -1.2 ] - [ 0. 0. 0. ] - [ 0.125 0. 0. ]] + # Creates a `LibSVMIter` with specified label file + >>> data_iter = mx.io.LibSVMIter(data_libsvm = 'data.t', data_shape = (3,), + label_libsvm = 'label.t', label_shape = (3,), batch_size = 3) + + # Both data and label are in csr storage type + >>> batch = data_iter.next() + >>> csr_data = batch.data[0] + + >>> csr_data.asnumpy() + [[ 0.5 0. 1.2 ] + [ 0. 0. 0. ] + [ 0.6 2.4 1.2 ]] + >>> csr_label = batch.label[0] + + >>> csr_label.asnumpy() + [[ 0. 0. 0. ] + [ 0.125 0. 0. ] + [ 0. 0. 1.2 ]] )code" ADD_FILELINE) .add_arguments(LibSVMIterParam::__FIELDS__()) diff --git a/tests/python/unittest/test_io.py b/tests/python/unittest/test_io.py index a543463f3663..fb8aa2aa20be 100644 --- a/tests/python/unittest/test_io.py +++ b/tests/python/unittest/test_io.py @@ -257,6 +257,26 @@ def check_libSVMIter_news_data(): check_libSVMIter_synthetic() check_libSVMIter_news_data() +def test_CSVIter(): + def check_CSVIter_synthetic(): + cwd = os.getcwd() + data_path = os.path.join(cwd, 'data.t') + label_path = os.path.join(cwd, 'label.t') + with open(data_path, 'w') as fout: + for i in range(1000): + fout.write(','.join(['1' for _ in range(8*8)]) + '\n') + with open(label_path, 'w') as fout: + for i in range(1000): + fout.write('0\n') + + data_train = mx.io.CSVIter(data_csv=data_path, data_shape=(8,8), + label_csv=label_path, batch_size=100) + expected = mx.nd.ones((100, 8, 8)) + for batch in iter(data_train): + assert_almost_equal(data_train.getdata().asnumpy(), expected.asnumpy()) + + check_CSVIter_synthetic() + if __name__ == "__main__": test_NDArrayIter() if h5py: @@ -265,3 +285,4 @@ def check_libSVMIter_news_data(): test_Cifar10Rec() test_LibSVMIter() test_NDArrayIter_csr() + test_CSVIter() From 1e48e1238609b94cae2af3d0d72b33882b9c5a24 Mon Sep 17 00:00:00 2001 From: dtmoodie Date: Sat, 26 Aug 2017 14:51:34 -0400 Subject: [PATCH 432/834] gpu access of ndarray (#7496) * gpu access of ndarray * gpu access from C++ api * gpu access fix * Update c_api.cc * Update c_api.cc --- cpp-package/include/mxnet-cpp/ndarray.hpp | 1 - src/c_api/c_api.cc | 8 +------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/cpp-package/include/mxnet-cpp/ndarray.hpp b/cpp-package/include/mxnet-cpp/ndarray.hpp index 5ed04a547b85..6bf26432359b 100644 --- a/cpp-package/include/mxnet-cpp/ndarray.hpp +++ b/cpp-package/include/mxnet-cpp/ndarray.hpp @@ -359,7 +359,6 @@ inline int NDArray::GetDType() const { inline const mx_float *NDArray::GetData() const { void *ret; - CHECK_NE(GetContext().GetDeviceType(), DeviceType::kGPU); MXNDArrayGetData(blob_ptr_->handle_, &ret); if (GetDType() != 0) { return NULL; diff --git a/src/c_api/c_api.cc b/src/c_api/c_api.cc index 0fe3fe3e302e..088e208c9cdc 100644 --- a/src/c_api/c_api.cc +++ b/src/c_api/c_api.cc @@ -437,13 +437,7 @@ int MXNDArrayGetData(NDArrayHandle handle, API_BEGIN(); NDArray *arr = static_cast(handle); if (!arr->is_none()) { - CHECK(arr->ctx().dev_mask() == cpu::kDevMask) - << "MXNDArrayGetData can only be called for NDArray on CPU"; - const TBlob &b = arr->data(); - CHECK(b.CheckContiguous()); - MSHADOW_REAL_TYPE_SWITCH(arr->dtype(), DType, { - *out_pdata = b.FlatTo2D().dptr_; - }); + *out_pdata = arr->data().dptr_; } else { *out_pdata = nullptr; } From cb36058dfc19aed092356db26ea4de4676c86f5e Mon Sep 17 00:00:00 2001 From: Eric Junyuan Xie Date: Sat, 26 Aug 2017 17:51:30 -0700 Subject: [PATCH 433/834] refactor cudnn algo reg to no use string (#7561) * refactor cudnn algo reg to no use string * refactor ctx list * fix * refactor save_inputs --- dmlc-core | 2 +- mshadow | 2 +- nnvm | 2 +- python/mxnet/gluon/parameter.py | 59 ++++++------ python/mxnet/metric.py | 8 +- src/c_api/c_api_common.h | 2 + src/c_api/c_api_function.cc | 4 +- src/c_api/c_api_ndarray.cc | 56 +++++++---- src/io/inst_vector.h | 10 +- src/io/iter_mnist.cc | 2 +- src/ndarray/autograd.cc | 111 ++++++++++++---------- src/ndarray/autograd.h | 30 +++--- src/ndarray/ndarray.cc | 6 +- src/operator/contrib/fft-inl.h | 6 +- src/operator/contrib/fft.cc | 10 +- src/operator/contrib/fft.cu | 11 ++- src/operator/contrib/ifft-inl.h | 7 +- src/operator/contrib/ifft.cc | 10 +- src/operator/contrib/ifft.cu | 11 ++- src/operator/contrib/multi_proposal-inl.h | 105 ++------------------ src/operator/contrib/multi_proposal.cu | 6 +- src/operator/contrib/proposal-inl.h | 105 ++------------------ src/operator/contrib/proposal.cc | 6 +- src/operator/contrib/proposal.cu | 6 +- src/operator/convolution-inl.h | 41 ++++++++ src/operator/cudnn_algoreg-inl.h | 90 ++++++++++++------ src/operator/cudnn_algoreg.cc | 13 ++- src/operator/cudnn_convolution-inl.h | 16 ++-- src/operator/cudnn_deconvolution-inl.h | 17 ++-- src/operator/deconvolution-inl.h | 45 +++++++++ 30 files changed, 386 insertions(+), 413 deletions(-) diff --git a/dmlc-core b/dmlc-core index 71bfbd3a9460..e880afeb932d 160000 --- a/dmlc-core +++ b/dmlc-core @@ -1 +1 @@ -Subproject commit 71bfbd3a946075cea66ca9e19bad86dd33c19b46 +Subproject commit e880afeb932d746e55eb92e8c6eb3ff1b3697c48 diff --git a/mshadow b/mshadow index 6d75df228978..380f825b84e2 160000 --- a/mshadow +++ b/mshadow @@ -1 +1 @@ -Subproject commit 6d75df228978ca5f182dd707578ef704099ab5ee +Subproject commit 380f825b84e28216516377e71199a8e14f12352f diff --git a/nnvm b/nnvm index bcfbf903429d..e842c098decf 160000 --- a/nnvm +++ b/nnvm @@ -1 +1 @@ -Subproject commit bcfbf903429d086f16b19b4d202788de06e45536 +Subproject commit e842c098decf9f5eb6bd84e307c58e50078596b7 diff --git a/python/mxnet/gluon/parameter.py b/python/mxnet/gluon/parameter.py index bef55d67e140..4bc2611a70a7 100644 --- a/python/mxnet/gluon/parameter.py +++ b/python/mxnet/gluon/parameter.py @@ -129,14 +129,22 @@ def grad_req(self, req): elif self._data is not None: self._init_grad() - def _check_initialized(self, ctx=None): - if self._data is not None: - if ctx is not None and ctx not in self._data: - raise RuntimeError( - "Parameter %s was not initialized on context %s. " - "It was only initialized on %s."%( - self.name, str(ctx), str(self.list_ctx()))) - return + def _check_and_get(self, arr_dict, ctx): + if arr_dict is not None: + if ctx is list: + return list(arr_dict.values()) + if ctx is None: + if len(self._ctx_list) == 1: + ctx = self._ctx_list[0] + else: + ctx = context.current_context() + ret = arr_dict.get(ctx, None) + if ret is not None: + return ret + raise RuntimeError( + "Parameter %s was not initialized on context %s. " + "It was only initialized on %s."%( + self.name, str(ctx), str(self._ctx_list))) if self._deferred_init: raise DeferredInitializationError raise RuntimeError( @@ -199,6 +207,7 @@ def _finish_deferred_init(self): def _init_impl(self, data, ctx): """Sets data and grad.""" self._data = OrderedDict() + self._ctx_list = list(ctx) for i in ctx: self._data[i] = data.copyto(i) self._init_grad() @@ -327,20 +336,12 @@ def data(self, ctx=None): ------- NDArray on ctx """ - if ctx is None: - list_ctx = self.list_ctx() - if len(list_ctx) == 1: - ctx = list_ctx[0] - else: - ctx = context.current_context() - self._check_initialized(ctx) - return self._data[ctx] + return self._check_and_get(self._data, ctx) def list_data(self): """Returns copies of this parameter on all contexts, in the same order as creation.""" - self._check_initialized() - return list(self._data.values()) + return self._check_and_get(self._data, list) def grad(self, ctx=None): """Returns a gradient buffer for this parameter on one context. @@ -350,26 +351,20 @@ def grad(self, ctx=None): ctx : Context Desired context. """ - if ctx is None: - list_ctx = self.list_ctx() - if len(list_ctx) == 1: - ctx = list_ctx[0] - else: - ctx = context.current_context() - self._check_initialized(ctx) - if self._grad is None: + if self._data is not None and self._grad is None: raise RuntimeError( "Cannot get gradient array for Parameter %s " \ "because grad_req='null'"%(self.name)) - return self._grad[ctx] + return self._check_and_get(self._grad, ctx) def list_grad(self): """Returns gradient buffers on all contexts, in the same order as `values`.""" - self._check_initialized() - assert self._grad is not None, \ - "Parameter %s does not have gradients because grad_req='null'"%self.name - return list(self._grad.values()) + if self._data is not None and self._grad is None: + raise RuntimeError( + "Cannot get gradient array for Parameter %s " \ + "because grad_req='null'"%(self.name)) + return self._check_and_get(self._grad, list) def list_ctx(self): """Returns a list of contexts this parameter is initialized on.""" @@ -377,7 +372,7 @@ def list_ctx(self): if self._deferred_init: return self._deferred_init[1] raise RuntimeError("Parameter %s has not been initialized"%self.name) - return list(self._data.keys()) + return self._ctx_list def zero_grad(self): """Sets gradient buffer on all contexts to 0. No action is taken if diff --git a/python/mxnet/metric.py b/python/mxnet/metric.py index 00cc2da61f3c..a33b00ae8ab3 100644 --- a/python/mxnet/metric.py +++ b/python/mxnet/metric.py @@ -390,13 +390,13 @@ def update(self, labels, preds): for label, pred_label in zip(labels, preds): if pred_label.shape != label.shape: pred_label = ndarray.argmax(pred_label, axis=self.axis) - pred_label = pred_label.asnumpy().astype('int32') - label = label.asnumpy().astype('int32') + label = label.astype('int32') + pred_label = pred_label.astype('int32').as_in_context(label.context) check_label_shapes(label, pred_label) - self.sum_metric += (pred_label.flat == label.flat).sum() - self.num_inst += len(pred_label.flat) + self.sum_metric += ndarray.sum(label == pred_label).asscalar() + self.num_inst += label.size @register diff --git a/src/c_api/c_api_common.h b/src/c_api/c_api_common.h index fee3f03f6db0..1ef385609239 100644 --- a/src/c_api/c_api_common.h +++ b/src/c_api/c_api_common.h @@ -84,6 +84,8 @@ struct MXAPIThreadLocalEntry { std::vector arg_shape_data, out_shape_data, aux_shape_data; /*! \brief uint32_t buffer for returning shape pointer */ std::vector arg_shape_buffer, out_shape_buffer, aux_shape_buffer; + /*! \brief bool buffer */ + std::vector save_inputs, save_outputs; // helper function to setup return value of shape array inline static void SetupShapeArrayReturnWithBuffer( const std::vector &shapes, diff --git a/src/c_api/c_api_function.cc b/src/c_api/c_api_function.cc index 3d8b5328c1a0..259c1331c7af 100644 --- a/src/c_api/c_api_function.cc +++ b/src/c_api/c_api_function.cc @@ -188,8 +188,8 @@ int MXCustomFunctionRecord(int num_inputs, NDArrayHandle *inputs, attrs.parsed = params; // TODO(piiswrong): remove state by using FComputeEx auto state = OpStatePtr::Create(params); - AutogradRuntime::Get()->RecordImperativeOperator( - state, attrs.op, attrs, &ndinputs, &ndoutputs); + AutogradRuntime::Get()->RecordOp( + std::move(attrs), &ndinputs, &ndoutputs, state); for (size_t i = 0; i < ndoutputs.size(); ++i) { *reinterpret_cast(outputs[i]) = ndoutputs[i]; diff --git a/src/c_api/c_api_ndarray.cc b/src/c_api/c_api_ndarray.cc index d392baf45d3e..64fa74d8b8c3 100644 --- a/src/c_api/c_api_ndarray.cc +++ b/src/c_api/c_api_ndarray.cc @@ -484,9 +484,11 @@ void PushOperator(const OpStatePtr& state, } void ImperativeInvokeImpl(const Context& default_ctx, - const nnvm::NodeAttrs& attrs, + nnvm::NodeAttrs&& attrs, std::vector* p_ndinputs, - std::vector* p_ndoutputs) { + std::vector* p_ndoutputs, + std::vector* p_save_inputs = nullptr, + std::vector* p_save_outputs = nullptr) { static auto& ndfunc = nnvm::Op::GetAttr("FNDArrayFunction"); static auto& createop = nnvm::Op::GetAttr("FCreateOpState"); MXAPIThreadLocalEntry *ret = MXAPIThreadLocalStore::Get(); @@ -514,29 +516,32 @@ void ImperativeInvokeImpl(const Context& default_ctx, FCompute fn = common::GetFCompute(op, "FCompute", ctx); FComputeEx fn_ex = common::GetFCompute(op, "FComputeEx", ctx); if (fn_ex && stype != kDefaultStorage) { - if (AutogradRuntime::Get()->IsRecording()) { - AutogradRuntime::Get()->RecordImperativeFCompute(op, - attrs, &ndinputs, &ndoutputs); - } PushFComputeEx(fn_ex, op, attrs, ctx, read_vars, write_vars, requested, ndinputs, ndoutputs); - } else if (fn) { if (AutogradRuntime::Get()->IsRecording()) { - AutogradRuntime::Get()->RecordImperativeFCompute(op, - attrs, &ndinputs, &ndoutputs); + AutogradRuntime::Get()->RecordOp( + std::move(attrs), &ndinputs, &ndoutputs, OpStatePtr(), + p_save_inputs, p_save_outputs); } + } else if (fn) { PushFCompute(fn, op, attrs, ctx, read_vars, write_vars, requested, ndinputs, ndoutputs, mutate_idx); + if (AutogradRuntime::Get()->IsRecording()) { + AutogradRuntime::Get()->RecordOp( + std::move(attrs), &ndinputs, &ndoutputs, OpStatePtr(), + p_save_inputs, p_save_outputs); + } } else if (createop.count(op)) { auto state = createop[op](attrs, ctx, ret->arg_shapes, ret->arg_types); - if (AutogradRuntime::Get()->IsRecording()) { - AutogradRuntime::Get()->RecordImperativeOperator(state, op, - attrs, &ndinputs, &ndoutputs); - } write_vars.push_back(state.get_var()); PushOperator(state, op, attrs, ctx, read_vars, write_vars, requested, ndinputs, ndoutputs, mutate_idx); + if (AutogradRuntime::Get()->IsRecording()) { + AutogradRuntime::Get()->RecordOp( + std::move(attrs), &ndinputs, &ndoutputs, state, + p_save_inputs, p_save_outputs); + } } else { LOG(FATAL) << "Operator " << op->name << " is not implemented for " @@ -569,7 +574,7 @@ int MXImperativeInvoke(AtomicSymbolCreator creator, SetNDInputsOutputs(op, &ndinputs, &ndoutputs, num_inputs, inputs, num_outputs, infered_num_outputs, num_visible_outputs, outarray); - ImperativeInvokeImpl(Context::CPU(), attrs, &ndinputs, &ndoutputs); + ImperativeInvokeImpl(Context::CPU(), std::move(attrs), &ndinputs, &ndoutputs); if (outarray == nullptr) { ret->ret_handles.clear(); @@ -618,6 +623,20 @@ int MXCreateCachedOp(SymbolHandle handle, auto vars = sym->ListInputs(nnvm::Symbol::kAll); CHECK_GE(vars.size(), 1) << "CachedOp must have at least 1 input."; g->attrs["vars"] = std::make_shared(std::move(vars)); + + const nnvm::IndexedGraph& idx = g->indexed_graph(); + std::vector > save_inputs(idx.num_nodes()); + std::vector > save_outputs(idx.num_nodes()); + for (size_t i = 0; i < idx.num_nodes(); ++i) { + nnvm::NodePtr node = nnvm::Node::Create(); + node->attrs = idx[i].source->attrs; + AutogradRuntime::Get()->GetBackwardDependency( + node, idx[i].source->num_inputs(), idx[i].source->num_outputs(), + &save_inputs[i], &save_outputs[i]); + } + g->attrs["save_inputs"] = std::make_shared(std::move(save_inputs)); + g->attrs["save_outputs"] = std::make_shared(std::move(save_outputs)); + *out = g; API_END(); } @@ -640,7 +659,11 @@ int MXInvokeCachedOp(CachedOpHandle handle, API_BEGIN(); const std::vector& vars = - g->GetAttr >("vars"); + g->GetAttr >("vars"); + std::vector > save_inputs = + g->GetAttr > >("save_inputs"); + std::vector > save_outputs = + g->GetAttr > >("save_outputs"); const nnvm::IndexedGraph& idx = g->indexed_graph(); CHECK_EQ(static_cast(num_inputs), vars.size()) << "Actually number of inputs differs from expected number of inputs"; @@ -661,7 +684,8 @@ int MXInvokeCachedOp(CachedOpHandle handle, in.emplace_back(buff[idx.entry_id(j)]); } std::vector out(node.source->num_outputs()); - ImperativeInvokeImpl(default_ctx, node.source->attrs, &in, &out); + ImperativeInvokeImpl(default_ctx, nnvm::NodeAttrs(node.source->attrs), &in, &out, + &save_inputs[i], &save_outputs[i]); for (size_t j = 0; j < node.source->num_outputs(); ++j) { buff[idx.entry_id(i, j)] = std::move(out[j]); diff --git a/src/io/inst_vector.h b/src/io/inst_vector.h index afa19e277653..7c4e894be924 100644 --- a/src/io/inst_vector.h +++ b/src/io/inst_vector.h @@ -30,7 +30,6 @@ #include #include #include -#include #include #include @@ -173,16 +172,16 @@ struct TBlobBatch { } }; // struct TBlobBatch -class TBlobContainer : public mshadow::TBlob { +class TBlobContainer : public TBlob { public: TBlobContainer(void) - : mshadow::TBlob(), tensor_container_(nullptr) {} + : TBlob(), tensor_container_(nullptr) {} ~TBlobContainer() { if (tensor_container_) { release(); } } - void resize(const mshadow::TShape &shape, int type_flag) { + void resize(const TShape &shape, int type_flag) { if (tensor_container_) { CHECK_EQ(this->type_flag_, type_flag); this->shape_ = shape; @@ -192,13 +191,12 @@ class TBlobContainer : public mshadow::TBlob { this->shape_ = shape; create(); } - this->stride_ = shape_[shape_.ndim() - 1]; } private: void create() { CHECK(tensor_container_ == nullptr); - CHECK_EQ(this->dev_mask_, mshadow::cpu::kDevMask); + CHECK_EQ(this->dev_mask(), mshadow::cpu::kDevMask); MSHADOW_TYPE_SWITCH(this->type_flag_, DType, { auto tensor_container = new mshadow::TensorContainer(false); tensor_container->Resize(mshadow::Shape1(shape_.Size())); diff --git a/src/io/iter_mnist.cc b/src/io/iter_mnist.cc index 055af52aaebd..9dbedbbba448 100644 --- a/src/io/iter_mnist.cc +++ b/src/io/iter_mnist.cc @@ -103,7 +103,7 @@ class MNISTIter: public IIterator { out_.batch_size = param_.batch_size; if (param_.shuffle) this->Shuffle(); if (param_.silent == 0) { - mshadow::TShape s; + TShape s; s = batch_data_.shape_; if (param_.flat) { LOG(INFO) << "MNISTIter: load " << (unsigned)img_.size(0) << " images, shuffle=" diff --git a/src/ndarray/autograd.cc b/src/ndarray/autograd.cc index 5ecea5decf03..421113f6edd7 100644 --- a/src/ndarray/autograd.cc +++ b/src/ndarray/autograd.cc @@ -29,6 +29,7 @@ #include #include "../executor/graph_executor.h" #include "./autograd.h" +#include "../c_api/c_api_common.h" namespace mxnet { namespace autograd { @@ -101,21 +102,6 @@ void AutogradRuntime::MarkVariables( } } -void AutogradRuntime::RecordImperativeFCompute(const nnvm::Op* op, - const nnvm::NodeAttrs& attrs, - std::vector *p_inputs, - std::vector *p_outputs) { - RecordOp(op, attrs, p_inputs, p_outputs, OpStatePtr()); -} - -void AutogradRuntime::RecordImperativeOperator(const OpStatePtr& state, - const nnvm::Op* op, - const nnvm::NodeAttrs& attrs, - std::vector *p_inputs, - std::vector *p_outputs) { - RecordOp(op, attrs, p_inputs, p_outputs, state); -} - std::shared_ptr AutogradRuntime::_GetSharedRef() { static std::shared_ptr inst(new AutogradRuntime()); return inst; @@ -126,12 +112,58 @@ AutogradRuntime* AutogradRuntime::Get() { return ptr; } -void AutogradRuntime::RecordOp(const nnvm::Op* op, - const nnvm::NodeAttrs& attrs, - std::vector *p_inputs, - std::vector *p_outputs, - const OpStatePtr& state) { +void AutogradRuntime::GetBackwardDependency(const nnvm::NodePtr& node, + uint32_t num_inputs, uint32_t num_outputs, + std::vector *p_save_inputs, + std::vector *p_save_outputs) { static auto& fgradient = nnvm::Op::GetAttr("FGradient"); + std::vector& save_inputs = *p_save_inputs; + std::vector& save_outputs = *p_save_outputs; + save_inputs.resize(num_inputs); + save_outputs.resize(num_outputs); + std::fill(save_inputs.begin(), save_inputs.end(), false); + std::fill(save_outputs.begin(), save_outputs.end(), false); + + node->inputs.clear(); + node->inputs.reserve(num_inputs); + for (uint32_t i = 0; i < num_inputs; ++i) { + node->inputs.emplace_back(NodeEntry{nullptr, i, 0}); + } + + if (fgradient.count(node->op())) { + std::vector ograd_entries; + ograd_entries.reserve(num_outputs); + for (uint32_t i = 0; i < num_outputs; ++i) { + ograd_entries.emplace_back(NodeEntry{nullptr, i, 1}); + } + auto igrad_entries = fgradient[node->op()](node, ograd_entries); + for (const auto& i : igrad_entries) { + if (i.node == nullptr && i.version == 0) { + save_inputs[i.index] = true; + } else if (i.node == node) { + save_outputs[i.index] = true; + } + } + DFSVisit(igrad_entries, [&](const NodePtr& gnode) { + if (!gnode || gnode == node) return; + for (const auto& i : gnode->inputs) { + if (i.node == nullptr && i.version == 0) { + save_inputs[i.index] = true; + } else if (i.node == node) { + save_outputs[i.index] = true; + } + } + }); + } +} + +void AutogradRuntime::RecordOp(nnvm::NodeAttrs&& attrs, + std::vector *p_inputs, + std::vector *p_outputs, + const OpStatePtr& state, + std::vector* p_save_inputs, + std::vector* p_save_outputs) { + MXAPIThreadLocalEntry *local_buff = MXAPIThreadLocalStore::Get(); std::vector& inputs = *p_inputs; std::vector& outputs = *p_outputs; @@ -144,7 +176,6 @@ void AutogradRuntime::RecordOp(const nnvm::Op* op, << "Please call backward first to clear the graph or do this out side of " << "a record section. "; } - if (!fgradient.count(attrs.op)) return; bool need_grad = false; for (const auto& i : inputs) { if (!i.entry_.is_none()) { @@ -155,36 +186,20 @@ void AutogradRuntime::RecordOp(const nnvm::Op* op, if (!need_grad) return; NodePtr nn_node = Node::Create(); - nn_node->attrs = attrs; + nn_node->attrs = std::move(attrs); nn_node->attrs.name = "node_" + std::to_string(node_count_++); - // Get backward dependency - std::vector save_inputs(inputs.size()), save_outputs(outputs.size()); - for (uint32_t i = 0; i < inputs.size(); ++i) { - nn_node->inputs.emplace_back(NodeEntry{nullptr, i, 0}); + if (p_save_inputs == nullptr) { + p_save_inputs = &(local_buff->save_inputs); + p_save_outputs = &(local_buff->save_outputs); + GetBackwardDependency( + nn_node, inputs.size(), outputs.size(), p_save_inputs, p_save_outputs); + } else { + nn_node->inputs.resize(inputs.size()); } - std::vector ograd_entries; - for (uint32_t i = 0; i < outputs.size(); ++i) { - ograd_entries.emplace_back(NodeEntry{nullptr, i, 1}); - } - auto igrad_entries = fgradient[nn_node->op()](nn_node, ograd_entries); - for (const auto& i : igrad_entries) { - if (i.node == nullptr && i.version == 0) { - save_inputs[i.index] = true; - } else if (i.node == nn_node) { - save_outputs[i.index] = true; - } - } - DFSVisit(igrad_entries, [&](const NodePtr& node) { - if (!node || node == nn_node) return; - for (const auto& i : node->inputs) { - if (i.node == nullptr && i.version == 0) { - save_inputs[i.index] = true; - } else if (i.node == nn_node) { - save_outputs[i.index] = true; - } - } - }); + + std::vector& save_inputs = *p_save_inputs; + std::vector& save_outputs = *p_save_outputs; AGNodePtr ag_node = AGNode::Create(nn_node); ag_node->state = state; diff --git a/src/ndarray/autograd.h b/src/ndarray/autograd.h index 199af350bf93..4632bc00ebf5 100644 --- a/src/ndarray/autograd.h +++ b/src/ndarray/autograd.h @@ -95,17 +95,19 @@ class AutogradRuntime { void MarkVariables(const std::vector& variables, const std::vector& grad_reqs, const std::vector& gradients); - /*! \brief record imperative operator which is executed by fcompute. */ - void RecordImperativeFCompute(const nnvm::Op* op, - const nnvm::NodeAttrs& attrs, - std::vector* p_inputs, - std::vector* p_outputs); - /*! \brief record imperative operator which is executed by operator. */ - void RecordImperativeOperator(const OpStatePtr& state, - const nnvm::Op* op, - const nnvm::NodeAttrs& attrs, - std::vector* p_inputs, - std::vector* p_outputs); + /*! \brief find the input/output ndarrays that are needed for backward */ + void GetBackwardDependency( + const nnvm::NodePtr& node, + uint32_t num_inputs, uint32_t num_outputs, + std::vector *p_save_inputs, + std::vector *p_save_outputs); + /*! \brief to record operator, return corresponding node. */ + void RecordOp(nnvm::NodeAttrs&& attrs, + std::vector* p_inputs, + std::vector* p_outputs, + const OpStatePtr& state = OpStatePtr(), + std::vector* p_save_inputs = nullptr, + std::vector* p_save_outputs = nullptr); /*! \brief compute the gradient of outputs w.r.t variables. */ void ComputeGradient(const std::vector& outputs, const std::vector& ograds, @@ -126,12 +128,6 @@ class AutogradRuntime { AutogradRuntime(); private: - /*! \brief to record operator, return corresponding node. */ - void RecordOp(const nnvm::Op* op, - const nnvm::NodeAttrs& attrs, - std::vector* p_inputs, - std::vector* p_outputs, - const OpStatePtr& state); /*! \brief AutogradRuntime singleton. */ static AutogradRuntime* instance_; /*! \brief indicate whether is training. */ diff --git a/src/ndarray/ndarray.cc b/src/ndarray/ndarray.cc index 139d97670bec..7b79d1051135 100644 --- a/src/ndarray/ndarray.cc +++ b/src/ndarray/ndarray.cc @@ -75,8 +75,7 @@ NDArray NDArray::Reshape(const TShape &shape) const { std::vector inputs, outputs; inputs.emplace_back(*this); outputs.emplace_back(std::move(ret)); - AutogradRuntime::Get()->RecordImperativeFCompute( - op, attrs, &inputs, &outputs); + AutogradRuntime::Get()->RecordOp(std::move(attrs), &inputs, &outputs); return outputs[0]; } else { CHECK_GE(shape_.Size(), shape.Size()) @@ -115,8 +114,7 @@ NDArray NDArray::Slice(index_t begin, index_t end) const { std::vector inputs, outputs; inputs.emplace_back(*this); outputs.emplace_back(std::move(ret)); - AutogradRuntime::Get()->RecordImperativeFCompute( - op, attrs, &inputs, &outputs); + AutogradRuntime::Get()->RecordOp(std::move(attrs), &inputs, &outputs); return outputs[0]; } else { return ret; diff --git a/src/operator/contrib/fft-inl.h b/src/operator/contrib/fft-inl.h index 5092f586fdf7..12474f183e84 100644 --- a/src/operator/contrib/fft-inl.h +++ b/src/operator/contrib/fft-inl.h @@ -54,6 +54,7 @@ struct FFTParam : public dmlc::Parameter { } }; +#if MXNET_USE_CUDA template class FFTOp : public Operator { public: @@ -102,7 +103,6 @@ class FFTOp : public Operator { Shape1(param_.compute_size*dim_*2), s); Tensor complex_data = Tensor(workspace.dptr_, Shape2(param_.compute_size, dim_*2), s); - #if MSHADOW_USE_CUDNN // start fft cufftHandle plan; cufftPlanMany(&plan, 1, &dim_, nullptr, 0, 0, nullptr, 0, 0, CUFFT_C2C, param_.compute_size); @@ -135,7 +135,6 @@ class FFTOp : public Operator { CHECK_EQ(cufftExecC2C(plan_remain, in_tmp, out_tmp, CUFFT_FORWARD), CUFFT_SUCCESS); cufftDestroy(plan_remain); } - #endif } virtual void Backward(const OpContext &ctx, @@ -170,7 +169,6 @@ class FFTOp : public Operator { // In this solution, out_grad must comes from a fft of real signal, // so that it is Hermitian symmetric, giving a real output // but if it is not, remember that we have implemented complex_take_real, and use this - #if MSHADOW_USE_CUDNN cufftHandle plan; cufftPlanMany(&plan, 1, &dim_, nullptr, 0, 0, nullptr, 0, 0, CUFFT_C2C, param_.compute_size); for (size_t idx = 0; idx < num_compute; ++idx) { @@ -203,7 +201,6 @@ class FFTOp : public Operator { req[fft::kData], complex_toreal(complex_data)); cufftDestroy(plan_remain); } - #endif // for bp, we should not divide it // but for comparison with np.fft.ifft, we should do it. // gdata /= dim_; @@ -214,6 +211,7 @@ class FFTOp : public Operator { int dim_, stride_, num_compute, n_ffts; bool init_cufft_; }; // class FFTOp +#endif // MXNET_USE_CUDA // Declare Factory Function, used for dispatch specialization template diff --git a/src/operator/contrib/fft.cc b/src/operator/contrib/fft.cc index 11f8425e07b1..6f78003baebb 100644 --- a/src/operator/contrib/fft.cc +++ b/src/operator/contrib/fft.cc @@ -28,17 +28,13 @@ namespace mxnet { namespace op { template<> Operator *CreateOp(FFTParam param, int dtype) { - LOG(FATAL) << "fft is only available for GPU."; - return NULL; + LOG(FATAL) << "fft is only available for GPU."; + return NULL; } Operator *FFTProp::CreateOperatorEx(Context ctx, std::vector *in_shape, std::vector *in_type) const { - std::vector out_shape, aux_shape; - std::vector out_type, aux_type; - CHECK(InferType(in_type, &out_type, &aux_type)); - CHECK(InferShape(in_shape, &out_shape, &aux_shape)); - DO_BIND_DISPATCH(CreateOp, param_, (*in_type)[0]); + DO_BIND_DISPATCH(CreateOp, param_, (*in_type)[0]); } DMLC_REGISTER_PARAMETER(FFTParam); diff --git a/src/operator/contrib/fft.cu b/src/operator/contrib/fft.cu index 3017ce76756b..dfe3fbba6124 100644 --- a/src/operator/contrib/fft.cu +++ b/src/operator/contrib/fft.cu @@ -29,11 +29,12 @@ namespace op { template<> Operator* CreateOp(FFTParam param, int dtype) { - Operator *op = NULL; - MSHADOW_REAL_TYPE_SWITCH(dtype, DType, { - op = new FFTOp(param); - }) - return op; + Operator *op = NULL; + MSHADOW_REAL_TYPE_SWITCH(dtype, DType, { + op = new FFTOp(param); + }) + return op; } + } // namespace op } // namespace mxnet diff --git a/src/operator/contrib/ifft-inl.h b/src/operator/contrib/ifft-inl.h index abd5bb22a389..5e89c5b644ce 100644 --- a/src/operator/contrib/ifft-inl.h +++ b/src/operator/contrib/ifft-inl.h @@ -54,6 +54,7 @@ struct IFFTParam : public dmlc::Parameter { } }; +#if MXNET_USE_CUDA template class IFFTOp : public Operator { public: @@ -98,7 +99,6 @@ class IFFTOp : public Operator { Shape1(param_.compute_size*dim_*2), s); Tensor complex_data = Tensor(workspace.dptr_, Shape2(param_.compute_size, dim_*2), s); - #if MSHADOW_USE_CUDNN // start ifft cufftHandle plan; cufftPlanMany(&plan, 1, &dim_, nullptr, 0, 0, nullptr, 0, 0, CUFFT_C2C, param_.compute_size); @@ -131,7 +131,6 @@ class IFFTOp : public Operator { req[ifft::kOut], complex_toreal(complex_data)); cufftDestroy(plan_remain); } - #endif // commenting this out to be consistant with caffe // out /= dim_; } @@ -162,7 +161,6 @@ class IFFTOp : public Operator { Shape1(param_.compute_size*dim_*2), s); Tensor complex_data = Tensor(workspace.dptr_, Shape2(param_.compute_size, dim_*2), s); - #if MSHADOW_USE_CUDNN // start fft cufftHandle plan; cufftPlanMany(&plan, 1, &dim_, nullptr, 0, 0, nullptr, 0, 0, CUFFT_C2C, param_.compute_size); @@ -194,7 +192,6 @@ class IFFTOp : public Operator { CHECK_EQ(cufftExecC2C(plan_remain, in_tmp, out_tmp, CUFFT_FORWARD), CUFFT_SUCCESS); cufftDestroy(plan_remain); } - #endif // commenting this out to be consistant with caffe // gdata /= dim_; } @@ -205,6 +202,8 @@ class IFFTOp : public Operator { bool init_cufft_; }; // class IFFTOp +#endif // MXNET_USE_CUDA + // Declare Factory Function, used for dispatch specialization template Operator* CreateOp(IFFTParam param, int dtype); diff --git a/src/operator/contrib/ifft.cc b/src/operator/contrib/ifft.cc index 0ea3a7ec112f..95c79a785a16 100644 --- a/src/operator/contrib/ifft.cc +++ b/src/operator/contrib/ifft.cc @@ -29,17 +29,13 @@ namespace op { template<> Operator *CreateOp(IFFTParam param, int dtype) { - LOG(FATAL) << "ifft is only available for GPU."; - return NULL; + LOG(FATAL) << "ifft is only available for GPU."; + return NULL; } Operator *IFFTProp::CreateOperatorEx(Context ctx, std::vector *in_shape, std::vector *in_type) const { - std::vector out_shape, aux_shape; - std::vector out_type, aux_type; - CHECK(InferType(in_type, &out_type, &aux_type)); - CHECK(InferShape(in_shape, &out_shape, &aux_shape)); - DO_BIND_DISPATCH(CreateOp, param_, (*in_type)[0]); + DO_BIND_DISPATCH(CreateOp, param_, (*in_type)[0]); } DMLC_REGISTER_PARAMETER(IFFTParam); diff --git a/src/operator/contrib/ifft.cu b/src/operator/contrib/ifft.cu index 79795d8561bf..35cdb4836b37 100644 --- a/src/operator/contrib/ifft.cu +++ b/src/operator/contrib/ifft.cu @@ -29,11 +29,12 @@ namespace op { template<> Operator* CreateOp(IFFTParam param, int dtype) { - Operator *op = NULL; - MSHADOW_REAL_TYPE_SWITCH(dtype, DType, { - op = new IFFTOp(param); - }) - return op; + Operator *op = NULL; + MSHADOW_REAL_TYPE_SWITCH(dtype, DType, { + op = new IFFTOp(param); + }) + return op; } + } // namespace op } // namespace mxnet diff --git a/src/operator/contrib/multi_proposal-inl.h b/src/operator/contrib/multi_proposal-inl.h index 7cd465e0b09e..ddfe0628f306 100644 --- a/src/operator/contrib/multi_proposal-inl.h +++ b/src/operator/contrib/multi_proposal-inl.h @@ -40,95 +40,6 @@ #include "../operator_common.h" #include "../mshadow_op.h" -// extend NumericalParam -namespace mxnet { -namespace op { - -/*! -* \brief structure for numerical tuple input -* \tparam VType data type of param -*/ -template -struct NumericalParam { - NumericalParam() {} - explicit NumericalParam(VType *begin, VType *end) { - int32_t size = static_cast(end - begin); - info.resize(size); - for (int i = 0; i < size; ++i) { - info[i] = *(begin + i); - } - } - inline size_t ndim() const { - return info.size(); - } - std::vector info; -}; - -template -inline std::istream &operator>>(std::istream &is, NumericalParam ¶m) { - while (true) { - char ch = is.get(); - if (ch == '(') break; - if (!isspace(ch)) { - is.setstate(std::ios::failbit); - return is; - } - } - VType idx; - std::vector tmp; - // deal with empty case - size_t pos = is.tellg(); - char ch = is.get(); - if (ch == ')') { - param.info = tmp; - return is; - } - is.seekg(pos); - // finish deal - while (is >> idx) { - tmp.push_back(idx); - char ch; - do { - ch = is.get(); - } while (isspace(ch)); - if (ch == ',') { - while (true) { - ch = is.peek(); - if (isspace(ch)) { - is.get(); continue; - } - if (ch == ')') { - is.get(); break; - } - break; - } - if (ch == ')') break; - } else if (ch == ')') { - break; - } else { - is.setstate(std::ios::failbit); - return is; - } - } - param.info = tmp; - return is; -} - -template -inline std::ostream &operator<<(std::ostream &os, const NumericalParam ¶m) { - os << '('; - for (index_t i = 0; i < param.info.size(); ++i) { - if (i != 0) os << ','; - os << param.info[i]; - } - // python style tuple - if (param.info.size() == 1) os << ','; - os << ')'; - return os; -} - -} // namespace op -} // namespace mxnet namespace mxnet { namespace op { @@ -144,8 +55,8 @@ struct MultiProposalParam : public dmlc::Parameter { int rpn_post_nms_top_n; float threshold; int rpn_min_size; - NumericalParam scales; - NumericalParam ratios; + nnvm::Tuple scales; + nnvm::Tuple ratios; int feature_stride; bool output_score; bool iou_loss; @@ -161,10 +72,10 @@ struct MultiProposalParam : public dmlc::Parameter { DMLC_DECLARE_FIELD(rpn_min_size).set_default(16) .describe("Minimum height or width in proposal"); tmp[0] = 4.0f; tmp[1] = 8.0f; tmp[2] = 16.0f; tmp[3] = 32.0f; - DMLC_DECLARE_FIELD(scales).set_default(NumericalParam(tmp, tmp + 4)) + DMLC_DECLARE_FIELD(scales).set_default(nnvm::Tuple(tmp, tmp + 4)) .describe("Used to generate anchor windows by enumerating scales"); tmp[0] = 0.5f; tmp[1] = 1.0f; tmp[2] = 2.0f; - DMLC_DECLARE_FIELD(ratios).set_default(NumericalParam(tmp, tmp + 3)) + DMLC_DECLARE_FIELD(ratios).set_default(nnvm::Tuple(tmp, tmp + 3)) .describe("Used to generate anchor windows by enumerating ratios"); DMLC_DECLARE_FIELD(feature_stride).set_default(16) .describe("The size of the receptive field each unit in the convolution layer of the rpn," @@ -302,11 +213,11 @@ inline void _Transform(float scale, // out_anchors must have shape (n, 5), where n is ratios.size() * scales.size() inline void GenerateAnchors(const std::vector& base_anchor, - const std::vector& ratios, - const std::vector& scales, + const nnvm::Tuple& ratios, + const nnvm::Tuple& scales, std::vector *out_anchors) { - for (size_t j = 0; j < ratios.size(); ++j) { - for (size_t k = 0; k < scales.size(); ++k) { + for (size_t j = 0; j < ratios.ndim(); ++j) { + for (size_t k = 0; k < scales.ndim(); ++k) { _Transform(scales[k], ratios[j], base_anchor, out_anchors); } } diff --git a/src/operator/contrib/multi_proposal.cu b/src/operator/contrib/multi_proposal.cu index cb9996344e3e..082de6a397a7 100644 --- a/src/operator/contrib/multi_proposal.cu +++ b/src/operator/contrib/multi_proposal.cu @@ -460,11 +460,11 @@ class MultiProposalGPUOp : public Operator{ base_anchor[1] = 0.0; base_anchor[2] = param_.feature_stride - 1.0; base_anchor[3] = param_.feature_stride - 1.0; - CHECK_EQ(num_anchors, param_.ratios.info.size() * param_.scales.info.size()); + CHECK_EQ(num_anchors, param_.ratios.ndim() * param_.scales.ndim()); std::vector anchors; utils::GenerateAnchors(base_anchor, - param_.ratios.info, - param_.scales.info, + param_.ratios, + param_.scales, &anchors); // Copy generated anchors to GPU diff --git a/src/operator/contrib/proposal-inl.h b/src/operator/contrib/proposal-inl.h index 3d1851cedbac..f989cdec3767 100644 --- a/src/operator/contrib/proposal-inl.h +++ b/src/operator/contrib/proposal-inl.h @@ -38,95 +38,6 @@ #include "../operator_common.h" #include "../mshadow_op.h" -// extend NumericalParam -namespace mxnet { -namespace op { - -/*! -* \brief structure for numerical tuple input -* \tparam VType data type of param -*/ -template -struct NumericalParam { - NumericalParam() {} - explicit NumericalParam(VType *begin, VType *end) { - int32_t size = static_cast(end - begin); - info.resize(size); - for (int i = 0; i < size; ++i) { - info[i] = *(begin + i); - } - } - inline size_t ndim() const { - return info.size(); - } - std::vector info; -}; - -template -inline std::istream &operator>>(std::istream &is, NumericalParam ¶m) { - while (true) { - char ch = is.get(); - if (ch == '(') break; - if (!isspace(ch)) { - is.setstate(std::ios::failbit); - return is; - } - } - VType idx; - std::vector tmp; - // deal with empty case - size_t pos = is.tellg(); - char ch = is.get(); - if (ch == ')') { - param.info = tmp; - return is; - } - is.seekg(pos); - // finish deal - while (is >> idx) { - tmp.push_back(idx); - char ch; - do { - ch = is.get(); - } while (isspace(ch)); - if (ch == ',') { - while (true) { - ch = is.peek(); - if (isspace(ch)) { - is.get(); continue; - } - if (ch == ')') { - is.get(); break; - } - break; - } - if (ch == ')') break; - } else if (ch == ')') { - break; - } else { - is.setstate(std::ios::failbit); - return is; - } - } - param.info = tmp; - return is; -} - -template -inline std::ostream &operator<<(std::ostream &os, const NumericalParam ¶m) { - os << '('; - for (index_t i = 0; i < param.info.size(); ++i) { - if (i != 0) os << ','; - os << param.info[i]; - } - // python style tuple - if (param.info.size() == 1) os << ','; - os << ')'; - return os; -} - -} // namespace op -} // namespace mxnet namespace mxnet { namespace op { @@ -142,8 +53,8 @@ struct ProposalParam : public dmlc::Parameter { int rpn_post_nms_top_n; float threshold; int rpn_min_size; - NumericalParam scales; - NumericalParam ratios; + nnvm::Tuple scales; + nnvm::Tuple ratios; int feature_stride; bool output_score; bool iou_loss; @@ -159,10 +70,10 @@ struct ProposalParam : public dmlc::Parameter { DMLC_DECLARE_FIELD(rpn_min_size).set_default(16) .describe("Minimum height or width in proposal"); tmp[0] = 4.0f; tmp[1] = 8.0f; tmp[2] = 16.0f; tmp[3] = 32.0f; - DMLC_DECLARE_FIELD(scales).set_default(NumericalParam(tmp, tmp + 4)) + DMLC_DECLARE_FIELD(scales).set_default(nnvm::Tuple(tmp, tmp + 4)) .describe("Used to generate anchor windows by enumerating scales"); tmp[0] = 0.5f; tmp[1] = 1.0f; tmp[2] = 2.0f; - DMLC_DECLARE_FIELD(ratios).set_default(NumericalParam(tmp, tmp + 3)) + DMLC_DECLARE_FIELD(ratios).set_default(nnvm::Tuple(tmp, tmp + 3)) .describe("Used to generate anchor windows by enumerating ratios"); DMLC_DECLARE_FIELD(feature_stride).set_default(16) .describe("The size of the receptive field each unit in the convolution layer of the rpn," @@ -300,11 +211,11 @@ inline void _Transform(float scale, // out_anchors must have shape (n, 5), where n is ratios.size() * scales.size() inline void GenerateAnchors(const std::vector& base_anchor, - const std::vector& ratios, - const std::vector& scales, + const nnvm::Tuple& ratios, + const nnvm::Tuple& scales, std::vector *out_anchors) { - for (size_t j = 0; j < ratios.size(); ++j) { - for (size_t k = 0; k < scales.size(); ++k) { + for (size_t j = 0; j < ratios.ndim(); ++j) { + for (size_t k = 0; k < scales.ndim(); ++k) { _Transform(scales[k], ratios[j], base_anchor, out_anchors); } } diff --git a/src/operator/contrib/proposal.cc b/src/operator/contrib/proposal.cc index ec539003b944..ccb541a403a2 100644 --- a/src/operator/contrib/proposal.cc +++ b/src/operator/contrib/proposal.cc @@ -335,11 +335,11 @@ class ProposalOp : public Operator{ base_anchor[1] = 0.0; base_anchor[2] = param_.feature_stride - 1.0; base_anchor[3] = param_.feature_stride - 1.0; - CHECK_EQ(num_anchors, param_.ratios.info.size() * param_.scales.info.size()); + CHECK_EQ(num_anchors, param_.ratios.ndim() * param_.scales.ndim()); std::vector anchors; utils::GenerateAnchors(base_anchor, - param_.ratios.info, - param_.scales.info, + param_.ratios, + param_.scales, &anchors); std::memcpy(workspace_proposals.dptr_, &anchors[0], sizeof(float) * anchors.size()); diff --git a/src/operator/contrib/proposal.cu b/src/operator/contrib/proposal.cu index 209ef79a2aaf..9f56685a7a7d 100644 --- a/src/operator/contrib/proposal.cu +++ b/src/operator/contrib/proposal.cu @@ -442,11 +442,11 @@ class ProposalGPUOp : public Operator{ base_anchor[1] = 0.0; base_anchor[2] = param_.feature_stride - 1.0; base_anchor[3] = param_.feature_stride - 1.0; - CHECK_EQ(num_anchors, param_.ratios.info.size() * param_.scales.info.size()); + CHECK_EQ(num_anchors, param_.ratios.ndim() * param_.scales.ndim()); std::vector anchors; utils::GenerateAnchors(base_anchor, - param_.ratios.info, - param_.scales.info, + param_.ratios, + param_.scales, &anchors); // Copy generated anchors to GPU diff --git a/src/operator/convolution-inl.h b/src/operator/convolution-inl.h index 0edaee1dae32..a9e2c1bd6e94 100644 --- a/src/operator/convolution-inl.h +++ b/src/operator/convolution-inl.h @@ -103,8 +103,49 @@ struct ConvolutionParam : public dmlc::Parameter { index_t DilatedKernelSize(int dim) const { return 1 + (kernel[dim] - 1) * dilate[dim]; } + + bool operator==(const ConvolutionParam& other) const { + return this->kernel == other.kernel && + this->stride == other.stride && + this->dilate == other.dilate && + this->pad == other.pad && + this->num_filter == other.num_filter && + this->num_group == other.num_group && + this->workspace == other.workspace && + this->no_bias == other.no_bias && + this->cudnn_tune == other.cudnn_tune && + this->cudnn_off == other.cudnn_off && + this->layout == other.layout; + } }; +} // namespace op +} // namespace mxnet + +namespace std { +template<> +struct hash { + size_t operator()(const mxnet::op::ConvolutionParam& val) { + size_t ret = 0; + ret = dmlc::HashCombine(ret, val.kernel); + ret = dmlc::HashCombine(ret, val.stride); + ret = dmlc::HashCombine(ret, val.dilate); + ret = dmlc::HashCombine(ret, val.pad); + ret = dmlc::HashCombine(ret, val.num_filter); + ret = dmlc::HashCombine(ret, val.num_group); + ret = dmlc::HashCombine(ret, val.workspace); + ret = dmlc::HashCombine(ret, val.no_bias); + ret = dmlc::HashCombine(ret, val.cudnn_tune); + ret = dmlc::HashCombine(ret, val.cudnn_off); + ret = dmlc::HashCombine(ret, val.layout); + return ret; + } +}; +} // namespace std + +namespace mxnet { +namespace op { + template class ConvolutionOp : public Operator { public: diff --git a/src/operator/cudnn_algoreg-inl.h b/src/operator/cudnn_algoreg-inl.h index dc5db6bbc8b7..b27d2be297fe 100644 --- a/src/operator/cudnn_algoreg-inl.h +++ b/src/operator/cudnn_algoreg-inl.h @@ -61,37 +61,22 @@ class CuDNNAlgo { bool is_tensor_core_algo_; }; +template class CuDNNAlgoReg { public: - template - std::string GetKey(const Param ¶m, const std::vector &in_shape, - const std::vector &out_shape, - cudnnDataType_t cudnn_data_type, - cudnnDataType_t cudnn_forward_compute_type, - cudnnDataType_t cudnn_backward_compute_type, - int sm_arch) { - std::ostringstream oss; - oss << "inputs="; - for (auto &i : in_shape) - oss << i << ";"; - oss << "outputs="; - for (auto &i : out_shape) - oss << i << ";"; - auto dict = param.__DICT__(); - for (auto &k : dict) - oss << k.first << "=" << k.second << ";"; - oss << "cudnn_data_type=" << cudnn_data_type << ";"; - oss << "cudnn_forward_compute_type=" << cudnn_forward_compute_type << ";"; - oss << "cudnn_backward_compute_type=" << cudnn_backward_compute_type << ";"; - // All GPUs of the same compute capability (SM arch) share an algo selection. - oss << "sm_arch=" << sm_arch << ";"; - return oss.str(); - } - - bool Find(std::string key, + bool Find(const ParamType ¶m, + const std::vector &in_shape, + const std::vector &out_shape, + cudnnDataType_t cudnn_data_type, + cudnnDataType_t cudnn_forward_compute_type, + cudnnDataType_t cudnn_backward_compute_type, + int sm_arch, CuDNNAlgo *fwd, CuDNNAlgo *bwd, CuDNNAlgo *flt) { + CHECK(in_shape.size() == 2 || in_shape.size() == 3); + ParamKey key{param, in_shape[0], in_shape[1], out_shape[0], cudnn_data_type, + cudnn_forward_compute_type, cudnn_backward_compute_type, sm_arch}; std::lock_guard guard(lock_); auto i = reg_.find(key); if (i != reg_.end()) { @@ -103,10 +88,19 @@ class CuDNNAlgoReg { return false; } - void Register(std::string key, + void Register(const ParamType ¶m, + const std::vector &in_shape, + const std::vector &out_shape, + cudnnDataType_t cudnn_data_type, + cudnnDataType_t cudnn_forward_compute_type, + cudnnDataType_t cudnn_backward_compute_type, + int sm_arch, const CuDNNAlgo &fwd, const CuDNNAlgo &bwd, const CuDNNAlgo &flt) { + CHECK(in_shape.size() == 2 || in_shape.size() == 3); + ParamKey key{param, in_shape[0], in_shape[1], out_shape[0], cudnn_data_type, + cudnn_forward_compute_type, cudnn_backward_compute_type, sm_arch}; std::lock_guard guard(lock_); if (reg_.size() % 50 == 0) { LOG(INFO) << "Running performance tests to find the best convolution " @@ -134,9 +128,49 @@ class CuDNNAlgoReg { CuDNNAlgo flt; }; + struct ParamKey { + ParamType param; + TShape data_shape, weight_shape, out_shape; + cudnnDataType_t cudnn_data_type; + cudnnDataType_t cudnn_forward_compute_type; + cudnnDataType_t cudnn_backward_compute_type; + int sm_arch; + + bool operator==(const ParamKey& other) const { + return this->param == other.param && + this->data_shape == other.data_shape && + this->weight_shape == other.weight_shape && + this->out_shape == other.out_shape && + this->cudnn_data_type == other.cudnn_data_type && + this->cudnn_forward_compute_type == other.cudnn_forward_compute_type && + this->cudnn_backward_compute_type == other.cudnn_backward_compute_type && + this->sm_arch == other.sm_arch; + } + }; + + struct ParamHash { + size_t operator()(const ParamKey& key) const { + std::hash hash_param; + size_t ret = hash_param(key.param); + ret = dmlc::HashCombine(ret, key.data_shape); + ret = dmlc::HashCombine(ret, key.weight_shape); + ret = dmlc::HashCombine(ret, key.out_shape); + for (const auto& i : key.out_shape) ret = dmlc::HashCombine(ret, i); + ret = dmlc::HashCombine(ret, static_cast(key.cudnn_data_type)); + ret = dmlc::HashCombine(ret, static_cast(key.cudnn_forward_compute_type)); + ret = dmlc::HashCombine(ret, static_cast(key.cudnn_backward_compute_type)); + ret = dmlc::HashCombine(ret, key.sm_arch); + return ret; + } + }; + std::mutex lock_; - std::unordered_map reg_; + std::unordered_map reg_; }; + +typedef CuDNNAlgoReg CuDNNConvAlgoReg; +typedef CuDNNAlgoReg CuDNNDeconvAlgoReg; + #endif // __CUDACC__ && CUDNN } // namespace op } // namespace mxnet diff --git a/src/operator/cudnn_algoreg.cc b/src/operator/cudnn_algoreg.cc index 5aa8688c8148..5b0e73f0b19d 100644 --- a/src/operator/cudnn_algoreg.cc +++ b/src/operator/cudnn_algoreg.cc @@ -32,9 +32,16 @@ namespace mxnet { namespace op { #if MXNET_USE_CUDNN == 1 -CuDNNAlgoReg *CuDNNAlgoReg::Get() { - static CuDNNAlgoReg *ptr = new CuDNNAlgoReg(); - return ptr; +template<> +CuDNNAlgoReg *CuDNNAlgoReg::Get() { + static CuDNNAlgoReg inst; + return &inst; +} + +template<> +CuDNNAlgoReg *CuDNNAlgoReg::Get() { + static CuDNNAlgoReg inst; + return &inst; } #endif // CUDNN } // namespace op diff --git a/src/operator/cudnn_convolution-inl.h b/src/operator/cudnn_convolution-inl.h index 428278498337..b2b59944e895 100644 --- a/src/operator/cudnn_convolution-inl.h +++ b/src/operator/cudnn_convolution-inl.h @@ -580,11 +580,10 @@ class CuDNNConvolutionOp : public Operator { const std::vector& out_shape, cudnnDataType_t cudnn_forward_compute_type, cudnnDataType_t cudnn_backward_compute_type) { - std::string key = CuDNNAlgoReg::Get()->GetKey(param_, in_shape, out_shape, dtype_, - cudnn_forward_compute_type, - cudnn_backward_compute_type, - SMArch(ctx.dev_id)); - if (!CuDNNAlgoReg::Get()->Find(key, &forward_algo_, &back_algo_, &back_algo_w_)) { + if (!CuDNNConvAlgoReg::Get()->Find(param_, in_shape, out_shape, dtype_, + cudnn_forward_compute_type, cudnn_backward_compute_type, + SMArch(ctx.dev_id), &forward_algo_, &back_algo_, + &back_algo_w_)) { // Not in algo registry, must determine via *Get*() or *Find*() Engine::VarHandle var = Engine::Get()->NewVariable(); Engine::Get()->PushSync([=](RunContext rctx) { @@ -772,8 +771,11 @@ class CuDNNConvolutionOp : public Operator { // convolution will match only if identically specified. // We're caching results of *Get* as well as *Find*, but these records // will be held distinctly because param_.cudnn_tune is part of the key. - CuDNNAlgoReg::Get()->Register(key, this->forward_algo_, this->back_algo_, - this->back_algo_w_); + CuDNNConvAlgoReg::Get()->Register(param_, in_shape, out_shape, dtype_, + cudnn_forward_compute_type, + cudnn_backward_compute_type, + SMArch(ctx.dev_id), this->forward_algo_, + this->back_algo_, this->back_algo_w_); }, ctx, {}, {var}); Engine::Get()->WaitForVar(var); Engine::Get()->DeleteVariable([](RunContext s) {}, ctx, var); diff --git a/src/operator/cudnn_deconvolution-inl.h b/src/operator/cudnn_deconvolution-inl.h index de3e70c7d6a7..5e9b7c5704d0 100644 --- a/src/operator/cudnn_deconvolution-inl.h +++ b/src/operator/cudnn_deconvolution-inl.h @@ -598,11 +598,11 @@ class CuDNNDeconvolutionOp : public Operator { const std::vector& out_shape, cudnnDataType_t cudnn_forward_compute_type, cudnnDataType_t cudnn_backward_compute_type) { - std::string key = CuDNNAlgoReg::Get()->GetKey(param_, in_shape, out_shape, dtype_, - cudnn_forward_compute_type, - cudnn_backward_compute_type, - SMArch(ctx.dev_id)); - if (!CuDNNAlgoReg::Get()->Find(key, &forward_algo_, &back_algo_, &back_algo_w_)) { + if (!CuDNNDeconvAlgoReg::Get()->Find(param_, in_shape, out_shape, dtype_, + cudnn_forward_compute_type, + cudnn_backward_compute_type, + SMArch(ctx.dev_id), &forward_algo_, + &back_algo_, &back_algo_w_)) { // Not in algo registry, must determine via *Get*() or *Find*() Engine::VarHandle var = Engine::Get()->NewVariable(); Engine::Get()->PushSync([=](RunContext rctx) { @@ -793,8 +793,11 @@ class CuDNNDeconvolutionOp : public Operator { // convolution will match only if identically specified. // We're caching results of *Get* as well as *Find*, but these records // will be held distinctly because param_.cudnn_tune is part of the key. - CuDNNAlgoReg::Get()->Register(key, this->forward_algo_, this->back_algo_, - this->back_algo_w_); + CuDNNDeconvAlgoReg::Get()->Register(param_, in_shape, out_shape, dtype_, + cudnn_forward_compute_type, + cudnn_backward_compute_type, + SMArch(ctx.dev_id), this->forward_algo_, + this->back_algo_, this->back_algo_w_); }, ctx, {}, {var}); Engine::Get()->WaitForVar(var); Engine::Get()->DeleteVariable([](RunContext s) {}, ctx, var); diff --git a/src/operator/deconvolution-inl.h b/src/operator/deconvolution-inl.h index dd77c150c970..a968ce44a800 100644 --- a/src/operator/deconvolution-inl.h +++ b/src/operator/deconvolution-inl.h @@ -144,8 +144,53 @@ struct DeconvolutionParam : public dmlc::Parameter { index_t DilatedKernelSize(int dim) const { return 1 + (kernel[dim] - 1) * dilate[dim]; } + + bool operator==(const DeconvolutionParam& other) const { + return this->kernel == other.kernel && + this->stride == other.stride && + this->dilate == other.dilate && + this->pad == other.pad && + this->adj == other.adj && + this->target_shape == other.target_shape && + this->num_filter == other.num_filter && + this->num_group == other.num_group && + this->workspace == other.workspace && + this->no_bias == other.no_bias && + this->cudnn_tune == other.cudnn_tune && + this->cudnn_off == other.cudnn_off && + this->layout == other.layout; + } }; +} // namespace op +} // namespace mxnet + +namespace std { +template<> +struct hash { + size_t operator()(const mxnet::op::DeconvolutionParam& val) { + size_t ret = 0; + ret = dmlc::HashCombine(ret, val.kernel); + ret = dmlc::HashCombine(ret, val.stride); + ret = dmlc::HashCombine(ret, val.dilate); + ret = dmlc::HashCombine(ret, val.pad); + ret = dmlc::HashCombine(ret, val.adj); + ret = dmlc::HashCombine(ret, val.target_shape); + ret = dmlc::HashCombine(ret, val.num_filter); + ret = dmlc::HashCombine(ret, val.num_group); + ret = dmlc::HashCombine(ret, val.workspace); + ret = dmlc::HashCombine(ret, val.no_bias); + ret = dmlc::HashCombine(ret, val.cudnn_tune); + ret = dmlc::HashCombine(ret, val.cudnn_off); + ret = dmlc::HashCombine(ret, val.layout); + return ret; + } +}; +} // namespace std + +namespace mxnet { +namespace op { + template class DeconvolutionOp : public Operator { public: From 39ff76494b6b1e7a6048ae80d66bb87c46263537 Mon Sep 17 00:00:00 2001 From: Kai Li <1196594711@qq.com> Date: Sun, 27 Aug 2017 13:17:16 +0800 Subject: [PATCH 434/834] Update io.md (#7634) --- docs/api/python/io.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/api/python/io.md b/docs/api/python/io.md index ce8245b73fe8..ecf3e75ac0d5 100644 --- a/docs/api/python/io.md +++ b/docs/api/python/io.md @@ -35,7 +35,7 @@ Let's see a complete example of how to use data iterator in model training. >>> data = mx.sym.Variable('data') >>> label = mx.sym.Variable('softmax_label') >>> fullc = mx.sym.FullyConnected(data=data, num_hidden=1) ->>> loss = mx.sym.SoftmaxOutput(data=data, label=label) +>>> loss = mx.sym.SoftmaxOutput(data=fullc, label=label) >>> mod = mx.mod.Module(loss, data_names=['data'], label_names=['softmax_label']) >>> mod.bind(data_shapes=nd_iter.provide_data, label_shapes=nd_iter.provide_label) >>> mod.fit(nd_iter, num_epoch=2) From 9aa051c2e87d41b4f2a61fb62728ecdf364f8997 Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Sun, 27 Aug 2017 00:14:28 -0700 Subject: [PATCH 435/834] fix tests (#7633) --- tests/python/gpu/test_operator_gpu.py | 4 ++-- tests/python/unittest/test_loss.py | 18 ++++++------------ 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/tests/python/gpu/test_operator_gpu.py b/tests/python/gpu/test_operator_gpu.py index 11d146cae840..0c5771ebffb6 100644 --- a/tests/python/gpu/test_operator_gpu.py +++ b/tests/python/gpu/test_operator_gpu.py @@ -1346,11 +1346,11 @@ def test_sequence_reverse(): def test_autograd_save_memory(): - x = mx.nd.zeros((128, 1024, 1024), ctx=mx.gpu(0)) + x = mx.nd.zeros((128, 512, 512), ctx=mx.gpu(0)) x.attach_grad() with mx.autograd.record(): - for i in range(50): + for i in range(200): x = x + 1 x.wait_to_read() x.backward() diff --git a/tests/python/unittest/test_loss.py b/tests/python/unittest/test_loss.py index b864215ca1d1..85875c604bf0 100644 --- a/tests/python/unittest/test_loss.py +++ b/tests/python/unittest/test_loss.py @@ -63,7 +63,6 @@ def get_net(num_hidden): def test_ce_loss(): - mx.random.seed(1234) np.random.seed(1234) nclass = 10 N = 20 @@ -83,7 +82,6 @@ def test_ce_loss(): def test_bce_loss(): - mx.random.seed(1234) np.random.seed(1234) N = 20 data = mx.random.uniform(-1, 1, shape=(N, 20)) @@ -111,7 +109,6 @@ def test_bce_equal_ce2(): def test_kl_loss(): - mx.random.seed(1234) np.random.seed(1234) N = 20 data = mx.random.uniform(-1, 1, shape=(N, 10)) @@ -129,12 +126,11 @@ def test_kl_loss(): def test_l2_loss(): - mx.random.seed(1234) np.random.seed(1234) N = 20 data = mx.random.uniform(-1, 1, shape=(N, 10)) label = mx.random.uniform(-1, 1, shape=(N, 1)) - data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label') + data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label', shuffle=True) output = get_net(1) l = mx.symbol.Variable('label') Loss = gluon.loss.L2Loss() @@ -142,26 +138,25 @@ def test_l2_loss(): loss = Loss(output, l) loss = mx.sym.make_loss(loss) mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',)) - mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 1.}, - eval_metric=mx.metric.Loss()) + mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 0.1, 'wd': 0.00045}, + initializer=mx.init.Xavier(magnitude=2), eval_metric=mx.metric.Loss()) assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.05 def test_l1_loss(): - mx.random.seed(1234) np.random.seed(1234) N = 20 data = mx.random.uniform(-1, 1, shape=(N, 10)) label = mx.random.uniform(-1, 1, shape=(N, 1)) - data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label') + data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label', shuffle=True) output = get_net(1) l = mx.symbol.Variable('label') Loss = gluon.loss.L1Loss() loss = Loss(output, l) loss = mx.sym.make_loss(loss) mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',)) - mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 0.1}, - initializer=mx.init.Uniform(0.5), eval_metric=mx.metric.Loss()) + mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 0.01}, + initializer=mx.init.Xavier(magnitude=3), eval_metric=mx.metric.Loss()) assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.1 @@ -196,7 +191,6 @@ def test_ctc_loss(): def test_sample_weight_loss(): - mx.random.seed(1234) np.random.seed(1234) nclass = 10 N = 20 From e05129774e76206fe890b511c346953107b05fce Mon Sep 17 00:00:00 2001 From: Nan Zhu Date: Sun, 27 Aug 2017 01:12:23 -0700 Subject: [PATCH 436/834] [build] explicitly install JDK8 (#7574) * explicitly install openjdk8 * handle earlier version of ubuntu * install software-properties-common * update -y * update commands --- docker/install/scala.sh | 10 +++++++++- docs/get_started/build_from_source.md | 8 +++++++- tests/ci_build/Dockerfile.ubuntu1404_cuda75_cudnn5 | 8 +++++++- tests/ci_build/install/ubuntu_install_scala.sh | 9 +++++++-- 4 files changed, 30 insertions(+), 5 deletions(-) diff --git a/docker/install/scala.sh b/docker/install/scala.sh index bb0bb9c900d4..c1d2de6c75b2 100755 --- a/docker/install/scala.sh +++ b/docker/install/scala.sh @@ -19,7 +19,15 @@ # install libraries for mxnet's scala package on ubuntu -apt-get install -y maven default-jdk + +apt-get install -y software-properties-common +add-apt-repository -y ppa:webupd8team/java +apt-get update +echo "oracle-java8-installer shared/accepted-oracle-license-v1-1 select true" | debconf-set-selections +apt-get install -y oracle-java8-installer +apt-get install -y oracle-java8-set-default + +apt-get install -y maven wget http://downloads.lightbend.com/scala/2.11.8/scala-2.11.8.deb dpkg -i scala-2.11.8.deb diff --git a/docs/get_started/build_from_source.md b/docs/get_started/build_from_source.md index 4ff2cc09aa82..9bf397bc9f14 100644 --- a/docs/get_started/build_from_source.md +++ b/docs/get_started/build_from_source.md @@ -367,7 +367,13 @@ Both JDK and Maven are required to build the Scala package.
    ```bash -sudo apt-get install -y maven default-jdk +apt-get install -y software-properties-common +add-apt-repository -y ppa:webupd8team/java +apt-get update +echo "oracle-java8-installer shared/accepted-oracle-license-v1-1 select true" | debconf-set-selections +apt-get install -y oracle-java8-installer +apt-get install -y oracle-java8-set-default +apt-get install -y maven ```
    diff --git a/tests/ci_build/Dockerfile.ubuntu1404_cuda75_cudnn5 b/tests/ci_build/Dockerfile.ubuntu1404_cuda75_cudnn5 index e9810af6b72c..88fd7cea6fcb 100644 --- a/tests/ci_build/Dockerfile.ubuntu1404_cuda75_cudnn5 +++ b/tests/ci_build/Dockerfile.ubuntu1404_cuda75_cudnn5 @@ -23,7 +23,13 @@ RUN cd /usr/src/gtest && cmake CMakeLists.txt && make && cp *.a /usr/lib RUN pip install nose cpplint 'pylint==1.4.4' 'astroid==1.3.6' # MAVEN -RUN apt-get install -y maven default-jdk +RUN apt-get install -y software-properties-common +RUN add-apt-repository ppa:webupd8team/java -y +RUN apt-get update +RUN echo "oracle-java8-installer shared/accepted-oracle-license-v1-1 select true" | debconf-set-selections +RUN apt-get install -y oracle-java8-installer +RUN apt-get install -y oracle-java8-set-default +RUN apt-get install -y maven # R RUN apt-get install -y software-properties-common r-base-core libcurl4-openssl-dev libssl-dev libxml2-dev diff --git a/tests/ci_build/install/ubuntu_install_scala.sh b/tests/ci_build/install/ubuntu_install_scala.sh index 712eff98b02a..169ece036d2f 100755 --- a/tests/ci_build/install/ubuntu_install_scala.sh +++ b/tests/ci_build/install/ubuntu_install_scala.sh @@ -19,5 +19,10 @@ # install libraries for mxnet's scala package on ubuntu -apt-get update && apt-get install -y \ - maven default-jdk +apt-get install -y software-properties-common +add-apt-repository -y ppa:webupd8team/java +apt-get update +echo "oracle-java8-installer shared/accepted-oracle-license-v1-1 select true" | debconf-set-selections +apt-get install -y oracle-java8-installer +apt-get install -y oracle-java8-set-default +apt-get update && apt-get install -y maven From aceef5abf3db968ee98333c1454a951dfbf07a43 Mon Sep 17 00:00:00 2001 From: Yao Wang Date: Mon, 28 Aug 2017 11:00:15 -0700 Subject: [PATCH 437/834] Add script to build doc files for all versions (#7636) * Add script to build doc files for all versions * Fix * Use add versipn script of each different version --- docs/build_version_doc/build_all_version.sh | 82 +++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100755 docs/build_version_doc/build_all_version.sh diff --git a/docs/build_version_doc/build_all_version.sh b/docs/build_version_doc/build_all_version.sh new file mode 100755 index 000000000000..140e51a3d3a1 --- /dev/null +++ b/docs/build_version_doc/build_all_version.sh @@ -0,0 +1,82 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This script is for locally building website for all versions +# Built files are stored in $built +# Version numbers are stored in $tag_list. +# Version numbers are ordered from latest to old and final one is master. +tag_list="0.11.0.rc3 master" + +mxnet_url="https://github.com/apache/incubator-mxnet.git" +mxnet_folder="apache_mxnet" +built="VersionedWeb" +mkdir $built +mkdir "$built/versions" + +git clone $mxnet_url $mxnet_folder --recursive +cd "$mxnet_folder/docs" +tag_file="tag_list.txt" + +# Write all version numbers into $tag_file +for tag in $tag_list; do + if [ $tag != 'master' ] + then + echo "$tag" >> "$tag_file" + fi +done + +# Build all versions and use latest version(First version number in $tag_list) as landing page. +version_num=0 +for tag in $tag_list; do + if [ $tag == 'master' ] + then + git checkout master + else + git checkout "tags/$tag" + fi + + git submodule update || exit 1 + cd .. + make clean + cd docs + make clean + make html USE_OPENMP=0 || exit 1 + python build_version_doc/AddVersion.py --file_path "_build/html/" --current_version "$tag" || exit 1 + + if [ $tag != 'master' ] + then + python build_version_doc/AddPackageLink.py --file_path "_build/html/get_started/install.html" \ + --current_version "$tag" || exit 1 + fi + + if [ $version_num == 0 ] + then + cp -a _build/html/. "../../$built" + else + file_loc="../../$built/versions/$tag" + mkdir "$file_loc" + cp -a _build/html/. "$file_loc" + fi + + ((++version_num)) +done + +mv "$tag_file" "../../$built/tag.txt" +cd ../.. +rm -rf "$mxnet_folder" From e845cec1c09626bd312f76d5b0ba56b1b986c57f Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Mon, 28 Aug 2017 11:05:50 -0700 Subject: [PATCH 438/834] add fashion mnist and move mnists to s3 (#7635) * add fashion mnist and move mnists to s3 * refactor --- python/mxnet/gluon/data/vision.py | 68 ++++++++++++++++++------ tests/python/unittest/test_gluon_data.py | 1 + 2 files changed, 53 insertions(+), 16 deletions(-) diff --git a/python/mxnet/gluon/data/vision.py b/python/mxnet/gluon/data/vision.py index b63624508124..24c060c54c84 100644 --- a/python/mxnet/gluon/data/vision.py +++ b/python/mxnet/gluon/data/vision.py @@ -40,6 +40,8 @@ def __init__(self, root, train, transform): self._data = None self._label = None + if not os.path.isdir(self._root): + os.makedirs(self._root) self._get_data() def __getitem__(self, idx): @@ -70,24 +72,29 @@ class MNIST(_DownloadedDataset): transform=lambda data, label: (data.astype(np.float32)/255, label) """ - def __init__(self, root='~/.mxnet/datasets/', train=True, + def __init__(self, root='~/.mxnet/datasets/mnist', train=True, transform=None): + self._base_url = 'https://apache-mxnet.s3.amazonaws.com/gluon/dataset/mnist/' + self._train_data = ('train-images-idx3-ubyte.gz', + '6c95f4b05d2bf285e1bfb0e7960c31bd3b3f8a7d') + self._train_label = ('train-labels-idx1-ubyte.gz', + '2a80914081dc54586dbdf242f9805a6b8d2a15fc') + self._test_data = ('t10k-images-idx3-ubyte.gz', + 'c3a25af1f52dad7f726cce8cacb138654b760d48') + self._test_label = ('t10k-labels-idx1-ubyte.gz', + '763e7fa3757d93b0cdec073cef058b2004252c17') super(MNIST, self).__init__(root, train, transform) def _get_data(self): - if not os.path.isdir(self._root): - os.makedirs(self._root) - url = 'http://data.mxnet.io/data/mnist/' if self._train: - data_file = download(url+'train-images-idx3-ubyte.gz', self._root, - sha1_hash='6c95f4b05d2bf285e1bfb0e7960c31bd3b3f8a7d') - label_file = download(url+'train-labels-idx1-ubyte.gz', self._root, - sha1_hash='2a80914081dc54586dbdf242f9805a6b8d2a15fc') + data, label = self._train_data, self._train_label else: - data_file = download(url+'t10k-images-idx3-ubyte.gz', self._root, - sha1_hash='c3a25af1f52dad7f726cce8cacb138654b760d48') - label_file = download(url+'t10k-labels-idx1-ubyte.gz', self._root, - sha1_hash='763e7fa3757d93b0cdec073cef058b2004252c17') + data, label = self._test_data, self._test_label + + data_file = download(self._base_url + data[0], self._root, + sha1_hash=data[1]) + label_file = download(self._base_url + label[0], self._root, + sha1_hash=label[1]) with gzip.open(label_file, 'rb') as fin: struct.unpack(">II", fin.read(8)) @@ -102,6 +109,38 @@ def _get_data(self): self._label = label +class FashionMNIST(MNIST): + """A dataset of Zalando's article images consisting of fashion products, + a drop-in replacement of the original MNIST dataset from + `https://github.com/zalandoresearch/fashion-mnist`_. + + Each sample is an image (in 3D NDArray) with shape (28, 28, 1). + + Parameters + ---------- + root : str + Path to temp folder for storing data. + train : bool + Whether to load the training or testing set. + transform : function + A user defined callback that transforms each instance. For example:: + + transform=lambda data, label: (data.astype(np.float32)/255, label) + """ + def __init__(self, root='~/.mxnet/datasets/fashion-mnist', train=True, + transform=None): + self._base_url = 'https://apache-mxnet.s3.amazonaws.com/gluon/dataset/fashion-mnist/' + self._train_data = ('train-images-idx3-ubyte.gz', + '0cf37b0d40ed5169c6b3aba31069a9770ac9043d') + self._train_label = ('train-labels-idx1-ubyte.gz', + '236021d52f1e40852b06a4c3008d8de8aef1e40b') + self._test_data = ('t10k-images-idx3-ubyte.gz', + '626ed6a7c06dd17c0eec72fa3be1740f146a2863') + self._test_label = ('t10k-labels-idx1-ubyte.gz', + '17f9ab60e7257a1620f4ad76bbbaf857c3920701') + super(FashionMNIST, self).__init__(root, train, transform) + + class CIFAR10(_DownloadedDataset): """CIFAR10 image classification dataset from `https://www.cs.toronto.edu/~kriz/cifar.html`_. @@ -118,7 +157,7 @@ class CIFAR10(_DownloadedDataset): transform=lambda data, label: (data.astype(np.float32)/255, label) """ - def __init__(self, root='~/.mxnet/datasets/', train=True, + def __init__(self, root='~/.mxnet/datasets/cifar10', train=True, transform=None): self._file_hashes = {'data_batch_1.bin': 'aadd24acce27caa71bf4b10992e9e7b2d74c2540', 'data_batch_2.bin': 'c0ba65cce70568cd57b4e03e9ac8d2a5367c1795', @@ -136,9 +175,6 @@ def _read_batch(self, filename): data[:, 0].astype(np.int32) def _get_data(self): - if not os.path.isdir(self._root): - os.makedirs(self._root) - file_paths = [(name, os.path.join(self._root, 'cifar-10-batches-bin/', name)) for name in self._file_hashes] if any(not os.path.exists(path) or not check_sha1(path, self._file_hashes[name]) diff --git a/tests/python/unittest/test_gluon_data.py b/tests/python/unittest/test_gluon_data.py index 32298fcd57d5..7f388be73cb3 100644 --- a/tests/python/unittest/test_gluon_data.py +++ b/tests/python/unittest/test_gluon_data.py @@ -71,6 +71,7 @@ def test_sampler(): def test_datasets(): assert len(gluon.data.vision.MNIST(root='data')) == 60000 + assert len(gluon.data.vision.FashionMNIST(root='data')) == 60000 assert len(gluon.data.vision.CIFAR10(root='data', train=False)) == 10000 def test_image_folder_dataset(): From 910b422ba77274ccc1c1ac2b27302212f79d6ad6 Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Mon, 28 Aug 2017 12:06:18 -0700 Subject: [PATCH 439/834] add doc for dataset (#7644) --- docs/api/python/gluon.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/api/python/gluon.md b/docs/api/python/gluon.md index ac637749f856..ed42a7d61120 100644 --- a/docs/api/python/gluon.md +++ b/docs/api/python/gluon.md @@ -228,6 +228,7 @@ in Python and then deploy with symbolic graph in C++ and Scala. :nosignatures: MNIST + FashionMNIST CIFAR10 ``` From 860dda2cc4741ac8167a7f81bd9d835364d5954a Mon Sep 17 00:00:00 2001 From: Yao Wang Date: Mon, 28 Aug 2017 12:06:38 -0700 Subject: [PATCH 440/834] Change apache package URL to https (#7622) --- docs/build_version_doc/AddPackageLink.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/build_version_doc/AddPackageLink.py b/docs/build_version_doc/AddPackageLink.py index 8fe04b50b5ce..e3cc19824ba4 100644 --- a/docs/build_version_doc/AddPackageLink.py +++ b/docs/build_version_doc/AddPackageLink.py @@ -30,13 +30,13 @@ args = parser.parse_args() tag = args.current_version - src_url = "http://www.apache.org/dyn/closer.cgi/incubator/" \ + src_url = "https://www.apache.org/dyn/closer.cgi/incubator/" \ "mxnet/%s-incubating/apache-mxnet-src-%s-incubating.tar.gz" % (tag, tag) - pgp_url = "http://www.apache.org/dyn/closer.cgi/incubator/" \ + pgp_url = "https://www.apache.org/dyn/closer.cgi/incubator/" \ "mxnet/%s-incubating/apache-mxnet-src-%s-incubating.tar.gz.asc" % (tag, tag) - sha_url = "http://www.apache.org/dyn/closer.cgi/incubator/" \ + sha_url = "https://www.apache.org/dyn/closer.cgi/incubator/" \ "mxnet/%s-incubating/apache-mxnet-src-%s-incubating.tar.gz.sha" % (tag, tag) - md5_url = "http://www.apache.org/dyn/closer.cgi/incubator/" \ + md5_url = "https://www.apache.org/dyn/closer.cgi/incubator/" \ "mxnet/%s-incubating/apache-mxnet-src-%s-incubating.tar.gz.md5" % (tag, tag) download_str = "
    " From 4e116740d1adf78e9e0c4ed6202965db08c2087b Mon Sep 17 00:00:00 2001 From: Pracheer Gupta Date: Mon, 28 Aug 2017 17:22:31 -0700 Subject: [PATCH 441/834] Pip installer for CoreML Converter: mxnet-to-coreml (#7624) * Fixing CoreML converter's README: typos/grammar/etc. * CoreML converter README update: Talk about layers first and then about models. * Providing examples on converting various standard models; calling out issues with InceptionV3. * Fixing CoreML converter's README: typos/grammar/etc. * CoreML converter README update: Talk about layers first and then about models. * Providing examples on converting various standard models; calling out issues with InceptionV3. * Pip installer for converter: mxnet-coreml-converter. Runs only on MacOS and python 2.7. Once inside the directory pip_package, user needs to run: python setup.py bdist_wheel twine upload dist/* Once uploaded it'll look like this: https://testpypi.python.org/pypi/mxnet-coreml-converter Also updated the README for converter to reflect this. Note that we are going with a package per tool for the time being. Please leave feedback if you think it is better to adopt the policy of all the tools in one single package. Unit tests continue to pass. * More informative pypi package documentation. * Updating MacOS in release notes to 10.11 after testing on it. * Changing the name to mxnet-to-coreml and version to 0.1.0. * Added license to setup.py * Updating readme files with the correct pip package name. --- tools/coreml/README.md | 39 +++++++-------- tools/coreml/{ => converter}/utils.py | 0 tools/coreml/mxnet_coreml_converter.py | 3 +- tools/coreml/pip_package/.gitignore | 10 ++++ tools/coreml/pip_package/MANIFEST.in | 5 ++ tools/coreml/pip_package/README.rst | 44 ++++++++++++++++ tools/coreml/pip_package/setup.py | 69 ++++++++++++++++++++++++++ tools/coreml/test/test_mxnet_image.py | 2 +- 8 files changed, 149 insertions(+), 23 deletions(-) rename tools/coreml/{ => converter}/utils.py (100%) create mode 100644 tools/coreml/pip_package/.gitignore create mode 100644 tools/coreml/pip_package/MANIFEST.in create mode 100644 tools/coreml/pip_package/README.rst create mode 100644 tools/coreml/pip_package/setup.py diff --git a/tools/coreml/README.md b/tools/coreml/README.md index e29eebe84bc1..45f19b608bdb 100644 --- a/tools/coreml/README.md +++ b/tools/coreml/README.md @@ -3,22 +3,23 @@ This tool helps convert MXNet models into [Apple CoreML](https://developer.apple.com/documentation/coreml) format which can then be run on Apple devices. ## Installation -In order to use this tool you need to have these installed: -* MacOS - High Sierra 10.13 -* Xcode 9 -* coremltools 0.5.0 or greater (pip install coremltools) -* mxnet 0.10.0 or greater. [Installation instructions](http://mxnet.io/get_started/install.html). -* yaml (pip install pyyaml) +In order to use this tool you need to have these: +* MacOS - 10.11 (El Capitan) or higher (for running inferences on the converted model MacOS 10.13 or higher (for phones: iOS 11 or above) is needed) * python 2.7 +* mxnet-to-coreml tool: + +```bash +pip install mxnet-to-coreml +``` ## How to use -Let's say you want to use your MXNet model in an iPhone App. For the purpose of this example, let's say you want to use squeezenet-v1.1. +Let's say you want to use your MXNet model in an iPhone App. For the purpose of this example, let's assume it is a squeezenet-v1.1 model. -1. Download the model into the directory where this converter resides. Squeezenet can be downloaded from [here](http://data.mxnet.io/models/imagenet/squeezenet/). +1. Download the model into the directory where this converter resides. Squeezenet can be downloaded from [here](http://data.mxnet.io/models/imagenet/squeezenet/). The synset.txt file which contains all the class-labels and can be downloaded from [here](http://data.mxnet.io/models/imagenet/synset.txt). 2. Run this command: ```bash -python mxnet_coreml_converter.py --model-prefix='squeezenet_v1.1' --epoch=0 --input-shape='{"data":"3,227,227"}' --mode=classifier --pre-processing-arguments='{"image_input_names":"data"}' --class-labels classLabels.txt --output-file="squeezenetv11.mlmodel" +mxnet_coreml_converter.py --model-prefix='squeezenet_v1.1' --epoch=0 --input-shape='{"data":"3,227,227"}' --mode=classifier --pre-processing-arguments='{"image_input_names":"data"}' --class-labels synset.txt --output-file="squeezenetv11.mlmodel" ``` The above command will save the converted model in CoreML format to file squeezenet-v11.mlmodel. Internally, the model is first loaded by MXNet recreating the entire symbolic graph in memory. The converter walks through this symbolic graph converting each operator into its CoreML equivalent. Some of the supplied arguments to the converter are used by MXNet to generate the graph while others are used by CoreML either to pre-process the input (before passing it to the neural network) or to process the output of the neural network in a particular way. @@ -40,20 +41,20 @@ python mxnet_coreml_converter.py --model-prefix='squeezenet_v1.1' --epoch=0 --in You could provide a file containing class labels (as above) so that CoreML will return the category a given image belongs to. The file should have a label per line and labels can have any special characters. The line number of the label in the file should correspond with the index of softmax output. E.g. ```bash -python mxnet_coreml_converter.py --model-prefix='squeezenet_v1.1' --epoch=0 --input-shape='{"data":"3,227,227"}' --mode=classifier --class-labels classLabels.txt --output-file="squeezenetv11.mlmodel" +mxnet_coreml_converter.py --model-prefix='squeezenet_v1.1' --epoch=0 --input-shape='{"data":"3,227,227"}' --mode=classifier --class-labels synset.txt --output-file="squeezenetv11.mlmodel" ``` ### Adding a pre-processing layer to CoreML model. You could ask CoreML to pre-process the images before passing them through the model. The following command provides image re-centering parameters for red, blue and green channel. ```bash -python mxnet_coreml_converter.py --model-prefix='squeezenet_v1.1' --epoch=0 --input-shape='{"data":"3,224,224"}' --pre-processing-arguments='{"red_bias":127,"blue_bias":117,"green_bias":103}' --output-file="squeezenet_v11.mlmodel" +mxnet_coreml_converter.py --model-prefix='squeezenet_v1.1' --epoch=0 --input-shape='{"data":"3,224,224"}' --pre-processing-arguments='{"red_bias":127,"blue_bias":117,"green_bias":103}' --output-file="squeezenet_v11.mlmodel" ``` If you are building an app for a model that takes "Image" as an input, you will have to provide image_input_names as pre-processing arguments. This tells CoreML that a particular input variable is of type Image. E.g.: ```bash -python mxnet_coreml_converter.py --model-prefix='squeezenet_v1.1' --epoch=0 --input-shape='{"data":"3,224,224"}' --pre-processing-arguments='{"red_bias":127,"blue_bias":117,"green_bias":103,"image_input_names":"data"}' --output-file="squeezenet_v11.mlmodel" +mxnet_coreml_converter.py --model-prefix='squeezenet_v1.1' --epoch=0 --input-shape='{"data":"3,224,224"}' --pre-processing-arguments='{"red_bias":127,"blue_bias":117,"green_bias":103,"image_input_names":"data"}' --output-file="squeezenet_v11.mlmodel" ``` ## Currently supported @@ -79,36 +80,32 @@ Any MXNet model that uses the above operators can be converted easily. For insta 1. [Inception-BN](http://data.mxnet.io/models/imagenet/inception-bn/) ```bash -python mxnet_coreml_converter.py --model-prefix='Inception-BN' --epoch=126 --input-shape='{"data":"3,224,224"}' --mode=classifier --pre-processing-arguments='{"image_input_names":"data"}' --class-labels classLabels.txt --output-file="InceptionBN.mlmodel" +mxnet_coreml_converter.py --model-prefix='Inception-BN' --epoch=126 --input-shape='{"data":"3,224,224"}' --mode=classifier --pre-processing-arguments='{"image_input_names":"data"}' --class-labels synset.txt --output-file="InceptionBN.mlmodel" ``` 2. [NiN](http://data.dmlc.ml/models/imagenet/nin/) ```bash -python mxnet_coreml_converter.py --model-prefix='nin' --epoch=0 --input-shape='{"data":"3,224,224"}' --mode=classifier --pre-processing-arguments='{"image_input_names":"data"}' --class-labels classLabels.txt --output-file="nin.mlmodel" +mxnet_coreml_converter.py --model-prefix='nin' --epoch=0 --input-shape='{"data":"3,224,224"}' --mode=classifier --pre-processing-arguments='{"image_input_names":"data"}' --class-labels synset.txt --output-file="nin.mlmodel" ``` 3. [Resnet](http://data.mxnet.io/models/imagenet/resnet/) ```bash -python mxnet_coreml_converter.py --model-prefix='resnet-50' --epoch=0 --input-shape='{"data":"3,224,224"}' --mode=classifier --pre-processing-arguments='{"image_input_names":"data"}' --class-labels classLabels.txt --output-file="resnet50.mlmodel" +mxnet_coreml_converter.py --model-prefix='resnet-50' --epoch=0 --input-shape='{"data":"3,224,224"}' --mode=classifier --pre-processing-arguments='{"image_input_names":"data"}' --class-labels synset.txt --output-file="resnet50.mlmodel" ``` 4. [Squeezenet](http://data.mxnet.io/models/imagenet/squeezenet/) ```bash -python mxnet_coreml_converter.py --model-prefix='squeezenet_v1.1' --epoch=0 --input-shape='{"data":"3,227,227"}' --mode=classifier --pre-processing-arguments='{"image_input_names":"data"}' --class-labels classLabels.txt --output-file="squeezenetv11.mlmodel" +mxnet_coreml_converter.py --model-prefix='squeezenet_v1.1' --epoch=0 --input-shape='{"data":"3,227,227"}' --mode=classifier --pre-processing-arguments='{"image_input_names":"data"}' --class-labels synset.txt --output-file="squeezenetv11.mlmodel" ``` 5. [Vgg](http://data.mxnet.io/models/imagenet/vgg/) ```bash -python mxnet_coreml_converter.py --model-prefix='vgg16' --epoch=0 --input-shape='{"data":"3,224,224"}' --mode=classifier --pre-processing-arguments='{"image_input_names":"data"}' --class-labels classLabels.txt --output-file="vgg16.mlmodel" +mxnet_coreml_converter.py --model-prefix='vgg16' --epoch=0 --input-shape='{"data":"3,224,224"}' --mode=classifier --pre-processing-arguments='{"image_input_names":"data"}' --class-labels synset.txt --output-file="vgg16.mlmodel" ``` ## Known issues * [Inception-V3](http://data.mxnet.io/models/imagenet/inception-v3.tar.gz) model can be converted into CoreML format but is unable to run on Xcode. - -## This tool has been tested with: -* MacOS - High Sierra 10.13 Beta. -* Xcode 9 beta 5. diff --git a/tools/coreml/utils.py b/tools/coreml/converter/utils.py similarity index 100% rename from tools/coreml/utils.py rename to tools/coreml/converter/utils.py diff --git a/tools/coreml/mxnet_coreml_converter.py b/tools/coreml/mxnet_coreml_converter.py index 502377eca864..ffa5008b3db4 100644 --- a/tools/coreml/mxnet_coreml_converter.py +++ b/tools/coreml/mxnet_coreml_converter.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -18,7 +19,7 @@ from __future__ import print_function import argparse from converter._mxnet_converter import convert -from utils import load_model +from converter.utils import load_model import yaml from ast import literal_eval diff --git a/tools/coreml/pip_package/.gitignore b/tools/coreml/pip_package/.gitignore new file mode 100644 index 000000000000..7c67bf467970 --- /dev/null +++ b/tools/coreml/pip_package/.gitignore @@ -0,0 +1,10 @@ +# Compiled python modules. +*.pyc + +# Setuptools distribution folder. +/dist/ + +# Python egg metadata, regenerated from source files by setuptools. +/*.egg-info +/*.egg + diff --git a/tools/coreml/pip_package/MANIFEST.in b/tools/coreml/pip_package/MANIFEST.in new file mode 100644 index 000000000000..6ecd97d57dc7 --- /dev/null +++ b/tools/coreml/pip_package/MANIFEST.in @@ -0,0 +1,5 @@ +# Include the license file +include LICENSE.txt + +# Documentation for pypi webpage +include README.rst diff --git a/tools/coreml/pip_package/README.rst b/tools/coreml/pip_package/README.rst new file mode 100644 index 000000000000..875d89fcd208 --- /dev/null +++ b/tools/coreml/pip_package/README.rst @@ -0,0 +1,44 @@ +MXNET -> CoreML Converter +========================= + +`Apache MXNet `_ (incubating) is a deep learning framework designed for both efficiency and flexibility. It allows you to mix `symbolic and imperative programming `_ to maximize efficiency and productivity. At its core, MXNet contains a dynamic dependency scheduler that automatically parallelizes both symbolic and imperative operations on the fly. A graph optimization layer on top of that makes symbolic execution fast and memory efficient. MXNet is portable and lightweight, scaling effectively to multiple GPUs and multiple machines. + +`Core ML `_ is an Apple framework which allows developers to simply and easily integrate machine learning (ML) models into apps running on Apple devices (including iOS, watchOS, macOS, and tvOS). Core ML introduces a public file format (.mlmodel) for a broad set of ML methods including deep neural networks (both convolutional and recurrent), tree ensembles with boosting, and generalized linear models. Models in this format can be directly integrated into apps through Xcode. + +This tool helps convert `MXNet models `_ into `Apple CoreML `_ format which can then be run on Apple devices. You can find more information about this tool on our `github `_ page. + +Prerequisites +------------- +This package can only be installed on MacOS X since it relies on Apple's CoreML SDK. It can be run on MacOS 10.11 or higher though for running inferences on the converted model MacOS 10.13 or higher is needed (or for phones, iOS 11 or above). + +Installation +------------ +The method for installing this tool follows the `standard python package installation steps `_. Once you have set up a python environment, run:: + + pip install mxnet-to-coreml + +The package `documentation `_ contains more details on how to use coremltools. + +Dependencies +------------ +This tool has the following dependencies: + +* mxnet (0.10.0+) +* coremltools (0.5.1+) +* pyyaml (3.12+) + +Sample Usage +------------ + +In order to convert, say a `Squeezenet model `_, with labels from `synset.txt `_, execute this :: + + mxnet_coreml_converter.py --model-prefix='squeezenet_v1.1' \ + --epoch=0 --input-shape='{"data":"3,227,227"}' \ + --mode=classifier --pre-processing-arguments='{"image_input_names":"data"}' \ + --class-labels synset.txt --output-file="squeezenetv11.mlmodel" + +More Information +---------------- +* `On Github `_ +* `MXNet framework `_ +* `Apple CoreML `_ diff --git a/tools/coreml/pip_package/setup.py b/tools/coreml/pip_package/setup.py new file mode 100644 index 000000000000..18c601d38166 --- /dev/null +++ b/tools/coreml/pip_package/setup.py @@ -0,0 +1,69 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from setuptools import setup +from setuptools import find_packages + +# We are overriding the default behavior of bdist_wheel which is generating +# pure python wheels while we need platform specific wheel since this tool +# can only work on MacOS. +try: + from wheel.bdist_wheel import bdist_wheel as _bdist_wheel + class bdist_wheel(_bdist_wheel): + def finalize_options(self): + _bdist_wheel.finalize_options(self) + self.root_is_pure = False +except ImportError: + bdist_wheel = None + + +def readme(): + """ + Reads README.rst file and allows us to provide + a better experience for pypi webpage. + """ + with open('README.rst') as f: + return f.read() + +setup(name='mxnet-to-coreml', + version='0.1.0', + description='Tool to convert MXNet models into Apple CoreML model format.', + long_description=readme(), + classifiers=[ + 'Development Status :: 4 - Beta', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: MacOS :: MacOS X', + 'Programming Language :: Python :: 2.7', + 'Topic :: Software Development :: Libraries :: Python Modules' + ], + keywords='Apache MXNet Apple CoreML Converter Deep Learning', + url='https://github.com/apache/incubator-mxnet/tree/master/tools/coreml', + author='pracheer', + author_email='pracheer_gupta@hotmail.com', + license='Apache 2.0', + package_dir = {'': '..'}, + packages=['converter'], + install_requires=[ + 'mxnet', + 'coremltools', + 'pyyaml', + ], + scripts=['../mxnet_coreml_converter.py'], + python_requires='~=2.7', + zip_safe=False, + cmdclass={'bdist_wheel': bdist_wheel},) diff --git a/tools/coreml/test/test_mxnet_image.py b/tools/coreml/test/test_mxnet_image.py index ac30ac7f5ad9..2bbf7b1e264b 100644 --- a/tools/coreml/test/test_mxnet_image.py +++ b/tools/coreml/test/test_mxnet_image.py @@ -24,7 +24,7 @@ sys.path.append(current_working_directory + "/..") sys.path.append(current_working_directory + "/../converter/") import _mxnet_converter as mxnet_converter -from utils import load_model +from converter.utils import load_model VAL_DATA = 'data/val-5k-256.rec' From 12b244dae7957715ca4bb77d76448c744b7730c4 Mon Sep 17 00:00:00 2001 From: Rahul Huilgol Date: Mon, 28 Aug 2017 17:31:45 -0700 Subject: [PATCH 442/834] Parallelize windows unit tests of python 2 and 3 in jenkins (#7646) * parallelize python windows tests * reordered for clarity --- Jenkinsfile | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index ac34e71a53f1..fe0151a879d6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -342,7 +342,7 @@ try { } } }, - 'Python2/3: CPU Win':{ + 'Python 2: CPU Win':{ node('mxnetwindows') { ws('workspace/ut-python-cpu') { init_git_win() @@ -351,20 +351,30 @@ try { 7z x -y vc14_cpu.7z''' bat """xcopy C:\\mxnet\\data data /E /I /Y xcopy C:\\mxnet\\model model /E /I /Y - call activate py3 + call activate py2 set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_cpu\\python del /S /Q ${env.WORKSPACE}\\pkg_vc14_cpu\\python\\*.pyc C:\\mxnet\\test_cpu.bat""" - bat """xcopy C:\\mxnet\\data data /E /I /Y + } + } + }, + 'Python 3: CPU Win': { + node('mxnetwindows') { + ws('workspace/ut-python-cpu') { + init_git_win() + unstash 'vc14_cpu' + bat '''rmdir /s/q pkg_vc14_cpu + 7z x -y vc14_cpu.7z''' + bat """xcopy C:\\mxnet\\data data /E /I /Y xcopy C:\\mxnet\\model model /E /I /Y - call activate py2 + call activate py3 set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_cpu\\python del /S /Q ${env.WORKSPACE}\\pkg_vc14_cpu\\python\\*.pyc C:\\mxnet\\test_cpu.bat""" } } }, - 'Python2/3: GPU Win':{ + 'Python 2: GPU Win':{ node('mxnetwindows') { ws('workspace/ut-python-gpu') { init_git_win() @@ -373,19 +383,29 @@ try { 7z x -y vc14_gpu.7z''' bat """xcopy C:\\mxnet\\data data /E /I /Y xcopy C:\\mxnet\\model model /E /I /Y - call activate py3 + call activate py2 set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_gpu\\python del /S /Q ${env.WORKSPACE}\\pkg_vc14_gpu\\python\\*.pyc C:\\mxnet\\test_gpu.bat""" + } + } + }, + 'Python 3: GPU Win':{ + node('mxnetwindows') { + ws('workspace/ut-python-gpu') { + init_git_win() + unstash 'vc14_gpu' + bat '''rmdir /s/q pkg_vc14_gpu + 7z x -y vc14_gpu.7z''' bat """xcopy C:\\mxnet\\data data /E /I /Y xcopy C:\\mxnet\\model model /E /I /Y - call activate py2 + call activate py3 set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_gpu\\python del /S /Q ${env.WORKSPACE}\\pkg_vc14_gpu\\python\\*.pyc C:\\mxnet\\test_gpu.bat""" } } - } + } } stage('Integration Test') { From 03b1d8de0959bcfe91bf3279660f50a0248021b9 Mon Sep 17 00:00:00 2001 From: Hagay Lupesko Date: Mon, 28 Aug 2017 20:08:50 -0700 Subject: [PATCH 443/834] Removed asset loaded insecurely and added the asset to be loaded from the origin securely (#7649) --- docs/_static/mxnet-theme/navbar.html | 45 +-------------------------- docs/_static/mxnet.png | Bin 0 -> 67645 bytes 2 files changed, 1 insertion(+), 44 deletions(-) create mode 100644 docs/_static/mxnet.png diff --git a/docs/_static/mxnet-theme/navbar.html b/docs/_static/mxnet-theme/navbar.html index c88fb58bb5c2..0d49eeb4dc89 100644 --- a/docs/_static/mxnet-theme/navbar.html +++ b/docs/_static/mxnet-theme/navbar.html @@ -1,51 +1,8 @@ - -