From e65687910afcffe2970550a9bc59d2ea68418133 Mon Sep 17 00:00:00 2001 From: yajiedesign Date: Fri, 16 Oct 2015 18:52:54 +0800 Subject: [PATCH 01/11] add begin_round set in checkpoint --- python/mxnet/model.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/mxnet/model.py b/python/mxnet/model.py index fc10c24f74e0..10b2e4b69866 100644 --- a/python/mxnet/model.py +++ b/python/mxnet/model.py @@ -510,6 +510,7 @@ def __init__(self, symbol, ctx=None, numpy_batch_size=128, arg_params=None, aux_params=None, allow_extra_params=False, + begin_round=0, **kwargs): # check if symbol contain duplicated names. _check_arguments(symbol) @@ -542,6 +543,7 @@ def __init__(self, symbol, ctx=None, # internal helper state self._pred_exec = None self._pred_exec_input = None + self.begin_round = begin_round @staticmethod def _is_data_arg(name): @@ -708,7 +710,7 @@ def fit(self, X, y=None, eval_data=None, eval_metric='acc', # do training _train_multi_device(self.symbol, self.ctx, input_shape, self.arg_params, self.aux_params, - begin_round=0, end_round=self.num_round, + begin_round=self.begin_round, end_round=self.num_round, optimizer=optimizer, train_data=X, eval_data=eval_data, eval_metric=eval_metric, @@ -774,6 +776,7 @@ def load(prefix, iteration, ctx=None, **kwargs): symbol, arg_params, aux_params = load_checkpoint(prefix, iteration) return FeedForward(symbol, ctx=ctx, arg_params=arg_params, aux_params=aux_params, + begin_round=iteration **kwargs) @staticmethod From dbada7ad8e72040f39aaa2ca81f1eb3c17f8fda7 Mon Sep 17 00:00:00 2001 From: yajiedesign Date: Fri, 16 Oct 2015 19:32:51 +0800 Subject: [PATCH 02/11] some bug fix --- python/mxnet/model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/mxnet/model.py b/python/mxnet/model.py index 10b2e4b69866..d58e1a6f0877 100644 --- a/python/mxnet/model.py +++ b/python/mxnet/model.py @@ -776,7 +776,7 @@ def load(prefix, iteration, ctx=None, **kwargs): symbol, arg_params, aux_params = load_checkpoint(prefix, iteration) return FeedForward(symbol, ctx=ctx, arg_params=arg_params, aux_params=aux_params, - begin_round=iteration + begin_round=iteration, **kwargs) @staticmethod From ca2afb67bb2d9f43aa8bc390e4d89fd48c6d1a1e Mon Sep 17 00:00:00 2001 From: yajiedesign Date: Sat, 17 Oct 2015 06:58:58 +0800 Subject: [PATCH 03/11] update the docstring --- python/mxnet/model.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python/mxnet/model.py b/python/mxnet/model.py index d58e1a6f0877..a7d7846bafde 100644 --- a/python/mxnet/model.py +++ b/python/mxnet/model.py @@ -501,6 +501,9 @@ class FeedForward(BASE_ESTIMATOR): If this is True, no error will be thrown when aux_params and arg_params contain extra parameters than needed. + begin_round : int,optional + The begining training iteration. + **kwargs : dict The additional keyword arguments passed to optimizer. """ From bad46c1a51721654887c99ea4c53cf27d6bd21cb Mon Sep 17 00:00:00 2001 From: tqchen Date: Sat, 17 Oct 2015 17:29:26 -0700 Subject: [PATCH 04/11] Upgrade mshadow --- mshadow | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mshadow b/mshadow index 3fb5e7365388..9aaa23b55999 160000 --- a/mshadow +++ b/mshadow @@ -1 +1 @@ -Subproject commit 3fb5e7365388293d5e75077fa068e9fdea02d4fb +Subproject commit 9aaa23b559990bb9354254e607e52e493fd8f7b9 From 3ae0c800c3dac7685f2d6472acfb008c6a8229cb Mon Sep 17 00:00:00 2001 From: Chiyuan Zhang Date: Sat, 17 Oct 2015 20:51:26 -0400 Subject: [PATCH 05/11] keep a shared_ptr to objectpool from threaded-engine --- src/common/object_pool.h | 19 +++++++++++++++++-- src/engine/threaded_engine.h | 7 +++++++ src/engine/threaded_engine_perdevice.cc | 3 +++ src/resource.cc | 3 +++ 4 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/common/object_pool.h b/src/common/object_pool.h index 5787f53ff497..a651a0f3ca57 100644 --- a/src/common/object_pool.h +++ b/src/common/object_pool.h @@ -41,6 +41,12 @@ class ObjectPool { */ static ObjectPool* Get(); + /*! + * \brief Get a shared ptr of the singleton instance of pool. + * \return Shared pointer to the Object Pool. + */ + static std::shared_ptr _GetSharedRef(); + private: /*! * \brief Internal structure to hold pointers. @@ -107,6 +113,8 @@ struct ObjectPoolAllocatable { template ObjectPool::~ObjectPool() { + printf("~ObjectPool() this = %p\n", this); + fflush(stdout); // TODO(hotpxl): mind destruction order // for (auto i : allocated_) { // free(i); @@ -141,8 +149,15 @@ void ObjectPool::Delete(T* ptr) { template ObjectPool* ObjectPool::Get() { - static ObjectPool inst; - return &inst; + //static ObjectPool inst; + //return &inst; + return _GetSharedRef().get(); +} + +template +std::shared_ptr > ObjectPool::_GetSharedRef() { + static std::shared_ptr > inst_ptr(new ObjectPool()); + return inst_ptr; } template diff --git a/src/engine/threaded_engine.h b/src/engine/threaded_engine.h index 7a8708484f9c..e2f5c39035f9 100644 --- a/src/engine/threaded_engine.h +++ b/src/engine/threaded_engine.h @@ -251,6 +251,7 @@ class ThreadedEngine : public Engine { ThreadedEngine() { engine_info_ = dmlc::GetEnv("MXNET_ENGINE_INFO", false); + objpool_ref_ = common::ObjectPool::_GetSharedRef(); } ~ThreadedEngine() { { @@ -329,6 +330,12 @@ class ThreadedEngine : public Engine { */ std::mutex finished_m_; std::condition_variable finished_cv_; + + /*! + * \brief Holding a shared_ptr to the object pool to prevent it from being destructed too early + * See also #309 (https://github.com/dmlc/mxnet/issues/309) + */ + std::shared_ptr > objpool_ref_; /*! * \brief Disallow copy construction and assignment. */ diff --git a/src/engine/threaded_engine_perdevice.cc b/src/engine/threaded_engine_perdevice.cc index 62738cbaedac..d34fa6cec59f 100644 --- a/src/engine/threaded_engine_perdevice.cc +++ b/src/engine/threaded_engine_perdevice.cc @@ -45,6 +45,9 @@ class ThreadedEnginePerDevice : public ThreadedEngine { // GPU tasks will be created lazily } ~ThreadedEnginePerDevice() noexcept(false) { + printf("~ThreadedEnginePerDevice()\n"); + fflush(stdout); + gpu_normal_workers_.Clear(); gpu_copy_workers_.Clear(); cpu_normal_workers_.Clear(); diff --git a/src/resource.cc b/src/resource.cc index 73e8856bf336..fd71a84bbef3 100644 --- a/src/resource.cc +++ b/src/resource.cc @@ -29,6 +29,9 @@ class ResourceManagerImpl : public ResourceManager { Context::CPU(), cpu_temp_space_copy_)); } ~ResourceManagerImpl() { + printf("~ResourceManagerImpl()\n"); + fflush(stdout); + // need explicit delete, before engine get killed cpu_rand_.reset(nullptr); cpu_space_.reset(nullptr); From eda0498cc5ac2cb20d4800623e0c5fd293d981a5 Mon Sep 17 00:00:00 2001 From: Chiyuan Zhang Date: Sat, 17 Oct 2015 21:02:24 -0400 Subject: [PATCH 06/11] more shared pointers needed to keep from crashing --- src/engine/threaded_engine.h | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/engine/threaded_engine.h b/src/engine/threaded_engine.h index e2f5c39035f9..c5437c370484 100644 --- a/src/engine/threaded_engine.h +++ b/src/engine/threaded_engine.h @@ -251,7 +251,12 @@ class ThreadedEngine : public Engine { ThreadedEngine() { engine_info_ = dmlc::GetEnv("MXNET_ENGINE_INFO", false); - objpool_ref_ = common::ObjectPool::_GetSharedRef(); + + objpool_opr_ref_ = common::ObjectPool::_GetSharedRef(); + objpool_blk_ref_ = common::ObjectPool::_GetSharedRef(); + objpool_varblk_ref_ = common::ObjectPool::_GetSharedRef(); + objpool_var_ref_ = common::ObjectPool::_GetSharedRef(); + } ~ThreadedEngine() { { @@ -335,7 +340,10 @@ class ThreadedEngine : public Engine { * \brief Holding a shared_ptr to the object pool to prevent it from being destructed too early * See also #309 (https://github.com/dmlc/mxnet/issues/309) */ - std::shared_ptr > objpool_ref_; + std::shared_ptr > objpool_opr_ref_; + std::shared_ptr > objpool_blk_ref_; + std::shared_ptr > objpool_varblk_ref_; + std::shared_ptr > objpool_var_ref_; /*! * \brief Disallow copy construction and assignment. */ From 2f3789a2193d6de181adb2ad98a710a86cf64ec4 Mon Sep 17 00:00:00 2001 From: Chiyuan Zhang Date: Sat, 17 Oct 2015 21:03:25 -0400 Subject: [PATCH 07/11] remove debug prints --- src/common/object_pool.h | 2 -- src/engine/threaded_engine_perdevice.cc | 3 --- src/resource.cc | 3 --- 3 files changed, 8 deletions(-) diff --git a/src/common/object_pool.h b/src/common/object_pool.h index a651a0f3ca57..86f2e1d81c84 100644 --- a/src/common/object_pool.h +++ b/src/common/object_pool.h @@ -113,8 +113,6 @@ struct ObjectPoolAllocatable { template ObjectPool::~ObjectPool() { - printf("~ObjectPool() this = %p\n", this); - fflush(stdout); // TODO(hotpxl): mind destruction order // for (auto i : allocated_) { // free(i); diff --git a/src/engine/threaded_engine_perdevice.cc b/src/engine/threaded_engine_perdevice.cc index d34fa6cec59f..62738cbaedac 100644 --- a/src/engine/threaded_engine_perdevice.cc +++ b/src/engine/threaded_engine_perdevice.cc @@ -45,9 +45,6 @@ class ThreadedEnginePerDevice : public ThreadedEngine { // GPU tasks will be created lazily } ~ThreadedEnginePerDevice() noexcept(false) { - printf("~ThreadedEnginePerDevice()\n"); - fflush(stdout); - gpu_normal_workers_.Clear(); gpu_copy_workers_.Clear(); cpu_normal_workers_.Clear(); diff --git a/src/resource.cc b/src/resource.cc index fd71a84bbef3..73e8856bf336 100644 --- a/src/resource.cc +++ b/src/resource.cc @@ -29,9 +29,6 @@ class ResourceManagerImpl : public ResourceManager { Context::CPU(), cpu_temp_space_copy_)); } ~ResourceManagerImpl() { - printf("~ResourceManagerImpl()\n"); - fflush(stdout); - // need explicit delete, before engine get killed cpu_rand_.reset(nullptr); cpu_space_.reset(nullptr); From 6d8f72b05860face462cdfab08c0589ffd257efb Mon Sep 17 00:00:00 2001 From: Chiyuan Zhang Date: Sat, 17 Oct 2015 21:06:21 -0400 Subject: [PATCH 08/11] cleanup --- src/common/object_pool.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/common/object_pool.h b/src/common/object_pool.h index 86f2e1d81c84..ac12e2b30c6e 100644 --- a/src/common/object_pool.h +++ b/src/common/object_pool.h @@ -147,8 +147,6 @@ void ObjectPool::Delete(T* ptr) { template ObjectPool* ObjectPool::Get() { - //static ObjectPool inst; - //return &inst; return _GetSharedRef().get(); } From f3b11659fe4309371c39d593fdf860d546fa573b Mon Sep 17 00:00:00 2001 From: Chiyuan Zhang Date: Sat, 17 Oct 2015 23:13:50 -0400 Subject: [PATCH 09/11] fix lint error --- src/engine/threaded_engine.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/engine/threaded_engine.h b/src/engine/threaded_engine.h index c5437c370484..9a7c32525961 100644 --- a/src/engine/threaded_engine.h +++ b/src/engine/threaded_engine.h @@ -256,7 +256,6 @@ class ThreadedEngine : public Engine { objpool_blk_ref_ = common::ObjectPool::_GetSharedRef(); objpool_varblk_ref_ = common::ObjectPool::_GetSharedRef(); objpool_var_ref_ = common::ObjectPool::_GetSharedRef(); - } ~ThreadedEngine() { { From 55948a77a7fe00a896d55b497138e69d20645815 Mon Sep 17 00:00:00 2001 From: tqchen Date: Sat, 17 Oct 2015 20:34:18 -0700 Subject: [PATCH 10/11] [R] Fix the shuffle --- CMakeLists.txt | 22 +++++++++++----------- R-package/R/io.R | 8 +++++++- R-package/demo/basic_training.R | 8 ++++---- R-package/src/base.h | 14 ++------------ R-package/src/executor.cc | 1 - R-package/src/executor.h | 11 +++++------ R-package/src/io.cc | 24 ++++++++++++++---------- R-package/src/io.h | 16 +++++++--------- R-package/src/kvstore.cc | 1 - R-package/src/kvstore.h | 7 ++++--- R-package/src/ndarray.h | 1 + R-package/src/symbol.cc | 1 - R-package/src/symbol.h | 23 +++++++++++------------ mshadow | 2 +- 14 files changed, 67 insertions(+), 72 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 078a2bfcb0a5..607d6ea909c1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 2.8.7) +cmake_minimum_required(VERSION 2.8.7) project(mxnet C CXX) @@ -47,13 +47,13 @@ if(USE_OPENCV) endif() if(USE_OPENMP) - FIND_PACKAGE( OpenMP REQUIRED) - if(OPENMP_FOUND) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") - set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}") - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}") - endif() + FIND_PACKAGE( OpenMP REQUIRED) + if(OPENMP_FOUND) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}") + endif() endif() # cudnn detection @@ -86,11 +86,11 @@ if(NOT MSVC) # Only add c++11 flags and definitions after cuda compiling add_definitions(-DDMLC_USE_CXX11) add_definitions(-DMSHADOW_IN_CXX11) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c++0x") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c++0x") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x") endif() -add_library(mxnet SHARED ${SOURCE}) +add_library(mxnet SHARED ${SOURCE}) target_link_libraries(mxnet ${mshadow_LINKER_LIBS}) target_link_libraries(mxnet dmlccore) target_link_libraries(mxnet pslite) diff --git a/R-package/R/io.R b/R-package/R/io.R index 37244ccbd7e8..938c501f689a 100644 --- a/R-package/R/io.R +++ b/R-package/R/io.R @@ -36,11 +36,17 @@ mx.io.extract <- function(iter, field) { #' @param shuffle Whether shuffle the data #' #' @export -mx.io.arrayiter <- function(data, label=NULL, +mx.io.arrayiter <- function(data, label, batch.size=128, shuffle=FALSE) { + if (shuffle) { + unif.rnds <- as.array(mx.runif(c(length(label)), ctx=mx.cpu())); + } else { + unif.rnds <- mx.array(0) + } mx.io.internal.arrayiter(as.array(data), as.array(label), + unif.rnds, batch.size, shuffle) } diff --git a/R-package/demo/basic_training.R b/R-package/demo/basic_training.R index 0b93b3ea4c92..6fa7cd766c93 100644 --- a/R-package/demo/basic_training.R +++ b/R-package/demo/basic_training.R @@ -12,23 +12,23 @@ act2 <- mx.symbol.Activation(fc2, name="relu2", act_type="relu") fc3 <- mx.symbol.FullyConnected(act2, name="fc3", num_hidden=10) softmax <- mx.symbol.Softmax(fc3, name = "sm") -dtrain = mx.varg.io.MNISTIter(list( +dtrain = mx.io.MNISTIter( image="data/train-images-idx3-ubyte", label="data/train-labels-idx1-ubyte", data.shape=c(784), batch.size=batch.size, flat=TRUE, silent=0, - seed=10)) + seed=10) -dtest = mx.varg.io.MNISTIter(list( +dtest = mx.io.MNISTIter( image="data/t10k-images-idx3-ubyte", label="data/t10k-labels-idx1-ubyte", data.shape=c(784), batch.size=batch.size, shuffle=FALSE, flat=TRUE, - silent=0)) + silent=0) # X is R's array, we load from mxnet's native iter structure, but you don't have to X = mx.io.extract(dtrain, "data") y = mx.io.extract(dtrain, "label") diff --git a/R-package/src/base.h b/R-package/src/base.h index 0ca33ccb91c7..fae6b005958e 100644 --- a/R-package/src/base.h +++ b/R-package/src/base.h @@ -23,8 +23,10 @@ namespace R { /*! \brief macro to be compatible with non c++11 env */ #if DMLC_USE_CXX11 == 0 +#ifndef nullptr #define nullptr NULL #endif +#endif /*! * \brief Log that enables Stop and print message to R console @@ -90,8 +92,6 @@ void SetSeed(int seed); * \brief Base Movable class of MXNet Module object. * This class will define several common functions. * \tparam Class The class name of subclass - * \tparam HandleType The type of handle the object have. - * \tparam finalizer The free function used to delete the handle. */ template class MXNetMovable { @@ -116,15 +116,6 @@ class MXNetMovable { protected: /*! \brief default constructor */ MXNetMovable() : moved_(false) {} - /*! - * \brief the finalizer for Rcpp - * \param self the pointer to the class. - */ - inline static void Finalizer(Class* self) { - if (!static_cast*>(self)->moved_) { - self->DoFinalize(); - } - } /*! * \brief Default implement to Move a existing R Class object to a new one. * \param src The source R Object. @@ -137,7 +128,6 @@ class MXNetMovable { return Rcpp::internal::make_new_object(moved); } - private: /*! \brief Whether the object has been moved */ bool moved_; }; diff --git a/R-package/src/executor.cc b/R-package/src/executor.cc index 31a8a1f1c9ef..33a28218f103 100644 --- a/R-package/src/executor.cc +++ b/R-package/src/executor.cc @@ -220,7 +220,6 @@ Executor::RObjectType Executor::Bind(const Symbol::RObjectType& symbol, void Executor::InitRcppModule() { using namespace Rcpp; // NOLINT(*) class_("MXExecutor") - .finalizer(&Executor::Finalizer) .method("update.aux.arrays", &Executor::UpdateAuxArray, "Update auxilary states array of executor, this will mutate the executor") diff --git a/R-package/src/executor.h b/R-package/src/executor.h index 0ddee19b40c8..31075b265161 100644 --- a/R-package/src/executor.h +++ b/R-package/src/executor.h @@ -120,23 +120,26 @@ class Executor : public MXNetMovable { static void InitRcppModule(); // destructor ~Executor() { - // delete can handle nullptr safely delete out_arrays_; delete arg_arrays_; delete grad_arrays_; delete aux_arrays_; + + if (!this->moved_) { + MX_CALL(MXExecutorFree(handle_)); + } } private: // friend with symbol friend class Symbol; - friend class MXNetMovable; // internal constructor, enable trivial operator= Executor() : out_arrays_(nullptr), arg_arrays_(nullptr), grad_arrays_(nullptr), aux_arrays_(nullptr) {} + /*! \return a new Object that is moved from current one */ inline Executor* CreateMoveObject() { Executor *moved = new Executor(); @@ -147,10 +150,6 @@ class Executor : public MXNetMovable { aux_arrays_ = nullptr; return moved; } - // finalizer that invoked on non-movable object - inline void DoFinalize() { - MX_CALL(MXExecutorFree(handle_)); - } /*! * \brief Clone src into a new space. * \param src source list of arrays to clone. diff --git a/R-package/src/io.cc b/R-package/src/io.cc index 5fbcd9451ba5..d0916b25b5ec 100644 --- a/R-package/src/io.cc +++ b/R-package/src/io.cc @@ -12,10 +12,6 @@ namespace mxnet { namespace R { -// Rcpp random wrapper -inline size_t RcppRandWrapper(const size_t n) { - return floor(unif_rand() * n); -} void MXDataIter::Reset() { MX_CALL(MXDataIterBeforeFirst(handle_)); @@ -44,14 +40,22 @@ Rcpp::List MXDataIter::Value() const { ArrayDataIter::ArrayDataIter(const Rcpp::NumericVector& data, const Rcpp::NumericVector& label, + const Rcpp::NumericVector& unif_rnds, int batch_size, bool shuffle) : counter_(0) { std::vector order(label.size()); for (size_t i = 0; i < order.size(); ++i) { order[i] = i; } + if (shuffle) { - std::random_shuffle(order.begin(), order.end(), RcppRandWrapper); + RCHECK(unif_rnds.size() == label.size()); + for (size_t i = order.size() - 1; i != 0; --i) { + size_t idx = static_cast(unif_rnds[i] * (i + 1)); + if (idx < i) { + std::swap(order[i], order[idx]); + } + } } ArrayDataIter::Convert(data, order, batch_size, &data_); ArrayDataIter::Convert(label, order, batch_size, &label_); @@ -118,9 +122,11 @@ int ArrayDataIter::NumPad() const { Rcpp::RObject ArrayDataIter::Create(const Rcpp::NumericVector& data, const Rcpp::NumericVector& label, + const Rcpp::NumericVector& unif_rnds, size_t batch_size, bool shuffle) { - return Rcpp::internal::make_new_object(new ArrayDataIter(data, label, batch_size, shuffle)); + return Rcpp::internal::make_new_object( + new ArrayDataIter(data, label, unif_rnds, batch_size, shuffle)); } DataIterCreateFunction::DataIterCreateFunction @@ -185,12 +191,10 @@ void DataIter::InitRcppModule() { .method("num.pad", &DataIter::NumPad); class_("MXNativeDataIter") - .derives("MXDataIter") - .finalizer(&MXDataIter::Finalizer); + .derives("MXDataIter"); class_("MXArrayDataIter") - .derives("MXDataIter") - .finalizer(&ArrayDataIter::Finalizer); + .derives("MXDataIter"); function("mx.io.internal.arrayiter", &ArrayDataIter::Create); } diff --git a/R-package/src/io.h b/R-package/src/io.h index 74c2f49d7d75..e643b958e944 100644 --- a/R-package/src/io.h +++ b/R-package/src/io.h @@ -61,6 +61,9 @@ class MXDataIter : public DataIter { virtual bool Next(); virtual int NumPad() const; virtual Rcpp::List Value() const; + virtual ~MXDataIter() { + MX_CALL(MXDataIterFree(handle_)); + } private: friend class DataIter; @@ -76,10 +79,6 @@ class MXDataIter : public DataIter { inline static Rcpp::RObject RObject(DataIterHandle handle) { return Rcpp::internal::make_new_object(new MXDataIter(handle)); } - // finalizer that invoked on non-movable object - static void Finalizer(MXDataIter *iter) { - MX_CALL(MXDataIterFree(iter->handle_)); - } /*! \brief internal data iter handle */ DataIterHandle handle_; }; @@ -100,11 +99,14 @@ class ArrayDataIter : public DataIter { * \brief Construct a ArrayDataIter from data and label. * \param data The data array. * \param label The label array. + * \param unif_rnds Uniform [0,1] random number of same length as label. + * Only needed when shuffle=TRUE * \param batch_size The size of the batch. * \param shuffle Whether shuffle the data. */ ArrayDataIter(const Rcpp::NumericVector& data, const Rcpp::NumericVector& label, + const Rcpp::NumericVector& unif_rnds, int batch_size, bool shuffle); virtual void Reset() { @@ -115,6 +117,7 @@ class ArrayDataIter : public DataIter { virtual Rcpp::List Value() const; static Rcpp::RObject Create(const Rcpp::NumericVector& data, const Rcpp::NumericVector& label, + const Rcpp::NumericVector& unif_rnds, size_t batch_size, bool shuffle); @@ -125,11 +128,6 @@ class ArrayDataIter : public DataIter { const std::vector &order, size_t batch_size, std::vector *out); - // finalizer that invoked on non-movable object - static void Finalizer(ArrayDataIter *iter) { - iter->data_.clear(); - iter->label_.clear(); - } /*! \brief The counter */ size_t counter_; /*! \brief number of pad instances*/ diff --git a/R-package/src/kvstore.cc b/R-package/src/kvstore.cc index 30571c62d6ea..9896a8762b94 100644 --- a/R-package/src/kvstore.cc +++ b/R-package/src/kvstore.cc @@ -152,7 +152,6 @@ Rcpp::RObject KVStore::Create(const char *type) { void KVStore::InitRcppModule() { using namespace Rcpp; // NOLINT(*) class_("MXKVStore") - .finalizer(&KVStore::Finalizer) .method("init", &KVStore::Init) .method("push", &KVStore::Push) .method("pull", &KVStore::Pull) diff --git a/R-package/src/kvstore.h b/R-package/src/kvstore.h index d82ef0e071c3..d4a92dfb7dad 100644 --- a/R-package/src/kvstore.h +++ b/R-package/src/kvstore.h @@ -68,13 +68,14 @@ class KVStore { static Rcpp::RObject Create(const char *type); /*! \brief initialize the R cpp Module */ static void InitRcppModule(); + // destructor + ~KVStore() { + MX_CALL(MXKVStoreFree(handle_)); + } private: explicit KVStore(KVStoreHandle handle) : handle_(handle), optimizer_set_(false) {} - static void Finalizer(KVStore *kv) { - MX_CALL(MXKVStoreFree(kv->handle_)); - } // the internal callback to kvstore. NDArray CreateState(int index, const NDArray& weight) const; /*! \brief internal KVStore handle */ diff --git a/R-package/src/ndarray.h b/R-package/src/ndarray.h index 377a7a39a1b1..deb0d27af881 100644 --- a/R-package/src/ndarray.h +++ b/R-package/src/ndarray.h @@ -91,6 +91,7 @@ class NDArray { // operator overloading inline NDArray& operator=(const NDArray& other) { ptr_ = other.ptr_; + return *this; } inline NDBlob* operator->() { return ptr_.get(); diff --git a/R-package/src/symbol.cc b/R-package/src/symbol.cc index 55f28f899587..26b5088bbd1b 100644 --- a/R-package/src/symbol.cc +++ b/R-package/src/symbol.cc @@ -296,7 +296,6 @@ SEXP SymbolFunction::operator() (SEXP* args) { void Symbol::InitRcppModule() { using namespace Rcpp; // NOLINT(*) class_("MXSymbol") - .finalizer(&Symbol::Finalizer) .method("debug.str", &Symbol::DebugStr, "Return the debug string of internals of symbol") .method("apply", &Symbol::Apply, diff --git a/R-package/src/symbol.h b/R-package/src/symbol.h index f944d12e672d..4bf2531a290a 100644 --- a/R-package/src/symbol.h +++ b/R-package/src/symbol.h @@ -18,8 +18,10 @@ namespace R { class SymbolFunction; /*! \brief The Rcpp Symbol class of MXNet */ -class Symbol : public MXNetMovable { +class Symbol { public: + // typedef RObjectType + typedef Rcpp::RObject RObjectType; /*! \return typename from R side. */ inline static const char* TypeName() { return "MXSymbol"; @@ -97,12 +99,19 @@ class Symbol : public MXNetMovable { static RObjectType Group(const Rcpp::List& symbols); /*! \brief static function to initialize the Rcpp functions */ static void InitRcppModule(); + // destructor + ~Symbol() { + MX_CALL(MXSymbolFree(handle_)); + } + // get external pointer of Symbol + inline static Symbol* XPtr(const Rcpp::RObject& obj) { + return Rcpp::as(obj); + } private: // friend with SymbolFunction friend class SymbolFunction; friend class Executor; - friend class MXNetMovable; // enable trivial copy constructors etc. Symbol() {} // constructor @@ -115,16 +124,6 @@ class Symbol : public MXNetMovable { inline static Rcpp::RObject RObject(SymbolHandle handle) { return Rcpp::internal::make_new_object(new Symbol(handle)); } - // Create a new Object that is moved from current one - inline Symbol* CreateMoveObject() { - Symbol* moved = new Symbol(); - *moved = *this; - return moved; - } - // finalizer that invoked on non-movable object - inline void DoFinalize() { - MX_CALL(MXSymbolFree(handle_)); - } /*! * \brief Return a clone of Symbol * Do not expose to R side diff --git a/mshadow b/mshadow index 9aaa23b55999..d2c27549571f 160000 --- a/mshadow +++ b/mshadow @@ -1 +1 @@ -Subproject commit 9aaa23b559990bb9354254e607e52e493fd8f7b9 +Subproject commit d2c27549571fb6a71e81d8b860b1484809d8922f From ced5486e8111744c4d1408ddef1486601c879186 Mon Sep 17 00:00:00 2001 From: tqchen Date: Sat, 17 Oct 2015 21:28:36 -0700 Subject: [PATCH 11/11] [DOC] Detail build info --- R-package/README.md | 17 ++++-- doc/R-package/index.md | 24 ++++---- doc/build.md | 132 ++++++++++++++++++++++++++++++----------- make/osx.mk | 84 ++++++++++++++++++++++++++ python/setup.py | 4 +- 5 files changed, 206 insertions(+), 55 deletions(-) create mode 100644 make/osx.mk diff --git a/R-package/README.md b/R-package/README.md index d9a79a36bddb..309a7874340f 100644 --- a/R-package/README.md +++ b/R-package/README.md @@ -1,18 +1,23 @@ MXNet R-Package =============== -The MXNet R packages brings flexible and efficient GPU computing and deep learning to R. + +You have find MXNet R Package! The MXNet R packages brings flexible and efficient GPU +computing and state-of-art deep learning to R. - It enables you to write seamless tensor/matrix computation with multiple GPUs in R. - It also enables you construct and customize the state-of-art deep learning models in R, and apply them to tasks such as image classification and data science challenges. -Document --------- +Sounds exciting? This page contains links to all the related documents on R package. + +Resources +--------- * [MXNet R Package Document](http://mxnet.readthedocs.org/en/latest/R-package/index.html) - - This is a online hosted documents for mxnet examples etc. + - Check this out for detailed documents, examples, installation guides. + Installation ------------ -- First build ```../lib/libmxnet.so``` by following [Build Instruction](../doc/build.md) -- Type ```R CMD INSTALL R-package``` in the root folder. +Follow [Installation Guide](http://mxnet.readthedocs.org/en/latest/build.html) + diff --git a/doc/R-package/index.md b/doc/R-package/index.md index 20d5e70f1ac3..055fd2284e06 100644 --- a/doc/R-package/index.md +++ b/doc/R-package/index.md @@ -1,24 +1,26 @@ MXNet R Package =============== -This page contains links to all the python related documents on R package. -The MXNet R packages brings flexible and efficient GPU computing and deep learning to R. +You have find MXNet R Package! The MXNet R packages brings flexible and efficient GPU +computing and state-of-art deep learning to R. - It enables you to write seamless tensor/matrix computation with multiple GPUs in R. - It also enables you construct and customize the state-of-art deep learning models in R, and apply them to tasks such as image classification and data science challenges. +Sounds exciting? This page contains links to all the related documents on R package. + +Get Started +----------- +There are several information to get you started +* [Installation Guide](../build.md) contains instructions to install mxnet. +* [Tutorials](#tutorials) contains various examples how how mxnet can be applied to different cool tasks :) +* [Contributor Guide](http://mxnet.readthedocs.org/en/latest/contribute.html#r-package) + - The R package section gives various guidelines on how to contribute code, tutorial, rmarkdown examples to mxnet. + - Your contribution is always welcomed! + Tutorials --------- * [Classify Realworld Images with Pretrained Model](classifyRealImageWithPretrainedModel.md) * [Handwritten Digits Classification Competition](mnistCompetition.md) * [Tutorial on NDArray and Symbol](ndarrayAndSymbolTutorial.md) -Installation ------------- -- First build ```../lib/libmxnet.so``` by following [Build Instruction](../doc/build.md) -- Type ```R CMD INSTALL R-package``` in the root folder. - -Contributor Guide ------------------ -* [Contributor Guide](http://mxnet.readthedocs.org/en/latest/contribute.html#r-package) R package section gives various guidelines on how to contribute code, tutorial, rmarkdown examples to mxnet. - diff --git a/doc/build.md b/doc/build.md index 137235dbdc69..db8b4c585504 100644 --- a/doc/build.md +++ b/doc/build.md @@ -1,74 +1,134 @@ -Build and Installation -====================== +Installation Guide +================== +This page gives the detail of how to install mxnet packages on various systems. +We tried to listed the detailed, but if the information on this page does not work for you. +Please ask questions at [mxnet/issues](https://github.com/dmlc/mxnet/issues), better still +if you have ideas to improve this page, please send a pull request! + +Contents +-------- +- [Build MXNet Library](#build-mxnet-library) + - Introduces how to build the mxnet core library for all packages. + - Supported platforms: linux, windows, osx +- [Advanced Build Configurations](#advanced-build-configuration) + - Introduces how to build mxnet with advanced features such as HDFS/S3 support, CUDNN +- [Python Package Installation](#python-package-installation) +- [R Package Installation](#r-package-installation) + +Build MXNet Library +------------------- +MXNet have a general runtime library that can be used by various packages such as python, R and Julia. +This section gives details about how to build the mxnet library. +- On Linux/OSX the target library will be ```libmxnet.so``` +- On Windows the target libary is ```mxnet.dll``` + +Things to do before get started: + +- Clone the project from github +```bash +git clone --recursive https://github.com/dmlc/mxnet +``` -Minimal system requirement: +The system dependency requirement for mxnet libraries are -- recent c++ compiler supporting C++ 11 such as `g++ >= 4.8` +- Recent c++ compiler supporting C++ 11 such as `g++ >= 4.8` - git - BLAS library. -- opencv +- opencv (optional if you do not need image augmentation, you can switch it off in config.mk) -On Ubuntu >= 13.10, one can install them by +### Linux + +On Ubuntu >= 13.10, one can install the dependencies by ```bash sudo apt-get update sudo apt-get install -y build-essential git libblas-dev libopencv-dev ``` -Then build mxnet - +Then build mxnet on the project root ```bash -git clone --recursive https://github.com/dmlc/mxnet -cd mxnet; make -j4 +make -j4 ``` +Then proceed to package installation instructions for python or R in this page. -To install the python package, first make sure `python >= 2.7` and `numpy >= ?` are installed, then +### OSX +On OSX, we can install the dependencies by ```bash -cd python; python setup.py install +brew update +brew tap homebrew/science +brew info opencv +brew install opencv ``` -If anything goes well, now we can train a multilayer perceptron on the hand -digit recognition dataset. +- Copy ```make/osx.mk``` to project root ```config.mk```. +```bash +cp make/osx.mk config.mk +``` +Then build mxnet on the project root ```bash -cd ..; python example/mnist/mlp.py +make -j4 ``` -Advanced Build --------------- +Then proceed to package installation instructions for python or R in this page. -- update the repo: +### Windows -```bash -git pull -git submodule update -``` +Firstly, we should make your Visual Studio 2013 support more C++11 features. -- install python package in developing model, + - Download and install [Visual C++ Compiler Nov 2013 CTP](http://www.microsoft.com/en-us/download/details.aspx?id=41151). + - Copy all files in `C:\Program Files (x86)\Microsoft Visual C++ Compiler Nov 2013 CTP` (or the folder where you extracted the zip archive) to `C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC` and overwrite all existed files. Don't forget to backup the original files before copying. -```bash -cd python; python setup.py develop --user -``` +Secondly, fetch the third-party libraries, including [OpenCV](http://sourceforge.net/projects/opencvlibrary/files/opencv-win/3.0.0/opencv-3.0.0.exe/download), [CuDNN](https://developer.nvidia.com/cudnn) and [OpenBlas](http://sourceforge.net/projects/openblas/files/v0.2.14/)(ignore this if you have MKL). + + - NOTICE: You need to register as a NVIDIA community user to get the download link of CuDNN. + +Finally, use CMake to create a Visual Studio solution in `./build/`. During configuration, you may need to set the path of each third-party library, until no error is reported. Open the solution and compile, you will get a `mxnet.dll` in `./build/Release` or `./build/Debug`. +Then proceed to package installation instructions for python or R in this page. + +Advanced Build Configurations +----------------------------- +The configuration of mxnet can be modified by ```config.mk``` - modify the compiling options such as compilers, CUDA, CUDNN, Intel MKL, various distributed filesystem such as HDFS/Amazon S3/... - - First copy [make/config.mk](../make/config.mk) to the project root, then +- First copy [make/config.mk](../make/config.mk) to the project root, then modify the according flags. -Build in Visual Studio 2013 +Python Package Installation --------------------------- +To install the python package. First finish the [Build MXNet Library](#build-mxnet-library) step. +Then use the following command to install mxnet. -Firstly, we should make your Visual Studio 2013 support more C++11 features. +```bash +cd python; python setup.py install +``` - - Download and install [Visual C++ Compiler Nov 2013 CTP](http://www.microsoft.com/en-us/download/details.aspx?id=41151). - - Copy all files in `C:\Program Files (x86)\Microsoft Visual C++ Compiler Nov 2013 CTP` (or the folder where you extracted the zip archive) to `C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC` and overwrite all existed files. Don't forget to backup the original files before copying. +If anything goes well, now we can train a multilayer perceptron on the hand +digit recognition dataset. -Secondly, fetch the third-party libraries, including [OpenCV](http://sourceforge.net/projects/opencvlibrary/files/opencv-win/3.0.0/opencv-3.0.0.exe/download), [CuDNN](https://developer.nvidia.com/cudnn) and [OpenBlas](http://sourceforge.net/projects/openblas/files/v0.2.14/)(ignore this if you have MKL). +```bash +cd ..; python example/mnist/mlp.py +``` - - NOTICE: You need to register as a NVIDIA community user to get the download link of CuDNN. +YOu can also install python to your user directory instead of root. -Finally, use CMake to create a Visual Studio solution in `./build/`. During configuration, you may need to set the path of each third-party library, until no error is reported. Open the solution and compile, you will get a `mxnet.dll` in `./build/Release` or `./build/Debug`. +```bash +cd python; python setup.py develop --user +``` + +R Package Installation +---------------------- +To install the python package. First finish the [Build MXNet Library](#build-mxnet-library) step. +Then use the following command to install mxnet at root folder + +```bash +R CMD INSTALL R-Package +``` + +Hopefully, we will now have mxnet on R! -The following steps are the same with Linux. +## Note on Library Build +We isolate the library build with Rcpp end to maximize the portability + - MSVC is needed on windows to build the mxnet library, because of CUDA compatiblity issue of toolchains. \ No newline at end of file diff --git a/make/osx.mk b/make/osx.mk new file mode 100644 index 000000000000..cf5d040732df --- /dev/null +++ b/make/osx.mk @@ -0,0 +1,84 @@ +#------------------------------------------------------------------------------- +# Template configuration for compiling mxnet +# +# If you want to change the configuration, please use the following +# steps. Assume you are on the root directory of mxnet. First copy the this +# file so that any local changes will be ignored by git +# +# $ cp make/config.mk . +# +# Next modify the according entries, and then compile by +# +# $ make +# +# or build in parallel with 8 threads +# +# $ make -j8 +#------------------------------------------------------------------------------- + +#--------------------- +# choice of compiler +#-------------------- + +export CC = gcc +export CXX = g++ +export NVCC = nvcc + +# whether compile with debug +DEBUG = 0 + +# the additional link flags you want to add +ADD_LDFLAGS = + +# the additional compile flags you want to add +ADD_CFLAGS = + +#--------------------------------------------- +# matrix computation libraries for CPU/GPU +#--------------------------------------------- + +# whether use CUDA during compile +USE_CUDA = 0 + +# add the path to CUDA libary to link and compile flag +# if you have already add them to enviroment variable, leave it as NONE +# USE_CUDA_PATH = /usr/local/cuda +USE_CUDA_PATH = NONE + +# whether use CUDNN R3 library +USE_CUDNN = 0 + +# whether use opencv during compilation +# you can disable it, however, you will not able to use +# imbin iterator +USE_OPENCV = 1 + +# use openmp for parallelization +USE_OPENMP = 0 + +# choose the version of blas you want to use +# can be: mkl, blas, atlas, openblas +USE_BLAS = apple + +# add path to intel libary, you may need it for MKL, if you did not add the path +# to enviroment variable +USE_INTEL_PATH = NONE + +#---------------------------- +# distributed computing +#---------------------------- + +# whether or not to enable mullti-machine supporting +USE_DIST_KVSTORE = 0 + +# whether or not allow to read and write HDFS directly. If yes, then hadoop is +# required +USE_HDFS = 0 + +# path to libjvm.so. required if USE_HDFS=1 +LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server + +# whether or not allow to read and write AWS S3 directly. If yes, then +# libcurl4-openssl-dev is required, it can be installed on Ubuntu by +# sudo apt-get install -y libcurl4-openssl-dev +USE_S3 = 0 diff --git a/python/setup.py b/python/setup.py index a41b2f0bd3d4..c3bfbbcd220a 100644 --- a/python/setup.py +++ b/python/setup.py @@ -18,8 +18,8 @@ version=__version__, description=open(os.path.join(CURRENT_DIR, 'README.md')).read(), install_requires=[ - 'numpy', - ], + 'numpy', + ], zip_safe=False, packages=['mxnet'], data_files=[('mxnet', [LIB_PATH[0]])],