Merge branch 'master' of https://github.com/apache/incubator-mxnet in…

…to refactor

CMakeLists.txt

@@ -186,7 +186,7 @@ endif()
 list(APPEND mxnet_LINKER_LIBS ${mshadow_LINKER_LIBS})
 
 foreach(var ${C_CXX_INCLUDE_DIRECTORIES})
- 	include_directories(${var})
+    include_directories(${var})
 endforeach()
 
 include_directories("include")
@@ -201,9 +201,13 @@ include_directories("dlpack/include")
 #  add_subdirectory(dlpack)
 #endif()
 
+# Prevent stripping out symbols (operator registrations, for example)
 if(NOT MSVC AND NOT APPLE)
   set(BEGIN_WHOLE_ARCHIVE -Wl,--whole-archive)
   set(END_WHOLE_ARCHIVE -Wl,--no-whole-archive)
+elseif(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
+  # using regular Clang or AppleClang
+  set(BEGIN_WHOLE_ARCHIVE -Wl,-force_load)
 endif()
 
 if(UNIX)
@@ -332,7 +336,7 @@ if(USE_CUDNN AND USE_CUDA)
     add_definitions(-DUSE_CUDNN)
     include_directories(SYSTEM ${CUDNN_INCLUDE})
     list(APPEND mxnet_LINKER_LIBS ${CUDNN_LIBRARY})
-	  add_definitions(-DMSHADOW_USE_CUDNN=1)
+      add_definitions(-DMSHADOW_USE_CUDNN=1)
   endif()
 endif()
 
@@ -372,17 +376,17 @@ assign_source_group("Include" ${GROUP_Include})
 assign_source_group("CUDA" ${GROUP_CUDA})
 
 if(USE_PLUGINS_WARPCTC)
-	set(WARPCTC_INCLUDE  "" CACHE PATH "WARPCTC include")
+    set(WARPCTC_INCLUDE  "" CACHE PATH "WARPCTC include")
     set(WARPCTC_LIB_DEBUG  "" CACHE FILEPATH "WARPCTC lib")
     set(WARPCTC_LIB_RELEASE  "" CACHE FILEPATH "WARPCTC lib")
 
 
-	include_directories(SYSTEM ${WARPCTC_INCLUDE})
-	list(APPEND mxnet_LINKER_LIBS ${WARPCTC_LIB})
-	FILE(GLOB_RECURSE PLUGINS_SOURCE "plugin/warpctc/*.cc" "plugin/warpctc/*.h")
-	FILE(GLOB_RECURSE PLUGINS_CUSRC "plugin/warpctc/*.cu")
-	list(APPEND SOURCE ${PLUGINS_SOURCE})
-	list(APPEND CUDA ${PLUGINS_CUSRC})
+    include_directories(SYSTEM ${WARPCTC_INCLUDE})
+    list(APPEND mxnet_LINKER_LIBS ${WARPCTC_LIB})
+    FILE(GLOB_RECURSE PLUGINS_SOURCE "plugin/warpctc/*.cc" "plugin/warpctc/*.h")
+    FILE(GLOB_RECURSE PLUGINS_CUSRC "plugin/warpctc/*.cu")
+    list(APPEND SOURCE ${PLUGINS_SOURCE})
+    list(APPEND CUDA ${PLUGINS_CUSRC})
 endif()
 
 if(USE_OPERATOR_TUNING)
@@ -425,11 +429,11 @@ if(USE_PLUGIN_CAFFE)
 endif()
 
 if (NOT (EXTRA_OPERATORS STREQUAL ""))
-	mxnet_source_group("Extra"   GLOB_RECURSE "${EXTRA_OPERATORS}/*.cc")
-	mxnet_source_group("Extra\\Cuda"   GLOB_RECURSE "${EXTRA_OPERATORS}/*.cu")
-	FILE(GLOB_RECURSE EXTRA_SRC "${EXTRA_OPERATORS}/*.cc")
-	FILE(GLOB_RECURSE EXTRA_CUSRC "${EXTRA_OPERATORS}/*.cu")
-	list(APPEND SOURCE ${EXTRA_SRC} ${EXTRA_CUSRC})
+    mxnet_source_group("Extra"   GLOB_RECURSE "${EXTRA_OPERATORS}/*.cc")
+    mxnet_source_group("Extra\\Cuda"   GLOB_RECURSE "${EXTRA_OPERATORS}/*.cu")
+    FILE(GLOB_RECURSE EXTRA_SRC "${EXTRA_OPERATORS}/*.cc")
+    FILE(GLOB_RECURSE EXTRA_CUSRC "${EXTRA_OPERATORS}/*.cu")
+    list(APPEND SOURCE ${EXTRA_SRC} ${EXTRA_CUSRC})
 endif()
 
 if(MSVC)
@@ -567,7 +571,7 @@ if(MSVC AND USE_MXNET_LIB_NAMING)
 endif()
 
 if(USE_PROFILER)
-	add_definitions(-DMXNET_USE_PROFILER)
+    add_definitions(-DMXNET_USE_PROFILER)
 endif()
 
 add_subdirectory(tests)
@@ -585,7 +589,7 @@ if (INSTALL_EXAMPLES)
 endif()
 
 if (USE_SIGNAL_HANDLER)
-	add_definitions(-DMXNET_USE_SIGNAL_HANDLER=1)
+    add_definitions(-DMXNET_USE_SIGNAL_HANDLER=1)
 endif()
 
 # AUTO_INSTALL_DIR -> Optional: specify post-build install direcory

Jenkinsfile

@@ -169,6 +169,42 @@ try {
         }
       }
     },
+    'CPU: Clang 3.9': {
+      node('mxnetlinux-cpu') {
+        ws('workspace/build-cpu-clang') {
+          init_git()
+          def flag = """ \
+            USE_PROFILER=1                \
+            USE_CPP_PACKAGE=1             \
+            USE_BLAS=openblas             \
+            USE_OPENMP=0                  \
+            CXX=clang++-3.9               \
+            CC=clang-3.9                  \
+            -j\$(nproc)
+            """
+          make("cpu_clang", flag)
+          pack_lib('cpu_clang')
+        }
+      }
+    },
+    'CPU: Clang 5': {
+      node('mxnetlinux-cpu') {
+        ws('workspace/build-cpu-clang') {
+          init_git()
+          def flag = """ \
+            USE_PROFILER=1                \
+            USE_CPP_PACKAGE=1             \
+            USE_BLAS=openblas             \
+            USE_OPENMP=1                  \
+            CXX=clang++-5.0               \
+            CC=clang-5.0                  \
+            -j\$(nproc)
+            """
+          make("cpu_clang", flag)
+          pack_lib('cpu_clang')
+        }
+      }
+    },
     'CPU: MKLDNN': {
       node('mxnetlinux-cpu') {
         ws('workspace/build-mkldnn-cpu') {

R-package/tests/testthat/test_model.R

@@ -172,7 +172,6 @@ test_that("Fine-tune", {
 })                                       
 
 test_that("Matrix Factorization", {
-  skip("Disabled due to an unavailible http server.  Tracked here: https://git.io/vNkrE")
   GetMovieLens()
   DF <- read.table("./data/ml-100k/u.data", header = F, sep = "\t")
   names(DF) <- c("user", "item", "score", "time")

cpp-package/include/mxnet-cpp/optimizer.h

@@ -146,6 +146,20 @@ class SGDOptimizer : public Optimizer {
   AtomicSymbolCreator mom_update_handle_;
 };
 
+class SignumOptimizer : public Optimizer {
+ public:
+  explicit SignumOptimizer(unsigned begin_num_update = 0);
+  std::string GetType() const override;
+  void Update(int index, NDArray weight, NDArray grad) override;
+ private:
+  virtual ~SignumOptimizer();
+  void CreateState_(int index, NDArray weight) override;
+  std::map<int, NDArray*> states_;
+  AtomicSymbolCreator update_handle_;
+  AtomicSymbolCreator mom_update_handle_;
+};
+
+
 class RMSPropOptimizer : public Optimizer {
  public:
   explicit RMSPropOptimizer(unsigned begin_num_update = 0);

cpp-package/include/mxnet-cpp/optimizer.hpp

@@ -131,6 +131,7 @@ inline Optimizer* OptimizerRegistry::Find(const std::string& name) {
   MXNETCPP_REGISTER_OPTIMIZER(adam, AdamOptimizer);
   MXNETCPP_REGISTER_OPTIMIZER(adagrad, AdaGradOptimizer);
   MXNETCPP_REGISTER_OPTIMIZER(adadelta, AdaDeltaOptimizer);
+  MXNETCPP_REGISTER_OPTIMIZER(signum, SignumOptimizer);
   auto it = cmap().find(name);
   if (it == cmap().end())
     return nullptr;
@@ -200,6 +201,69 @@ inline void SGDOptimizer::CreateState_(int index, NDArray weight) {
   }
 }
 
+// inplementing Signum optimizer
+
+inline SignumOptimizer::SignumOptimizer(unsigned begin_num_update)
+  : Optimizer(begin_num_update) {
+  update_handle_ = op_map()->GetSymbolCreator("signsgd_update");
+  mom_update_handle_ = op_map()->GetSymbolCreator("signum_update");
+}
+
+inline std::string SignumOptimizer::GetType() const {
+  return "signum";
+}
+
+inline SignumOptimizer::~SignumOptimizer() {
+  for (auto &it : states_) {
+    delete it.second;
+  }
+}
+
+inline void SignumOptimizer::Update(int index, NDArray weight, NDArray grad) {
+  if (states_.count(index) == 0) {
+    CreateState_(index, weight);
+  }
+
+  params_["lr"] = std::to_string(GetLR_(index));
+  params_["wd"] = std::to_string(GetWD_(index));
+  UpdateCount_(index);
+  auto keys = GetParamKeys_();
+  auto values = GetParamValues_();
+  CHECK_EQ(keys.size(), values.size());
+
+  NDArrayHandle inputs[3];
+  inputs[0] = weight.GetHandle();
+  inputs[1] = grad.GetHandle();
+
+  int num_outputs = 1;
+  NDArrayHandle output = weight.GetHandle();
+  NDArrayHandle *outputs = &output;
+
+  if (states_[index] == nullptr) {
+    MXImperativeInvoke(update_handle_, 2, inputs,
+        &num_outputs, &outputs,
+        keys.size(), keys.data(), values.data());
+  } else {
+    inputs[2] = states_[index]->GetHandle();
+    MXImperativeInvoke(mom_update_handle_, 3, inputs,
+        &num_outputs, &outputs,
+        keys.size(), keys.data(), values.data());
+  }
+}
+
+inline void SignumOptimizer::CreateState_(int index, NDArray weight) {
+  if (params_.count("momentum") == 0) {
+    states_[index] = nullptr;
+  } else {
+    states_[index] = new NDArray(weight.GetShape(), weight.GetContext());
+    *states_[index] = 0;
+  }
+}
+
+// finish implementing Signum
+
+
+
 inline RMSPropOptimizer::RMSPropOptimizer(unsigned begin_num_update)
   : Optimizer(begin_num_update) {
   update_handle_ = op_map()->GetSymbolCreator("rmsprop_update");

docs/tutorials/speech_recognition/ctc.md

@@ -0,0 +1,15 @@
+# Connectionist Temporal Classification
+
+[Connectionist Temporal Classification](https://www.cs.toronto.edu/~graves/icml_2006.pdf) (CTC) is a cost function that is used to train Recurrent Neural Networks (RNNs) to label unsegmented input sequence data in supervised learning. For example, in a speech recognition application, using a typical cross-entropy loss, the input signal needs to be segmented into words or sub-words. However, using CTC-loss, it suffices to provide one label sequence for input sequence and the network learns both the alignment as well labeling. Baidu's warp-ctc page contains a more detailed [introduction to CTC-loss](https://github.com/baidu-research/warp-ctc#introduction).
+
+## CTC-loss in MXNet
+MXNet supports two CTC-loss layers in Symbol API:
+
+* `mxnet.symbol.contrib.ctc_loss` is implemented in MXNet and included as part of the standard package.
+* `mxnet.symbol.WarpCTC` uses Baidu's warp-ctc library and requires building warp-ctc library and mxnet library both from source.
+
+## LSTM OCR Example
+MXNet's example folder contains a [CTC example](https://github.com/apache/incubator-mxnet/tree/master/example/ctc) for using CTC loss with an LSTM network to perform Optical Character Recognition (OCR) prediction on CAPTCHA images. The example demonstrates use of both CTC loss options, as well as inference after training using network symbol and parameter checkpoints.
+
+## Next Steps
+* [MXNet tutorials index](http://mxnet.io/tutorials/index.html)