diff --git a/.gitignore b/.gitignore index 5ada46a483c1..04153e1f7136 100644 --- a/.gitignore +++ b/.gitignore @@ -62,9 +62,11 @@ __pycache__ *.json *.d build +cmake-build-debug +cmake-build-release +cmake-build data recommonmark -bin deps # R @@ -85,6 +87,7 @@ R-package/R/mxnet_generated.R *ubyte *.bin *.txt +!CMakeLists.txt # ipython notebook *_pb2.py @@ -141,7 +144,12 @@ tools/pip_package/mxnet.egg-info tools/pip_package/mxnet # temporary path for building dependencies when building wheel -deps/ - +./deps/ +bld +./tmp/* *.jar -target \ No newline at end of file +target +cmake-build + + +bin/im2rec diff --git a/.gitmodules b/.gitmodules index 63d0c4cbeb04..08f2bc99f2aa 100644 --- a/.gitmodules +++ b/.gitmodules @@ -13,4 +13,4 @@ [submodule "cub"] path = cub url = https://github.com/NVlabs/cub - shallow = true + shallow=true diff --git a/.travis.yml b/.travis.yml index 0cb5b33f6870..c8ba0b1e645b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,8 @@ os: # - linux - osx +osx_image: xcode8 + env: # code analysis # - TASK=lint @@ -21,9 +23,19 @@ env: # TODO, R test, distributed test, clang, more g++ versions -# matrix: -# exclude: -# - os: osx +matrix: + include: + - # os: linux + # dist: trusty + # env: TASK=perl_test + - os: osx + ## sudo is required because + ## prexexisting packages conflict + ## with new ones. + ## would be nice to have macports + ## on travis osx, it has all needed perl packages + sudo: required + env: TASK=perl_test # env: TASK=julia JULIA_VER=0.4 # - os: linux # env: TASK=build @@ -57,6 +69,11 @@ addons: - python3-dev - python3-nose - graphviz + - libmouse-perl + - pdl + - cpanminus + - swig + - libgraphviz-perl before_install: - export NVCC_PREFIX=${HOME} diff --git a/CMakeLists.txt b/CMakeLists.txt index fe5cf32ae68a..4c94073fe419 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 2.8.7) +cmake_minimum_required(VERSION 3.0.2) project(mxnet C CXX) @@ -9,18 +9,29 @@ endif() set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake/Modules;${CMAKE_MODULE_PATH}") include(cmake/Utils.cmake) -mxnet_option(USE_OPENCV "Build with OpenCV support" ON) -mxnet_option(USE_OPENMP "Build with Openmp support" ON) -mxnet_option(USE_CUDNN "Build with cudnn support" ON) # one could set CUDNN_ROOT for search path -mxnet_option(USE_CUDA "Build with CUDA support" ON) -mxnet_option(USE_PROFILER "Build with Profiler support" OFF) -mxnet_option(USE_DIST_KVSTORE "Build with DIST_KVSTORE support" OFF) +mxnet_option(USE_OPENCV "Build with OpenCV support" ON) +mxnet_option(USE_OPENMP "Build with Openmp support" ON) +mxnet_option(USE_CUDA "Build with CUDA support" ON) +mxnet_option(USE_CUDNN "Build with cudnn support" ON) # one could set CUDNN_ROOT for search path +mxnet_option(USE_MKL_IF_AVAILABLE "Use MKL if found" ON) +mxnet_option(USE_MKLML_MKL "Use MKLML variant of MKL (if MKL found)" ON IF USE_MKL_IF_AVAILABLE AND UNIX AND (NOT APPLE)) +mxnet_option(USE_MKL_EXPERIMENTAL "Use experimental MKL (if MKL enabled and found)" OFF) +mxnet_option(USE_JEMALLOC "Build with Jemalloc support" OFF) +mxnet_option(USE_PROFILER "Build with Profiler support" OFF) +mxnet_option(USE_DIST_KVSTORE "Build with DIST_KVSTORE support" OFF) mxnet_option(USE_PLUGINS_WARPCTC "Use WARPCTC Plugins" OFF) -mxnet_option(USE_PLUGIN_CAFFE "Use Caffe Plugin" OFF) +mxnet_option(USE_PLUGIN_CAFFE "Use Caffe Plugin" OFF) +mxnet_option(USE_CPP_PACKAGE "Build C++ Package" OFF) mxnet_option(USE_MXNET_LIB_NAMING "Use MXNet library naming conventions." ON) SET(EXTRA_OPERATORS "" CACHE PATH "EXTRA OPERATORS PATH") +if("$ENV{VERBOSE}" STREQUAL "1") + message(STATUS " Verbose Makefile ACTIVATED") + set(CMAKE_VERBOISE_MAKEFILE ON) +endif() + + if(MSVC) add_definitions(-DWIN32_LEAN_AND_MEAN) add_definitions(-DDMLC_USE_CXX11) @@ -51,6 +62,37 @@ else(MSVC) endif() endif(MSVC) +set(mxnet_LINKER_LIBS "") + +if(USE_MKL_IF_AVAILABLE) + if(USE_MKL_EXPERIMENTAL AND NOT USE_MKLML_MKL) + message(ERROR " USE_MKL_EXPERIMENTAL can only be used when USE_MKL_EXPERIMENTAL is enabled") + endif() + find_package(MKL) + if(MKL_FOUND) + include_directories(${MKL_INCLUDE_DIR}) + include_directories(${CMAKE_CURRENT_SOURCE_DIR}/src/operator/mkl) + add_definitions(-DMXNET_USE_MKL2017=1) + add_definitions(-DUSE_MKL=1) + add_definitions(-DCUB_MKL=1) + list(APPEND mxnet_LINKER_LIBS ${MKL_LIBRARIES}) + if(NOT MSVC) + list(APPEND mxnet_LINKER_LIBS dl) + endif() + if(USE_MKL_EXPERIMENTAL) + add_definitions(-DMKL_EXPERIMENTAL=1) + else() + add_definitions(-DMKL_EXPERIMENTAL=0) + endif() + else() + message(STATUS " MKL not found") + endif() +endif() + +# Allow Cuda compiles outside of src tree to find things in 'src' and 'include' +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/src) + if(EXISTS ${PROJECT_SOURCE_DIR}/mshadow/cmake) include(mshadow/cmake/mshadow.cmake) include(mshadow/cmake/Utils.cmake) @@ -61,21 +103,40 @@ else() include(mshadow) endif() +list(APPEND mxnet_LINKER_LIBS ${mshadow_LINKER_LIBS}) + foreach(var ${C_CXX_INCLUDE_DIRECTORIES}) include_directories(${var}) endforeach() -set(mxnet_LINKER_LIBS "") -set(mxnet_LINKER_LIBS_DEBUG "") -set(mxnet_LINKER_LIBS_RELEASE "") -list(APPEND mxnet_LINKER_LIBS ${mshadow_LINKER_LIBS}) - include_directories("include") include_directories("mshadow") include_directories("cub") include_directories("nnvm/include") include_directories("dmlc-core/include") +if(NOT MSVC) + set(BEGIN_WHOLE_ARCHIVE -Wl,--whole-archive) + set(END_WHOLE_ARCHIVE -Wl,--no-whole-archive) +endif() + +if(UNIX) + find_library(RTLIB rt) + if(RTLIB) + list(APPEND mxnet_LINKER_LIBS ${RTLIB}) + endif() +endif() + +# ---[ jemalloc +if(USE_JEMALLOC) + find_package(JeMalloc) + if(JEMALLOC_FOUND) + add_definitions(-DUSE_JEMALLOC) + include_directories(${JEMALLOC_INCLUDE_DIRS}) + set(mxnet_LINKER_LIBS ${mxnet_LINKER_LIBS} ${JEMALLOC_LIBRARIES}) + endif() +endif() + if(USE_OPENCV) find_package(OpenCV QUIET COMPONENTS core highgui imgproc imgcodecs) if(NOT OpenCV_FOUND) # if not OpenCV 3.x, then imgcodecs are not found @@ -83,15 +144,19 @@ if(USE_OPENCV) endif() include_directories(SYSTEM ${OpenCV_INCLUDE_DIRS}) list(APPEND mxnet_LINKER_LIBS ${OpenCV_LIBS}) + message(STATUS " OpenCV_LIBS=${OpenCV_LIBS}") message(STATUS "OpenCV found (${OpenCV_CONFIG_PATH})") add_definitions(-DMXNET_USE_OPENCV=1) + if(NOT MSVC) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,--no-undefined") + endif() else(USE_OPENCV) message(STATUS "OpenCV Disabled") add_definitions(-DMXNET_USE_OPENCV=0) endif() if(USE_OPENMP) - FIND_PACKAGE( OpenMP REQUIRED) + find_package(OpenMP REQUIRED) if(OPENMP_FOUND) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") @@ -100,6 +165,25 @@ if(USE_OPENMP) endif() endif() +if(UNIX) + find_library(RTLIB rt) + if(RTLIB) + list(APPEND mxnet_LINKER_LIBS ${RTLIB}) + endif() +endif() + +# ---[ jemalloc +if(USE_JEMALLOC) + find_package(JeMalloc) + if(JEMALLOC_FOUND) + add_definitions(-DUSE_JEMALLOC) + include_directories(${JEMALLOC_INCLUDE_DIRS}) + set(mxnet_LINKER_LIBS ${mxnet_LINKER_LIBS} ${JEMALLOC_LIBRARIES}) + endif() +endif() + +include(CTest) + # cudnn detection if(USE_CUDNN AND USE_CUDA) detect_cuDNN() @@ -111,15 +195,10 @@ if(USE_CUDNN AND USE_CUDA) endif() endif() -if(EXISTS ${PROJECT_SOURCE_DIR}/dmlc-core/cmake) +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/dmlc-core/cmake) add_subdirectory("dmlc-core") endif() -if(USE_DIST_KVSTORE) - if(EXISTS ${PROJECT_SOURCE_DIR}/ps-lite/CMakeLists.txt) - add_subdirectory("ps-lite") - endif() -endif() mxnet_source_group("Include\\common" GLOB "src/common/*.h") mxnet_source_group("Include\\c_api" GLOB "src/c_api/*.h") @@ -199,8 +278,7 @@ if(USE_PLUGINS_WARPCTC) set(WARPCTC_INCLUDE "" CACHE PATH "WARPCTC include") set(WARPCTC_LIB_DEBUG "" CACHE FILEPATH "WARPCTC lib") set(WARPCTC_LIB_RELEASE "" CACHE FILEPATH "WARPCTC lib") - set(mxnet_LINKER_LIBS_RELEASE ${WARPCTC_LIB_RELEASE}) - set(mxnet_LINKER_LIBS_DEBUG ${WARPCTC_LIB_DEBUG}) + include_directories(SYSTEM ${WARPCTC_INCLUDE}) list(APPEND mxnet_LINKER_LIBS ${WARPCTC_LIB}) @@ -214,8 +292,12 @@ if(USE_PLUGINS_WARPCTC) endif() if(USE_PLUGIN_CAFFE) + if(NOT USE_CUDA) + set(CPU_ONLY ON) + add_definitions(-DCPU_ONLY=1) + endif() if(NOT DEFINED CAFFE_PATH) - if(EXISTS ${PROJECT_SOURCE_DIR}/caffe) + if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/caffe) # Need newer FindCUDA.cmake that correctly handles -std=c++11 cmake_minimum_required(VERSION 3.3) set(CAFFE_PATH ${PROJECT_SOURCE_DIR}/caffe) @@ -237,6 +319,7 @@ if(USE_PLUGIN_CAFFE) list(APPEND SOURCE ${PLUGINS_SOURCE}) list(APPEND CUDA ${PLUGINS_CUSRC}) include_directories(${CMAKE_BINARY_DIR}/include) + add_definitions(-DMXNET_USE_CAFFE=1) list(APPEND mxnet_LINKER_LIBS protobuf boost_system boost_thread boost_filesystem gflags glog caffe @@ -271,27 +354,39 @@ if(USE_CUDA) list(APPEND mxnet_LINKER_LIBS ${CUDA_nvrtc_LIBRARY}) set(CUDA_cuda_LIBRARY "${CUDA_nvrtc_LIBRARY}/../cuda.lib") list(APPEND mxnet_LINKER_LIBS ${CUDA_cuda_LIBRARY}) + FIND_LIBRARY(CUDA_cufft_LIBRARY nvrtc "${CUDA_TOOLKIT_ROOT_DIR}/lib/x64" "${CUDA_TOOLKIT_ROOT_DIR}/lib/win32") + list(APPEND mxnet_LINKER_LIBS "${CUDA_cufft_LIBRARY}/../cufft.lib") # For fft operator else(MSVC) - list(APPEND mxnet_LINKER_LIBS nvrtc cuda) + list(APPEND mxnet_LINKER_LIBS nvrtc cuda cufft) link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") endif() list(APPEND SOURCE ${cuda_objs} ${CUDA}) add_definitions(-DMXNET_USE_CUDA=1) add_definitions(-DMXNET_USE_NVRTC=1) if(CUDA_LIBRARY_PATH) - link_directories(${CUDA_LIBRARY_PATH}/stubs) + if(IS_CONTAINER_BUILD) + # In case of building on a production-like build container which may not have Cuda installed + if(NOT CMAKE_SYSTEM_HAS_CUDA) + # Assuming building in a container that doesn't have CUDA installed (ie CPU-only build machine) + # so use the stub cuda driver shared library + if(EXISTS ${CUDA_LIBRARY_PATH}/stubs/libcuda.so) + link_directories(${CUDA_LIBRARY_PATH}/stubs) + endif() + endif() + endif() endif() endif() # unsupported: if caffe is a subdirectory of mxnet, load its CMakeLists.txt as well if(USE_PLUGIN_CAFFE) - if(EXISTS ${PROJECT_SOURCE_DIR}/caffe) + if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/caffe) add_subdirectory(caffe) endif() endif() if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/nnvm/CMakeLists.txt") - list(APPEND mxnet_LINKER_LIBS nnvm) + set(nnvm_LINKER_LIBS nnvm) + list(APPEND mxnet_LINKER_LIBS ${nnvm_LINKER_LIBS}) endif() if(NOT MSVC) @@ -307,44 +402,98 @@ endif() if(${CMAKE_SYSTEM_NAME} STREQUAL "Darwin" AND USE_MXNET_LIB_NAMING) add_library(mxnet MODULE ${SOURCE}) else() - add_library(mxnet SHARED ${SOURCE}) + if(UNIX) + set(MXNET_DYNAMIC_ONLY ON) + if(MXNET_DYNAMIC_ONLY) + add_library(mxnet SHARED ${SOURCE}) + else() + set(INITIALIZE_SOURCE_FILE ${CMAKE_CURRENT_SOURCE_DIR}/src/initialize.cc) + list(REMOVE_ITEM SOURCE ${INITIALIZE_SOURCE_FILE}) + add_library(mxnet_static STATIC ${INITIALIZE_SOURCE_FILE} ${SOURCE}) + # Need an arbitrary source file to trigger CMake to build the library + add_library(mxnet SHARED ${INITIALIZE_SOURCE_FILE}) + # This has prolems, as it adds libmxnet_static to INTERFACE_LINK_LIBRARIES + target_link_libraries(mxnet "-Wl,--whole-archive $ -Wl,--no-whole-archive") + #target_link_libraries(mxnet mxnet_static) + add_custom_target( + StaticallyLinkStaticMXNetLibrary ALL + BYPRODUCTS ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY}/libmxnet.a + WORKING_DIRECTORY ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY} + COMMAND ln -sf libmxnet_static.a libmxnet.a + DEPENDS mxnet_static + ) + endif() + else() + add_library(mxnet SHARED ${SOURCE}) + endif() endif() target_link_libraries(mxnet ${mxnet_LINKER_LIBS}) if(USE_PLUGINS_WARPCTC) - target_link_libraries(mxnet debug ${mxnet_LINKER_LIBS_DEBUG}) - target_link_libraries(mxnet optimized ${mxnet_LINKER_LIBS_RELEASE}) + target_link_libraries(mxnet debug ${WARPCTC_LIB_DEBUG}) + target_link_libraries(mxnet optimized ${WARPCTC_LIB_RELEASE}) endif() target_link_libraries(mxnet dmlccore) - if(MSVC AND USE_MXNET_LIB_NAMING) set_target_properties(mxnet PROPERTIES OUTPUT_NAME "libmxnet") - endif() + if(USE_DIST_KVSTORE) - add_definitions(-DMXNET_USE_DIST_KVSTORE) - target_link_libraries(mxnet pslite) - target_link_libraries(mxnet ${pslite_LINKER_LIBS}) - include_directories(SYSTEM ${pslite_INCLUDE_DIR}) + if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/ps-lite/CMakeLists.txt) + add_subdirectory("ps-lite") + list(APPEND pslite_LINKER_LIBS pslite) + target_link_libraries(mxnet debug ${pslite_LINKER_LIBS_DEBUG}) + target_link_libraries(mxnet optimized ${pslite_LINKER_LIBS_RELEASE}) + else() + set(pslite_LINKER_LIBS protobuf zmq-static ) + endif() + add_definitions(-DMXNET_USE_DIST_KVSTORE) + target_link_libraries(mxnet ${pslite_LINKER_LIBS}) + include_directories(SYSTEM ${pslite_INCLUDE_DIR}) endif() if(USE_PROFILER) add_definitions(-DMXNET_USE_PROFILER) endif() +# Do tests after chrpath so that we use the "real" cuda driver +add_subdirectory(tests) + +# AUTO_INSTALL_DIR -> Optional: specify post-build install direcory +if(AUTO_INSTALL_DIR) + # ---[ Install Includes + add_custom_command(TARGET mxnet POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_directory + ${PROJECT_SOURCE_DIR}/include ${AUTO_INSTALL_DIR}/include + ) + + # ---[ Install Examples + add_custom_command(TARGET mxnet POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_directory + ${PROJECT_SOURCE_DIR}/example ${AUTO_INSTALL_DIR}/example + ) +endif() + if(INSTALL_PYTHON_VERSIONS) + message(STATUS "Installing for python versions: ${INSTALL_PYTHON_VERSIONS}") foreach(version ${INSTALL_PYTHON_VERSIONS}) set(outdir ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/python${version}/site-packages/mxnet) add_custom_command(TARGET mxnet POST_BUILD COMMAND mkdir -p ${outdir} - COMMAND cp -rul ${CMAKE_CURRENT_SOURCE_DIR}/python/mxnet/* ${outdir} + COMMAND cp -ru ${CMAKE_CURRENT_SOURCE_DIR}/python/mxnet/* ${outdir} ) endforeach() endif() +if(USE_CPP_PACKAGE) + add_subdirectory(cpp-package) +endif() + +add_subdirectory(example/image-classification/predict-cpp) + # ---[ Linter target if(MSVC) find_package(PythonInterp) @@ -353,5 +502,3 @@ endif() set(LINT_DIRS include src scripts python) add_custom_target(mxnet_lint COMMAND ${CMAKE_COMMAND} -DMSVC=${MSVC} -DPYTHON_EXECUTABLE=${PYTHON_EXECUTABLE} -DLINT_DIRS=${LINT_DIRS} -DPROJECT_SOURCE_DIR=${PROJECT_SOURCE_DIR} -DPROJECT_NAME=mxnet -P ${PROJECT_SOURCE_DIR}/dmlc-core/cmake/lint.cmake) -add_subdirectory(tests/cpp) -add_subdirectory(example/image-classification/predict-cpp) \ No newline at end of file diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 3db3e4e448d1..7c01c62e433f 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -40,11 +40,11 @@ The committers are the granted write access to the project. * [Yuan Tang](https://github.com/terrytangyuan) - Yuan is one of major maintainers of mxnet scala package. -### Become a Comitter -MXNet is a opensource project and we are actively looking for new comitters -who are willing to help maintaining and lead the project. Committers comes from contributors who: +### Become a Committer +MXNet is a opensource project and we are actively looking for new committers +who are willing to help maintaining and leading the project. Committers come from contributors who: * Made substantial contribution to the project. -* Willing to actively spent time on maintaining and lead the project. +* Willing to actively spend time on maintaining and leading the project. New committers will be proposed by current committers, with support from more than two of current committers. @@ -122,3 +122,12 @@ List of Contributors * [Yu Du](https://github.com/Answeror) * [Xu Dong](https://github.com/dsqx71) * [Chihiro Komaki](https://github.com/ckomaki) +* [Piyush Singh](https://github.com/Piyush3dB) +* [Freddy Chua](https://github.com/freddycct) +* [Jie Zhang](https://github.com/luoyetx) +* [Leonard Lausen](https://github.com/leezu) +* [Sergey Kolychev](https://github.com/sergeykolychev) + - Sergey is original author and current maintainer of Perl5 interface. +* [Robert Stone](https://github.com/tlby) +* [Pedro Larroy](https://github.com/larroy) +* [Jun Wu](https://github.com/reminisce) diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 000000000000..35fb2d70881a --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,322 @@ +// -*- mode: groovy -*- +// Jenkins pipeline +// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ + +// mxnet libraries +mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, dmlc-core/libdmlc.a, nnvm/lib/libnnvm.a' +// command to start a docker container +docker_run = 'tests/ci_build/ci_build.sh' +// timeout in minutes +max_time = 60 + +// initialize source codes +def init_git() { + checkout scm + retry(5) { + timeout(time: 2, unit: 'MINUTES') { + sh 'git submodule update --init' + } + } +} + +def init_git_win() { + checkout scm + retry(5) { + timeout(time: 2, unit: 'MINUTES') { + bat 'git submodule update --init' + } + } +} + +stage("Sanity Check") { + timeout(time: max_time, unit: 'MINUTES') { + node('linux') { + ws('workspace/sanity') { + init_git() + make('lint', 'cpplint rcpplint jnilint') + make('lint', 'pylint') + } + } + } +} + +// Run make. First try to do an incremental make from a previous workspace in hope to +// accelerate the compilation. If something wrong, clean the workspace and then +// build from scratch. +def make(docker_type, make_flag) { + timeout(time: max_time, unit: 'MINUTES') { + try { + sh "${docker_run} ${docker_type} make ${make_flag}" + } catch (exc) { + echo 'Incremental compilation failed. Fall back to build from scratch' + sh "${docker_run} ${docker_type} make clean" + sh "${docker_run} ${docker_type} make ${make_flag}" + } + } +} + +// pack libraries for later use +def pack_lib(name, libs=mx_lib) { + sh """ +echo "Packing ${libs} into ${name}" +echo ${libs} | sed -e 's/,/ /g' | xargs md5sum +""" + stash includes: libs, name: name +} + + +// unpack libraries saved before +def unpack_lib(name, libs=mx_lib) { + unstash name + sh """ +echo "Unpacked ${libs} from ${name}" +echo ${libs} | sed -e 's/,/ /g' | xargs md5sum +""" +} + +stage('Build') { + parallel 'CPU: Openblas': { + node('linux') { + ws('workspace/build-cpu') { + init_git() + def flag = """ \ +USE_PROFILER=1 \ +USE_BLAS=openblas \ +-j\$(nproc) +""" + make("cpu", flag) + pack_lib('cpu') + } + } + }, + 'GPU: CUDA7.5+cuDNN5': { + node('GPU' && 'linux') { + ws('workspace/build-gpu') { + init_git() + def flag = """ \ +USE_PROFILER=1 \ +USE_BLAS=openblas \ +USE_CUDA=1 \ +USE_CUDA_PATH=/usr/local/cuda \ +USE_CUDNN=1 \ +-j\$(nproc) +""" + make('gpu', flag) + pack_lib('gpu') + } + } + }, + 'Amalgamation': { + node('linux') { + ws('workspace/amalgamation') { + init_git() + make('cpu', '-C amalgamation/ USE_BLAS=openblas MIN=1') + } + } + }, + 'GPU: MKLML': { + node('GPU' && 'linux') { + ws('workspace/build-mklml') { + init_git() + def flag = """ \ +USE_PROFILER=1 \ +USE_BLAS=openblas \ +USE_MKL2017=1 \ +USE_MKL2017_EXPERIMENTAL=1 \ +USE_CUDA=1 \ +USE_CUDA_PATH=/usr/local/cuda \ +USE_CUDNN=1 \ +-j\$(nproc) +""" + make('mklml_gpu', flag) + pack_lib('mklml') + } + } + }, + 'CPU windows':{ + node('windows') { + ws('workspace/build-cpu') { + withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas', 'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0']) { + init_git_win() + bat """mkdir build_vc14_cpu +cd build_vc14_cpu +cmake -G \"Visual Studio 14 2015 Win64\" -DUSE_CUDA=0 -DUSE_CUDNN=0 -DUSE_NVRTC=0 -DUSE_OPENCV=1 -DUSE_OPENMP=1 -DUSE_PROFILER=1 -DUSE_BLAS=open -DUSE_DIST_KVSTORE=0 ${env.WORKSPACE}""" + bat 'C:\\mxnet\\build_vc14_cpu.bat' + + bat '''rmdir /s/q pkg_vc14_gpu +mkdir pkg_vc14_cpu\\lib +mkdir pkg_vc14_cpu\\python +mkdir pkg_vc14_cpu\\include +mkdir pkg_vc14_cpu\\build +copy build_vc14_cpu\\Release\\libmxnet.lib pkg_vc14_cpu\\lib +copy build_vc14_cpu\\Release\\libmxnet.dll pkg_vc14_cpu\\build +xcopy python pkg_vc14_cpu\\python /E /I /Y +xcopy include pkg_vc14_cpu\\include /E /I /Y +xcopy dmlc-core\\include pkg_vc14_cpu\\include /E /I /Y +xcopy mshadow\\mshadow pkg_vc14_cpu\\include\\mshadow /E /I /Y +xcopy nnvm\\include pkg_vc14_cpu\\nnvm\\include /E /I /Y +del /Q *.7z +7z.exe a vc14_cpu.7z pkg_vc14_cpu\\ +''' + stash includes: 'vc14_cpu.7z', name: 'vc14_cpu' + } + } + } + }, + 'GPU windows':{ + node('windows') { + ws('workspace/build-gpu') { + withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas', 'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0']) { + init_git_win() + bat """mkdir build_vc14_gpu +call "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\x86_amd64\\vcvarsx86_amd64.bat" +cd build_vc14_gpu +cmake -G \"NMake Makefiles JOM\" -DUSE_CUDA=1 -DUSE_CUDNN=1 -DUSE_NVRTC=1 -DUSE_OPENCV=1 -DUSE_OPENMP=1 -DUSE_PROFILER=1 -DUSE_BLAS=open -DUSE_DIST_KVSTORE=0 -DCUDA_ARCH_NAME=All -DCMAKE_CXX_FLAGS_RELEASE="/FS /MD /O2 /Ob2 /DNDEBUG" -DCMAKE_BUILD_TYPE=Release ${env.WORKSPACE}""" + bat 'C:\\mxnet\\build_vc14_gpu.bat' + bat '''rmdir /s/q pkg_vc14_gpu +mkdir pkg_vc14_gpu\\lib +mkdir pkg_vc14_gpu\\python +mkdir pkg_vc14_gpu\\include +mkdir pkg_vc14_gpu\\build +copy build_vc14_gpu\\libmxnet.lib pkg_vc14_gpu\\lib +copy build_vc14_gpu\\libmxnet.dll pkg_vc14_gpu\\build +xcopy python pkg_vc14_gpu\\python /E /I /Y +xcopy include pkg_vc14_gpu\\include /E /I /Y +xcopy dmlc-core\\include pkg_vc14_gpu\\include /E /I /Y +xcopy mshadow\\mshadow pkg_vc14_gpu\\include\\mshadow /E /I /Y +xcopy nnvm\\include pkg_vc14_gpu\\nnvm\\include /E /I /Y +del /Q *.7z +7z.exe a vc14_gpu.7z pkg_vc14_gpu\\ +''' + stash includes: 'vc14_gpu.7z', name: 'vc14_gpu' + } + } + } + } +} + +// Python unittest for CPU +def python_ut(docker_type) { + timeout(time: max_time, unit: 'MINUTES') { + sh "${docker_run} ${docker_type} PYTHONPATH=./python/ nosetests --with-timer --verbose tests/python/unittest" + sh "${docker_run} ${docker_type} PYTHONPATH=./python/ nosetests-3.4 --with-timer --verbose tests/python/unittest" + } +} + +// GPU test has two parts. 1) run unittest on GPU, 2) compare the results on +// both CPU and GPU +def python_gpu_ut(docker_type) { + timeout(time: max_time, unit: 'MINUTES') { + sh "${docker_run} ${docker_type} PYTHONPATH=./python/ nosetests --with-timer --verbose tests/python/gpu" + sh "${docker_run} ${docker_type} PYTHONPATH=./python/ nosetests-3.4 --with-timer --verbose tests/python/gpu" + } +} + +stage('Unit Test') { + parallel 'Python2/3: CPU': { + node('linux') { + ws('workspace/ut-python-cpu') { + init_git() + unpack_lib('cpu') + python_ut('cpu') + } + } + }, + 'Python2/3: GPU': { + node('GPU' && 'linux') { + ws('workspace/ut-python-gpu') { + init_git() + unpack_lib('gpu', mx_lib) + python_gpu_ut('gpu') + } + } + }, + 'Python2/3: MKLML': { + node('GPU' && 'linux') { + ws('workspace/ut-python-mklml') { + init_git() + unpack_lib('mklml') + python_ut('mklml_gpu') + python_gpu_ut('mklml_gpu') + } + } + }, + 'Scala: CPU': { + node('linux') { + ws('workspace/ut-scala-cpu') { + init_git() + unpack_lib('cpu') + timeout(time: max_time, unit: 'MINUTES') { + sh "${docker_run} cpu make scalapkg USE_BLAS=openblas" + sh "${docker_run} cpu make scalatest USE_BLAS=openblas" + } + } + } + }, + 'Python2/3: CPU Win':{ + node('windows') { + ws('workspace/ut-python-cpu') { + init_git_win() + unstash 'vc14_cpu' + bat '''rmdir /s/q pkg_vc14_cpu +7z x -y vc14_cpu.7z''' + bat """xcopy C:\\mxnet\\data data /E /I /Y +xcopy C:\\mxnet\\model model /E /I /Y +call activate py3 +set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_cpu\\python +C:\\mxnet\\test_cpu.bat""" + bat """xcopy C:\\mxnet\\data data /E /I /Y +xcopy C:\\mxnet\\model model /E /I /Y +call activate py2 +set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_cpu\\python +C:\\mxnet\\test_cpu.bat""" + } + } + }, + 'Python2/3: GPU Win':{ + node('windows') { + ws('workspace/ut-python-gpu') { + init_git_win() + unstash 'vc14_gpu' + bat '''rmdir /s/q pkg_vc14_gpu +7z x -y vc14_gpu.7z''' + bat """xcopy C:\\mxnet\\data data /E /I /Y +xcopy C:\\mxnet\\model model /E /I /Y +call activate py3 +set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_gpu\\python +C:\\mxnet\\test_gpu.bat""" + bat """xcopy C:\\mxnet\\data data /E /I /Y +xcopy C:\\mxnet\\model model /E /I /Y +call activate py2 +set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_gpu\\python +C:\\mxnet\\test_gpu.bat""" + } + } + } +} + + +stage('Integration Test') { + parallel 'Python': { + node('GPU' && 'linux') { + ws('workspace/it-python-gpu') { + init_git() + unpack_lib('gpu') + timeout(time: max_time, unit: 'MINUTES') { + sh "${docker_run} gpu PYTHONPATH=./python/ python example/image-classification/test_score.py" + } + } + } + }, + 'Caffe': { + node('GPU' && 'linux') { + ws('workspace/it-caffe') { + init_git() + unpack_lib('gpu') + timeout(time: max_time, unit: 'MINUTES') { + sh "${docker_run} caffe_gpu PYTHONPATH=/caffe/python:./python python tools/caffe_converter/test_converter.py" + } + } + } + } +} diff --git a/Makefile b/Makefile old mode 100755 new mode 100644 index 6af45a9bda12..cad58d974494 --- a/Makefile +++ b/Makefile @@ -22,21 +22,21 @@ ifneq ($(USE_OPENMP), 1) export NO_OPENMP = 1 endif - # use customized config file include $(config) ifeq ($(USE_MKL2017), 1) - RETURN_STRING=$(shell ./prepare_mkl.sh $(MKLML_ROOT)) - MKLROOT=$(firstword $(RETURN_STRING)) - export USE_MKLML=$(lastword $(RETURN_STRING)) +# must run ./prepare_mkl before including mshadow.mk + RETURN_STRING = $(shell ./prepare_mkl.sh $(MKLML_ROOT)) + MKLROOT = $(firstword $(RETURN_STRING)) + export USE_MKLML = $(lastword $(RETURN_STRING)) endif include mshadow/make/mshadow.mk include $(DMLC_CORE)/make/dmlc.mk # all tge possible warning tread -WARNFLAGS= -Wall +WARNFLAGS= -Wall -Wsign-compare CFLAGS = -DMSHADOW_FORCE_STREAM $(WARNFLAGS) ifeq ($(DEV), 1) @@ -62,6 +62,11 @@ ifeq ($(USE_PROFILER), 1) CFLAGS += -DMXNET_USE_PROFILER=1 endif +# Caffe Plugin +ifdef CAFFE_PATH + CFLAGS += -DMXNET_USE_CAFFE=1 +endif + ifndef LINT_LANG LINT_LANG="all" endif @@ -69,7 +74,7 @@ endif # setup opencv ifeq ($(USE_OPENCV), 1) CFLAGS += -DMXNET_USE_OPENCV=1 $(shell pkg-config --cflags opencv) - LDFLAGS += $(shell pkg-config --libs opencv) + LDFLAGS += $(filter-out -lopencv_ts, $(shell pkg-config --libs opencv)) BIN += bin/im2rec else CFLAGS+= -DMXNET_USE_OPENCV=0 @@ -88,6 +93,8 @@ ifeq ($(USE_MKL2017), 1) CFLAGS += -DMXNET_USE_MKL2017=1 CFLAGS += -DUSE_MKL=1 CFLAGS += -I$(ROOTDIR)/src/operator/mkl/ + CFLAGS += -I$(MKLML_ROOT)/include + LDFLAGS += -L$(MKLML_ROOT)/lib ifeq ($(USE_MKL2017_EXPERIMENTAL), 1) CFLAGS += -DMKL_EXPERIMENTAL=1 else @@ -100,6 +107,8 @@ ifeq ($(USE_CUDNN), 1) LDFLAGS += -lcudnn endif + + ifeq ($(USE_THREADED_ENGINE), 1) CFLAGS += -DMXNET_USE_THREADED_ENGINE endif @@ -126,10 +135,10 @@ ifeq ($(USE_DIST_KVSTORE), 1) LDFLAGS += $(PS_LDFLAGS_A) endif -.PHONY: clean all test lint doc clean_all rcpplint rcppexport roxygen\ +.PHONY: clean all extra-packages test lint doc clean_all rcpplint rcppexport roxygen\ cython2 cython3 cython cyclean -all: lib/libmxnet.a lib/libmxnet.so $(BIN) +all: lib/libmxnet.a lib/libmxnet.so $(BIN) extra-packages SRC = $(wildcard src/*/*/*.cc src/*/*.cc src/*.cc) OBJ = $(patsubst %.cc, build/%.o, $(SRC)) @@ -178,7 +187,7 @@ ALL_DEP = $(OBJ) $(EXTRA_OBJ) $(PLUGIN_OBJ) $(LIB_DEP) ifeq ($(USE_CUDA), 1) CFLAGS += -I$(ROOTDIR)/cub ALL_DEP += $(CUOBJ) $(EXTRA_CUOBJ) $(PLUGIN_CUOBJ) - LDFLAGS += -lcuda + LDFLAGS += -lcuda -lcufft SCALA_PKG_PROFILE := $(SCALA_PKG_PROFILE)-gpu else SCALA_PKG_PROFILE := $(SCALA_PKG_PROFILE)-cpu @@ -255,12 +264,25 @@ $(BIN) : @mkdir -p $(@D) $(CXX) $(CFLAGS) -std=c++11 -o $@ $(filter %.cpp %.o %.c %.a %.cc, $^) $(LDFLAGS) +# CPP Package +ifeq ($(USE_CPP_PACKAGE), 1) +include cpp-package/cpp-package.mk +endif + include tests/cpp/unittest.mk +extra-packages: $(EXTRA_PACKAGES) + test: $(TEST) -lint: rcpplint jnilint - python2 dmlc-core/scripts/lint.py mxnet ${LINT_LANG} include src plugin scripts python predict/python +lint: cpplint rcpplint jnilint pylint + +cpplint: + python2 dmlc-core/scripts/lint.py mxnet cpp include src plugin cpp-package + +pylint: +# ideally we want to check all, such as: python tools example tests + pylint python/mxnet --rcfile=$(ROOTDIR)/tests/ci_build/pylintrc doc: doxygen @@ -307,31 +329,32 @@ scalapkg: (cd $(ROOTDIR)/scala-package; \ mvn clean package -P$(SCALA_PKG_PROFILE) -Dcxx="$(CXX)" \ -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ - -Dlddeps="$(LIB_DEP)") + -Dcurrent_libdir="$(ROOTDIR)/lib" \ + -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a") scalatest: (cd $(ROOTDIR)/scala-package; \ mvn verify -P$(SCALA_PKG_PROFILE) -Dcxx="$(CXX)" \ -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ - -Dlddeps="$(LIB_DEP)" $(SCALA_TEST_ARGS)) + -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a" $(SCALA_TEST_ARGS)) scalainstall: (cd $(ROOTDIR)/scala-package; \ mvn install -P$(SCALA_PKG_PROFILE) -DskipTests -Dcxx="$(CXX)" \ -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ - -Dlddeps="$(LIB_DEP)") + -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a") scaladeploy: (cd $(ROOTDIR)/scala-package; \ mvn deploy -Prelease,$(SCALA_PKG_PROFILE) -DskipTests -Dcxx="$(CXX)" \ -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ - -Dlddeps="$(LIB_DEP)") + -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a") jnilint: python2 dmlc-core/scripts/lint.py mxnet-jnicpp cpp scala-package/native/src ifneq ($(EXTRA_OPERATORS),) -clean: cyclean +clean: cyclean $(EXTRA_PACKAGES_CLEAN) $(RM) -r build lib bin *~ */*~ */*/*~ */*/*/*~ R-package/NAMESPACE R-package/man R-package/R/mxnet_generated.R \ R-package/inst R-package/src/*.o R-package/src/*.so mxnet_*.tar.gz cd $(DMLC_CORE); $(MAKE) clean; cd - @@ -340,7 +363,7 @@ clean: cyclean $(RM) -r $(patsubst %, %/*.d, $(EXTRA_OPERATORS)) $(patsubst %, %/*/*.d, $(EXTRA_OPERATORS)) $(RM) -r $(patsubst %, %/*.o, $(EXTRA_OPERATORS)) $(patsubst %, %/*/*.o, $(EXTRA_OPERATORS)) else -clean: cyclean +clean: cyclean $(EXTRA_PACKAGES_CLEAN) $(RM) -r build lib bin *~ */*~ */*/*~ */*/*/*~ R-package/NAMESPACE R-package/man R-package/R/mxnet_generated.R \ R-package/inst R-package/src/*.o R-package/src/*.so mxnet_*.tar.gz cd $(DMLC_CORE); $(MAKE) clean; cd - diff --git a/NEWS.md b/NEWS.md index 3bc76381bdfe..f29119be897e 100644 --- a/NEWS.md +++ b/NEWS.md @@ -3,13 +3,13 @@ MXNet Change Log ## 0.9.3 - Move symbolic API to NNVM @tqchen - Most front-end C API are backward compatible - - Removed symbolic api in MXNet and relies on NNVM + - Removed symbolic API in MXNet and relies on NNVM - New features: - - MXNet profiler for profiling operator level executions + - MXNet profiler for profiling operator-level executions - mxnet.image package for fast image loading and processing - Change of JSON format - param and attr field are merged to attr - - New code is backward compatible can load old json format + - New code is backward-compatible can load old json format - OpProperty registration now is deprecated - New operators are encouraged to register their property to NNVM op registry attribute - Known features removed limitations to be fixed diff --git a/R-package/DESCRIPTION b/R-package/DESCRIPTION index 895d7d65127d..1ad56e33daa8 100644 --- a/R-package/DESCRIPTION +++ b/R-package/DESCRIPTION @@ -1,7 +1,7 @@ Package: mxnet Type: Package Title: MXNet -Version: 0.9.4 +Version: 0.9.5 Date: 2015-12-23 Author: Tianqi Chen, Qiang Kou, Tong He Maintainer: Qiang Kou diff --git a/README.md b/README.md index fb9809023911..62117f1eb6ff 100644 --- a/README.md +++ b/README.md @@ -8,15 +8,15 @@ ![banner](https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/image/banner.png) MXNet is a deep learning framework designed for both *efficiency* and *flexibility*. -It allows you to ***mix*** the [flavours](http://mxnet.io/architecture/index.html#deep-learning-system-design-concepts) of symbolic -programming and imperative programming to ***maximize*** efficiency and productivity. -In its core, a dynamic dependency scheduler that automatically parallelizes both symbolic and imperative operations on the fly. +It allows you to ***mix*** [symbolic and imperative programming](http://mxnet.io/architecture/index.html#deep-learning-system-design-concepts) +to ***maximize*** efficiency and productivity. +At its core, MXNet contains a dynamic dependency scheduler that automatically parallelizes both symbolic and imperative operations on the fly. A graph optimization layer on top of that makes symbolic execution fast and memory efficient. -The library is portable and lightweight, and it scales to multiple GPUs and multiple machines. +MXNet is portable and lightweight, scaling effectively to multiple GPUs and multiple machines. MXNet is also more than a deep learning project. It is also a collection of [blue prints and guidelines](http://mxnet.io/architecture/index.html#deep-learning-system-design-concepts) for building -deep learning system, and interesting insights of DL systems for hackers. +deep learning systems, and interesting insights of DL systems for hackers. [![Join the chat at https://gitter.im/dmlc/mxnet](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/dmlc/mxnet?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) @@ -55,15 +55,15 @@ Features -------- * Design notes providing useful insights that can re-used by other DL projects * Flexible configuration for arbitrary computation graph -* Mix and match good flavours of programming to maximize flexibility and efficiency +* Mix and match imperative and symbolic programming to maximize flexibility and efficiency * Lightweight, memory efficient and portable to smart devices * Scales up to multi GPUs and distributed setting with auto parallelism -* Support for python, R, C++ and Julia +* Support for Python, R, C++ and Julia * Cloud-friendly and directly compatible with S3, HDFS, and Azure Ask Questions ------------- -* Please use [mxnet/issues](https://github.com/dmlc/mxnet/issues) for how to use mxnet and reporting bugs +* Please use [mxnet/issues](https://github.com/dmlc/mxnet/issues) for how to use mxnet and reporting bugs License ------- @@ -79,4 +79,4 @@ In Neural Information Processing Systems, Workshop on Machine Learning Systems, History ------- -MXNet is initiated and designed in collaboration by the authors of [cxxnet](https://github.com/dmlc/cxxnet), [minerva](https://github.com/dmlc/minerva) and [purine2](https://github.com/purine/purine2). The project reflects what we have learnt from the past projects. It combines important flavours of the existing projects for efficiency, flexibility and memory efficiency. +MXNet emerged from a collaboration by the authors of [cxxnet](https://github.com/dmlc/cxxnet), [minerva](https://github.com/dmlc/minerva), and [purine2](https://github.com/purine/purine2). The project reflects what we have learned from the past projects. MXNet combines aspects of each of these projects to achieve flexibility, speed, and memory efficiency. diff --git a/amalgamation/Makefile b/amalgamation/Makefile index 37c88f17a51c..2446667c1e9e 100644 --- a/amalgamation/Makefile +++ b/amalgamation/Makefile @@ -1,6 +1,9 @@ export MXNET_ROOT=`pwd`/.. -# Change this to path of openblas -export OPENBLAS_ROOT=/usr/local/opt/openblas + +# Change this to path or specify in make command +ifndef OPENBLAS_ROOT + export OPENBLAS_ROOT=/usr/local/opt/openblas +endif # Whether use minimum build without blas and SSE, this will make the library super slow ifndef MIN @@ -16,15 +19,32 @@ else DEFS+=-DMSHADOW_USE_SSE=0 endif +# Use locally installed emscripten if not specified +ifndef EMCC + EMCC=emcc +endif + +ifndef DISABLE_OPENMP + DEFS+=-DDISABLE_OPENMP=1 +endif .PHONY: all clean DEFS+=-DMSHADOW_USE_CUDA=0 -DMSHADOW_USE_MKL=0 -DMSHADOW_RABIT_PS=0 -DMSHADOW_DIST_PS=0 -DDMLC_LOG_STACK_TRACE=0 -DEFS+=-DMSHADOW_FORCE_STREAM -DMXNET_USE_OPENCV=0 -DMXNET_PREDICT_ONLY=1 -DDISABLE_OPENMP=1 +DEFS+=-DMSHADOW_FORCE_STREAM -DMXNET_USE_OPENCV=0 -DMXNET_PREDICT_ONLY=1 CFLAGS=-std=c++11 -Wno-unknown-pragmas -Wall $(DEFS) ifneq ($(MIN), 1) CFLAGS += -I${OPENBLAS_ROOT} -I${OPENBLAS_ROOT}/include - LDFLAGS+= -L${OPENBLAS_ROOT} -L${OPENBLAS_ROOT}/lib -lopenblas + LDFLAGS+= -L${OPENBLAS_ROOT} -L${OPENBLAS_ROOT}/lib + + # Define which blas is installed. Uses OpenBLAS by default. + ifeq ($(USE_BLAS), atlas) + LDFLAGS += -lcblas + else ifeq ($(USE_BLAS), blas) + LDFLAGS += -lblas + else + LDFLAGS += -lopenblas + endif endif @@ -58,7 +78,7 @@ mxnet_predict-all.o: mxnet_predict-all.cc libmxnet_predict.a: mxnet_predict-all.o ar rcs libmxnet_predict.a $+ -jni_libmxnet_predict.o: mxnet_predict-all.cc +jni_libmxnet_predict.o: mxnet_predict-all.cc jni/predictor.cc ${CXX} ${CFLAGS} -fPIC -o $@ -c jni/predictor.cc jni_libmxnet_predict.so: jni_libmxnet_predict.o @@ -73,15 +93,23 @@ else endif libmxnet_predict.js: mxnet_predict-all.cc - emcc -std=c++11 -O2 $(DEFS) -DMSHADOW_USE_SSE=0 -D__MXNET_JS__ -o $@ $+ \ - -s EXPORTED_FUNCTIONS="['_MXPredCreate', '_MXPredGetOutputShape', '_MXPredSetInput', '_MXPredForward', '_MXPredPartialForward', '_MXPredGetOutput', '_MXPredFree', '_MXNDListCreate', '_MXNDListGet', '_MXNDListFree']" \ + ${EMCC} -std=c++11 -O2 $(DEFS) -DMSHADOW_USE_SSE=0 -D__MXNET_JS__ -o $@ $+ \ + -s EXPORTED_FUNCTIONS="['_MXPredCreate', \ + '_MXPredGetOutputShape', \ + '_MXPredSetInput', \ + '_MXPredForward', \ + '_MXPredPartialForward', \ + '_MXPredGetOutput', \ + '_MXPredFree', \ + '_MXNDListCreate', \ + '_MXNDListGet', \ + '_MXNDListFree']" \ -s ALLOW_MEMORY_GROWTH=1 - ${MXNET_ROOT}/lib/libmxnet_predict.so: mxnet_predict-all.o @mkdir -p ${MXNET_ROOT}/lib ${CXX} ${CFLAGS} -shared -o $@ $(filter %.o %.a, $^) $(LDFLAGS) ls -alh $@ clean: - rm -f *.d *.o *.so *.a mxnet_predict-all.cc nnvm.cc + rm -f *.d *.o *.so *.a *.js *.js.mem mxnet_predict-all.cc nnvm.cc diff --git a/amalgamation/README.md b/amalgamation/README.md index 5c6522c55eaa..3faf4d1e18a4 100644 --- a/amalgamation/README.md +++ b/amalgamation/README.md @@ -45,10 +45,15 @@ You can use generated library in [Leliana WhatsThis Android app](https://github. Javascript --------------- -JS version works without OpenBLAS. You need [emscripten](http://kripken.github.io/emscripten-site/) to build it. -Type ```make MIN=1 libmxnet_predict.js``` +JS version uses [emscripten](http://kripken.github.io/emscripten-site/) to cross-compile the amalgamation source file into a Javascript library that can be integrated into client side applications. If you already have emanscripten installed then -You can use generated library in [mxnet.js](https://github.com/dmlc/mxnet.js) +```make clean libmxnet_predict.js MIN=1``` + +otherwise you can use [emscripten docker image](https://hub.docker.com/r/apiaryio/emcc/) to compile in the following way + +```make clean libmxnet_predict.js MIN=1 EMCC="docker run -v ${PWD}:/src apiaryio/emcc emcc"``` + +An example WebApp that uses the generated JS library can be found at [mxnet.js](https://github.com/dmlc/mxnet.js) iOS --------------- diff --git a/amalgamation/amalgamation.py b/amalgamation/amalgamation.py index 88fda297f475..da3b60ac8399 100644 --- a/amalgamation/amalgamation.py +++ b/amalgamation/amalgamation.py @@ -8,7 +8,7 @@ 'kvstore_dist.h', 'mach/clock.h', 'mach/mach.h', 'malloc.h', 'mkl.h', 'mkl_cblas.h', 'mkl_vsl.h', 'mkl_vsl_functions.h', 'nvml.h', 'opencv2/opencv.hpp', 'sys/stat.h', 'sys/types.h', 'cuda.h', 'cuda_fp16.h', - 'omp.h', 'execinfo.h', 'packet/sse-inl.h' + 'omp.h', 'execinfo.h', 'packet/sse-inl.h', 'emmintrin.h', 'thrust/device_vector.h' ] minimum = int(sys.argv[6]) if len(sys.argv) > 5 else 0 @@ -65,6 +65,8 @@ def find_source(name, start, stage): out = StringIO.StringIO() + + def expand(x, pending, stage): if x in history and x not in ['mshadow/mshadow/expr_scalar-inl.h']: # MULTIPLE includes return @@ -73,7 +75,10 @@ def expand(x, pending, stage): #print 'loop found: %s in ' % x, pending return - print >>out, "//===== EXPANDING: %s =====\n" %x + whtspace = ' '*expand.treeDepth + expand.fileCount+=1 + print >>out, "//=====[%3d] STAGE:%4s %sEXPANDING: %s =====\n" % (expand.fileCount, stage, whtspace, x) + print "//=====[%3d] STAGE:%4s %sEXPANDING: %s " % (expand.fileCount, stage, whtspace, x) for line in open(x): if line.find('#include') < 0: out.write(line) @@ -95,19 +100,28 @@ def expand(x, pending, stage): 'nnpack' not in h and not h.endswith('.cuh')): sysheaders.append(h) else: + expand.treeDepth+=1 expand(source, pending + [x], stage) - print >>out, "//===== EXPANDED: %s =====\n" %x + expand.treeDepth-=1 + print >>out, "//===== EXPANDED : %s =====\n" %x history.add(x) + +# Vars to keep track of number of files expanded. +# Used in printing informative comments. +expand.treeDepth = 0 +expand.fileCount = 0 + +# Expand the stages expand(sys.argv[2], [], "dmlc") expand(sys.argv[3], [], "nnvm") expand(sys.argv[4], [], "src") - - +# Write to amalgamation file f = open(sys.argv[5], 'wb') if minimum != 0: + sysheaders.remove('cblas.h') print >>f, "#define MSHADOW_STAND_ALONE 1" print >>f, "#define MSHADOW_USE_SSE 0" print >>f, "#define MSHADOW_USE_CBLAS 0" diff --git a/amalgamation/jni/predictor.cc b/amalgamation/jni/predictor.cc index 6ee9547b34a5..2687d1d9d93e 100644 --- a/amalgamation/jni/predictor.cc +++ b/amalgamation/jni/predictor.cc @@ -19,14 +19,17 @@ JNIEXPORT jlong JNICALL Java_org_dmlc_mxnet_Predictor_createPredictor track.emplace_back(js, s); } - std::vector index{0}; + std::vector index; std::vector shapes; + mx_uint prev = 0; + index.emplace_back(prev); for (int i=0; iGetArrayLength(jshapes); i++) { jintArray jshape = (jintArray) env->GetObjectArrayElement(jshapes, i); jsize shape_len = env->GetArrayLength(jshape); jint *shape = env->GetIntArrayElements(jshape, 0); - index.emplace_back(shape_len); + prev += shape_len; + index.emplace_back(prev); for (int j=0; jReleaseIntArrayElements(jshape, shape, 0); } diff --git a/cmake/Modules/FindJeMalloc.cmake b/cmake/Modules/FindJeMalloc.cmake new file mode 100644 index 000000000000..8b965cf6c3bb --- /dev/null +++ b/cmake/Modules/FindJeMalloc.cmake @@ -0,0 +1,45 @@ + +# Copyright (c) 2014 Thomas Heller +# Copyright (c) 2007-2012 Hartmut Kaiser +# Copyright (c) 2010-2011 Matt Anderson +# Copyright (c) 2011 Bryce Lelbach +# +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +find_package(PkgConfig) +pkg_check_modules(PC_JEMALLOC QUIET jemalloc) + +find_path(JEMALLOC_INCLUDE_DIR jemalloc/jemalloc.h + HINTS + ${JEMALLOC_ROOT} ENV JEMALLOC_ROOT + ${PC_JEMALLOC_MINIMAL_INCLUDEDIR} + ${PC_JEMALLOC_MINIMAL_INCLUDE_DIRS} + ${PC_JEMALLOC_INCLUDEDIR} + ${PC_JEMALLOC_INCLUDE_DIRS} + PATH_SUFFIXES include) + +find_library(JEMALLOC_LIBRARY NAMES jemalloc libjemalloc + HINTS + ${JEMALLOC_ROOT} ENV JEMALLOC_ROOT + ${PC_JEMALLOC_MINIMAL_LIBDIR} + ${PC_JEMALLOC_MINIMAL_LIBRARY_DIRS} + ${PC_JEMALLOC_LIBDIR} + ${PC_JEMALLOC_LIBRARY_DIRS} + PATH_SUFFIXES lib lib64) + +set(JEMALLOC_LIBRARIES ${JEMALLOC_LIBRARY}) +set(JEMALLOC_INCLUDE_DIRS ${JEMALLOC_INCLUDE_DIR}) + +find_package_handle_standard_args(Jemalloc DEFAULT_MSG + JEMALLOC_LIBRARY JEMALLOC_INCLUDE_DIR) + +get_property(_type CACHE JEMALLOC_ROOT PROPERTY TYPE) +if(_type) + set_property(CACHE JEMALLOC_ROOT PROPERTY ADVANCED 1) + if("x${_type}" STREQUAL "xUNINITIALIZED") + set_property(CACHE JEMALLOC_ROOT PROPERTY TYPE PATH) + endif() +endif() + +mark_as_advanced(JEMALLOC_ROOT JEMALLOC_LIBRARY JEMALLOC_INCLUDE_DIR) diff --git a/cmake/Modules/FindMKL.cmake b/cmake/Modules/FindMKL.cmake index b3cf9e279663..9679f3d72e60 100644 --- a/cmake/Modules/FindMKL.cmake +++ b/cmake/Modules/FindMKL.cmake @@ -2,6 +2,8 @@ # # Options: # +# USE_MKLML_MKL : Search for MKL:ML library variant +# # MKL_USE_SINGLE_DYNAMIC_LIBRARY : use single dynamic library interface # MKL_USE_STATIC_LIBS : use static libraries # MKL_MULTI_THREADED : use multi-threading @@ -13,110 +15,168 @@ # MKL_FOUND : True mkl is found # MKL_INCLUDE_DIR : unclude directory # MKL_LIBRARIES : the libraries to link against. +# +# cjolivier01: Changed to also look for MKLML library (subset of mkl) instead of standard MKL package +# - -# ---[ Options -mxnet_option(MKL_USE_SINGLE_DYNAMIC_LIBRARY "Use single dynamic library interface" ON) -mxnet_option(MKL_USE_STATIC_LIBS "Use static libraries" OFF IF NOT MKL_USE_SINGLE_DYNAMIC_LIBRARY) -mxnet_option(MKL_MULTI_THREADED "Use multi-threading" ON IF NOT MKL_USE_SINGLE_DYNAMIC_LIBRARY) -mxnet_option(MKL_USE_ILP64 "Use ilp64 data model" OFF) -mxnet_option(MKL_USE_CLUSTER "Use cluster functions" OFF IF CMAKE_SIZEOF_VOID_P EQUAL 4) +if(MKL_FOUND) + return() +endif() # ---[ Root folders set(INTEL_ROOT "/opt/intel" CACHE PATH "Folder contains intel libs") -find_path(MKL_ROOT include/mkl.h PATHS $ENV{MKL_ROOT} ${INTEL_ROOT}/mkl - DOC "Folder contains MKL") - -# ---[ Find include dir -find_path(MKL_INCLUDE_DIR mkl.h PATHS ${MKL_ROOT} PATH_SUFFIXES include) -set(__looked_for MKL_INCLUDE_DIR) - -# ---[ Find libraries -if(CMAKE_SIZEOF_VOID_P EQUAL 4) - set(__path_suffixes lib lib/ia32) -else() - set(__path_suffixes lib lib/intel64) -endif() -set(__mkl_libs "") -if(MKL_USE_SINGLE_DYNAMIC_LIBRARY) - list(APPEND __mkl_libs rt) -else() +if(USE_MKLML_MKL) + + find_path(MKL_ROOT include/mkl_blas.h + PATHS $ENV{MKL_ROOT} + ${INTEL_ROOT}/mklml + ${DIRECT_DEPENDENCY_ROOTS} + DOC "Folder contains MKL" + ) + + # ---[ Find include dir + find_path(MKL_INCLUDE_DIR mkl_blas.h PATHS ${MKL_ROOT} PATH_SUFFIXES include) + set(__looked_for MKL_INCLUDE_DIR) + + # ---[ Find libraries if(CMAKE_SIZEOF_VOID_P EQUAL 4) - if(WIN32) - list(APPEND __mkl_libs intel_c) - else() - list(APPEND __mkl_libs intel) - if(CMAKE_COMPILER_IS_GNUFORTRAN) - list(APPEND __mkl_libs gf) - endif() - endif() + set(__path_suffixes lib lib/ia32) else() - set(__mkl_lib64_suffix "lp64") - if(MKL_USE_ILP64) - set(__mkl_lib64_suffix "ilp64") - add_definitions(-DMKL_ILP64) - endif() - list(APPEND __mkl_libs "intel_${__mkl_lib64_suffix}") - if(CMAKE_COMPILER_IS_GNUFORTRAN) - list(APPEND __mkl_libs "gf_${__mkl_lib64_suffix}") - endif() + set(__path_suffixes lib lib/intel64) endif() - if(MKL_MULTI_THREADED) - list(APPEND __mkl_libs intel_thread) + set(__mkl_libs "") + + if(WIN32) + list(APPEND __mkl_libs intel) else() - list(APPEND __mkl_libs sequential) + list(APPEND __mkl_libs gnu) endif() - list(APPEND __mkl_libs core) - if(CMAKE_SIZEOF_VOID_P EQUAL 8 AND MKL_USE_CLUSTER) - list(APPEND __mkl_libs cdft_core) - endif() -endif() + foreach (__lib ${__mkl_libs}) + set(__mkl_lib "mklml_${__lib}") + string(TOUPPER ${__mkl_lib} __mkl_lib_upper) + if(MKL_USE_STATIC_LIBS) + set(__mkl_lib "lib${__mkl_lib}.a") + endif() -foreach (__lib ${__mkl_libs}) - set(__mkl_lib "mkl_${__lib}") - string(TOUPPER ${__mkl_lib} __mkl_lib_upper) + find_library(${__mkl_lib_upper}_LIBRARY + NAMES ${__mkl_lib} + PATHS ${MKL_ROOT} "${MKL_INCLUDE_DIR}/.." + PATH_SUFFIXES ${__path_suffixes} + DOC "The path to Intel(R) MKL ${__mkl_lib} library") + mark_as_advanced(${__mkl_lib_upper}_LIBRARY) - if(MKL_USE_STATIC_LIBS) - set(__mkl_lib "lib${__mkl_lib}.a") - endif() + list(APPEND __looked_for ${__mkl_lib_upper}_LIBRARY) + list(APPEND MKL_LIBRARIES ${${__mkl_lib_upper}_LIBRARY}) + endforeach() + + +else(USE_MKLML_MKL) - find_library(${__mkl_lib_upper}_LIBRARY - NAMES ${__mkl_lib} - PATHS ${MKL_ROOT} "${MKL_INCLUDE_DIR}/.." - PATH_SUFFIXES ${__path_suffixes} - DOC "The path to Intel(R) MKL ${__mkl_lib} library") - mark_as_advanced(${__mkl_lib_upper}_LIBRARY) + # ---[ Options + mxnet_option(MKL_USE_SINGLE_DYNAMIC_LIBRARY "Use single dynamic library interface" ON) + mxnet_option(MKL_USE_STATIC_LIBS "Use static libraries" OFF IF NOT MKL_USE_SINGLE_DYNAMIC_LIBRARY) + mxnet_option(MKL_MULTI_THREADED "Use multi-threading" ON IF NOT MKL_USE_SINGLE_DYNAMIC_LIBRARY) + mxnet_option(MKL_USE_ILP64 "Use ilp64 data model" OFF) + mxnet_option(MKL_USE_CLUSTER "Use cluster functions" OFF IF CMAKE_SIZEOF_VOID_P EQUAL 4) - list(APPEND __looked_for ${__mkl_lib_upper}_LIBRARY) - list(APPEND MKL_LIBRARIES ${${__mkl_lib_upper}_LIBRARY}) -endforeach() + find_path(MKL_ROOT include/mkl.h PATHS $ENV{MKL_ROOT} ${INTEL_ROOT}/mkl + DOC "Folder contains MKL") + # ---[ Find include dir + find_path(MKL_INCLUDE_DIR mkl.h PATHS ${MKL_ROOT} PATH_SUFFIXES include) + set(__looked_for MKL_INCLUDE_DIR) -if(NOT MKL_USE_SINGLE_DYNAMIC_LIBRARY) - if (MKL_USE_STATIC_LIBS) - set(__iomp5_libs iomp5 libiomp5mt.lib) + # ---[ Find libraries + if(CMAKE_SIZEOF_VOID_P EQUAL 4) + set(__path_suffixes lib lib/ia32) else() - set(__iomp5_libs iomp5 libiomp5md.lib) + set(__path_suffixes lib lib/intel64) endif() - if(WIN32) - find_path(INTEL_INCLUDE_DIR omp.h PATHS ${INTEL_ROOT} PATH_SUFFIXES include) - list(APPEND __looked_for INTEL_INCLUDE_DIR) + set(__mkl_libs "") + if(MKL_USE_SINGLE_DYNAMIC_LIBRARY) + list(APPEND __mkl_libs rt) + else() + if(CMAKE_SIZEOF_VOID_P EQUAL 4) + if(WIN32) + list(APPEND __mkl_libs intel_c) + else() + list(APPEND __mkl_libs intel) + if(CMAKE_COMPILER_IS_GNUFORTRAN) + list(APPEND __mkl_libs gf) + endif() + endif() + else() + set(__mkl_lib64_suffix "lp64") + if(MKL_USE_ILP64) + set(__mkl_lib64_suffix "ilp64") + add_definitions(-DMKL_ILP64) + endif() + list(APPEND __mkl_libs "intel_${__mkl_lib64_suffix}") + if(CMAKE_COMPILER_IS_GNUFORTRAN) + list(APPEND __mkl_libs "gf_${__mkl_lib64_suffix}") + endif() + endif() + + if(MKL_MULTI_THREADED) + list(APPEND __mkl_libs intel_thread) + else() + list(APPEND __mkl_libs sequential) + endif() + + list(APPEND __mkl_libs core) + if(CMAKE_SIZEOF_VOID_P EQUAL 8 AND MKL_USE_CLUSTER) + list(APPEND __mkl_libs cdft_core) + endif() endif() - find_library(MKL_RTL_LIBRARY ${__iomp5_libs} - PATHS ${INTEL_RTL_ROOT} ${INTEL_ROOT}/compiler ${MKL_ROOT}/.. ${MKL_ROOT}/../compiler - PATH_SUFFIXES ${__path_suffixes} - DOC "Path to Path to OpenMP runtime library") - list(APPEND __looked_for MKL_RTL_LIBRARY) - list(APPEND MKL_LIBRARIES ${MKL_RTL_LIBRARY}) -endif() + foreach (__lib ${__mkl_libs}) + set(__mkl_lib "mkl_${__lib}") + string(TOUPPER ${__mkl_lib} __mkl_lib_upper) + + if(MKL_USE_STATIC_LIBS) + set(__mkl_lib "lib${__mkl_lib}.a") + endif() + find_library(${__mkl_lib_upper}_LIBRARY + NAMES ${__mkl_lib} + PATHS ${MKL_ROOT} "${MKL_INCLUDE_DIR}/.." + PATH_SUFFIXES ${__path_suffixes} + DOC "The path to Intel(R) MKL ${__mkl_lib} library") + mark_as_advanced(${__mkl_lib_upper}_LIBRARY) + + list(APPEND __looked_for ${__mkl_lib_upper}_LIBRARY) + list(APPEND MKL_LIBRARIES ${${__mkl_lib_upper}_LIBRARY}) + endforeach() + + + if(NOT MKL_USE_SINGLE_DYNAMIC_LIBRARY) + if (MKL_USE_STATIC_LIBS) + set(__iomp5_libs iomp5 libiomp5mt.lib) + else() + set(__iomp5_libs iomp5 libiomp5md.lib) + endif() + + if(WIN32) + find_path(INTEL_INCLUDE_DIR omp.h PATHS ${INTEL_ROOT} PATH_SUFFIXES include) + list(APPEND __looked_for INTEL_INCLUDE_DIR) + endif() + + find_library(MKL_RTL_LIBRARY ${__iomp5_libs} + PATHS ${INTEL_RTL_ROOT} ${INTEL_ROOT}/compiler ${MKL_ROOT}/.. ${MKL_ROOT}/../compiler + PATH_SUFFIXES ${__path_suffixes} + DOC "Path to Path to OpenMP runtime library") + + list(APPEND __looked_for MKL_RTL_LIBRARY) + list(APPEND MKL_LIBRARIES ${MKL_RTL_LIBRARY}) + endif() + +endif(USE_MKLML_MKL) include(FindPackageHandleStandardArgs) find_package_handle_standard_args(MKL DEFAULT_MSG ${__looked_for}) @@ -126,3 +186,4 @@ if(MKL_FOUND) endif() mxnet_clear_vars(__looked_for __mkl_libs __path_suffixes __lib_suffix __iomp5_libs) + diff --git a/cmake/Modules/FindOpenBLAS.cmake b/cmake/Modules/FindOpenBLAS.cmake index 67a130cb4bc3..a0c96c3f02f7 100644 --- a/cmake/Modules/FindOpenBLAS.cmake +++ b/cmake/Modules/FindOpenBLAS.cmake @@ -1,3 +1,6 @@ +if(MKL_FOUND) + message(ERROR " OpenBLAS is not required since MKL is enabled") +endif() file(TO_CMAKE_PATH "$ENV{OpenBLAS_HOME}" OpenBLAS_HOME) file(TO_CMAKE_PATH "$ENV{OpenBLAS}" OpenBLAS_DIR) diff --git a/cpp-package/.travis.yml b/cpp-package/.travis.yml new file mode 100644 index 000000000000..e7a332d09125 --- /dev/null +++ b/cpp-package/.travis.yml @@ -0,0 +1,48 @@ +sudo: false + +language: cpp + +os: + - linux +# disable for now since clang doesn't support openmp +# - osx + +env: + # code analysis + - TASK=lint + # TODO: build example + - TASK=build + +# dependent apt packages +addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - gcc-4.8 + - g++-4.8 +# - wget +# - git +# - libcurl4-openssl-dev +# - unzip +# - libatlas-dev +# - libopencv-dev + +before_install: + +install: + - source tests/travis/setup.sh + +script: + - tests/travis/run_test.sh + +cache: + directories: + - ${HOME}/.cache/usr + +notifications: +# Emails are sent to the committer's git-configured email address by default, + email: + on_success: change + on_failure: always + #slack: dmlc:NmroCzntCiWOuxUZpii40USd diff --git a/cpp-package/CMakeLists.txt b/cpp-package/CMakeLists.txt new file mode 100644 index 000000000000..2cc322bdd2a4 --- /dev/null +++ b/cpp-package/CMakeLists.txt @@ -0,0 +1,18 @@ + +if(USE_CPP_PACKAGE AND NOT MSVC) + +set(CPP_PACKAGE_OP_H_HEADER ${CMAKE_CURRENT_LIST_DIR}/include/mxnet-cpp/op.h) + +add_custom_target( + cpp_package_op_h ALL + BYPRODUCTS ${CPP_PACKAGE_OP_H_HEADER} + MAIN_DEPENDENCY mxnet + DEPENDS mxnet ${CMAKE_SOURCE_DIR}/cpp-package/src/OpWrapperGenerator/OpWrapperGenerator.py + COMMAND echo "Running: OpWrapperGenerator.py" + COMMAND python OpWrapperGenerator.py ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY}/$ + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/cpp-package/src/OpWrapperGenerator/ +) + +add_subdirectory(example) + +endif() \ No newline at end of file diff --git a/cpp-package/LICENSE b/cpp-package/LICENSE new file mode 100644 index 000000000000..2525650c621b --- /dev/null +++ b/cpp-package/LICENSE @@ -0,0 +1,13 @@ +Copyright (c) 2015 by Contributors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/cpp-package/README.md b/cpp-package/README.md new file mode 100644 index 000000000000..dcfcbc81f3a7 --- /dev/null +++ b/cpp-package/README.md @@ -0,0 +1,8 @@ +# MxNet C++ Package + +[![Build Status](https://travis-ci.org/dmlc/MXNet.cpp.svg?branch=master)](https://travis-ci.org/dmlc/MXNet.cpp) +[![Build status](https://ci.appveyor.com/api/projects/status/ckfq6j53sg5ll01d/branch/master?svg=true)](https://ci.appveyor.com/project/lx75249/mxnet-cpp/branch/master) + +The examples dir containers examples for you to get started. +The lib dir should contain the compiled mxnet library. +Windows dir contains Visual C++ solution files and project files. diff --git a/cpp-package/cpp-package.mk b/cpp-package/cpp-package.mk new file mode 100644 index 000000000000..54f4cce4fa7f --- /dev/null +++ b/cpp-package/cpp-package.mk @@ -0,0 +1,28 @@ +ifndef LINT_LANG + LINT_LANG="all" +endif + +ifdef CAFFE_PATH +export LD_LIBRARY_PATH=$(CAFFE_PATH)/lib +endif + +CPP_PACKAGE_OP_H_FILE = cpp-package/include/mxnet-cpp/op.h + +EXTRA_PACKAGES += cpp-package-all +EXTRA_PACKAGES_CLEAN += cpp-package-clean + +.PHONY: cpp-package-all cpp-package-lint cpp-package-clean + +cpp-package-all: $(CPP_PACKAGE_OP_H_FILE) + +cpp-package-clean: + rm -f $(CPP_PACKAGE_OP_H_FILE) + +$(CPP_PACKAGE_OP_H_FILE): lib/libmxnet.so cpp-package/src/OpWrapperGenerator/OpWrapperGenerator.py + (cd cpp-package/src/OpWrapperGenerator; python OpWrapperGenerator.py $(ROOTDIR)/lib/libmxnet.so) + +cpp-package-lint: + (cd cpp-package; python scripts/lint.py dmlc ${LINT_LANG} include example) + +include cpp-package/example/example.mk + diff --git a/cpp-package/example/CMakeLists.txt b/cpp-package/example/CMakeLists.txt new file mode 100644 index 000000000000..2f596678415a --- /dev/null +++ b/cpp-package/example/CMakeLists.txt @@ -0,0 +1,54 @@ + +if(NOT MSVC) + set(UNITTEST_STATIC_LINK ON) +endif() + +set(CPP_EXAMPLE_LIBS + rt + ${BEGIN_WHOLE_ARCHIVE} mxnet_static ${END_WHOLE_ARCHIVE} + dmlccore + ${mxnet_LINKER_LIBS} + ) + +set(CPP_PACKAGE_INCLUDE_DIR ${CMAKE_CURRENT_LIST_DIR}/../include/mxnet-cpp/) + +set(CPPEX_DEPS cpp_package_op_h) + +file(GLOB_RECURSE CPP_PACKAGE_HEADERS + "${CPP_PACKAGE_INCLUDE_DIR}/*.h" + "${CPP_PACKAGE_INCLUDE_DIR}/*.hpp" + ) + +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../include) + +add_executable(lenet lenet.cpp ${CPP_PACKAGE_HEADERS}) +target_link_libraries(lenet ${CPP_EXAMPLE_LIBS}) +add_dependencies(lenet ${CPPEX_DEPS}) + +add_executable(lenet_with_mxdataiter lenet_with_mxdataiter.cpp ${CPP_PACKAGE_HEADERS}) +target_link_libraries(lenet_with_mxdataiter ${CPP_EXAMPLE_LIBS}) +add_dependencies(lenet_with_mxdataiter ${CPPEX_DEPS}) + +add_executable(alexnet alexnet.cpp ${CPP_PACKAGE_HEADERS}) +target_link_libraries(alexnet ${CPP_EXAMPLE_LIBS}) +add_dependencies(alexnet ${CPPEX_DEPS}) + +add_executable(charRNN charRNN.cpp ${CPP_PACKAGE_HEADERS}) +target_link_libraries(charRNN ${CPP_EXAMPLE_LIBS}) +add_dependencies(charRNN ${CPPEX_DEPS}) + +add_executable(googlenet googlenet.cpp ${CPP_PACKAGE_HEADERS}) +target_link_libraries(googlenet ${CPP_EXAMPLE_LIBS}) +add_dependencies(googlenet ${CPPEX_DEPS}) + +add_executable(inception_bn inception_bn.cpp ${CPP_PACKAGE_HEADERS}) +target_link_libraries(inception_bn ${CPP_EXAMPLE_LIBS}) +add_dependencies(inception_bn ${CPPEX_DEPS}) + +add_executable(mlp mlp.cpp ${CPP_PACKAGE_HEADERS}) +target_link_libraries(mlp ${CPP_EXAMPLE_LIBS}) +add_dependencies(mlp ${CPPEX_DEPS}) + +add_executable(resnet resnet.cpp ${CPP_PACKAGE_HEADERS}) +target_link_libraries(resnet ${CPP_EXAMPLE_LIBS}) +add_dependencies(resnet ${CPPEX_DEPS}) diff --git a/cpp-package/example/alexnet.cpp b/cpp-package/example/alexnet.cpp new file mode 100644 index 000000000000..c0d8273d559b --- /dev/null +++ b/cpp-package/example/alexnet.cpp @@ -0,0 +1,303 @@ +/*! + * Copyright (c) 2016 by Contributors + */ +#include +#include +#include +#include "mxnet-cpp/MxNetCpp.h" +// Allow IDE to parse the types +#include "../include/mxnet-cpp/op.h" + +using namespace std; +using namespace mxnet::cpp; + +Symbol AlexnetSymbol(int num_classes) { + auto input_data = Symbol::Variable("data"); + auto target_label = Symbol::Variable("label"); + /*stage 1*/ + auto conv1 = Operator("Convolution") + .SetParam("kernel", Shape(11, 11)) + .SetParam("num_filter", 96) + .SetParam("stride", Shape(4, 4)) + .SetParam("dilate", Shape(1, 1)) + .SetParam("pad", Shape(0, 0)) + .SetParam("num_group", 1) + .SetParam("workspace", 512) + .SetParam("no_bias", false) + .SetInput("data", input_data) + .CreateSymbol("conv1"); + auto relu1 = Operator("Activation") + .SetParam("act_type", "relu") /*relu,sigmoid,softrelu,tanh */ + .SetInput("data", conv1) + .CreateSymbol("relu1"); + auto pool1 = Operator("Pooling") + .SetParam("kernel", Shape(3, 3)) + .SetParam("pool_type", "max") /*avg,max,sum */ + .SetParam("global_pool", false) + .SetParam("stride", Shape(2, 2)) + .SetParam("pad", Shape(0, 0)) + .SetInput("data", relu1) + .CreateSymbol("pool1"); + auto lrn1 = Operator("LRN") + .SetParam("nsize", 5) + .SetParam("alpha", 0.0001) + .SetParam("beta", 0.75) + .SetParam("knorm", 1) + .SetInput("data", pool1) + .CreateSymbol("lrn1"); + /*stage 2*/ + auto conv2 = Operator("Convolution") + .SetParam("kernel", Shape(5, 5)) + .SetParam("num_filter", 256) + .SetParam("stride", Shape(1, 1)) + .SetParam("dilate", Shape(1, 1)) + .SetParam("pad", Shape(2, 2)) + .SetParam("num_group", 1) + .SetParam("workspace", 512) + .SetParam("no_bias", false) + .SetInput("data", lrn1) + .CreateSymbol("conv2"); + auto relu2 = Operator("Activation") + .SetParam("act_type", "relu") /*relu,sigmoid,softrelu,tanh */ + .SetInput("data", conv2) + .CreateSymbol("relu2"); + auto pool2 = Operator("Pooling") + .SetParam("kernel", Shape(3, 3)) + .SetParam("pool_type", "max") /*avg,max,sum */ + .SetParam("global_pool", false) + .SetParam("stride", Shape(2, 2)) + .SetParam("pad", Shape(0, 0)) + .SetInput("data", relu2) + .CreateSymbol("pool2"); + auto lrn2 = Operator("LRN") + .SetParam("nsize", 5) + .SetParam("alpha", 0.0001) + .SetParam("beta", 0.75) + .SetParam("knorm", 1) + .SetInput("data", pool2) + .CreateSymbol("lrn2"); + /*stage 3*/ + auto conv3 = Operator("Convolution") + .SetParam("kernel", Shape(3, 3)) + .SetParam("num_filter", 384) + .SetParam("stride", Shape(1, 1)) + .SetParam("dilate", Shape(1, 1)) + .SetParam("pad", Shape(1, 1)) + .SetParam("num_group", 1) + .SetParam("workspace", 512) + .SetParam("no_bias", false) + .SetInput("data", lrn2) + .CreateSymbol("conv3"); + auto relu3 = Operator("Activation") + .SetParam("act_type", "relu") /*relu,sigmoid,softrelu,tanh */ + .SetInput("data", conv3) + .CreateSymbol("relu3"); + auto conv4 = Operator("Convolution") + .SetParam("kernel", Shape(3, 3)) + .SetParam("num_filter", 384) + .SetParam("stride", Shape(1, 1)) + .SetParam("dilate", Shape(1, 1)) + .SetParam("pad", Shape(1, 1)) + .SetParam("num_group", 1) + .SetParam("workspace", 512) + .SetParam("no_bias", false) + .SetInput("data", relu3) + .CreateSymbol("conv4"); + auto relu4 = Operator("Activation") + .SetParam("act_type", "relu") /*relu,sigmoid,softrelu,tanh */ + .SetInput("data", conv4) + .CreateSymbol("relu4"); + auto conv5 = Operator("Convolution") + .SetParam("kernel", Shape(3, 3)) + .SetParam("num_filter", 256) + .SetParam("stride", Shape(1, 1)) + .SetParam("dilate", Shape(1, 1)) + .SetParam("pad", Shape(1, 1)) + .SetParam("num_group", 1) + .SetParam("workspace", 512) + .SetParam("no_bias", false) + .SetInput("data", relu4) + .CreateSymbol("conv5"); + auto relu5 = Operator("Activation") + .SetParam("act_type", "relu") + .SetInput("data", conv5) + .CreateSymbol("relu5"); + auto pool3 = Operator("Pooling") + .SetParam("kernel", Shape(3, 3)) + .SetParam("pool_type", "max") + .SetParam("global_pool", false) + .SetParam("stride", Shape(2, 2)) + .SetParam("pad", Shape(0, 0)) + .SetInput("data", relu5) + .CreateSymbol("pool3"); + /*stage4*/ + auto flatten = + Operator("Flatten").SetInput("data", pool3).CreateSymbol("flatten"); + auto fc1 = Operator("FullyConnected") + .SetParam("num_hidden", 4096) + .SetParam("no_bias", false) + .SetInput("data", flatten) + .CreateSymbol("fc1"); + auto relu6 = Operator("Activation") + .SetParam("act_type", "relu") + .SetInput("data", fc1) + .CreateSymbol("relu6"); + auto dropout1 = Operator("Dropout") + .SetParam("p", 0.5) + .SetInput("data", relu6) + .CreateSymbol("dropout1"); + /*stage5*/ + auto fc2 = Operator("FullyConnected") + .SetParam("num_hidden", 4096) + .SetParam("no_bias", false) + .SetInput("data", dropout1) + .CreateSymbol("fc2"); + auto relu7 = Operator("Activation") + .SetParam("act_type", "relu") + .SetInput("data", fc2) + .CreateSymbol("relu7"); + auto dropout2 = Operator("Dropout") + .SetParam("p", 0.5) + .SetInput("data", relu7) + .CreateSymbol("dropout2"); + /*stage6*/ + auto fc3 = Operator("FullyConnected") + .SetParam("num_hidden", num_classes) + .SetParam("no_bias", false) + .SetInput("data", dropout2) + .CreateSymbol("fc3"); + auto softmax = Operator("SoftmaxOutput") + .SetParam("grad_scale", 1) + .SetParam("ignore_label", -1) + .SetParam("multi_output", false) + .SetParam("use_ignore", false) + .SetParam("normalization", "null") /*batch,null,valid */ + .SetInput("data", fc3) + .SetInput("label", target_label) + .CreateSymbol("softmax"); + return softmax; +} + +int main(int argc, char const *argv[]) { + /*basic config*/ + int batch_size = 256; + int max_epo = 100; + float learning_rate = 1e-4; + float weight_decay = 1e-4; + + /*context and net symbol*/ + auto ctx = Context::gpu(); + auto Net = AlexnetSymbol(10); + + /*args_map and aux_map is used for parameters' saving*/ + map args_map; + map aux_map; + + /*we should tell mxnet the shape of data and label*/ + args_map["data"] = NDArray(Shape(batch_size, 3, 256, 256), ctx); + args_map["label"] = NDArray(Shape(batch_size), ctx); + + /*with data and label, executor can be generated automatically*/ + auto *exec = Net.SimpleBind(ctx, args_map); + aux_map = exec->aux_dict(); + args_map = exec->arg_dict(); + + /*if fine tune from some pre-trained model, we should load the parameters*/ + // NDArray::Load("./model/alex_params_3", nullptr, &args_map); + /*else, we should use initializer Xavier to init the params*/ + Xavier xavier = Xavier(Xavier::gaussian, Xavier::in, 2.34); + for (auto &arg : args_map) { + /*be careful here, the arg's name must has some specific ends or starts for + * initializer to call*/ + xavier(arg.first, &arg.second); + } + /*print out to check the shape of the net*/ + for (const auto &s : Net.ListArguments()) { + LG << s; + const auto &k = args_map[s].GetShape(); + for (const auto &i : k) { + cout << i << " "; + } + cout << endl; + } + + /*these binary files should be generated using im2rc tools, which can be found + * in mxnet/bin*/ + auto train_iter = MXDataIter("ImageRecordIter") + .SetParam("path_imglist", "./data/train_rec.lst") + .SetParam("path_imgrec", "./data/train_rec.bin") + .SetParam("data_shape", Shape(3, 256, 256)) + .SetParam("batch_size", batch_size) + .SetParam("shuffle", 1) + .CreateDataIter(); + auto val_iter = MXDataIter("ImageRecordIter") + .SetParam("path_imglist", "./data/val_rec.lst") + .SetParam("path_imgrec", "./data/val_rec.bin") + .SetParam("data_shape", Shape(3, 256, 256)) + .SetParam("batch_size", batch_size) + .CreateDataIter(); + + Optimizer* opt = OptimizerRegistry::Find("ccsgd"); + opt->SetParam("momentum", 0.9) + ->SetParam("rescale_grad", 1.0 / batch_size) + ->SetParam("clip_gradient", 10); + + Accuracy acu_train, acu_val; + LogLoss logloss_val; + for (int iter = 0; iter < max_epo; ++iter) { + LG << "Train Epoch: " << iter; + /*reset the metric every epoch*/ + acu_train.Reset(); + /*reset the data iter every epoch*/ + train_iter.Reset(); + while (train_iter.Next()) { + auto batch = train_iter.GetDataBatch(); + LG << train_iter.GetDataBatch().index.size(); + /*use copyto to feed new data and label to the executor*/ + batch.data.CopyTo(&args_map["data"]); + batch.label.CopyTo(&args_map["label"]); + exec->Forward(true); + exec->Backward(); + exec->UpdateAll(opt, learning_rate, weight_decay); + NDArray::WaitAll(); + acu_train.Update(batch.label, exec->outputs[0]); + } + LG << "ITER: " << iter << " Train Accuracy: " << acu_train.Get(); + + LG << "Val Epoch: " << iter; + acu_val.Reset(); + val_iter.Reset(); + logloss_val.Reset(); + while (val_iter.Next()) { + auto batch = val_iter.GetDataBatch(); + LG << val_iter.GetDataBatch().index.size(); + batch.data.CopyTo(&args_map["data"]); + batch.label.CopyTo(&args_map["label"]); + exec->Forward(false); + NDArray::WaitAll(); + acu_val.Update(batch.label, exec->outputs[0]); + logloss_val.Update(batch.label, exec->outputs[0]); + } + LG << "ITER: " << iter << " Val Accuracy: " << acu_val.Get(); + LG << "ITER: " << iter << " Val LogLoss: " << logloss_val.Get(); + + /*save the parameters*/ + stringstream ss; + ss << iter; + string iter_str; + ss >> iter_str; + string save_path_param = "./model/alex_param_" + iter_str; + auto save_args = args_map; + /*we do not want to save the data and label*/ + save_args.erase(save_args.find("data")); + save_args.erase(save_args.find("label")); + /*the alexnet does not get any aux array, so we do not need to save + * aux_map*/ + LG << "ITER: " << iter << " Saving to..." << save_path_param; + NDArray::Save(save_path_param, save_args); + } + /*don't foget to release the executor*/ + delete exec; + MXNotifyShutdown(); + return 0; +} diff --git a/cpp-package/example/charRNN.cpp b/cpp-package/example/charRNN.cpp new file mode 100644 index 000000000000..daf31a4fe69c --- /dev/null +++ b/cpp-package/example/charRNN.cpp @@ -0,0 +1,719 @@ +/*! + * Copyright (c) 2016 by Contributors + * Hua Zhang mz24cn@hotmail.com + * The code implements C++ version charRNN for mxnet\example\rnn\char-rnn.ipynb with MXNet.cpp API. + * The generated params file is compatiable with python version. + * train() and predict() has been verified with original data samples. + * 2017/1/23: + * Add faster version charRNN based on built-in cuDNN RNN operator, 10 times faster. + * Add time major computation graph, although no substantial performance difference. + * Support continuing training from last params file. + * Rename params file epoch number starts from zero. + */ + +#pragma warning(disable: 4996) // VS2015 complains on 'std::copy' ... +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "mxnet-cpp/MxNetCpp.h" + +// Allow IDE to parse the types +#include "../include/mxnet-cpp/op.h" + +using namespace std; +using namespace mxnet::cpp; + +struct LSTMState { + Symbol C; + Symbol h; +}; + +struct LSTMParam { + Symbol i2h_weight; + Symbol i2h_bias; + Symbol h2h_weight; + Symbol h2h_bias; +}; + +bool TIME_MAJOR = true; + +// LSTM Cell symbol +LSTMState LSTM(int num_hidden, const Symbol& indata, const LSTMState& prev_state, + const LSTMParam& param, int seqidx, int layeridx, mx_float dropout = 0) { + auto input = dropout > 0? Dropout(indata, dropout) : indata; + auto prefix = string("t") + to_string(seqidx) + "_l" + to_string(layeridx); + auto i2h = FullyConnected(prefix + "_i2h", input, param.i2h_weight, param.i2h_bias, + num_hidden * 4); + auto h2h = FullyConnected(prefix + "_h2h", prev_state.h, param.h2h_weight, param.h2h_bias, + num_hidden * 4); + auto gates = i2h + h2h; + auto slice_gates = SliceChannel(prefix + "_slice", gates, 4); + auto in_gate = Activation(slice_gates[0], ActivationActType::sigmoid); + auto in_transform = Activation(slice_gates[1], ActivationActType::tanh); + auto forget_gate = Activation(slice_gates[2], ActivationActType::sigmoid); + auto out_gate = Activation(slice_gates[3], ActivationActType::sigmoid); + + LSTMState state; + state.C = (forget_gate * prev_state.C) + (in_gate * in_transform); + state.h = out_gate * Activation(state.C, ActivationActType::tanh); + return state; +} + +Symbol LSTMUnroll(int num_lstm_layer, int sequence_length, int input_dim, + int num_hidden, int num_embed, mx_float dropout = 0) { + auto isTrain = sequence_length > 1; + auto data = Symbol::Variable("data"); + if (TIME_MAJOR && isTrain) + data = transpose(data); + auto embed_weight = Symbol::Variable("embed_weight"); + auto embed = Embedding("embed", data, embed_weight, input_dim, num_embed); + auto wordvec = isTrain? SliceChannel(embed, sequence_length, TIME_MAJOR? 0 : 1, true) : embed; + + vector last_states; + vector param_cells; + for (int l = 0; l < num_lstm_layer; l++) { + string layer = "l" + to_string(l); + LSTMParam param; + param.i2h_weight = Symbol::Variable(layer + "_i2h_weight"); + param.i2h_bias = Symbol::Variable(layer + "_i2h_bias"); + param.h2h_weight = Symbol::Variable(layer + "_h2h_weight"); + param.h2h_bias = Symbol::Variable(layer + "_h2h_bias"); + param_cells.push_back(param); + LSTMState state; + state.C = Symbol::Variable(layer + "_init_c"); + state.h = Symbol::Variable(layer + "_init_h"); + last_states.push_back(state); + } + + vector hidden_all; + for (int i = 0; i < sequence_length; i++) { + auto hidden = wordvec[i]; + for (int layer = 0; layer < num_lstm_layer; layer++) { + double dp_ratio = layer == 0? 0 : dropout; + auto next_state = LSTM(num_hidden, hidden, last_states[layer], param_cells[layer], + i, layer, dp_ratio); + hidden = next_state.h; + last_states[layer] = next_state; + } + if (dropout > 0) + hidden = Dropout(hidden, dropout); + hidden_all.push_back(hidden); + } + + auto hidden_concat = isTrain? Concat(hidden_all, hidden_all.size(), 0) : hidden_all[0]; + auto cls_weight = Symbol::Variable("cls_weight"); + auto cls_bias = Symbol::Variable("cls_bias"); + auto pred = FullyConnected("pred", hidden_concat, cls_weight, cls_bias, input_dim); + + auto label = Symbol::Variable("softmax_label"); + label = transpose(label); + label = Reshape(label, Shape(), false, false, Shape(-1)); // -1: infer from graph + auto sm = SoftmaxOutput("softmax", pred, label); + if (isTrain) + return sm; + + vector outputs = { sm }; + for (auto& state : last_states) { + outputs.push_back(state.C); + outputs.push_back(state.h); + } + return Symbol::Group(outputs); +} + +// Currently mxnet GPU version RNN operator is implemented via *fast* NVIDIA cuDNN. +Symbol LSTMWithBuiltInRNNOp(int num_lstm_layer, int sequence_length, int input_dim, + int num_hidden, int num_embed, mx_float dropout = 0) { + auto isTrain = sequence_length > 1; + auto data = Symbol::Variable("data"); + if (TIME_MAJOR && isTrain) + data = transpose(data); + + auto embed_weight = Symbol::Variable("embed_weight"); + auto embed = Embedding("embed", data, embed_weight, input_dim, num_embed); + auto label = Symbol::Variable("softmax_label"); + label = transpose(label); + label = Reshape(label, Shape(), false, + false, Shape(-1)); // FullyConnected requires one dimension + if (!TIME_MAJOR && isTrain) + embed = SwapAxis(embed, 0, 1); // Change to time-major as cuDNN requires + + // We need not do the SwapAxis op as python version does. Direct and better performance in C++! + auto rnn_h_init = Symbol::Variable("LSTM_init_h"); + auto rnn_c_init = Symbol::Variable("LSTM_init_c"); + auto rnn_params = Symbol::Variable("LSTM_parameters"); // See explanations near RNNXavier class + auto rnn = RNN(embed, rnn_params, rnn_h_init, rnn_c_init, num_hidden, num_lstm_layer, + RNNMode::lstm, false, dropout, !isTrain); + auto hidden = Reshape(rnn[0], Shape(), false, false, Shape(-1, num_hidden)); + + auto cls_weight = Symbol::Variable("cls_weight"); + auto cls_bias = Symbol::Variable("cls_bias"); + auto pred = FullyConnected("pred", hidden, cls_weight, cls_bias, input_dim); + /*In rnn-time-major/rnn_cell_demo.py, the author claimed time-major version speeds up + * 1.5~2 times versus batch version. I doubts on the conclusion. In my test, the performance + * of both codes are almost same. In fact, there are no substantially differences between + * two codes. They are both based on time major cuDNN, the computation graph only differs + * slightly on the choices of where to put Reshape/SwapAxis/transpose operation. Here I don't + * use Reshape on pred and keep label shape on SoftmaxOutput like time major version code, + * but Reshape on label for simplification. It doesn't make influence on performacne. */ + + auto sm = SoftmaxOutput("softmax", pred, label); + if (isTrain) + return sm; + else + return Symbol::Group({ sm, rnn[1/*RNNOpOutputs::kStateOut=1*/], + rnn[2/*RNNOpOutputs::kStateCellOut=2*/] }); +} + +class Shuffler { + vector sequence; + public: + explicit Shuffler(int size) : sequence(size) { + int* p = sequence.data(); + for (int i = 0; i < size; i++) + *p++ = i; + } + void shuffle(function lambda = nullptr) { + random_shuffle(sequence.begin(), sequence.end()); + int n = 0; + if (lambda != nullptr) + for (int i : sequence) + lambda(n++, i); + } + const int* data() { + return sequence.data(); + } +}; + +class BucketSentenceIter : public DataIter { + Shuffler* random; + int batch, current, end, sequence_length; + Context device; + vector> sequences; + vector index2chars; + unordered_map charIndices; + + public: + BucketSentenceIter(string filename, int minibatch, Context context) : batch(minibatch), + current(-1), device(context) { + auto content = readContent(filename); + buildCharIndex(content); + sequences = convertTextToSequences(content, '\n'); + + int N = sequences.size() / batch * batch; // total used samples + sequences.resize(N); + sort(sequences.begin(), sequences.end(), [](const vector& a, + const vector& b) { return a.size() < b.size(); }); + + sequence_length = sequences.back().size(); + random = new Shuffler(N); + // We still can get random results if call Reset() firstly +// vector>* target = &sequences; +// random->shuffle([target](int n, int i) { (*target)[n].swap((*target)[i]); }); + end = N / batch; + } + virtual ~BucketSentenceIter() { + delete random; + } + + unsigned int maxSequenceLength() { + return sequence_length; + } + + size_t characterSize() { + return charIndices.size(); + } + + virtual bool Next(void) { + return ++current < end; + } + virtual NDArray GetData(void) { + const int* indices = random->data(); + mx_float *data = new mx_float[sequence_length * batch], *pdata = data; + + for (int i = current * batch, end = i + batch; i < end; i++) { + memcpy(pdata, sequences[indices[i]].data(), sequences[indices[i]].size() * sizeof(mx_float)); + if (sequences[indices[i]].size() < sequence_length) + memset(pdata + sequences[indices[i]].size(), 0, + (sequence_length - sequences[indices[i]].size()) * sizeof(mx_float)); + pdata += sequence_length; + } + NDArray array(Shape(batch, sequence_length), device, false); + array.SyncCopyFromCPU(data, batch * sequence_length); + return array; + } + virtual NDArray GetLabel(void) { + const int* indices = random->data(); + mx_float *label = new mx_float[sequence_length * batch], *plabel = label; + + for (int i = current * batch, end = i + batch; i < end; i++) { + memcpy(plabel, sequences[indices[i]].data() + 1, + (sequences[indices[i]].size() - 1) * sizeof(mx_float)); + memset(plabel + sequences[indices[i]].size() - 1, 0, + (sequence_length - sequences[indices[i]].size() + 1) * sizeof(mx_float)); + plabel += sequence_length; + } + NDArray array(Shape(batch, sequence_length), device, false); + array.SyncCopyFromCPU(label, batch * sequence_length); + return array; + } + virtual int GetPadNum(void) { + return sequence_length - sequences[random->data()[current * batch]].size(); + } + virtual std::vector GetIndex(void) { + const int* indices = random->data(); + vector list(indices + current * batch, indices + current * batch + batch); + return list; + } + virtual void BeforeFirst(void) { + current = -1; + random->shuffle(nullptr); + } + + wstring readContent(const string file) { + wifstream ifs(file, ios::binary); + if (ifs) { + wostringstream os; + os << ifs.rdbuf(); + return os.str(); + } + return L""; + } + + void buildCharIndex(const wstring& content) { + // This version buildCharIndex() Compatiable with python version char_rnn dictionary + int n = 1; + charIndices['\0'] = 0; // padding character + index2chars.push_back(0); // padding character index + for (auto c : content) + if (charIndices.find(c) == charIndices.end()) { + charIndices[c] = n++; + index2chars.push_back(c); + } + } +// void buildCharIndex(wstring& content) { +// for (auto c : content) +// charIndices[c]++; // char-frequency map; then char-index map +// vector> characters; +// for (auto& iter : charIndices) +// characters.push_back(make_tuple(iter.first, iter.second)); +// sort(characters.begin(), characters.end(), [](const tuple& a, +// const tuple& b) { return get<1>(a) > get<1>(b); }); +// mx_float index = 1; //0 is left for zero-padding +// index2chars.clear(); +// index2chars.push_back(0); //zero-padding +// for (auto& t : characters) { +// charIndices[get<0>(t)] = index++; +// index2chars.push_back(get<0>(t)); +// } +// } + + inline wchar_t character(int i) { + return index2chars[i]; + } + + inline mx_float index(wchar_t c) { + return charIndices[c]; + } + + void saveCharIndices(const string file) { + wofstream ofs(file, ios::binary); + if (ofs) { + ofs.write(index2chars.data() + 1, index2chars.size() - 1); + ofs.close(); + } + } + + static tuple, vector> loadCharIndices( + const string file) { + wifstream ifs(file, ios::binary); + unordered_map map; + vector chars; + if (ifs) { + wostringstream os; + os << ifs.rdbuf(); + int n = 1; + map[L'\0'] = 0; + chars.push_back(L'\0'); + for (auto c : os.str()) { + map[c] = (mx_float) n++; + chars.push_back(c); + } + } + // Note: Can't use {} because this would hit the explicit constructor + return tuple, vector>(map, chars); + } + + vector> convertTextToSequences(const wstring& content, wchar_t spliter) { + vector> sequences; + sequences.push_back(vector()); + for (auto c : content) + if (c == spliter && !sequences.back().empty()) + sequences.push_back(vector()); + else + sequences.back().push_back(charIndices[c]); + return sequences; + } +}; + +void OutputPerplexity(NDArray* labels, NDArray* output) { + vector charIndices, a; + labels->SyncCopyToCPU(&charIndices, 0L); // 0L indicates all + output->SyncCopyToCPU(&a, 0L)/*4128*84*/; + mx_float loss = 0; + int batchSize = labels->GetShape()[0]/*32*/, sequenceLength = labels->GetShape()[1]/*129*/, + nSamples = output->GetShape()[0]/*4128*/, vocabSize = output->GetShape()[1]/*84*/; + for (int n = 0; n < nSamples; n++) { + int row = n % batchSize, column = n / batchSize, labelOffset = column + + row * sequenceLength; // Search based on column storage: labels.T + mx_float safe_value = max(1e-10f, a[vocabSize * n + + static_cast(charIndices[labelOffset])]); + loss += -log(safe_value); // Calculate negative log-likelihood + } + loss = exp(loss / nSamples); + cout << "Train-Perplexity=" << loss << endl; +} + +void SaveCheckpoint(const string filepath, Symbol net, Executor* exe) { + map params; + for (auto iter : exe->arg_dict()) + if (iter.first.find("_init_") == string::npos + && iter.first.rfind("data") != iter.first.length() - 4 + && iter.first.rfind("label") != iter.first.length() - 5) + params.insert({"arg:" + iter.first, iter.second}); + for (auto iter : exe->aux_dict()) + params.insert({"aux:" + iter.first, iter.second}); + NDArray::Save(filepath, params); +} + +void LoadCheckpoint(const string filepath, Executor* exe) { + map params = NDArray::LoadToMap(filepath); + for (auto iter : params) { + string type = iter.first.substr(0, 4); + string name = iter.first.substr(4); + NDArray target; + if (type == "arg:") + target = exe->arg_dict()[name]; + else if (type == "aux:") + target = exe->aux_dict()[name]; + else + continue; + iter.second.CopyTo(&target); + } +} + +int input_dim = 0;/*84*/ +int sequence_length_max = 0;/*129*/ +int num_embed = 256; +int num_lstm_layer = 3; +int num_hidden = 512; +mx_float dropout = 0.2; +void train(const string file, int batch_size, int max_epoch, int start_epoch) { + Context device(DeviceType::kGPU, 0); + BucketSentenceIter dataIter(file, batch_size, device); + string prefix = file.substr(0, file.rfind(".")); + dataIter.saveCharIndices(prefix + ".dictionary"); + + input_dim = static_cast(dataIter.characterSize()); + sequence_length_max = dataIter.maxSequenceLength(); + + auto RNN = LSTMUnroll(num_lstm_layer, sequence_length_max, input_dim, num_hidden, + num_embed, dropout); + map args_map; + args_map["data"] = NDArray(Shape(batch_size, sequence_length_max), device, false); + args_map["softmax_label"] = NDArray(Shape(batch_size, sequence_length_max), device, false); + for (int i = 0; i < num_lstm_layer; i++) { + string key = "l" + to_string(i) + "_init_"; + args_map[key + "c"] = NDArray(Shape(batch_size, num_hidden), device, false); + args_map[key + "h"] = NDArray(Shape(batch_size, num_hidden), device, false); + } + vector zeros(batch_size * num_hidden, 0); + // RNN.SimpleBind(device, args_map, {}, {{"data", kNullOp}}); + Executor* exe = RNN.SimpleBind(device, args_map); + + if (start_epoch == -1) { + Xavier xavier = Xavier(Xavier::gaussian, Xavier::in, 2.34); + for (auto &arg : exe->arg_dict()) + xavier(arg.first, &arg.second); + } else { + LoadCheckpoint(prefix + "-" + to_string(start_epoch) + ".params", exe); + } + start_epoch++; + + mx_float learning_rate = 0.0002; + mx_float weight_decay = 0.000002; + Optimizer* opt = OptimizerRegistry::Find("ccsgd"); +// opt->SetParam("momentum", 0.9)->SetParam("rescale_grad", 1.0 / batch_size) +// ->SetParam("clip_gradient", 10); + + for (int epoch = start_epoch; epoch < max_epoch; ++epoch) { + dataIter.Reset(); + auto tic = chrono::system_clock::now(); + while (dataIter.Next()) { + auto data_batch = dataIter.GetDataBatch(); + data_batch.data.CopyTo(&exe->arg_dict()["data"]); + data_batch.label.CopyTo(&exe->arg_dict()["softmax_label"]); + for (int l = 0; l < num_lstm_layer; l++) { + string key = "l" + to_string(l) + "_init_"; + exe->arg_dict()[key + "c"].SyncCopyFromCPU(zeros); + exe->arg_dict()[key + "h"].SyncCopyFromCPU(zeros); + } + NDArray::WaitAll(); + + exe->Forward(true); + exe->Backward(); + exe->UpdateAll(opt, learning_rate, weight_decay); + NDArray::WaitAll(); + } + auto toc = chrono::system_clock::now(); + cout << "Epoch[" << epoch << "] Time Cost:" << + chrono::duration_cast(toc - tic).count() << " seconds "; + OutputPerplexity(&exe->arg_dict()["softmax_label"], &exe->outputs[0]); + string filepath = prefix + "-" + to_string(epoch) + ".params"; + SaveCheckpoint(filepath, RNN, exe); + } +} + +/*The original example, rnn_cell_demo.py, uses default Xavier as initalizer, which relies on + * variable name, cannot initialize LSTM_parameters. Thus it was renamed to LSTM_bias, + * which can be initialized as zero. But it cannot converge after 100 epochs in this corpus + * example. Using RNNXavier, after 15 oscillating epochs, it rapidly converges like old + * LSTMUnroll version. */ +class RNNXavier : public Xavier { + public: + RNNXavier(RandType rand_type = gaussian, FactorType factor_type = avg, + float magnitude = 3) : Xavier(rand_type, factor_type, magnitude) { + } + virtual ~RNNXavier() {} + protected: + virtual void InitDefault(NDArray* arr) { + Xavier::InitWeight(arr); + } +}; + +void trainWithBuiltInRNNOp(const string file, int batch_size, int max_epoch, int start_epoch) { + Context device(DeviceType::kGPU, 0); + BucketSentenceIter dataIter(file, batch_size, device); + string prefix = file.substr(0, file.rfind(".")); + dataIter.saveCharIndices(prefix + ".dictionary"); + + input_dim = static_cast(dataIter.characterSize()); + sequence_length_max = dataIter.maxSequenceLength(); + + auto RNN = LSTMWithBuiltInRNNOp(num_lstm_layer, sequence_length_max, input_dim, num_hidden, + num_embed, dropout); + map args_map; + args_map["data"] = NDArray(Shape(batch_size, sequence_length_max), device, false); + // Avoiding SwapAxis, batch_size is of second dimension. + args_map["LSTM_init_c"] = NDArray(Shape(num_lstm_layer, batch_size, num_hidden), device, false); + args_map["LSTM_init_h"] = NDArray(Shape(num_lstm_layer, batch_size, num_hidden), device, false); + args_map["softmax_label"] = NDArray(Shape(batch_size, sequence_length_max), device, false); + vector zeros(batch_size * num_lstm_layer * num_hidden, 0); + Executor* exe = RNN.SimpleBind(device, args_map); + + if (start_epoch == -1) { + RNNXavier xavier = RNNXavier(Xavier::gaussian, Xavier::in, 2.34); + for (auto &arg : exe->arg_dict()) + xavier(arg.first, &arg.second); + } else { + LoadCheckpoint(prefix + "-" + to_string(start_epoch) + ".params", exe); + } + start_epoch++; + + mx_float learning_rate = 0.0002; + mx_float weight_decay = 0.000002; + Optimizer* opt = OptimizerRegistry::Find("ccsgd"); +// opt->SetParam("momentum", 0.9)->SetParam("rescale_grad", 1.0 / batch_size) +// ->SetParam("clip_gradient", 10); + + for (int epoch = start_epoch; epoch < max_epoch; ++epoch) { + dataIter.Reset(); + auto tic = chrono::system_clock::now(); + while (dataIter.Next()) { + auto data_batch = dataIter.GetDataBatch(); + data_batch.data.CopyTo(&exe->arg_dict()["data"]); + data_batch.label.CopyTo(&exe->arg_dict()["softmax_label"]); + exe->arg_dict()["LSTM_init_c"].SyncCopyFromCPU(zeros); + exe->arg_dict()["LSTM_init_h"].SyncCopyFromCPU(zeros); + NDArray::WaitAll(); + + exe->Forward(true); + exe->Backward(); + exe->UpdateAll(opt, learning_rate, weight_decay); + NDArray::WaitAll(); + } + auto toc = chrono::system_clock::now(); + cout << "Epoch[" << epoch << "] Time Cost:" << + chrono::duration_cast(toc - tic).count() << " seconds "; + OutputPerplexity(&exe->arg_dict()["softmax_label"], &exe->outputs[0]); + string filepath = prefix + "-" + to_string(epoch) + ".params"; + SaveCheckpoint(filepath, RNN, exe); + } +} + +void predict(wstring* ptext, int sequence_length, const string param_file, + const string dictionary_file) { + Context device(DeviceType::kGPU, 0); + auto results = BucketSentenceIter::loadCharIndices(dictionary_file); + auto dictionary = get<0>(results); + auto charIndices = get<1>(results); + input_dim = static_cast(charIndices.size()); + auto RNN = LSTMUnroll(num_lstm_layer, 1, input_dim, num_hidden, num_embed, 0); + + map args_map; + args_map["data"] = NDArray(Shape(1, 1), device, false); + args_map["softmax_label"] = NDArray(Shape(1, 1), device, false); + vector zeros(1 * num_hidden, 0); + for (int l = 0; l < num_lstm_layer; l++) { + string key = "l" + to_string(l) + "_init_"; + args_map[key + "c"] = NDArray(Shape(1, num_hidden), device, false); + args_map[key + "h"] = NDArray(Shape(1, num_hidden), device, false); + args_map[key + "c"].SyncCopyFromCPU(zeros); + args_map[key + "h"].SyncCopyFromCPU(zeros); + } + Executor* exe = RNN.SimpleBind(device, args_map); + LoadCheckpoint(param_file, exe); + + mx_float index; + wchar_t next; + vector softmax; + softmax.resize(input_dim); + for (auto c : *ptext) { + exe->arg_dict()["data"].SyncCopyFromCPU(&dictionary[c], 1); + exe->Forward(false); + + exe->outputs[0].SyncCopyToCPU(softmax.data(), input_dim); + for (int l = 0; l < num_lstm_layer; l++) { + string key = "l" + to_string(l) + "_init_"; + exe->outputs[l * 2 + 1].CopyTo(&args_map[key + "c"]); + exe->outputs[l * 2 + 2].CopyTo(&args_map[key + "h"]); + } + + size_t n = max_element(softmax.begin(), softmax.end()) - softmax.begin(); + index = (mx_float) n; + next = charIndices[n]; + } + ptext->push_back(next); + + for (int i = 0; i < sequence_length; i++) { + exe->arg_dict()["data"].SyncCopyFromCPU(&index, 1); + exe->Forward(false); + + exe->outputs[0].SyncCopyToCPU(softmax.data(), input_dim); + for (int l = 0; l < num_lstm_layer; l++) { + string key = "l" + to_string(l) + "_init_"; + exe->outputs[l * 2 + 1].CopyTo(&args_map[key + "c"]); + exe->outputs[l * 2 + 2].CopyTo(&args_map[key + "h"]); + } + + size_t n = max_element(softmax.begin(), softmax.end()) - softmax.begin(); + index = (mx_float) n; + next = charIndices[n]; + ptext->push_back(next); + } +} + +void predictWithBuiltInRNNOp(wstring* ptext, int sequence_length, const string param_file, + const string dictionary_file) { + Context device(DeviceType::kGPU, 0); + auto results = BucketSentenceIter::loadCharIndices(dictionary_file); + auto dictionary = get<0>(results); + auto charIndices = get<1>(results); + input_dim = static_cast(charIndices.size()); + auto RNN = LSTMWithBuiltInRNNOp(num_lstm_layer, 1, input_dim, num_hidden, num_embed, 0); + + map args_map; + args_map["data"] = NDArray(Shape(1, 1), device, false); + args_map["softmax_label"] = NDArray(Shape(1, 1), device, false); + vector zeros(1 * num_lstm_layer * num_hidden, 0); + // Avoiding SwapAxis, batch_size=1 is of second dimension. + args_map["LSTM_init_c"] = NDArray(Shape(num_lstm_layer, 1, num_hidden), device, false); + args_map["LSTM_init_h"] = NDArray(Shape(num_lstm_layer, 1, num_hidden), device, false); + args_map["LSTM_init_c"].SyncCopyFromCPU(zeros); + args_map["LSTM_init_h"].SyncCopyFromCPU(zeros); + Executor* exe = RNN.SimpleBind(device, args_map); + LoadCheckpoint(param_file, exe); + + mx_float index; + wchar_t next; + vector softmax; + softmax.resize(input_dim); + for (auto c : *ptext) { + exe->arg_dict()["data"].SyncCopyFromCPU(&dictionary[c], 1); + exe->Forward(false); + + exe->outputs[0].SyncCopyToCPU(softmax.data(), input_dim); + exe->outputs[1].CopyTo(&args_map["LSTM_init_h"]); + exe->outputs[2].CopyTo(&args_map["LSTM_init_c"]); + + size_t n = max_element(softmax.begin(), softmax.end()) - softmax.begin(); + index = (mx_float) n; + next = charIndices[n]; + } + ptext->push_back(next); + + for (int i = 0; i < sequence_length; i++) { + exe->arg_dict()["data"].SyncCopyFromCPU(&index, 1); + exe->Forward(false); + + exe->outputs[0].SyncCopyToCPU(softmax.data(), input_dim); + exe->outputs[1].CopyTo(&args_map["LSTM_init_h"]); + exe->outputs[2].CopyTo(&args_map["LSTM_init_c"]); + + size_t n = max_element(softmax.begin(), softmax.end()) - softmax.begin(); + index = (mx_float) n; + next = charIndices[n]; + ptext->push_back(next); + } +} + +int main(int argc, char** argv) { + if (argc < 5) { + cout << "Usage for training: charRNN train[BuiltIn][TimeMajor] {corpus file}" + " {batch size} {max epoch} [{starting epoch}]" << endl; + cout <<"Usage for prediction: charRNN predict[BuiltIn][TimeMajor] {params file}" + " {dictionary file} {beginning of text}" << endl; + cout <<"Note: The {params file} of train/trainBuiltIn/trainTimeMajor/trainBuiltInTimeMajor" + " are not compatible with each other." << endl; + return 0; + } + + string task = argv[1]; + bool builtIn = task.find("BuiltIn") != string::npos; + TIME_MAJOR = task.find("TimeMajor") != string::npos; + cout << "use BuiltIn cuDNN RNN: " << builtIn << endl + << "use data as TimeMajor: " << TIME_MAJOR << endl; + if (task.find("train") == 0) { + cout << "train batch size: " << argv[3] << endl + << "train max epoch: " << argv[4] << endl; + int start_epoch = argc > 5? atoi(argv[5]) : -1; + // this function will generate dictionary file and params file. + if (builtIn) + trainWithBuiltInRNNOp(argv[2], atoi(argv[3]), atoi(argv[4]), start_epoch); + else + train(argv[2], atoi(argv[3]), atoi(argv[4]), start_epoch); // ditto + } else if (task.find("predict") == 0) { + wstring text; // = L"If there is anyone out there who still doubts "; + // Considering of extending to Chinese samples in future, use wchar_t instead of char + for (char c : string(argv[4])) + text.push_back((wchar_t) c); + /*Python version predicts text default to random selecltions. Here I didn't write the random + code, always choose the 'best' character. So the text length reduced to 600. Longer size often + leads to repeated sentances, since training sequence length is only 129 for obama corpus.*/ + if (builtIn) + predictWithBuiltInRNNOp(&text, 600, argv[2], argv[3]); + else + predict(&text, 600, argv[2], argv[3]); + wcout << text << endl; + } + + MXNotifyShutdown(); + return 0; +} diff --git a/cpp-package/example/example.mk b/cpp-package/example/example.mk new file mode 100644 index 000000000000..3f3016a5aa24 --- /dev/null +++ b/cpp-package/example/example.mk @@ -0,0 +1,22 @@ +CPPEX_SRC = $(wildcard cpp-package/example/*.cpp) +CPPEX_EXE = $(patsubst cpp-package/example/%.cpp, build/cpp-package/example/%, $(CPPEX_SRC)) + +CPPEX_CFLAGS += -Icpp-package/include -Ibuild/cpp-package/include +CPPEX_EXTRA_LDFLAGS := -L$(ROOTDIR)/lib -lmxnet + +EXTRA_PACKAGES += cpp-package-example-all +EXTRA_PACKAGES_CLEAN += cpp-package-example-clean + +.PHONY: cpp-package-example-all cpp-package-example-clean + +cpp-package-example-all: cpp-package-all $(CPPEX_EXE) + +build/cpp-package/example/% : cpp-package/example/%.cpp lib/libmxnet.so $(CPP_PACKAGE_OP_H_FILE) + @mkdir -p $(@D) + $(CXX) -std=c++0x $(CFLAGS) $(CPPEX_CFLAGS) -MM -MT cpp-package/example/$* $< >build/cpp-package/example//$*.d + $(CXX) -std=c++0x $(CFLAGS) $(CPPEX_CFLAGS) -o $@ $(filter %.cpp %.a, $^) $(LDFLAGS) $(CPPEX_EXTRA_LDFLAGS) + +cpp-package-example-clean: + rm -rf build/cpp-package/example/* + +-include build/cpp-package/example/*.d diff --git a/cpp-package/example/feature_extract/Makefile b/cpp-package/example/feature_extract/Makefile new file mode 100644 index 000000000000..808f2613b001 --- /dev/null +++ b/cpp-package/example/feature_extract/Makefile @@ -0,0 +1,26 @@ +CXX=g++ +BLAS=-L /opt/openblas/lib -lopenblas -DMSHADOW_USE_CBLAS=1 -DMSHADOW_USE_MKL=0 +CUDA=-DMSHADOW_USE_CUDA=1 +OPENCV_CFLAGS=`pkg-config --cflags opencv` +OPENCV_LDFLAGS=`pkg-config --libs opencv` + +#COMMFLAGS=-static -static-libgcc -static-libstdc++ + +CFLAGS=$(COMMFLAGS) -I ../../include -Wall -O3 -msse3 -funroll-loops -Wno-unused-parameter -Wno-unknown-pragmas -fopenmp +LDFLAGS=$(COMMFLAGS) -L ../../lib/linux -lmxnet $(BLAS) $(CUDA) -lgomp -pthread + +all: feature_extract prepare_data_with_opencv + +feature_extract: ./feature_extract.cpp + $(CXX) -c -std=c++0x $(CFLAGS) $^ + $(CXX) $(basename $@).o -o $@ $(LDFLAGS) + -rm -f $(basename $@).o + +prepare_data_with_opencv: ./prepare_data_with_opencv.cpp + $(CXX) -c -std=c++0x $(OPENCV_CFLAGS) $^ + $(CXX) $(basename $@).o -o $@ $(OPENCV_LDFLAGS) + -rm -f $(basename $@).o + +clean: + -rm -f feature_extract + -rm -f prepare_data_with_opencv diff --git a/cpp-package/example/feature_extract/feature_extract.cpp b/cpp-package/example/feature_extract/feature_extract.cpp new file mode 100644 index 000000000000..21853a3912e7 --- /dev/null +++ b/cpp-package/example/feature_extract/feature_extract.cpp @@ -0,0 +1,120 @@ +/*! + * Copyright (c) 2015 by Contributors + */ +#include +#include +#include +#include +#include +#include "mxnet-cpp/MxNetCpp.h" +using namespace std; +using namespace mxnet::cpp; + +/* + * This example shows how to extract features with a pretrained model. + * Get the model here: + * https://github.com/dmlc/mxnet-model-gallery + * */ + +/*The global context, change them if necessary*/ +Context global_ctx(kGPU, 0); +// Context global_ctx(kCPU,0); + +class FeatureExtractor { + private: + /*the mean image, get from the pretrained model*/ + NDArray mean_img; + /*the following two maps store all the paramters need by the model*/ + map args_map; + map aux_map; + Symbol net; + Executor *executor; + /*Get the feature layer we want to extract*/ + void GetFeatureSymbol() { + /* + * use the following to check all the layers' names: + * */ + /* + net=Symbol::Load("./model/Inception_BN-symbol.json").GetInternals(); + for(const auto & layer_name:net.ListOutputs()){ + LG< paramters; + NDArray::Load("./model/Inception_BN-0039.params", 0, ¶mters); + for (const auto &k : paramters) { + if (k.first.substr(0, 4) == "aux:") { + auto name = k.first.substr(4, k.first.size() - 4); + aux_map[name] = k.second.Copy(global_ctx); + } + if (k.first.substr(0, 4) == "arg:") { + auto name = k.first.substr(4, k.first.size() - 4); + args_map[name] = k.second.Copy(global_ctx); + } + } + /*WaitAll is need when we copy data between GPU and the main memory*/ + NDArray::WaitAll(); + } + void GetMeanImg() { + mean_img = NDArray(Shape(1, 3, 224, 224), global_ctx, false); + mean_img.SyncCopyFromCPU( + NDArray::LoadToMap("./model/mean_224.nd")["mean_img"].GetData(), + 1 * 3 * 224 * 224); + NDArray::WaitAll(); + } + + public: + FeatureExtractor() { + /*prepare the model, fill the pretrained parameters, get the mean image*/ + GetFeatureSymbol(); + LoadParameters(); + GetMeanImg(); + } + + void Extract(NDArray data) { + /*Normalize the pictures*/ + data.Slice(0, 1) -= mean_img; + data.Slice(1, 2) -= mean_img; + args_map["data"] = data; + /*bind the excutor*/ + executor = net.SimpleBind(global_ctx, args_map, map(), + map(), aux_map); + executor->Forward(false); + /*print out the features*/ + auto array = executor->outputs[0].Copy(Context(kCPU, 0)); + NDArray::WaitAll(); + for (int i = 0; i < 1024; ++i) { + cout << array.At(0, i) << ","; + } + cout << endl; + } +}; + +NDArray Data2NDArray() { + NDArray ret(Shape(2, 3, 224, 224), global_ctx, false); + ifstream inf("./img.dat", ios::binary); + vector data(2 * 3 * 224 * 224); + inf.read(reinterpret_castdata.data(), 2 * 3 * 224 * 224 * sizeof(float)); + inf.close(); + ret.SyncCopyFromCPU(data.data(), 2 * 3 * 224 * 224); + NDArray::WaitAll(); + return ret; +} + +int main() { + /* + * get the data from a binary file ./img.data + * this file is generated by ./prepare_data_with_opencv + * it stores 2 pictures in NDArray format + * + */ + auto data = Data2NDArray(); + FeatureExtractor fe; + fe.Extract(data); + return 0; +} diff --git a/cpp-package/example/feature_extract/prepare_data_with_opencv.cpp b/cpp-package/example/feature_extract/prepare_data_with_opencv.cpp new file mode 100644 index 000000000000..20cbe140fc09 --- /dev/null +++ b/cpp-package/example/feature_extract/prepare_data_with_opencv.cpp @@ -0,0 +1,37 @@ +/*! + * Copyright (c) 2015 by Contributors + */ +#include +#include +#include +#include +#include + +using namespace std; + +/*read images and store them the NDArray format that MXNet.cpp can handle*/ +void Mat2Array() { + string file_name_list[] = {"./1.jpg", "./2.jpg"}; + + std::vector array; + for (auto &t : file_name_list) { + cv::Mat mat = cv::imread(t); + /*resize pictures to (224, 224) according to the pretrained model*/ + cv::resize(mat, mat, cv::Size(224, 224)); + for (int c = 0; c < 3; ++c) { + for (int i = 0; i < 224; ++i) { + for (int j = 0; j < 224; ++j) { + array.push_back(static_cast(mat.data[(i * 224 + j) * 3 + c])); + } + } + } + } + ofstream outf("./img.dat", ios::binary); + outf.write(reinterpret_castarray.data(), array.size() * sizeof(float)); + outf.close(); +} + +int main(int argc, char *argv[]) { + Mat2Array(); + return 0; +} diff --git a/cpp-package/example/feature_extract/run.sh b/cpp-package/example/feature_extract/run.sh new file mode 100755 index 000000000000..afac492b0a9d --- /dev/null +++ b/cpp-package/example/feature_extract/run.sh @@ -0,0 +1,12 @@ +### To run the this example, +### +### 1. +### Get Inseption-BN model first, from here +### https://github.com/dmlc/mxnet-model-gallery +### +### 2. +### Then Prepare 2 pictures, 1.jpg 2.jpg to extract + +make +./prepare_data_with_opencv +LD_LIBRARY_PATH=../../lib/linux ./feature_extract diff --git a/cpp-package/example/googlenet.cpp b/cpp-package/example/googlenet.cpp new file mode 100644 index 000000000000..51ff9f8b7197 --- /dev/null +++ b/cpp-package/example/googlenet.cpp @@ -0,0 +1,165 @@ +/*! + * Copyright (c) 2016 by Contributors + */ +#include +#include +#include + +#include "mxnet-cpp/MxNetCpp.h" +// Allow IDE to parse the types +#include "../include/mxnet-cpp/op.h" + +using namespace mxnet::cpp; + +Symbol ConvFactory(Symbol data, int num_filter, + Shape kernel, + Shape stride = Shape(1, 1), + Shape pad = Shape(0, 0), + const std::string & name = "", + const std::string & suffix = "") { + Symbol conv_w("conv_" + name + suffix + "_w"), conv_b("conv_" + name + suffix + "_b"); + + Symbol conv = Convolution("conv_" + name + suffix, data, + conv_w, conv_b, kernel, + num_filter, stride, Shape(1, 1), pad); + return Activation("relu_" + name + suffix, conv, "relu"); +} + +Symbol InceptionFactory(Symbol data, int num_1x1, int num_3x3red, + int num_3x3, int num_d5x5red, int num_d5x5, + PoolingPoolType pool, int proj, const std::string & name) { + Symbol c1x1 = ConvFactory(data, num_1x1, Shape(1, 1), + Shape(1, 1), Shape(0, 0), name + "_1x1"); + + Symbol c3x3r = ConvFactory(data, num_3x3red, Shape(1, 1), + Shape(1, 1), Shape(0, 0), name + "_3x3", "_reduce"); + + Symbol c3x3 = ConvFactory(c3x3r, num_3x3, Shape(3, 3), + Shape(1, 1), Shape(1, 1), name + "_3x3"); + + Symbol cd5x5r = ConvFactory(data, num_d5x5red, Shape(1, 1), + Shape(1, 1), Shape(0, 0), name + "_5x5", "_reduce"); + + Symbol cd5x5 = ConvFactory(cd5x5r, num_d5x5, Shape(5, 5), + Shape(1, 1), Shape(2, 2), name + "_5x5"); + + Symbol pooling = Pooling(name + "_pool", data, Shape(3, 3), pool, + false, false, PoolingPoolingConvention::valid, + Shape(1, 1), Shape(1, 1)); + + Symbol cproj = ConvFactory(pooling, proj, Shape(1, 1), + Shape(1, 1), Shape(0, 0), name + "_proj"); + + std::vector lst; + lst.push_back(c1x1); + lst.push_back(c3x3); + lst.push_back(cd5x5); + lst.push_back(cproj); + return Concat("ch_concat_" + name + "_chconcat", lst, lst.size()); +} + +Symbol GoogleNetSymbol(int num_classes) { + // data and label + Symbol data = Symbol::Variable("data"); + Symbol data_label = Symbol::Variable("data_label"); + + Symbol conv1 = ConvFactory(data, 64, Shape(7, 7), Shape(2, 2), Shape(3, 3), "conv1"); + Symbol pool1 = Pooling("pool1", conv1, Shape(3, 3), PoolingPoolType::max, + false, false, PoolingPoolingConvention::valid, Shape(2, 2)); + Symbol conv2 = ConvFactory(pool1, 64, Shape(1, 1), Shape(1, 1), + Shape(0, 0), "conv2"); + Symbol conv3 = ConvFactory(conv2, 192, Shape(3, 3), Shape(1, 1), Shape(1, 1), "conv3"); + Symbol pool3 = Pooling("pool3", conv3, Shape(3, 3), PoolingPoolType::max, + false, false, PoolingPoolingConvention::valid, Shape(2, 2)); + + Symbol in3a = InceptionFactory(pool3, 64, 96, 128, 16, 32, PoolingPoolType::max, 32, "in3a"); + Symbol in3b = InceptionFactory(in3a, 128, 128, 192, 32, 96, PoolingPoolType::max, 64, "in3b"); + Symbol pool4 = Pooling("pool4", in3b, Shape(3, 3), PoolingPoolType::max, + false, false, PoolingPoolingConvention::valid, Shape(2, 2)); + Symbol in4a = InceptionFactory(pool4, 192, 96, 208, 16, 48, PoolingPoolType::max, 64, "in4a"); + Symbol in4b = InceptionFactory(in4a, 160, 112, 224, 24, 64, PoolingPoolType::max, 64, "in4b"); + Symbol in4c = InceptionFactory(in4b, 128, 128, 256, 24, 64, PoolingPoolType::max, 64, "in4c"); + Symbol in4d = InceptionFactory(in4c, 112, 144, 288, 32, 64, PoolingPoolType::max, 64, "in4d"); + Symbol in4e = InceptionFactory(in4d, 256, 160, 320, 32, 128, PoolingPoolType::max, 128, "in4e"); + Symbol pool5 = Pooling("pool5", in4e, Shape(3, 3), PoolingPoolType::max, + false, false, PoolingPoolingConvention::valid, Shape(2, 2)); + Symbol in5a = InceptionFactory(pool5, 256, 160, 320, 32, 128, PoolingPoolType::max, 128, "in5a"); + Symbol in5b = InceptionFactory(in5a, 384, 192, 384, 48, 128, PoolingPoolType::max, 128, "in5b"); + Symbol pool6 = Pooling("pool6", in5b, Shape(7, 7), PoolingPoolType::avg, + false, false, PoolingPoolingConvention::valid, Shape(1, 1)); + + Symbol flatten = Flatten("flatten", pool6); + + Symbol fc1_w("fc1_w"), fc1_b("fc1_b"); + Symbol fc1 = FullyConnected("fc1", flatten, fc1_w, fc1_b, num_classes); + + return SoftmaxOutput("softmax", fc1, data_label); +} + +int main(int argc, char const *argv[]) { + int batch_size = 50; + int max_epoch = 100; + float learning_rate = 1e-4; + float weight_decay = 1e-4; + + auto googlenet = GoogleNetSymbol(10); + std::map args_map; + std::map aux_map; + + args_map["data"] = NDArray(Shape(batch_size, 3, 256, 256), Context::gpu()); + args_map["data_label"] = NDArray(Shape(batch_size), Context::gpu()); + googlenet.InferArgsMap(Context::gpu(), &args_map, args_map); + + auto train_iter = MXDataIter("ImageRecordIter") + .SetParam("path_imglist", "./train.lst") + .SetParam("path_imgrec", "./train.rec") + .SetParam("data_shape", Shape(3, 256, 256)) + .SetParam("batch_size", batch_size) + .SetParam("shuffle", 1) + .CreateDataIter(); + + auto val_iter = MXDataIter("ImageRecordIter") + .SetParam("path_imglist", "./val.lst") + .SetParam("path_imgrec", "./_val.rec") + .SetParam("data_shape", Shape(3, 256, 256)) + .SetParam("batch_size", batch_size) + .CreateDataIter(); + + Optimizer* opt = OptimizerRegistry::Find("ccsgd"); + opt->SetParam("momentum", 0.9) + ->SetParam("rescale_grad", 1.0 / batch_size) + ->SetParam("clip_gradient", 10); + + for (int iter = 0; iter < max_epoch; ++iter) { + LG << "Epoch: " << iter; + train_iter.Reset(); + while (train_iter.Next()) { + auto data_batch = train_iter.GetDataBatch(); + args_map["data"] = data_batch.data.Copy(Context::gpu()); + args_map["data_label"] = data_batch.label.Copy(Context::gpu()); + NDArray::WaitAll(); + auto *exec = googlenet.SimpleBind(Context::gpu(), args_map); + exec->Forward(true); + exec->Backward(); + exec->UpdateAll(opt, learning_rate, weight_decay); + delete exec; + } + + Accuracy acu; + val_iter.Reset(); + while (val_iter.Next()) { + auto data_batch = val_iter.GetDataBatch(); + args_map["data"] = data_batch.data.Copy(Context::gpu()); + args_map["data_label"] = data_batch.label.Copy(Context::gpu()); + NDArray::WaitAll(); + auto *exec = googlenet.SimpleBind(Context::gpu(), args_map); + exec->Forward(false); + NDArray::WaitAll(); + acu.Update(data_batch.label, exec->outputs[0]); + delete exec; + } + LG << "Accuracy: " << acu.Get(); + } + MXNotifyShutdown(); + return 0; +} diff --git a/cpp-package/example/inception_bn.cpp b/cpp-package/example/inception_bn.cpp new file mode 100644 index 000000000000..b65611215b7a --- /dev/null +++ b/cpp-package/example/inception_bn.cpp @@ -0,0 +1,192 @@ +/*! + * Copyright (c) 2016 by Contributors + */ +#include +#include +#include +#include +#include "mxnet-cpp/MxNetCpp.h" +// Allow IDE to parse the types +#include "../include/mxnet-cpp/op.h" + +using namespace mxnet::cpp; + +static const Symbol BN_BETA; +static const Symbol BN_GAMMA; + +Symbol ConvFactoryBN(Symbol data, int num_filter, + Shape kernel, Shape stride, Shape pad, + const std::string & name, + const std::string & suffix = "") { + Symbol conv_w("conv_" + name + suffix + "_w"), conv_b("conv_" + name + suffix + "_b"); + + Symbol conv = Convolution("conv_" + name + suffix, data, + conv_w, conv_b, kernel, + num_filter, stride, Shape(1, 1), pad); + Symbol bn = BatchNorm("bn_" + name + suffix, conv, BN_GAMMA, BN_BETA); + return Activation("relu_" + name + suffix, bn, "relu"); +} + +Symbol InceptionFactoryA(Symbol data, int num_1x1, int num_3x3red, + int num_3x3, int num_d3x3red, int num_d3x3, + PoolingPoolType pool, int proj, + const std::string & name) { + Symbol c1x1 = ConvFactoryBN(data, num_1x1, Shape(1, 1), Shape(1, 1), + Shape(0, 0), name + "1x1"); + Symbol c3x3r = ConvFactoryBN(data, num_3x3red, Shape(1, 1), Shape(1, 1), + Shape(0, 0), name + "_3x3r"); + Symbol c3x3 = ConvFactoryBN(c3x3r, num_3x3, Shape(3, 3), Shape(1, 1), + Shape(1, 1), name + "_3x3"); + Symbol cd3x3r = ConvFactoryBN(data, num_d3x3red, Shape(1, 1), Shape(1, 1), + Shape(0, 0), name + "_double_3x3", "_reduce"); + Symbol cd3x3 = ConvFactoryBN(cd3x3r, num_d3x3, Shape(3, 3), Shape(1, 1), + Shape(1, 1), name + "_double_3x3_0"); + cd3x3 = ConvFactoryBN(data = cd3x3, num_d3x3, Shape(3, 3), Shape(1, 1), + Shape(1, 1), name + "_double_3x3_1"); + Symbol pooling = Pooling(name + "_pool", data, + Shape(3, 3), pool, false, false, + PoolingPoolingConvention::valid, + Shape(1, 1), Shape(1, 1)); + Symbol cproj = ConvFactoryBN(pooling, proj, Shape(1, 1), Shape(1, 1), + Shape(0, 0), name + "_proj"); + std::vector lst; + lst.push_back(c1x1); + lst.push_back(c3x3); + lst.push_back(cd3x3); + lst.push_back(cproj); + return Concat("ch_concat_" + name + "_chconcat", lst, lst.size()); +} + +Symbol InceptionFactoryB(Symbol data, int num_3x3red, int num_3x3, + int num_d3x3red, int num_d3x3, const std::string & name) { + Symbol c3x3r = ConvFactoryBN(data, num_3x3red, Shape(1, 1), + Shape(1, 1), Shape(0, 0), + name + "_3x3", "_reduce"); + Symbol c3x3 = ConvFactoryBN(c3x3r, num_3x3, Shape(3, 3), Shape(2, 2), + Shape(1, 1), name + "_3x3"); + Symbol cd3x3r = ConvFactoryBN(data, num_d3x3red, Shape(1, 1), Shape(1, 1), + Shape(0, 0), name + "_double_3x3", "_reduce"); + Symbol cd3x3 = ConvFactoryBN(cd3x3r, num_d3x3, Shape(3, 3), Shape(1, 1), + Shape(1, 1), name + "_double_3x3_0"); + cd3x3 = ConvFactoryBN(cd3x3, num_d3x3, Shape(3, 3), Shape(2, 2), + Shape(1, 1), name + "_double_3x3_1"); + Symbol pooling = Pooling("max_pool_" + name + "_pool", data, + Shape(3, 3), PoolingPoolType::max, + false, false, PoolingPoolingConvention::valid, Shape(2, 2)); + std::vector lst; + lst.push_back(c3x3); + lst.push_back(cd3x3); + lst.push_back(pooling); + return Concat("ch_concat_" + name + "_chconcat", lst, lst.size()); +} + +Symbol InceptionSymbol(int num_classes) { + // data and label + Symbol data = Symbol::Variable("data"); + Symbol data_label = Symbol::Variable("data_label"); + + // stage 1 + Symbol conv1 = ConvFactoryBN(data, 64, Shape(7, 7), Shape(2, 2), Shape(3, 3), "conv1"); + Symbol pool1 = Pooling("pool1", conv1, Shape(3, 3), PoolingPoolType::max, + false, false, PoolingPoolingConvention::valid, Shape(2, 2)); + + // stage 2 + Symbol conv2red = ConvFactoryBN(pool1, 64, Shape(1, 1), Shape(1, 1), Shape(0, 0), "conv2red"); + Symbol conv2 = ConvFactoryBN(conv2red, 192, Shape(3, 3), Shape(1, 1), Shape(1, 1), "conv2"); + Symbol pool2 = Pooling("pool2", conv2, Shape(3, 3), PoolingPoolType::max, + false, false, PoolingPoolingConvention::valid, Shape(2, 2)); + + // stage 3 + Symbol in3a = InceptionFactoryA(pool2, 64, 64, 64, 64, 96, PoolingPoolType::avg, 32, "3a"); + Symbol in3b = InceptionFactoryA(in3a, 64, 64, 96, 64, 96, PoolingPoolType::avg, 64, "3b"); + Symbol in3c = InceptionFactoryB(in3b, 128, 160, 64, 96, "3c"); + + // stage 4 + Symbol in4a = InceptionFactoryA(in3c, 224, 64, 96, 96, 128, PoolingPoolType::avg, 128, "4a"); + Symbol in4b = InceptionFactoryA(in4a, 192, 96, 128, 96, 128, PoolingPoolType::avg, 128, "4b"); + Symbol in4c = InceptionFactoryA(in4b, 160, 128, 160, 128, 160, PoolingPoolType::avg, 128, "4c"); + Symbol in4d = InceptionFactoryA(in4c, 96, 128, 192, 160, 192, PoolingPoolType::avg, 128, "4d"); + Symbol in4e = InceptionFactoryB(in4d, 128, 192, 192, 256, "4e"); + + // stage 5 + Symbol in5a = InceptionFactoryA(in4e, 352, 192, 320, 160, 224, PoolingPoolType::avg, 128, "5a"); + Symbol in5b = InceptionFactoryA(in5a, 352, 192, 320, 192, 224, PoolingPoolType::max, 128, "5b"); + + // average pooling + Symbol avg = Pooling("global_pool", in5b, Shape(7, 7), PoolingPoolType::avg); + + // classifier + Symbol flatten = Flatten("flatten", avg); + Symbol conv1_w("conv1_w"), conv1_b("conv1_b"); + Symbol fc1 = FullyConnected("fc1", flatten, conv1_w, conv1_b, num_classes); + return SoftmaxOutput("softmax", fc1, data_label); +} + +int main(int argc, char const *argv[]) { + int batch_size = 40; + int max_epoch = 100; + float learning_rate = 1e-4; + float weight_decay = 1e-4; + + auto inception_bn_net = InceptionSymbol(10); + std::map args_map; + std::map aux_map; + + args_map["data"] = NDArray(Shape(batch_size, 3, 224, 224), Context::gpu()); + args_map["data_label"] = NDArray(Shape(batch_size), Context::gpu()); + inception_bn_net.InferArgsMap(Context::gpu(), &args_map, args_map); + + auto train_iter = MXDataIter("ImageRecordIter") + .SetParam("path_imglist", "./train.lst") + .SetParam("path_imgrec", "./train.rec") + .SetParam("data_shape", Shape(3, 224, 224)) + .SetParam("batch_size", batch_size) + .SetParam("shuffle", 1) + .CreateDataIter(); + + auto val_iter = MXDataIter("ImageRecordIter") + .SetParam("path_imglist", "./val.lst") + .SetParam("path_imgrec", "./val.rec") + .SetParam("data_shape", Shape(3, 224, 224)) + .SetParam("batch_size", batch_size) + .CreateDataIter(); + + Optimizer* opt = OptimizerRegistry::Find("ccsgd"); + opt->SetParam("momentum", 0.9) + ->SetParam("rescale_grad", 1.0 / batch_size) + ->SetParam("clip_gradient", 10); + + auto *exec = inception_bn_net.SimpleBind(Context::gpu(), args_map); + + for (int iter = 0; iter < max_epoch; ++iter) { + LG << "Epoch: " << iter; + train_iter.Reset(); + while (train_iter.Next()) { + auto data_batch = train_iter.GetDataBatch(); + data_batch.data.CopyTo(&args_map["data"]); + data_batch.label.CopyTo(&args_map["data_label"]); + NDArray::WaitAll(); + + exec->Forward(true); + exec->Backward(); + exec->UpdateAll(opt, learning_rate, weight_decay); + NDArray::WaitAll(); + } + + Accuracy acu; + val_iter.Reset(); + while (val_iter.Next()) { + auto data_batch = val_iter.GetDataBatch(); + data_batch.data.CopyTo(&args_map["data"]); + data_batch.label.CopyTo(&args_map["data_label"]); + NDArray::WaitAll(); + exec->Forward(false); + NDArray::WaitAll(); + acu.Update(data_batch.label, exec->outputs[0]); + } + LG << "Accuracy: " << acu.Get(); + } + delete exec; + MXNotifyShutdown(); + return 0; +} diff --git a/cpp-package/example/lenet.cpp b/cpp-package/example/lenet.cpp new file mode 100644 index 000000000000..d0101a038333 --- /dev/null +++ b/cpp-package/example/lenet.cpp @@ -0,0 +1,236 @@ +/*! + * Copyright (c) 2015 by Contributors + */ +#include +#include +#include +#include +#include +#include "mxnet-cpp/MxNetCpp.h" +// Allow IDE to parse the types +#include "../include/mxnet-cpp/op.h" + +using namespace std; +using namespace mxnet::cpp; + +class Lenet { + public: + Lenet() + : ctx_cpu(Context(DeviceType::kCPU, 0)), + ctx_dev(Context(DeviceType::kGPU, 0)) {} + void Run() { + /* + * LeCun, Yann, Leon Bottou, Yoshua Bengio, and Patrick Haffner. + * "Gradient-based learning applied to document recognition." + * Proceedings of the IEEE (1998) + * */ + + /*define the symbolic net*/ + Symbol data = Symbol::Variable("data"); + Symbol data_label = Symbol::Variable("data_label"); + Symbol conv1_w("conv1_w"), conv1_b("conv1_b"); + Symbol conv2_w("conv2_w"), conv2_b("conv2_b"); + Symbol conv3_w("conv3_w"), conv3_b("conv3_b"); + Symbol fc1_w("fc1_w"), fc1_b("fc1_b"); + Symbol fc2_w("fc2_w"), fc2_b("fc2_b"); + + Symbol conv1 = + Convolution("conv1", data, conv1_w, conv1_b, Shape(5, 5), 20); + Symbol tanh1 = Activation("tanh1", conv1, ActivationActType::tanh); + Symbol pool1 = Pooling("pool1", tanh1, Shape(2, 2), PoolingPoolType::max, + false, false, PoolingPoolingConvention::valid, Shape(2, 2)); + + Symbol conv2 = Convolution("conv2", pool1, conv2_w, conv2_b, + Shape(5, 5), 50); + Symbol tanh2 = Activation("tanh2", conv2, ActivationActType::tanh); + Symbol pool2 = Pooling("pool2", tanh2, Shape(2, 2), PoolingPoolType::max, + false, false, PoolingPoolingConvention::valid, Shape(2, 2)); + + Symbol conv3 = Convolution("conv3", pool2, conv3_w, conv3_b, + Shape(2, 2), 500); + Symbol tanh3 = Activation("tanh3", conv3, ActivationActType::tanh); + Symbol pool3 = Pooling("pool3", tanh3, Shape(2, 2), PoolingPoolType::max, + false, false, PoolingPoolingConvention::valid, Shape(1, 1)); + + Symbol flatten = Flatten("flatten", pool3); + Symbol fc1 = FullyConnected("fc1", flatten, fc1_w, fc1_b, 500); + Symbol tanh4 = Activation("tanh4", fc1, ActivationActType::tanh); + Symbol fc2 = FullyConnected("fc2", tanh4, fc2_w, fc2_b, 10); + + Symbol lenet = SoftmaxOutput("softmax", fc2, data_label); + + for (auto s : lenet.ListArguments()) { + LG << s; + } + + /*setup basic configs*/ + int val_fold = 1; + int W = 28; + int H = 28; + int batch_size = 42; + int max_epoch = 100000; + float learning_rate = 1e-4; + float weight_decay = 1e-4; + + /*prepare the data*/ + vector data_vec, label_vec; + size_t data_count = GetData(&data_vec, &label_vec); + const float *dptr = data_vec.data(); + const float *lptr = label_vec.data(); + NDArray data_array = NDArray(Shape(data_count, 1, W, H), ctx_cpu, + false); // store in main memory, and copy to + // device memory while training + NDArray label_array = + NDArray(Shape(data_count), ctx_cpu, + false); // it's also ok if just store them all in device memory + data_array.SyncCopyFromCPU(dptr, data_count * W * H); + label_array.SyncCopyFromCPU(lptr, data_count); + data_array.WaitToRead(); + label_array.WaitToRead(); + + size_t train_num = data_count * (1 - val_fold / 10.0); + train_data = data_array.Slice(0, train_num); + train_label = label_array.Slice(0, train_num); + val_data = data_array.Slice(train_num, data_count); + val_label = label_array.Slice(train_num, data_count); + + LG << "here read fin"; + + /*init some of the args*/ + // map args_map; + args_map["data"] = data_array.Slice(0, batch_size).Copy(ctx_dev); + args_map["data_label"] = label_array.Slice(0, batch_size).Copy(ctx_dev); + NDArray::WaitAll(); + + LG << "here slice fin"; + /* + * we can also feed in some of the args other than the input all by + * ourselves, + * fc2-w , fc1-b for example: + * */ + // args_map["fc2_w"] = + // NDArray(mshadow::Shape2(500, 4 * 4 * 50), ctx_dev, false); + // NDArray::SampleGaussian(0, 1, &args_map["fc2_w"]); + // args_map["fc1_b"] = NDArray(mshadow::Shape1(10), ctx_dev, false); + // args_map["fc1_b"] = 0; + + lenet.InferArgsMap(ctx_dev, &args_map, args_map); + Optimizer* opt = OptimizerRegistry::Find("ccsgd"); + opt->SetParam("momentum", 0.9) + ->SetParam("rescale_grad", 1.0) + ->SetParam("clip_gradient", 10); + + for (int ITER = 0; ITER < max_epoch; ++ITER) { + size_t start_index = 0; + while (start_index < train_num) { + if (start_index + batch_size > train_num) { + start_index = train_num - batch_size; + } + args_map["data"] = + train_data.Slice(start_index, start_index + batch_size) + .Copy(ctx_dev); + args_map["data_label"] = + train_label.Slice(start_index, start_index + batch_size) + .Copy(ctx_dev); + start_index += batch_size; + NDArray::WaitAll(); + + Executor *exe = lenet.SimpleBind(ctx_dev, args_map); + exe->Forward(true); + exe->Backward(); + exe->UpdateAll(opt, learning_rate, weight_decay); + + delete exe; + } + + LG << "Iter " << ITER + << ", accuracy: " << ValAccuracy(batch_size * 10, lenet); + } + } + + private: + Context ctx_cpu; + Context ctx_dev; + map args_map; + NDArray train_data; + NDArray train_label; + NDArray val_data; + NDArray val_label; + + size_t GetData(vector *data, vector *label) { + const char *train_data_path = "./train.csv"; + ifstream inf(train_data_path); + string line; + inf >> line; // ignore the header + size_t _N = 0; + while (inf >> line) { + for (auto &c : line) c = (c == ',') ? ' ' : c; + stringstream ss; + ss << line; + float _data; + ss >> _data; + label->push_back(_data); + while (ss >> _data) data->push_back(_data / 256.0); + _N++; + } + inf.close(); + return _N; + } + + float ValAccuracy(int batch_size, Symbol lenet) { + size_t val_num = val_data.GetShape()[0]; + + size_t correct_count = 0; + size_t all_count = 0; + + size_t start_index = 0; + while (start_index < val_num) { + if (start_index + batch_size > val_num) { + start_index = val_num - batch_size; + } + args_map["data"] = + val_data.Slice(start_index, start_index + batch_size).Copy(ctx_dev); + args_map["data_label"] = + val_label.Slice(start_index, start_index + batch_size).Copy(ctx_dev); + start_index += batch_size; + NDArray::WaitAll(); + + Executor *exe = lenet.SimpleBind(ctx_dev, args_map); + exe->Forward(false); + + const auto &out = exe->outputs; + NDArray out_cpu = out[0].Copy(ctx_cpu); + NDArray label_cpu = + val_label.Slice(start_index - batch_size, start_index).Copy(ctx_cpu); + + NDArray::WaitAll(); + + const mx_float *dptr_out = out_cpu.GetData(); + const mx_float *dptr_label = label_cpu.GetData(); + for (int i = 0; i < batch_size; ++i) { + float label = dptr_label[i]; + int cat_num = out_cpu.GetShape()[1]; + float p_label = 0, max_p = dptr_out[i * cat_num]; + for (int j = 0; j < cat_num; ++j) { + float p = dptr_out[i * cat_num + j]; + if (max_p < p) { + p_label = j; + max_p = p; + } + } + if (label == p_label) correct_count++; + } + all_count += batch_size; + + delete exe; + } + return correct_count * 1.0 / all_count; + } +}; + +int main(int argc, char const *argv[]) { + Lenet lenet; + lenet.Run(); + MXNotifyShutdown(); + return 0; +} diff --git a/cpp-package/example/lenet_with_mxdataiter.cpp b/cpp-package/example/lenet_with_mxdataiter.cpp new file mode 100644 index 000000000000..889b9f083fe7 --- /dev/null +++ b/cpp-package/example/lenet_with_mxdataiter.cpp @@ -0,0 +1,122 @@ +/*! + * Copyright (c) 2016 by Contributors + */ +#include +#include +#include +#include +#include +#include "mxnet-cpp/MxNetCpp.h" +// Allow IDE to parse the types +#include "../include/mxnet-cpp/op.h" + +using namespace std; +using namespace mxnet::cpp; + +Symbol LenetSymbol() { + /* + * LeCun, Yann, Leon Bottou, Yoshua Bengio, and Patrick Haffner. + * "Gradient-based learning applied to document recognition." + * Proceedings of the IEEE (1998) + * */ + + /*define the symbolic net*/ + Symbol data = Symbol::Variable("data"); + Symbol data_label = Symbol::Variable("data_label"); + Symbol conv1_w("conv1_w"), conv1_b("conv1_b"); + Symbol conv2_w("conv2_w"), conv2_b("conv2_b"); + Symbol conv3_w("conv3_w"), conv3_b("conv3_b"); + Symbol fc1_w("fc1_w"), fc1_b("fc1_b"); + Symbol fc2_w("fc2_w"), fc2_b("fc2_b"); + + Symbol conv1 = Convolution("conv1", data, conv1_w, conv1_b, Shape(5, 5), 20); + Symbol tanh1 = Activation("tanh1", conv1, ActivationActType::tanh); + Symbol pool1 = Pooling("pool1", tanh1, Shape(2, 2), PoolingPoolType::max, + false, false, PoolingPoolingConvention::valid, Shape(2, 2)); + + Symbol conv2 = Convolution("conv2", pool1, conv2_w, conv2_b, Shape(5, 5), 50); + Symbol tanh2 = Activation("tanh2", conv2, ActivationActType::tanh); + Symbol pool2 = Pooling("pool2", tanh2, Shape(2, 2), PoolingPoolType::max, + false, false, PoolingPoolingConvention::valid, Shape(2, 2)); + + Symbol flatten = Flatten("flatten", pool2); + Symbol fc1 = FullyConnected("fc1", flatten, fc1_w, fc1_b, 500); + Symbol tanh3 = Activation("tanh3", fc1, ActivationActType::tanh); + Symbol fc2 = FullyConnected("fc2", tanh3, fc2_w, fc2_b, 10); + + Symbol lenet = SoftmaxOutput("softmax", fc2, data_label); + + return lenet; +} + +int main(int argc, char const *argv[]) { + /*setup basic configs*/ + int W = 28; + int H = 28; + int batch_size = 128; + int max_epoch = 100; + float learning_rate = 1e-4; + float weight_decay = 1e-4; + + auto lenet = LenetSymbol(); + std::map args_map; + + args_map["data"] = NDArray(Shape(batch_size, 1, W, H), Context::gpu()); + args_map["data_label"] = NDArray(Shape(batch_size), Context::gpu()); + lenet.InferArgsMap(Context::gpu(), &args_map, args_map); + + args_map["fc1_w"] = NDArray(Shape(500, 4 * 4 * 50), Context::gpu()); + NDArray::SampleGaussian(0, 1, &args_map["fc1_w"]); + args_map["fc2_b"] = NDArray(Shape(10), Context::gpu()); + args_map["fc2_b"] = 0; + + auto train_iter = MXDataIter("MNISTIter") + .SetParam("image", "./train-images-idx3-ubyte") + .SetParam("label", "./train-labels-idx1-ubyte") + .SetParam("batch_size", batch_size) + .SetParam("shuffle", 1) + .SetParam("flat", 0) + .CreateDataIter(); + auto val_iter = MXDataIter("MNISTIter") + .SetParam("image", "./t10k-images-idx3-ubyte") + .SetParam("label", "./t10k-labels-idx1-ubyte") + .CreateDataIter(); + + Optimizer* opt = OptimizerRegistry::Find("ccsgd"); + opt->SetParam("momentum", 0.9) + ->SetParam("rescale_grad", 1.0) + ->SetParam("clip_gradient", 10); + + for (int iter = 0; iter < max_epoch; ++iter) { + LG << "Epoch: " << iter; + train_iter.Reset(); + while (train_iter.Next()) { + auto data_batch = train_iter.GetDataBatch(); + args_map["data"] = data_batch.data.Copy(Context::gpu()); + args_map["data_label"] = data_batch.label.Copy(Context::gpu()); + NDArray::WaitAll(); + auto *exec = lenet.SimpleBind(Context::gpu(), args_map); + exec->Forward(true); + exec->Backward(); + exec->UpdateAll(opt, learning_rate, weight_decay); + delete exec; + } + + Accuracy acu; + val_iter.Reset(); + while (val_iter.Next()) { + auto data_batch = val_iter.GetDataBatch(); + args_map["data"] = data_batch.data.Copy(Context::gpu()); + args_map["data_label"] = data_batch.label.Copy(Context::gpu()); + NDArray::WaitAll(); + auto *exec = lenet.SimpleBind(Context::gpu(), args_map); + exec->Forward(false); + NDArray::WaitAll(); + acu.Update(data_batch.label, exec->outputs[0]); + delete exec; + } + LG << "Accuracy: " << acu.Get(); + } + MXNotifyShutdown(); + return 0; +} diff --git a/cpp-package/example/mlp.cpp b/cpp-package/example/mlp.cpp new file mode 100644 index 000000000000..347a196835d2 --- /dev/null +++ b/cpp-package/example/mlp.cpp @@ -0,0 +1,164 @@ +/*! + * Copyright (c) 2015 by Contributors + */ + +#include +#include +#include +#include "mxnet-cpp/MxNetCpp.h" +// Allow IDE to parse the types +#include "../include/mxnet-cpp/op.h" + +using namespace std; +using namespace mxnet::cpp; + +/* + * In this example, + * we make by hand some data in 10 classes with some pattern + * and try to use MLP to recognize the pattern. + */ + +void OutputAccuracy(mx_float* pred, mx_float* target) { + int right = 0; + for (int i = 0; i < 128; ++i) { + float mx_p = pred[i * 10 + 0]; + float p_y = 0; + for (int j = 0; j < 10; ++j) { + if (pred[i * 10 + j] > mx_p) { + mx_p = pred[i * 10 + j]; + p_y = j; + } + } + if (p_y == target[i]) right++; + } + cout << "Accuracy: " << right / 128.0 << endl; +} + +void MLP() { + auto sym_x = Symbol::Variable("X"); + auto sym_label = Symbol::Variable("label"); + + const int nLayers = 2; + vector layerSizes({512, 10}); + vector weights(nLayers); + vector biases(nLayers); + vector outputs(nLayers); + + for (int i = 0; i < nLayers; i++) { + string istr = to_string(i); + weights[i] = Symbol::Variable(string("w") + istr); + biases[i] = Symbol::Variable(string("b") + istr); + Symbol fc = FullyConnected(string("fc") + istr, + i == 0? sym_x : outputs[i-1], + weights[i], biases[i], layerSizes[i]); + outputs[i] = LeakyReLU(string("act") + istr, fc, LeakyReLUActType::leaky); + } + auto sym_out = SoftmaxOutput("softmax", outputs[nLayers - 1], sym_label); + + Context ctx_dev(DeviceType::kCPU, 0); + + NDArray array_x(Shape(128, 28), ctx_dev, false); + NDArray array_y(Shape(128), ctx_dev, false); + + mx_float* aptr_x = new mx_float[128 * 28]; + mx_float* aptr_y = new mx_float[128]; + + // we make the data by hand, in 10 classes, with some pattern + for (int i = 0; i < 128; i++) { + for (int j = 0; j < 28; j++) { + aptr_x[i * 28 + j] = i % 10 * 1.0f; + } + aptr_y[i] = i % 10; + } + array_x.SyncCopyFromCPU(aptr_x, 128 * 28); + array_x.WaitToRead(); + array_y.SyncCopyFromCPU(aptr_y, 128); + array_y.WaitToRead(); + + // init the parameters + NDArray array_w_1(Shape(512, 28), ctx_dev, false); + NDArray array_b_1(Shape(512), ctx_dev, false); + NDArray array_w_2(Shape(10, 512), ctx_dev, false); + NDArray array_b_2(Shape(10), ctx_dev, false); + + // the parameters should be initialized in some kind of distribution, + // so it learns fast + // but here just give a const value by hand + array_w_1 = 0.5f; + array_b_1 = 0.0f; + array_w_2 = 0.5f; + array_b_2 = 0.0f; + + // the grads + NDArray array_w_1_g(Shape(512, 28), ctx_dev, false); + NDArray array_b_1_g(Shape(512), ctx_dev, false); + NDArray array_w_2_g(Shape(10, 512), ctx_dev, false); + NDArray array_b_2_g(Shape(10), ctx_dev, false); + + // Bind the symolic network with the ndarray + // all the input args + std::vector in_args; + in_args.push_back(array_x); + in_args.push_back(array_w_1); + in_args.push_back(array_b_1); + in_args.push_back(array_w_2); + in_args.push_back(array_b_2); + in_args.push_back(array_y); + // all the grads + std::vector arg_grad_store; + arg_grad_store.push_back(NDArray()); // we don't need the grad of the input + arg_grad_store.push_back(array_w_1_g); + arg_grad_store.push_back(array_b_1_g); + arg_grad_store.push_back(array_w_2_g); + arg_grad_store.push_back(array_b_2_g); + arg_grad_store.push_back( + NDArray()); // neither do we need the grad of the loss + // how to handle the grad + std::vector grad_req_type; + grad_req_type.push_back(kNullOp); + grad_req_type.push_back(kWriteTo); + grad_req_type.push_back(kWriteTo); + grad_req_type.push_back(kWriteTo); + grad_req_type.push_back(kWriteTo); + grad_req_type.push_back(kNullOp); + std::vector aux_states; + + cout << "make the Executor" << endl; + Executor* exe = new Executor(sym_out, ctx_dev, in_args, arg_grad_store, + grad_req_type, aux_states); + + cout << "Training" << endl; + int max_iters = 20000; + mx_float learning_rate = 0.0001; + for (int iter = 0; iter < max_iters; ++iter) { + exe->Forward(true); + + if (iter % 100 == 0) { + cout << "epoch " << iter << endl; + std::vector& out = exe->outputs; + float* cptr = new float[128 * 10]; + out[0].SyncCopyToCPU(cptr, 128 * 10); + NDArray::WaitAll(); + OutputAccuracy(cptr, aptr_y); + delete[] cptr; + } + + // update the parameters + exe->Backward(); + for (int i = 1; i < 5; ++i) { + in_args[i] -= arg_grad_store[i] * learning_rate; + } + NDArray::WaitAll(); + } + + delete exe; + delete[] aptr_x; + delete[] aptr_y; +} + +int main(int argc, char** argv) { + MLP(); + MXNotifyShutdown(); + return 0; +} + diff --git a/cpp-package/example/resnet.cpp b/cpp-package/example/resnet.cpp new file mode 100644 index 000000000000..5d3131223ef3 --- /dev/null +++ b/cpp-package/example/resnet.cpp @@ -0,0 +1,196 @@ +/*! + * Copyright (c) 2016 by Contributors + */ +#include +#include +#include +#include +#include "mxnet-cpp/MxNetCpp.h" +// Allow IDE to parse the types +#include "../include/mxnet-cpp/op.h" + +using namespace mxnet::cpp; + +Symbol ConvolutionNoBias(const std::string& symbol_name, + Symbol data, + Symbol weight, + Shape kernel, + int num_filter, + Shape stride = Shape(1, 1), + Shape dilate = Shape(1, 1), + Shape pad = Shape(0, 0), + int num_group = 1, + int64_t workspace = 512) { + return Operator("Convolution") + .SetParam("kernel", kernel) + .SetParam("num_filter", num_filter) + .SetParam("stride", stride) + .SetParam("dilate", dilate) + .SetParam("pad", pad) + .SetParam("num_group", num_group) + .SetParam("workspace", workspace) + .SetParam("no_bias", true) + .SetInput("data", data) + .SetInput("weight", weight) + .CreateSymbol(symbol_name); +} + +static const Symbol BN_BETA; +static const Symbol BN_GAMMA; + +Symbol getConv(const std::string & name, Symbol data, + int num_filter, + Shape kernel, Shape stride, Shape pad, + bool with_relu, + mx_float bn_momentum) { + Symbol conv_w(name + "_w"); + Symbol conv = ConvolutionNoBias(name, data, conv_w, + kernel, num_filter, stride, Shape(1, 1), + pad, 1, 512); + + Symbol bn = BatchNorm(name + "_bn", conv, BN_GAMMA, BN_BETA, 2e-5, bn_momentum, false); + + if (with_relu) { + return Activation(name + "_relu", bn, "relu"); + } else { + return bn; + } +} + +Symbol makeBlock(const std::string & name, Symbol data, int num_filter, + bool dim_match, mx_float bn_momentum) { + Shape stride; + if (dim_match) { + stride = Shape(1, 1); + } else { + stride = Shape(2, 2); + } + + Symbol conv1 = getConv(name + "_conv1", data, num_filter, + Shape(3, 3), stride, Shape(1, 1), + true, bn_momentum); + + Symbol conv2 = getConv(name + "_conv2", conv1, num_filter, + Shape(3, 3), Shape(1, 1), Shape(1, 1), + false, bn_momentum); + + Symbol shortcut; + + if (dim_match) { + shortcut = data; + } else { + Symbol shortcut_w(name + "_proj_w"); + shortcut = ConvolutionNoBias(name + "_proj", data, shortcut_w, + Shape(2, 2), num_filter, + Shape(2, 2), Shape(1, 1), Shape(0, 0), + 1, 512); + } + + Symbol fused = shortcut + conv2; + return Activation(name + "_relu", fused, "relu"); +} + +Symbol getBody(Symbol data, int num_level, int num_block, int num_filter, mx_float bn_momentum) { + for (int level = 0; level < num_level; level++) { + for (int block = 0; block < num_block; block++) { + data = makeBlock("level" + std::to_string(level + 1) + "_block" + std::to_string(block + 1), + data, num_filter * (std::pow(2, level)), + (level == 0 || block > 0), bn_momentum); + } + } + return data; +} + +Symbol ResNetSymbol(int num_class, int num_level = 3, int num_block = 9, + int num_filter = 16, mx_float bn_momentum = 0.9, + mxnet::cpp::Shape pool_kernel = mxnet::cpp::Shape(8, 8)) { + // data and label + Symbol data = Symbol::Variable("data"); + Symbol data_label = Symbol::Variable("data_label"); + + Symbol zscore = BatchNorm("zscore", data, BN_GAMMA, BN_BETA, 0.001, bn_momentum); + + Symbol conv = getConv("conv0", zscore, num_filter, + Shape(3, 3), Shape(1, 1), Shape(1, 1), + true, bn_momentum); + + Symbol body = getBody(conv, num_level, num_block, num_filter, bn_momentum); + + Symbol pool = Pooling("pool", body, pool_kernel, PoolingPoolType::avg); + + Symbol flat = Flatten("flatten", pool); + + Symbol fc_w("fc_w"), fc_b("fc_b"); + Symbol fc = FullyConnected("fc", flat, fc_w, fc_b, num_class); + + return SoftmaxOutput("softmax", fc, data_label); +} + +int main(int argc, char const *argv[]) { + int batch_size = 50; + int max_epoch = 100; + float learning_rate = 1e-4; + float weight_decay = 1e-4; + + auto resnet = ResNetSymbol(10); + std::map args_map; + std::map aux_map; + + args_map["data"] = NDArray(Shape(batch_size, 3, 256, 256), Context::gpu()); + args_map["data_label"] = NDArray(Shape(batch_size), Context::gpu()); + resnet.InferArgsMap(Context::gpu(), &args_map, args_map); + + auto train_iter = MXDataIter("ImageRecordIter") + .SetParam("path_imglist", "./sf1_train.lst") + .SetParam("path_imgrec", "./sf1_train.rec") + .SetParam("data_shape", Shape(3, 256, 256)) + .SetParam("batch_size", batch_size) + .SetParam("shuffle", 1) + .CreateDataIter(); + + auto val_iter = MXDataIter("ImageRecordIter") + .SetParam("path_imglist", "./sf1_val.lst") + .SetParam("path_imgrec", "./sf1_val.rec") + .SetParam("data_shape", Shape(3, 256, 256)) + .SetParam("batch_size", batch_size) + .CreateDataIter(); + + Optimizer* opt = OptimizerRegistry::Find("ccsgd"); + opt->SetParam("momentum", 0.9) + ->SetParam("rescale_grad", 1.0 / batch_size) + ->SetParam("clip_gradient", 10); + + auto *exec = resnet.SimpleBind(Context::gpu(), args_map); + + for (int iter = 0; iter < max_epoch; ++iter) { + LG << "Epoch: " << iter; + train_iter.Reset(); + while (train_iter.Next()) { + auto data_batch = train_iter.GetDataBatch(); + data_batch.data.CopyTo(&args_map["data"]); + data_batch.label.CopyTo(&args_map["data_label"]); + NDArray::WaitAll(); + + exec->Forward(true); + exec->Backward(); + exec->UpdateAll(opt, learning_rate, weight_decay); + NDArray::WaitAll(); + } + + Accuracy acu; + val_iter.Reset(); + while (val_iter.Next()) { + auto data_batch = val_iter.GetDataBatch(); + data_batch.data.CopyTo(&args_map["data"]); + data_batch.label.CopyTo(&args_map["data_label"]); + NDArray::WaitAll(); + exec->Forward(false); + NDArray::WaitAll(); + acu.Update(data_batch.label, exec->outputs[0]); + } + LG << "Accuracy: " << acu.Get(); + } + delete exec; + MXNotifyShutdown(); + return 0; +} diff --git a/cpp-package/example/run_lenet_with_mxdataiter.sh b/cpp-package/example/run_lenet_with_mxdataiter.sh new file mode 100755 index 000000000000..fffc355865bc --- /dev/null +++ b/cpp-package/example/run_lenet_with_mxdataiter.sh @@ -0,0 +1,6 @@ +if [ ! -f "./mnist.zip" ]; then + wget http://webdocs.cs.ualberta.ca/~bx3/data/mnist.zip + unzip -u mnist.zip +fi +make lenet_with_mxdataiter +LD_LIBRARY_PATH=../lib/linux ./lenet_with_mxdataiter diff --git a/cpp-package/include/mxnet-cpp/.gitignore b/cpp-package/include/mxnet-cpp/.gitignore new file mode 100644 index 000000000000..995efdd6f07b --- /dev/null +++ b/cpp-package/include/mxnet-cpp/.gitignore @@ -0,0 +1,2 @@ +# Rebuildable file(s) +op.h diff --git a/cpp-package/include/mxnet-cpp/CPPLINT.cfg b/cpp-package/include/mxnet-cpp/CPPLINT.cfg new file mode 100644 index 000000000000..2f2b772b465b --- /dev/null +++ b/cpp-package/include/mxnet-cpp/CPPLINT.cfg @@ -0,0 +1,2 @@ +filter=-runtime/references +exclude_files=op.h diff --git a/cpp-package/include/mxnet-cpp/MxNetCpp.h b/cpp-package/include/mxnet-cpp/MxNetCpp.h new file mode 100644 index 000000000000..dc5d7750a70d --- /dev/null +++ b/cpp-package/include/mxnet-cpp/MxNetCpp.h @@ -0,0 +1,23 @@ +/*! + * Copyright (c) 2016 by Contributors + * \file MxNetCpp.h + * \brief meta include file for mxnet.cpp + * \author Chuntao Hong, Zhang Chen + */ + +#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_MXNETCPP_H_ +#define CPP_PACKAGE_INCLUDE_MXNET_CPP_MXNETCPP_H_ + +#include "mxnet-cpp/executor.hpp" +#include "mxnet-cpp/symbol.hpp" +#include "mxnet-cpp/ndarray.hpp" +#include "mxnet-cpp/operator.hpp" +#include "mxnet-cpp/optimizer.hpp" +#include "mxnet-cpp/kvstore.hpp" +#include "mxnet-cpp/op.h" +#include "mxnet-cpp/op_suppl.h" +#include "mxnet-cpp/io.hpp" +#include "mxnet-cpp/metric.h" +#include "mxnet-cpp/initializer.h" + +#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_MXNETCPP_H_ diff --git a/cpp-package/include/mxnet-cpp/base.h b/cpp-package/include/mxnet-cpp/base.h new file mode 100644 index 000000000000..18f268a8a85a --- /dev/null +++ b/cpp-package/include/mxnet-cpp/base.h @@ -0,0 +1,38 @@ +/*! +* Copyright (c) 2016 by Contributors +* \file base.h +* \brief base definitions for mxnetcpp +* \author Chuntao Hong, Zhang Chen +*/ + +#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_BASE_H_ +#define CPP_PACKAGE_INCLUDE_MXNET_CPP_BASE_H_ + +#include +#include "mxnet/c_api.h" +#include "nnvm/c_api.h" + +namespace mxnet { +namespace cpp { + +typedef unsigned index_t; + +enum OpReqType { + /*! \brief no operation, do not write anything */ + kNullOp, + /*! \brief write gradient to provided space */ + kWriteTo, + /*! + * \brief perform an inplace write, + * Target shares memory with one of input arguments. + * This option only happen when + */ + kWriteInplace, + /*! \brief add to the provided space */ + kAddTo +}; + +} // namespace cpp +} // namespace mxnet + +#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_BASE_H_ diff --git a/cpp-package/include/mxnet-cpp/executor.h b/cpp-package/include/mxnet-cpp/executor.h new file mode 100644 index 000000000000..9b358f15fde7 --- /dev/null +++ b/cpp-package/include/mxnet-cpp/executor.h @@ -0,0 +1,137 @@ +/*! +* Copyright (c) 2016 by Contributors +* \file executor.h +* \brief executor definition +* \author Chuntao Hong, Zhang Chen +*/ + +#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_EXECUTOR_H_ +#define CPP_PACKAGE_INCLUDE_MXNET_CPP_EXECUTOR_H_ + +#include +#include +#include +#include +#include "mxnet-cpp/base.h" +#include "mxnet-cpp/symbol.h" + +namespace mxnet { +namespace cpp { + +class Optimizer; + +/*! +* \brief Executor interface +*/ +class Executor { + public: + Executor(const Symbol &symbol, Context context, + const std::vector &arg_arrays, + const std::vector &grad_arrays, + const std::vector &grad_reqs, + const std::vector &aux_arrays, + const std::map &group_to_ctx = + std::map(), + Executor *shared_exec = nullptr); + explicit Executor(const ExecutorHandle &h) { handle_ = h; } + /*! + * \brief Perform a Forward operation of Operator + * After this operation, user can get the result by using function head. + */ + void Forward(bool is_train) { + MXExecutorForward(handle_, is_train ? 1 : 0); + mx_uint out_size; + NDArrayHandle *out_array; + CHECK_EQ(MXExecutorOutputs(handle_, &out_size, &out_array), 0); + for (mx_uint i = 0; i < out_size; ++i) { + outputs[i] = NDArray(out_array[i]); + } + } + /*! + * \brief Perform a Backward operation of the Operator. + * This must be called after Forward. + * After this operation, NDArrays specified by grad_in_args_store will be + *updated accordingly. + * User is allowed to pass in an empty Array if the head node is + * loss function and head gradeitn is not needed. + * + * \param head_grads the gradient of head nodes to be backproped. + */ + void Backward(const std::vector &head_grads = + std::vector()) { + std::vector head_grads_; + for (auto d : head_grads) { + head_grads_.push_back(d.GetHandle()); + } + if (head_grads_.size() > 0) { + MXExecutorBackward(handle_, head_grads_.size(), head_grads_.data()); + } else { + MXExecutorBackward(handle_, 0, nullptr); + } + } + // TODO(zhangchen-qinyinghua) + // To implement reshape function + void Reshape(); + /*! + * \brief update the arguments with given learning rate and optimizer + * \return the SymbolHandle + */ + std::string DebugStr(); + /*! + * \brief update the arguments with given learning rate and optimizer + * \param opt the pointer to the optimizer + * \param lr learning rate + * \param wd weight decay + * \param arg_update_begin begin index of the arguments to be updated, it + * starts after the input data by default + * \param arg_update_end end index of the arguments to be updated, it ends + * before the label data by default + */ + void UpdateAll(Optimizer *opt, float lr, float wd, int arg_update_begin = 1, + int arg_update_end = -1); + /*! + * \brief destructor, free the handle + */ + ~Executor() { MXExecutorFree(handle_); } + std::vector arg_arrays; + std::vector grad_arrays; + std::vector aux_arrays; + /*! + * \brief arrays store the outputs of forward + */ + std::vector outputs; + std::map arg_dict() { + return GetDict(symbol_.ListArguments(), arg_arrays); + } + std::map grad_dict() { + return GetDict(symbol_.ListArguments(), grad_arrays); + } + std::map aux_dict() { + return GetDict(symbol_.ListAuxiliaryStates(), aux_arrays); + } + + private: + Executor(const Executor &e); + Executor &operator=(const Executor &e); + ExecutorHandle handle_; + Symbol symbol_; + std::map GetDict(const std::vector &names, + const std::vector &arrays) { + std::map ret; + std::set name_set; + for (const auto &s : names) { + CHECK(name_set.find(s) == name_set.end()) << "Duplicate names detected, " + << s; + name_set.insert(s); + } + CHECK_EQ(name_set.size(), arrays.size()) + << "names size not equal to arrays size"; + for (size_t i = 0; i < names.size(); ++i) { + ret[names[i]] = arrays[i]; + } + return ret; + } +}; +} // namespace cpp +} // namespace mxnet +#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_EXECUTOR_H_ diff --git a/cpp-package/include/mxnet-cpp/executor.hpp b/cpp-package/include/mxnet-cpp/executor.hpp new file mode 100644 index 000000000000..4cae684f8881 --- /dev/null +++ b/cpp-package/include/mxnet-cpp/executor.hpp @@ -0,0 +1,92 @@ +/*! + * Copyright (c) 2016 by Contributors + * \file executor.hpp + * \brief implementation of the executor + * \author Zhang Chen, Chuntao Hong + */ + +#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_EXECUTOR_HPP_ +#define CPP_PACKAGE_INCLUDE_MXNET_CPP_EXECUTOR_HPP_ + +#include +#include +#include +#include "mxnet-cpp/executor.h" +#include "mxnet-cpp/optimizer.h" + +namespace mxnet { +namespace cpp { +inline Executor::Executor(const Symbol &symbol, Context context, + const std::vector &arg_arrays, + const std::vector &grad_arrays, + const std::vector &grad_reqs, + const std::vector &aux_arrays, + const std::map &group_to_ctx, + Executor *shared_exec) { + this->arg_arrays = arg_arrays; + this->grad_arrays = grad_arrays; + this->aux_arrays = aux_arrays; + this->symbol_ = symbol; + + std::vector arg_handles; + std::vector grad_handles; + std::vector aux_handles; + + for (const auto &array : arg_arrays) { + arg_handles.push_back(array.GetHandle()); + } + for (const auto &array : grad_arrays) { + grad_handles.push_back(array.GetHandle()); + } + for (const auto &array : aux_arrays) { + aux_handles.push_back(array.GetHandle()); + } + + std::vector grad_reqs_uint; + for (auto s : grad_reqs) grad_reqs_uint.push_back(s); + + std::vector map_keys; + std::vector dev_types, dev_ids; + for (const auto &s : group_to_ctx) { + map_keys.push_back(s.first.c_str()); + dev_types.push_back(s.second.GetDeviceType()); + dev_ids.push_back(s.second.GetDeviceId()); + } + + ExecutorHandle *shared_exec_handle = + shared_exec == nullptr ? nullptr : &shared_exec->handle_; + + CHECK_EQ(MXExecutorBindEX(symbol.GetHandle(), context.GetDeviceType(), + context.GetDeviceId(), group_to_ctx.size(), + map_keys.data(), dev_types.data(), dev_ids.data(), + arg_handles.size(), arg_handles.data(), + grad_handles.data(), grad_reqs_uint.data(), + aux_handles.size(), aux_handles.data(), + shared_exec_handle, &handle_), + 0); + + mx_uint out_size; + NDArrayHandle *out_array; + CHECK_EQ(MXExecutorOutputs(handle_, &out_size, &out_array), 0); + for (mx_uint i = 0; i < out_size; ++i) { + outputs.push_back(NDArray(out_array[i])); + } +} + +inline std::string Executor::DebugStr() { + const char *output; + MXExecutorPrint(handle_, &output); + return std::string(output); +} + +inline void Executor::UpdateAll(Optimizer *opt, float lr, float wd, + int arg_update_begin, int arg_update_end) { + arg_update_end = arg_update_end < 0 ? arg_arrays.size() - 1 : arg_update_end; + for (int i = arg_update_begin; i < arg_update_end; ++i) { + opt->Update(i, arg_arrays[i], grad_arrays[i], lr, wd); + } +} +} // namespace cpp +} // namespace mxnet + +#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_EXECUTOR_HPP_ diff --git a/cpp-package/include/mxnet-cpp/initializer.h b/cpp-package/include/mxnet-cpp/initializer.h new file mode 100644 index 000000000000..cdcc1a8a8fc6 --- /dev/null +++ b/cpp-package/include/mxnet-cpp/initializer.h @@ -0,0 +1,130 @@ +/*! + * Copyright (c) 2016 by Contributors + * \file initializer.h + * \brief random initializer + * \author Zhang Chen + */ + +#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_INITIALIZER_H_ +#define CPP_PACKAGE_INCLUDE_MXNET_CPP_INITIALIZER_H_ + +#include +#include +#include +#include "mxnet-cpp/ndarray.h" + +namespace mxnet { +namespace cpp { + +class Initializer { + public: + static bool StringStartWith(const std::string& name, + const std::string& check_str) { + return (name.size() >= check_str.size() && + name.substr(0, check_str.size()) == check_str); + } + static bool StringEndWith(const std::string& name, + const std::string& check_str) { + return (name.size() >= check_str.size() && + name.substr(name.size() - check_str.size(), check_str.size()) == + check_str); + } + virtual void operator()(const std::string& name, NDArray* arr) { + if (StringStartWith(name, "upsampling")) { + InitBilinear(arr); + } else if (StringEndWith(name, "bias")) { + InitBias(arr); + } else if (StringEndWith(name, "gamma")) { + InitGamma(arr); + } else if (StringEndWith(name, "beta")) { + InitBeta(arr); + } else if (StringEndWith(name, "weight")) { + InitWeight(arr); + } else if (StringEndWith(name, "moving_mean")) { + InitZero(arr); + } else if (StringEndWith(name, "moving_var")) { + InitOne(arr); + } else if (StringEndWith(name, "moving_inv_var")) { + InitZero(arr); + } else if (StringEndWith(name, "moving_avg")) { + InitZero(arr); + } else { + InitDefault(arr); + } + } + + protected: + virtual void InitBilinear(NDArray* arr) { + Shape shape(arr->GetShape()); + std::vector weight(shape.Size(), 0); + int f = std::ceil(shape[3] / 2.0); + float c = (2 * f - 1 - f % 2) / (2. * f); + for (size_t i = 0; i < shape.Size(); ++i) { + int x = i % shape[3]; + int y = (i / shape[3]) % shape[2]; + weight[i] = (1 - std::abs(x / f - c)) * (1 - std::abs(y / f - c)); + } + (*arr).SyncCopyFromCPU(weight); + } + virtual void InitZero(NDArray* arr) { (*arr) = 0.0f; } + virtual void InitOne(NDArray* arr) { (*arr) = 1.0f; } + virtual void InitBias(NDArray* arr) { (*arr) = 0.0f; } + virtual void InitGamma(NDArray* arr) { (*arr) = 1.0f; } + virtual void InitBeta(NDArray* arr) { (*arr) = 0.0f; } + virtual void InitWeight(NDArray* arr) {} + virtual void InitDefault(NDArray* arr) {} +}; + +class Xavier : public Initializer { + public: + enum RandType { + gaussian, + uniform + } rand_type; + enum FactorType { + avg, + in, + out + } factor_type; + float magnitude; + Xavier(RandType rand_type = gaussian, FactorType factor_type = avg, + float magnitude = 3) + : rand_type(rand_type), factor_type(factor_type), magnitude(magnitude) {} + + protected: + virtual void InitWeight(NDArray* arr) { + Shape shape(arr->GetShape()); + float hw_scale = 1.0f; + if (shape.ndim() > 2) { + for (size_t i = 2; i < shape.ndim(); ++i) { + hw_scale *= shape[i]; + } + } + float fan_in = shape[1] * hw_scale, fan_out = shape[0] * hw_scale; + float factor = 1.0f; + switch (factor_type) { + case avg: + factor = (fan_in + fan_out) / 2.0; + break; + case in: + factor = fan_in; + break; + case out: + factor = fan_out; + } + float scale = std::sqrt(magnitude / factor); + switch (rand_type) { + case uniform: + NDArray::SampleUniform(-scale, scale, arr); + break; + case gaussian: + NDArray::SampleGaussian(0, scale, arr); + break; + } + } +}; + +} // namespace cpp +} // namespace mxnet + +#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_INITIALIZER_H_ diff --git a/cpp-package/include/mxnet-cpp/io.h b/cpp-package/include/mxnet-cpp/io.h new file mode 100644 index 000000000000..171803831109 --- /dev/null +++ b/cpp-package/include/mxnet-cpp/io.h @@ -0,0 +1,128 @@ +/*! +* Copyright (c) 2016 by Contributors +* \file operator.h +* \brief definition of io, such as DataIter +* \author Zhang Chen +*/ +#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_IO_H_ +#define CPP_PACKAGE_INCLUDE_MXNET_CPP_IO_H_ + +#include +#include +#include +#include +#include "mxnet-cpp/base.h" +#include "mxnet-cpp/ndarray.h" +#include "dmlc/logging.h" + +namespace mxnet { +namespace cpp { +/*! +* \brief Default object for holding a mini-batch of data and related +* information. +*/ +class DataBatch { + public: + NDArray data; + NDArray label; + int pad_num; + std::vector index; +}; +class DataIter { + public: + virtual void BeforeFirst(void) = 0; + virtual bool Next(void) = 0; + virtual NDArray GetData(void) = 0; + virtual NDArray GetLabel(void) = 0; + virtual int GetPadNum(void) = 0; + virtual std::vector GetIndex(void) = 0; + + DataBatch GetDataBatch() { + return DataBatch{GetData(), GetLabel(), GetPadNum(), GetIndex()}; + } + void Reset() { BeforeFirst(); } +}; + +class MXDataIterMap { + public: + inline MXDataIterMap() { + mx_uint num_data_iter_creators = 0; + DataIterCreator *data_iter_creators = nullptr; + int r = MXListDataIters(&num_data_iter_creators, &data_iter_creators); + CHECK_EQ(r, 0); + for (mx_uint i = 0; i < num_data_iter_creators; i++) { + const char *name; + const char *description; + mx_uint num_args; + const char **arg_names; + const char **arg_type_infos; + const char **arg_descriptions; + r = MXDataIterGetIterInfo(data_iter_creators[i], &name, &description, + &num_args, &arg_names, &arg_type_infos, + &arg_descriptions); + CHECK_EQ(r, 0); + mxdataiter_creators_[name] = data_iter_creators[i]; + } + } + inline DataIterCreator GetMXDataIterCreator(const std::string &name) { + return mxdataiter_creators_[name]; + } + + private: + std::map mxdataiter_creators_; +}; + +struct MXDataIterBlob { + public: + MXDataIterBlob() : handle_(nullptr) {} + explicit MXDataIterBlob(DataIterHandle handle) : handle_(handle) {} + ~MXDataIterBlob() { MXDataIterFree(handle_); } + DataIterHandle handle_; + + private: + MXDataIterBlob &operator=(const MXDataIterBlob &); +}; + +class MXDataIter : public DataIter { + public: + explicit MXDataIter(const std::string &mxdataiter_type); + MXDataIter(const MXDataIter &other) { + creator_ = other.creator_; + params_ = other.params_; + blob_ptr_ = other.blob_ptr_; + } + void BeforeFirst(); + bool Next(); + NDArray GetData(); + NDArray GetLabel(); + int GetPadNum(); + std::vector GetIndex(); + MXDataIter CreateDataIter(); + /*! + * \brief set config parameters + * \param name name of the config parameter + * \param value value of the config parameter + * \return reference of self + */ + template + MXDataIter &SetParam(const std::string &name, const T &value) { + std::string value_str; + std::stringstream ss; + ss << value; + ss >> value_str; + + params_[name] = value_str; + return *this; + } + + private: + DataIterCreator creator_; + std::map params_; + std::shared_ptr blob_ptr_; + static MXDataIterMap*& mxdataiter_map(); +}; +} // namespace cpp +} // namespace mxnet + +#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_IO_H_ + diff --git a/cpp-package/include/mxnet-cpp/io.hpp b/cpp-package/include/mxnet-cpp/io.hpp new file mode 100644 index 000000000000..61e575e949a9 --- /dev/null +++ b/cpp-package/include/mxnet-cpp/io.hpp @@ -0,0 +1,90 @@ +/*! +* Copyright (c) 2016 by Contributors +* \file operator.hpp +* \brief implementation of data iter +* \author Zhang Chen +*/ +#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_IO_HPP_ +#define CPP_PACKAGE_INCLUDE_MXNET_CPP_IO_HPP_ + +#include +#include +#include "mxnet-cpp/io.h" + +namespace mxnet { +namespace cpp { + +inline MXDataIterMap*& MXDataIter::mxdataiter_map() { + static MXDataIterMap* mxdataiter_map_ = new MXDataIterMap; + return mxdataiter_map_; +} + +inline MXDataIter::MXDataIter(const std::string &mxdataiter_type) { + creator_ = mxdataiter_map()->GetMXDataIterCreator(mxdataiter_type); + blob_ptr_ = std::make_shared(nullptr); +} + +inline void MXDataIter::BeforeFirst() { + int r = MXDataIterBeforeFirst(blob_ptr_->handle_); + CHECK_EQ(r, 0); +} + +inline bool MXDataIter::Next() { + int out; + int r = MXDataIterNext(blob_ptr_->handle_, &out); + CHECK_EQ(r, 0); + return out; +} + +inline NDArray MXDataIter::GetData() { + NDArrayHandle handle; + int r = MXDataIterGetData(blob_ptr_->handle_, &handle); + CHECK_EQ(r, 0); + return NDArray(handle); +} + +inline NDArray MXDataIter::GetLabel() { + NDArrayHandle handle; + int r = MXDataIterGetLabel(blob_ptr_->handle_, &handle); + CHECK_EQ(r, 0); + return NDArray(handle); +} + +inline int MXDataIter::GetPadNum() { + int out; + int r = MXDataIterGetPadNum(blob_ptr_->handle_, &out); + CHECK_EQ(r, 0); + return out; +} +inline std::vector MXDataIter::GetIndex() { + uint64_t *out_index, out_size; + int r = MXDataIterGetIndex(blob_ptr_->handle_, &out_index, &out_size); + CHECK_EQ(r, 0); + std::vector ret; + for (uint64_t i = 0; i < out_size; ++i) { + ret.push_back(out_index[i]); + } + return ret; +} + +inline MXDataIter MXDataIter::CreateDataIter() { + std::vector param_keys; + std::vector param_values; + + for (auto &data : params_) { + param_keys.push_back(data.first.c_str()); + param_values.push_back(data.second.c_str()); + } + + MXDataIterCreateIter(creator_, param_keys.size(), param_keys.data(), + param_values.data(), &blob_ptr_->handle_); + return *this; +} + +// MXDataIter MNIst + +} // namespace cpp +} // namespace mxnet + +#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_IO_HPP_ + diff --git a/cpp-package/include/mxnet-cpp/kvstore.h b/cpp-package/include/mxnet-cpp/kvstore.h new file mode 100644 index 000000000000..6d3987ecf030 --- /dev/null +++ b/cpp-package/include/mxnet-cpp/kvstore.h @@ -0,0 +1,49 @@ +/*! +* Copyright (c) 2016 by Contributors +* \file kvstore.h +* \brief definition of kvstore +* \author Chuntao Hong +*/ + +#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_KVSTORE_H_ +#define CPP_PACKAGE_INCLUDE_MXNET_CPP_KVSTORE_H_ + +#include +#include +#include "mxnet-cpp/ndarray.h" + +namespace mxnet { +namespace cpp { + +class KVStore { + public: + static void SetType(const std::string& type); + static void RunServer(); + static void Init(int key, const NDArray& val); + static void Init(const std::vector& keys, const std::vector& vals); + static void Push(int key, const NDArray& val, int priority = 0); + static void Push(const std::vector& keys, + const std::vector& vals, int priority = 0); + static void Pull(int key, NDArray* out, int priority = 0); + static void Pull(const std::vector& keys, std::vector* outs, int priority = 0); + // TODO(lx): put lr in optimizer or not? + static void SetOptimizer(std::unique_ptr optimizer, bool local = false); + static std::string GetType(); + static int GetRank(); + static int GetNumWorkers(); + static void Barrier(); + static std::string GetRole(); + + private: + KVStore(); + static KVStoreHandle& get_handle(); + static std::unique_ptr& get_optimizer(); + static KVStore*& get_kvstore(); + static void Controller(int head, const char* body, void* controller_handle); + static void Updater(int key, NDArrayHandle recv, NDArrayHandle local, void* handle_); +}; + +} // namespace cpp +} // namespace mxnet + +#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_KVSTORE_H_ diff --git a/cpp-package/include/mxnet-cpp/kvstore.hpp b/cpp-package/include/mxnet-cpp/kvstore.hpp new file mode 100644 index 000000000000..d9effcf82f3c --- /dev/null +++ b/cpp-package/include/mxnet-cpp/kvstore.hpp @@ -0,0 +1,178 @@ +/*! + * Copyright (c) 2016 by Contributors + * \file kvstore.hpp + * \brief implementation of kvstore + * \author Xin Li + */ + +#include +#include +#include +#include +#include + +#include "mxnet-cpp/kvstore.h" +#include "mxnet-cpp/optimizer.h" + +#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_KVSTORE_HPP_ +#define CPP_PACKAGE_INCLUDE_MXNET_CPP_KVSTORE_HPP_ + +namespace mxnet { +namespace cpp { + +inline void KVStore::Controller(int head, const char* body, void* controller_handle) { + if (head == 0) { + std::map params; + std::istringstream sin(body); + std::string line; + while (getline(sin, line)) { + size_t n = line.find('='); + params.emplace(line.substr(0, n), line.substr(n+1)); + } + std::unique_ptr opt(OptimizerRegistry::Find(params.at("opt_type"))); + params.erase("opt_type"); + for (const auto& pair : params) { + opt->SetParam(pair.first, pair.second); + } + get_kvstore()->SetOptimizer(std::move(opt), true); + } +} + +inline KVStoreHandle& KVStore::get_handle() { + static KVStoreHandle handle_ = nullptr; + return handle_; +} + +inline std::unique_ptr& KVStore::get_optimizer() { + static std::unique_ptr optimizer_; + return optimizer_; +} + +inline KVStore*& KVStore::get_kvstore() { + static KVStore* kvstore_ = new KVStore; + return kvstore_; +} + +inline KVStore::KVStore() {} + +inline void KVStore::SetType(const std::string& type) { + CHECK_EQ(MXKVStoreCreate(type.c_str(), &(get_kvstore()->get_handle())), 0); +} + +inline void KVStore::RunServer() { + CHECK_NE(GetRole(), "worker"); + CHECK_EQ(MXKVStoreRunServer(get_kvstore()->get_handle(), &Controller, 0), 0); +} + +inline void KVStore::Init(int key, const NDArray& val) { + NDArrayHandle val_handle = val.GetHandle(); + CHECK_EQ(MXKVStoreInit(get_kvstore()->get_handle(), 1, &key, &val_handle), 0); +} + +inline void KVStore::Init(const std::vector& keys, const std::vector& vals) { + CHECK_EQ(keys.size(), vals.size()); + std::vector val_handles(vals.size()); + std::transform(vals.cbegin(), vals.cend(), val_handles.begin(), + [](const NDArray& val) { + return val.GetHandle(); + }); + + CHECK_EQ(MXKVStoreInit(get_kvstore()->get_handle(), keys.size(), keys.data(), + val_handles.data()), 0); +} + +inline void KVStore::Push(int key, const NDArray& val, int priority) { + NDArrayHandle val_handle = val.GetHandle(); + CHECK_EQ(MXKVStorePush(get_kvstore()->get_handle(), 1, &key, &val_handle, priority), 0); +} + +inline void KVStore::Push(const std::vector& keys, + const std::vector& vals, + int priority) { + CHECK_EQ(keys.size(), vals.size()); + std::vector val_handles(vals.size()); + std::transform(vals.cbegin(), vals.cend(), val_handles.begin(), + [](const NDArray& val) { + return val.GetHandle(); + }); + + CHECK_EQ(MXKVStorePush(get_kvstore()->get_handle(), keys.size(), keys.data(), + val_handles.data(), priority), 0); +} + +inline void KVStore::Pull(int key, NDArray* out, int priority) { + NDArrayHandle out_handle = out->GetHandle(); + CHECK_EQ(MXKVStorePull(get_kvstore()->get_handle(), 1, &key, &out_handle, priority), 0); +} + +inline void KVStore::Pull(const std::vector& keys, std::vector* outs, int priority) { + CHECK_EQ(keys.size(), outs->size()); + + std::vector out_handles(keys.size()); + std::transform(outs->cbegin(), outs->cend(), out_handles.begin(), + [](const NDArray& val) { + return val.GetHandle(); + }); + + CHECK_EQ(MXKVStorePull(get_kvstore()->get_handle(), keys.size(), keys.data(), + out_handles.data(), priority), 0); +} + +inline void KVStore::Updater(int key, NDArrayHandle recv, NDArrayHandle local, + void* handle_) { + Optimizer *opt = static_cast(handle_); + opt->Update(key, NDArray(local), NDArray(recv)); +} + +inline void KVStore::SetOptimizer(std::unique_ptr optimizer, bool local) { + if (local) { + get_kvstore()->get_optimizer() = std::move(optimizer); + CHECK_EQ(MXKVStoreSetUpdater(get_kvstore()->get_handle(), + &Updater, get_kvstore()->get_optimizer().get()), 0); + } else { + CHECK_EQ(MXKVStoreSendCommmandToServers(get_kvstore()->get_handle(), 0, + (*optimizer).Serialize().c_str()), 0); + } +} + +inline std::string KVStore::GetType() { + const char *type; + CHECK_EQ(MXKVStoreGetType(get_kvstore()->get_handle(), &type), 0); + return type; +} + +inline int KVStore::GetRank() { + int rank; + CHECK_EQ(MXKVStoreGetRank(get_kvstore()->get_handle(), &rank), 0); + return rank; +} + +inline int KVStore::GetNumWorkers() { + int num_workers; + CHECK_EQ(MXKVStoreGetGroupSize(get_kvstore()->get_handle(), &num_workers), 0); + return num_workers; +} + +inline void KVStore::Barrier() { + CHECK_EQ(MXKVStoreBarrier(get_kvstore()->get_handle()), 0); +} + +inline std::string KVStore::GetRole() { + int ret; + CHECK_EQ(MXKVStoreIsSchedulerNode(&ret), 0); + if (ret) { + return "scheduler"; + } + CHECK_EQ(MXKVStoreIsServerNode(&ret), 0); + if (ret) { + return "server"; + } + CHECK_EQ(MXKVStoreIsWorkerNode(&ret), 0); + CHECK(ret); + return "worker"; +} + +} // namespace cpp +} // namespace mxnet + +#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_KVSTORE_HPP_ diff --git a/cpp-package/include/mxnet-cpp/metric.h b/cpp-package/include/mxnet-cpp/metric.h new file mode 100644 index 000000000000..da4b7fc42716 --- /dev/null +++ b/cpp-package/include/mxnet-cpp/metric.h @@ -0,0 +1,91 @@ +/*! +* Copyright (c) 2016 by Contributors +* \file base.h +* \brief metrics defined +* \author Zhang Chen +*/ + +#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_METRIC_H_ +#define CPP_PACKAGE_INCLUDE_MXNET_CPP_METRIC_H_ + +#include +#include +#include +#include +#include "mxnet-cpp/ndarray.h" +#include "dmlc/logging.h" + +namespace mxnet { +namespace cpp { + +class EvalMetric { + public: + explicit EvalMetric(const std::string& name, int num = 0) + : name(name), num(num) {} + virtual void Update(NDArray labels, NDArray preds) = 0; + void Reset() { + num_inst = 0; + sum_metric = 0.0f; + } + float Get() { return sum_metric / num_inst; } + void GetNameValue(); + + protected: + std::string name; + int num; + float sum_metric = 0.0f; + int num_inst = 0; + + static bool CheckLabelShapes(NDArray labels, NDArray preds, + Shape shape = Shape(0)) { + // TODO(zhangchen-qinyinghua) + // inplement this + return true; + } +}; + +class Accuracy : public EvalMetric { + public: + Accuracy() : EvalMetric("accuracy") {} + + void Update(NDArray labels, NDArray preds) { + CHECK_EQ(labels.GetShape().size(), 1); + mx_uint len = labels.GetShape()[0]; + std::vector pred_data(len); + std::vector label_data(len); + preds.ArgmaxChannel().SyncCopyToCPU(&pred_data, len); + labels.SyncCopyToCPU(&label_data, len); + NDArray::WaitAll(); + for (mx_uint i = 0; i < len; ++i) { + sum_metric += (pred_data[i] == label_data[i]) ? 1 : 0; + num_inst += 1; + } + } +}; + +class LogLoss : public EvalMetric { + public: + LogLoss() : EvalMetric("logloss") {} + + void Update(NDArray labels, NDArray preds) { + static const float epsilon = 1e-15; + mx_uint len = labels.GetShape()[0]; + mx_uint m = preds.GetShape()[1]; + std::vector pred_data(len * m); + std::vector label_data(len); + preds.SyncCopyToCPU(&pred_data, pred_data.size()); + labels.SyncCopyToCPU(&label_data, len); + NDArray::WaitAll(); + for (mx_uint i = 0; i < len; ++i) { + sum_metric += + -std::log(std::max(pred_data[i * m + label_data[i]], epsilon)); + num_inst += 1; + } + } +}; + +} // namespace cpp +} // namespace mxnet + +#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_METRIC_H_ + diff --git a/cpp-package/include/mxnet-cpp/model.h b/cpp-package/include/mxnet-cpp/model.h new file mode 100644 index 000000000000..7bfe1980f095 --- /dev/null +++ b/cpp-package/include/mxnet-cpp/model.h @@ -0,0 +1,58 @@ +/*! +* Copyright (c) 2016 by Contributors +* \file model.h +* \brief MXNET.cpp model module +* \author Zhang Chen +*/ + +#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_MODEL_H_ +#define CPP_PACKAGE_INCLUDE_MXNET_CPP_MODEL_H_ + +#include +#include +#include "mxnet-cpp/base.h" +#include "mxnet-cpp/symbol.h" +#include "mxnet-cpp/ndarray.h" + +namespace mxnet { +namespace cpp { + +struct FeedForwardConfig { + Symbol symbol; + std::vector ctx = {Context::cpu()}; + int num_epoch = 0; + int epoch_size = 0; + std::string optimizer = "sgd"; + // TODO(zhangchen-qinyinghua) More implement + // initializer=Uniform(0.01), + // numpy_batch_size=128, + // arg_params=None, aux_params=None, + // allow_extra_params=False, + // begin_epoch=0, + // **kwargs): + FeedForwardConfig(const FeedForwardConfig &other) {} + FeedForwardConfig() {} +}; +class FeedForward { + public: + explicit FeedForward(const FeedForwardConfig &conf) : conf_(conf) {} + void Predict(); + void Score(); + void Fit(); + void Save(); + void Load(); + static FeedForward Create(); + + private: + void InitParams(); + void InitPredictor(); + void InitIter(); + void InitEvalIter(); + FeedForwardConfig conf_; +}; + +} // namespace cpp +} // namespace mxnet + +#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_MODEL_H_ + diff --git a/cpp-package/include/mxnet-cpp/ndarray.h b/cpp-package/include/mxnet-cpp/ndarray.h new file mode 100644 index 000000000000..b03a1f3809d1 --- /dev/null +++ b/cpp-package/include/mxnet-cpp/ndarray.h @@ -0,0 +1,411 @@ +/*! +* Copyright (c) 2016 by Contributors +* \file ndarray.h +* \brief definition of ndarray +* \author Chuntao Hong, Zhang Chen +*/ + +#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_NDARRAY_H_ +#define CPP_PACKAGE_INCLUDE_MXNET_CPP_NDARRAY_H_ + +#include +#include +#include +#include +#include "mxnet-cpp/base.h" +#include "mxnet-cpp/shape.h" + +namespace mxnet { +namespace cpp { + +enum DeviceType { + kCPU = 1, + kGPU = 2, + kCPUPinned = 3 +}; + +/*! +* \brief Context interface +*/ +class Context { + public: + /*! + * \brief Context constructor + * \param type type of the device + * \param id id of the device + */ + Context(const DeviceType &type, int id) : type_(type), id_(id) {} + /*! + * \return the type of the device + */ + DeviceType GetDeviceType() const { return type_; } + /*! + * \return the id of the device + */ + int GetDeviceId() const { return id_; } + + /*! + * \brief Return a GPU context + * \param device_id id of the device + * \return the corresponding GPU context + */ + static Context gpu(int device_id = 0) { + return Context(DeviceType::kGPU, device_id); + } + + /*! + * \brief Return a CPU context + * \param device_id id of the device. this is not needed by CPU + * \return the corresponding CPU context + */ + static Context cpu(int device_id = 0) { + return Context(DeviceType::kCPU, device_id); + } + + private: + DeviceType type_; + int id_; +}; + +/*! +* \brief struct to store NDArrayHandle +*/ +struct NDBlob { + public: + /*! + * \brief default constructor + */ + NDBlob() : handle_(nullptr) {} + /*! + * \brief construct with a NDArrayHandle + * \param handle NDArrayHandle to store + */ + explicit NDBlob(NDArrayHandle handle) : handle_(handle) {} + /*! + * \brief destructor, free the NDArrayHandle + */ + ~NDBlob() { MXNDArrayFree(handle_); } + /*! + * \brief the NDArrayHandle + */ + NDArrayHandle handle_; + + private: + NDBlob(const NDBlob &); + NDBlob &operator=(const NDBlob &); +}; + +/*! +* \brief NDArray interface +*/ +class NDArray { + public: + /*! + * \brief construct with a none handle + */ + NDArray(); + /*! + * \brief construct with a NDArrayHandle + */ + explicit NDArray(const NDArrayHandle &handle); + /*! + * \brief construct a new dynamic NDArray + * \param shape the shape of array + * \param constext context of NDArray + * \param delay_alloc whether delay the allocation + */ + NDArray(const std::vector &shape, const Context &context, + bool delay_alloc = true); + /*! + * \brief construct a new dynamic NDArray + * \param shape the shape of array + * \param constext context of NDArray + * \param delay_alloc whether delay the allocation + */ + NDArray(const Shape &shape, const Context &context, bool delay_alloc = true); + NDArray(const mx_float *data, size_t size); + /*! + * \brief construct a new dynamic NDArray + * \param data the data to create NDArray from + * \param shape the shape of array + * \param constext context of NDArray + */ + NDArray(const mx_float *data, const Shape &shape, const Context &context); + /*! + * \brief construct a new dynamic NDArray + * \param data the data to create NDArray from + * \param shape the shape of array + * \param constext context of NDArray + */ + NDArray(const std::vector &data, const Shape &shape, + const Context &context); + explicit NDArray(const std::vector &data); + NDArray operator+(mx_float scalar); + NDArray operator-(mx_float scalar); + NDArray operator*(mx_float scalar); + NDArray operator/(mx_float scalar); + NDArray operator+(const NDArray &); + NDArray operator-(const NDArray &); + NDArray operator*(const NDArray &); + NDArray operator/(const NDArray &); + /*! + * \brief set all the elements in ndarray to be scalar + * \param scalar the scalar to set + * \return reference of self + */ + NDArray &operator=(mx_float scalar); + /*! + * \brief elementwise add to current space + * this mutate the current NDArray + * \param scalar the data to add + * \return reference of self + */ + NDArray &operator+=(mx_float scalar); + /*! + * \brief elementwise subtract from current ndarray + * this mutate the current NDArray + * \param scalar the data to substract + * \return reference of self + */ + NDArray &operator-=(mx_float scalar); + /*! + * \brief elementwise multiplication to current ndarray + * this mutate the current NDArray + * \param scalar the data to substract + * \return reference of self + */ + NDArray &operator*=(mx_float scalar); + /*! + * \brief elementwise division from current ndarray + * this mutate the current NDArray + * \param scalar the data to substract + * \return reference of self + */ + NDArray &operator/=(mx_float scalar); + /*! + * \brief elementwise add to current space + * this mutate the current NDArray + * \param src the data to add + * \return reference of self + */ + NDArray &operator+=(const NDArray &src); + /*! + * \brief elementwise subtract from current ndarray + * this mutate the current NDArray + * \param src the data to substract + * \return reference of self + */ + NDArray &operator-=(const NDArray &src); + /*! + * \brief elementwise multiplication to current ndarray + * this mutate the current NDArray + * \param src the data to substract + * \return reference of self + */ + NDArray &operator*=(const NDArray &src); + /*! + * \brief elementwise division from current ndarray + * this mutate the current NDArray + * \param src the data to substract + * \return reference of self + */ + NDArray &operator/=(const NDArray &src); + NDArray ArgmaxChannel(); + /*! + * \brief Do a synchronize copy from a continugous CPU memory region. + * + * This function will call WaitToWrite before the copy is performed. + * This is useful to copy data from existing memory region that are + * not wrapped by NDArray(thus dependency not being tracked). + * + * \param data the data source to copy from. + * \param size the memory size we want to copy from. + */ + void SyncCopyFromCPU(const mx_float *data, size_t size); + /*! + * \brief Do a synchronize copy from a continugous CPU memory region. + * + * This function will call WaitToWrite before the copy is performed. + * This is useful to copy data from existing memory region that are + * not wrapped by NDArray(thus dependency not being tracked). + * + * \param data the data source to copy from, int the form of mx_float vector + */ + void SyncCopyFromCPU(const std::vector &data); + /*! + * \brief Do a synchronize copy to a continugous CPU memory region. + * + * This function will call WaitToRead before the copy is performed. + * This is useful to copy data from existing memory region that are + * not wrapped by NDArray(thus dependency not being tracked). + * + * \param data the data source to copyinto. + * \param size the memory size we want to copy into. Defualt value is Size() + */ + void SyncCopyToCPU(mx_float *data, size_t size = 0); + /*! + * \brief Do a synchronize copy to a continugous CPU memory region. + * + * This function will call WaitToRead before the copy is performed. + * This is useful to copy data from existing memory region that are + * not wrapped by NDArray(thus dependency not being tracked). + * + * \param data the data source to copyinto. + * \param size the memory size we want to copy into. Defualt value is Size() + */ + void SyncCopyToCPU(std::vector *data, size_t size = 0); + /*! + * \brief Copy the content of current array to other. + * \param other the new context of this NDArray + * \return the new copy + */ + NDArray CopyTo(NDArray * other) const; + /*! + * \brief return a new copy this NDArray + * \param other the target NDArray + * \return the copy target NDarray + */ + NDArray Copy(const Context &) const; + /*! + * \brief return offset of the element at (h, w) + * \param h height position + * \param w width position + * \return offset of two dimensions array + */ + size_t Offset(size_t h = 0, size_t w = 0) const; + /*! + * \brief return offset of three dimensions array + * \param c channel position + * \param h height position + * \param w width position + * \return offset of three dimensions array + */ + size_t Offset(size_t c, size_t h, size_t w) const; + /*! + * \brief return value of the element at (h, w) + * \param h height position + * \param w width position + * \return value of two dimensions array + */ + mx_float At(size_t h, size_t w) const; + /*! + * \brief return value of three dimensions array + * \param c channel position + * \param h height position + * \param w width position + * \return value of three dimensions array + */ + mx_float At(size_t c, size_t h, size_t w) const; + /*! + * \brief Slice a NDArray + * \param begin begin index in first dim + * \param end end index in first dim + * \return sliced NDArray + */ + NDArray Slice(mx_uint begin, mx_uint end) const; + /*! + * \brief Return a reshaped NDArray that shares memory with current one + * \param new_shape the new shape + * \return reshaped NDarray + */ + NDArray Reshape(const Shape &new_shape) const; + /*! + * \brief Block until all the pending write operations with respect + * to current NDArray are finished, and read can be performed. + */ + void WaitToRead() const; + /*! + * \brief Block until all the pending read/write operations with respect + * to current NDArray are finished, and write can be performed. + */ + void WaitToWrite(); + /*! + * \brief Block until all the pending read/write operations with respect + * to current NDArray are finished, and read/write can be performed. + */ + static void WaitAll(); + /*! + * \brief Sample gaussian distribution for each elements of out. + * \param mu mean of gaussian distribution. + * \param sigma standard deviation of gaussian distribution. + * \param out output NDArray. + */ + static void SampleGaussian(mx_float mu, mx_float sigma, NDArray *out); + /*! + * \brief Sample uniform distribution for each elements of out. + * \param begin lower bound of distribution. + * \param end upper bound of distribution. + * \param out output NDArray. + */ + static void SampleUniform(mx_float begin, mx_float end, NDArray *out); + /*! + * \brief Load NDArrays from binary file. + * \param file_name name of the binary file. + * \param array_list a list of NDArrays returned, do not fill the list if + * nullptr is given. + * \param array_map a map from names to NDArrays returned, do not fill the map + * if nullptr is given or no names is stored in binary file. + */ + static void Load(const std::string &file_name, + std::vector *array_list = nullptr, + std::map *array_map = nullptr); + /*! + * \brief Load map of NDArrays from binary file. + * \param file_name name of the binary file. + * \return a list of NDArrays. + */ + static std::map LoadToMap(const std::string &file_name); + /*! + * \brief Load list of NDArrays from binary file. + * \param file_name name of the binary file. + * \return a map from names to NDArrays. + */ + static std::vector LoadToList(const std::string &file_name); + /*! + * \brief save a map of string->NDArray to binary file. + * \param file_name name of the binary file. + * \param array_map a map from names to NDArrays. + */ + static void Save(const std::string &file_name, + const std::map &array_map); + /*! + * \brief save a list of NDArrays to binary file. + * \param file_name name of the binary file. + * \param array_list a list of NDArrays. + */ + static void Save(const std::string &file_name, + const std::vector &array_list); + /*! + * \return the size of current NDArray, a.k.a. the production of all shape dims + */ + size_t Size() const; + /*! + * \return the shape of current NDArray, in the form of mx_uint vector + */ + std::vector GetShape() const; + /*! + * \return the data type of current NDArray + */ + int GetDType() const; + /*! + * \return the data pointer to the current NDArray + */ + const mx_float *GetData() const; + + /*! + * \return the context of NDArray + */ + Context GetContext() const; + + /*! + * \return the NDArrayHandle of the current NDArray + */ + NDArrayHandle GetHandle() const { return blob_ptr_->handle_; } + + private: + std::shared_ptr blob_ptr_; +}; +} // namespace cpp +} // namespace mxnet + +#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_NDARRAY_H_ diff --git a/cpp-package/include/mxnet-cpp/ndarray.hpp b/cpp-package/include/mxnet-cpp/ndarray.hpp new file mode 100644 index 000000000000..addf0d3870a5 --- /dev/null +++ b/cpp-package/include/mxnet-cpp/ndarray.hpp @@ -0,0 +1,341 @@ +/*! + * Copyright (c) 2016 by Contributors + * \file ndarray.hpp + * \brief implementation of the ndarray + * \author Zhang Chen, Chuntao Hong + */ + +#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_NDARRAY_HPP_ +#define CPP_PACKAGE_INCLUDE_MXNET_CPP_NDARRAY_HPP_ + +#include +#include +#include +#include "dmlc/logging.h" +#include "mxnet-cpp/ndarray.h" + +namespace mxnet { +namespace cpp { + +inline NDArray::NDArray() { + NDArrayHandle handle; + CHECK_EQ(MXNDArrayCreateNone(&handle), 0); + blob_ptr_ = std::make_shared(handle); +} +inline NDArray::NDArray(const NDArrayHandle &handle) { + blob_ptr_ = std::make_shared(handle); +} +inline NDArray::NDArray(const std::vector &shape, const Context &context, + bool delay_alloc) { + NDArrayHandle handle; + CHECK_EQ(MXNDArrayCreate(shape.data(), shape.size(), context.GetDeviceType(), + context.GetDeviceId(), delay_alloc, &handle), + 0); + blob_ptr_ = std::make_shared(handle); +} +inline NDArray::NDArray(const Shape &shape, const Context &context, bool delay_alloc) { + NDArrayHandle handle; + CHECK_EQ(MXNDArrayCreate(shape.data(), shape.ndim(), context.GetDeviceType(), + context.GetDeviceId(), delay_alloc, &handle), + 0); + blob_ptr_ = std::make_shared(handle); +} +inline NDArray::NDArray(const mx_float *data, size_t size) { + NDArrayHandle handle; + CHECK_EQ(MXNDArrayCreateNone(&handle), 0); + MXNDArraySyncCopyFromCPU(handle, data, size); + blob_ptr_ = std::make_shared(handle); +} +inline NDArray::NDArray(const mx_float *data, const Shape &shape, + const Context &context) { + NDArrayHandle handle; + CHECK_EQ(MXNDArrayCreate(shape.data(), shape.ndim(), context.GetDeviceType(), + context.GetDeviceId(), false, &handle), + 0); + MXNDArraySyncCopyFromCPU(handle, data, shape.Size()); + blob_ptr_ = std::make_shared(handle); +} +inline NDArray::NDArray(const std::vector &data, const Shape &shape, + const Context &context) { + NDArrayHandle handle; + CHECK_EQ(MXNDArrayCreate(shape.data(), shape.ndim(), context.GetDeviceType(), + context.GetDeviceId(), false, &handle), + 0); + MXNDArraySyncCopyFromCPU(handle, data.data(), shape.Size()); + blob_ptr_ = std::make_shared(handle); +} +inline NDArray::NDArray(const std::vector &data) { + NDArrayHandle handle; + CHECK_EQ(MXNDArrayCreateNone(&handle), 0); + MXNDArraySyncCopyFromCPU(handle, data.data(), data.size()); + blob_ptr_ = std::make_shared(handle); +} + +inline NDArray NDArray::operator+(mx_float scalar) { + NDArray ret; + Operator("_plus_scalar")(*this, scalar).Invoke(ret); + return ret; +} +inline NDArray NDArray::operator-(mx_float scalar) { + NDArray ret; + Operator("_minus_scalar")(*this, scalar).Invoke(ret); + return ret; +} +inline NDArray NDArray::operator*(mx_float scalar) { + NDArray ret; + Operator("_mul_scalar")(*this, scalar).Invoke(ret); + return ret; +} +inline NDArray NDArray::operator/(mx_float scalar) { + NDArray ret; + Operator("_div_scalar")(*this, scalar).Invoke(ret); + return ret; +} +inline NDArray NDArray::operator+(const NDArray &rhs) { + NDArray ret; + Operator("_plus")(*this, rhs).Invoke(ret); + return ret; +} +inline NDArray NDArray::operator-(const NDArray &rhs) { + NDArray ret; + Operator("_minus")(*this, rhs).Invoke(ret); + return ret; +} +inline NDArray NDArray::operator*(const NDArray &rhs) { + NDArray ret; + Operator("_mul")(*this, rhs).Invoke(ret); + return ret; +} +inline NDArray NDArray::operator/(const NDArray &rhs) { + NDArray ret; + Operator("_div")(*this, rhs).Invoke(ret); + return ret; +} +inline NDArray &NDArray::operator=(mx_float scalar) { + Operator("_set_value")(scalar).Invoke(*this); + return *this; +} +inline NDArray &NDArray::operator+=(mx_float scalar) { + Operator("_plus_scalar")(*this, scalar).Invoke(*this); + return *this; +} +inline NDArray &NDArray::operator-=(mx_float scalar) { + Operator("_minus_scalar")(*this, scalar).Invoke(*this); + return *this; +} +inline NDArray &NDArray::operator*=(mx_float scalar) { + Operator("_mul_scalar")(*this, scalar).Invoke(*this); + return *this; +} +inline NDArray &NDArray::operator/=(mx_float scalar) { + Operator("_div_scalar")(*this, scalar).Invoke(*this); + return *this; +} +inline NDArray &NDArray::operator+=(const NDArray &rhs) { + Operator("_plus")(*this, rhs).Invoke(*this); + return *this; +} +inline NDArray &NDArray::operator-=(const NDArray &rhs) { + Operator("_minus")(*this, rhs).Invoke(*this); + return *this; +} +inline NDArray &NDArray::operator*=(const NDArray &rhs) { + Operator("_mul")(*this, rhs).Invoke(*this); + return *this; +} +inline NDArray &NDArray::operator/=(const NDArray &rhs) { + Operator("_div")(*this, rhs).Invoke(*this); + return *this; +} + +inline NDArray NDArray::ArgmaxChannel() { + NDArray ret; + Operator("argmax_channel")(*this).Invoke(ret); + return ret; +} + +inline void NDArray::SyncCopyFromCPU(const mx_float *data, size_t size) { + MXNDArraySyncCopyFromCPU(blob_ptr_->handle_, data, size); +} +inline void NDArray::SyncCopyFromCPU(const std::vector &data) { + MXNDArraySyncCopyFromCPU(blob_ptr_->handle_, data.data(), data.size()); +} +inline void NDArray::SyncCopyToCPU(mx_float *data, size_t size) { + MXNDArraySyncCopyToCPU(blob_ptr_->handle_, data, size > 0 ? size : Size()); +} +inline void NDArray::SyncCopyToCPU(std::vector *data, size_t size) { + size = size > 0 ? size : Size(); + data->resize(size); + MXNDArraySyncCopyToCPU(blob_ptr_->handle_, data->data(), size); +} +inline NDArray NDArray::Copy(const Context &ctx) const { + NDArray ret(GetShape(), ctx); + Operator("_copyto")(*this).Invoke(ret); + return ret; +} +inline NDArray NDArray::CopyTo(NDArray * other) const { + Operator("_copyto")(*this).Invoke(*other); + return *other; +} +inline NDArray NDArray::Slice(mx_uint begin, mx_uint end) const { + NDArrayHandle handle; + CHECK_EQ(MXNDArraySlice(GetHandle(), begin, end, &handle), 0); + return NDArray(handle); +} +inline NDArray NDArray::Reshape(const Shape &new_shape) const { + NDArrayHandle handle; + std::vector dims(new_shape.ndim()); + for (index_t i = 0; i < new_shape.ndim(); ++i) { + dims[i] = new_shape[i]; + } + new_shape.data(); + CHECK_EQ( + MXNDArrayReshape(GetHandle(), new_shape.ndim(), dims.data(), &handle), 0); + return NDArray(handle); +} +inline void NDArray::WaitToRead() const { + CHECK_EQ(MXNDArrayWaitToRead(blob_ptr_->handle_), 0); +} +inline void NDArray::WaitToWrite() { + CHECK_EQ(MXNDArrayWaitToWrite(blob_ptr_->handle_), 0); +} +inline void NDArray::WaitAll() { CHECK_EQ(MXNDArrayWaitAll(), 0); } +inline void NDArray::SampleGaussian(mx_float mu, mx_float sigma, NDArray *out) { + Operator("_sample_normal")(mu, sigma).Invoke(*out); +} +inline void NDArray::SampleUniform(mx_float begin, mx_float end, NDArray *out) { + Operator("_sample_uniform")(begin, end).Invoke(*out); +} +inline void NDArray::Load(const std::string &file_name, + std::vector *array_list, + std::map *array_map) { + mx_uint out_size, out_name_size; + NDArrayHandle *out_arr; + const char **out_names; + CHECK_EQ(MXNDArrayLoad(file_name.c_str(), &out_size, &out_arr, &out_name_size, + &out_names), + 0); + if (array_list != nullptr) { + for (mx_uint i = 0; i < out_size; ++i) { + array_list->push_back(NDArray(out_arr[i])); + } + } + if (array_map != nullptr && out_name_size > 0) { + CHECK_EQ(out_name_size, out_size); + for (mx_uint i = 0; i < out_size; ++i) { + (*array_map)[out_names[i]] = NDArray(out_arr[i]); + } + } +} +inline std::map NDArray::LoadToMap( + const std::string &file_name) { + std::map array_map; + mx_uint out_size, out_name_size; + NDArrayHandle *out_arr; + const char **out_names; + CHECK_EQ(MXNDArrayLoad(file_name.c_str(), &out_size, &out_arr, &out_name_size, + &out_names), + 0); + if (out_name_size > 0) { + CHECK_EQ(out_name_size, out_size); + for (mx_uint i = 0; i < out_size; ++i) { + array_map[out_names[i]] = NDArray(out_arr[i]); + } + } + return array_map; +} +inline std::vector NDArray::LoadToList(const std::string &file_name) { + std::vector array_list; + mx_uint out_size, out_name_size; + NDArrayHandle *out_arr; + const char **out_names; + CHECK_EQ(MXNDArrayLoad(file_name.c_str(), &out_size, &out_arr, &out_name_size, + &out_names), + 0); + for (mx_uint i = 0; i < out_size; ++i) { + array_list.push_back(NDArray(out_arr[i])); + } + return array_list; +} +inline void NDArray::Save(const std::string &file_name, + const std::map &array_map) { + std::vector args; + std::vector keys; + for (const auto &t : array_map) { + args.push_back(t.second.GetHandle()); + keys.push_back(t.first.c_str()); + } + CHECK_EQ( + MXNDArraySave(file_name.c_str(), args.size(), args.data(), keys.data()), + 0); +} +inline void NDArray::Save(const std::string &file_name, + const std::vector &array_list) { + std::vector args; + for (const auto &t : array_list) { + args.push_back(t.GetHandle()); + } + CHECK_EQ(MXNDArraySave(file_name.c_str(), args.size(), args.data(), nullptr), + 0); +} + +inline size_t NDArray::Offset(size_t h, size_t w) const { + return (h * GetShape()[1]) + w; +} + +inline size_t NDArray::Offset(size_t c, size_t h, size_t w) const { + auto const shape = GetShape(); + return h * shape[0] * shape[2] + w * shape[0] + c; +} + +inline mx_float NDArray::At(size_t h, size_t w) const { + return GetData()[Offset(h, w)]; +} + +inline mx_float NDArray::At(size_t c, size_t h, size_t w) const { + return GetData()[Offset(c, h, w)]; +} + +inline size_t NDArray::Size() const { + size_t ret = 1; + for (auto &i : GetShape()) ret *= i; + return ret; +} + +inline std::vector NDArray::GetShape() const { + const mx_uint *out_pdata; + mx_uint out_dim; + MXNDArrayGetShape(blob_ptr_->handle_, &out_dim, &out_pdata); + std::vector ret; + for (mx_uint i = 0; i < out_dim; ++i) { + ret.push_back(out_pdata[i]); + } + return ret; +} + +inline int NDArray::GetDType() const { + int ret; + MXNDArrayGetDType(blob_ptr_->handle_, &ret); + return ret; +} + +inline const mx_float *NDArray::GetData() const { + void *ret; + CHECK_NE(GetContext().GetDeviceType(), DeviceType::kGPU); + MXNDArrayGetData(blob_ptr_->handle_, &ret); + if (GetDType() != 0) { + return NULL; + } + return static_cast(ret); +} + +inline Context NDArray::GetContext() const { + int out_dev_type; + int out_dev_id; + MXNDArrayGetContext(blob_ptr_->handle_, &out_dev_type, &out_dev_id); + return Context((DeviceType)out_dev_type, out_dev_id); +} +} // namespace cpp +} // namespace mxnet + +#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_NDARRAY_HPP_ diff --git a/cpp-package/include/mxnet-cpp/op_map.h b/cpp-package/include/mxnet-cpp/op_map.h new file mode 100644 index 000000000000..2a2ae50a4e84 --- /dev/null +++ b/cpp-package/include/mxnet-cpp/op_map.h @@ -0,0 +1,92 @@ +/*! +* Copyright (c) 2016 by Contributors +* \file op_map.h +* \brief definition of OpMap +* \author Chuntao Hong +*/ + +#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_OP_MAP_H_ +#define CPP_PACKAGE_INCLUDE_MXNET_CPP_OP_MAP_H_ + +#include +#include +#include "mxnet-cpp/base.h" +#include "dmlc/logging.h" + +namespace mxnet { +namespace cpp { + +/*! +* \brief OpMap instance holds a map of all the symbol creators so we can +* get symbol creators by name. +* This is used internally by Symbol and Operator. +*/ +class OpMap { + public: + /*! + * \brief Create an Mxnet instance + */ + inline OpMap() { + mx_uint num_symbol_creators = 0; + AtomicSymbolCreator *symbol_creators = nullptr; + int r = + MXSymbolListAtomicSymbolCreators(&num_symbol_creators, &symbol_creators); + CHECK_EQ(r, 0); + for (mx_uint i = 0; i < num_symbol_creators; i++) { + const char *name; + const char *description; + mx_uint num_args; + const char **arg_names; + const char **arg_type_infos; + const char **arg_descriptions; + const char *key_var_num_args; + r = MXSymbolGetAtomicSymbolInfo(symbol_creators[i], &name, &description, + &num_args, &arg_names, &arg_type_infos, + &arg_descriptions, &key_var_num_args); + CHECK_EQ(r, 0); + symbol_creators_[name] = symbol_creators[i]; + } + + nn_uint num_ops; + const char **op_names; + r = NNListAllOpNames(&num_ops, &op_names); + CHECK_EQ(r, 0); + for (nn_uint i = 0; i < num_ops; i++) { + OpHandle handle; + r = NNGetOpHandle(op_names[i], &handle); + CHECK_EQ(r, 0); + op_handles_[op_names[i]] = handle; + } + } + + /*! + * \brief Get a symbol creator with its name. + * + * \param name name of the symbol creator + * \return handle to the symbol creator + */ + inline AtomicSymbolCreator GetSymbolCreator(const std::string &name) { + if (symbol_creators_.count(name) == 0) + return GetOpHandle(name); + return symbol_creators_[name]; + } + + /*! + * \brief Get an op handle with its name. + * + * \param name name of the op + * \return handle to the op + */ + inline OpHandle GetOpHandle(const std::string &name) { + return op_handles_[name]; + } + + private: + std::map symbol_creators_; + std::map op_handles_; +}; + +} // namespace cpp +} // namespace mxnet + +#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_OP_MAP_H_ diff --git a/cpp-package/include/mxnet-cpp/op_suppl.h b/cpp-package/include/mxnet-cpp/op_suppl.h new file mode 100644 index 000000000000..5eb86d8ef275 --- /dev/null +++ b/cpp-package/include/mxnet-cpp/op_suppl.h @@ -0,0 +1,188 @@ +/*! +* Copyright (c) 2016 by Contributors +* \file op_suppl.h +* \brief A supplement and amendment of the operators from op.h +* \author Zhang Chen, zhubuntu, Xin Li +*/ + +#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_OP_SUPPL_H_ +#define CPP_PACKAGE_INCLUDE_MXNET_CPP_OP_SUPPL_H_ + +#include +#include +#include +#include "mxnet-cpp/base.h" +#include "mxnet-cpp/shape.h" +#include "mxnet-cpp/operator.h" +#include "mxnet-cpp/MxNetCpp.h" + +namespace mxnet { +namespace cpp { + +inline Symbol _Plus(Symbol lhs, Symbol rhs) { + return Operator("_Plus")(lhs, rhs) + .CreateSymbol(); +} +inline Symbol _Mul(Symbol lhs, Symbol rhs) { + return Operator("_Mul")(lhs, rhs) + .CreateSymbol(); +} +inline Symbol _Minus(Symbol lhs, Symbol rhs) { + return Operator("_Minus")(lhs, rhs) + .CreateSymbol(); +} +inline Symbol _Div(Symbol lhs, Symbol rhs) { + return Operator("_Div")(lhs, rhs) + .CreateSymbol(); +} +inline Symbol _Power(Symbol lhs, Symbol rhs) { + return Operator("_Power")(lhs, rhs) + .CreateSymbol(); +} +inline Symbol _Maximum(Symbol lhs, Symbol rhs) { + return Operator("_Maximum")(lhs, rhs) + .CreateSymbol(); +} +inline Symbol _Minimum(Symbol lhs, Symbol rhs) { + return Operator("_Minimum")(lhs, rhs) + .CreateSymbol(); +} +inline Symbol _PlusScalar(Symbol lhs, mx_float scalar) { + return Operator("_PlusScalar")(lhs) + .SetParam("scalar", scalar) + .CreateSymbol(); +} +inline Symbol _MinusScalar(Symbol lhs, mx_float scalar) { + return Operator("_MinusScalar")(lhs) + .SetParam("scalar", scalar) + .CreateSymbol(); +} +inline Symbol _RMinusScalar(mx_float scalar, Symbol rhs) { + return Operator("_RMinusScalar")(rhs) + .SetParam("scalar", scalar) + .CreateSymbol(); +} +inline Symbol _MulScalar(Symbol lhs, mx_float scalar) { + return Operator("_MulScalar")(lhs) + .SetParam("scalar", scalar) + .CreateSymbol(); +} +inline Symbol _DivScalar(Symbol lhs, mx_float scalar) { + return Operator("_DivScalar")(lhs) + .SetParam("scalar", scalar) + .CreateSymbol(); +} +inline Symbol _RDivScalar(mx_float scalar, Symbol rhs) { + return Operator("_RDivScalar")(rhs) + .SetParam("scalar", scalar) + .CreateSymbol(); +} +inline Symbol _PowerScalar(Symbol lhs, mx_float scalar) { + return Operator("_PowerScalar")(lhs) + .SetParam("scalar", scalar) + .CreateSymbol(); +} +inline Symbol _RPowerScalar(mx_float scalar, Symbol rhs) { + return Operator("_RPowerScalar")(rhs) + .SetParam("scalar", scalar) + .CreateSymbol(); +} +inline Symbol _MaximumScalar(Symbol lhs, mx_float scalar) { + return Operator("_MaximumScalar")(lhs) + .SetParam("scalar", scalar) + .CreateSymbol(); +} +inline Symbol _MinimumScalar(Symbol lhs, mx_float scalar) { + return Operator("_MinimumScalar")(lhs) + .SetParam("scalar", scalar) + .CreateSymbol(); +} +// TODO(zhangcheng-qinyinghua) +// make crop function run in op.h +// This function is due to [zhubuntu](https://github.com/zhubuntu) +inline Symbol Crop(const std::string& symbol_name, + int num_args, + Symbol data, + Symbol crop_like, + Shape offset = Shape(0, 0), + Shape h_w = Shape(0, 0), + bool center_crop = false) { + return Operator("Crop") + .SetParam("num_args", num_args) + .SetParam("offset", offset) + .SetParam("h_w", h_w) + .SetParam("center_crop", center_crop) + .SetInput("arg0", data) + .SetInput("arg1", crop_like) + .CreateSymbol(symbol_name); +} + + +/*! + * \breif Slice input equally along specified axis. + * \param data input symbol. + * \param num_outputs Number of outputs to be sliced. + * \param axis Dimension along which to slice. + * \param squeeze_axis If true AND the sliced dimension becomes 1, squeeze that dimension. + * \return new symbol + */ +inline Symbol SliceChannel(Symbol data, + int num_outputs, + int axis = 1, + bool squeeze_axis = false) { + return Operator("SliceChannel") + .SetParam("num_outputs", num_outputs) + .SetParam("axis", axis) + .SetParam("squeeze_axis", squeeze_axis) (data) + .CreateSymbol(); +} + + +/*! + * \breif Slice input equally along specified axis. + * \param symbol_name name of the resulting symbol. + * \param data input symbol. + * \param num_outputs Number of outputs to be sliced. + * \param axis Dimension along which to slice. + * \param squeeze_axis If true AND the sliced dimension becomes 1, squeeze that dimension. + * \return new symbol + */ +inline Symbol SliceChannel(const std::string& symbol_name, + Symbol data, + int num_outputs, + int axis = 1, + bool squeeze_axis = false) { + return Operator("SliceChannel") + .SetParam("num_outputs", num_outputs) + .SetParam("axis", axis) + .SetParam("squeeze_axis", squeeze_axis) (data) + .CreateSymbol(symbol_name); +} + +/*! + * \breif Apply activation function to input. + * Softmax Activation is only available with CUDNN on GPUand will be + * computed at each location across channel if input is 4D. + * \param symbol_name name of the resulting symbol. + * \param data Input data to activation function. + * \param act_type Activation function to be applied. + * \return new symbol + */ +inline Symbol Activation(const std::string& symbol_name, + Symbol data, + const std::string& act_type) { + assert(act_type == "relu" || + act_type == "sigmoid" || + act_type == "softrelu" || + act_type == "tanh"); + return Operator("Activation") + .SetParam("act_type", act_type.c_str()) + .SetInput("data", data) + .CreateSymbol(symbol_name); +} + +} // namespace cpp +} // namespace mxnet + +#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_OP_SUPPL_H_ + diff --git a/cpp-package/include/mxnet-cpp/op_util.h b/cpp-package/include/mxnet-cpp/op_util.h new file mode 100644 index 000000000000..bf67eab4c1ae --- /dev/null +++ b/cpp-package/include/mxnet-cpp/op_util.h @@ -0,0 +1,46 @@ +/*! +* Copyright (c) 2017 by Contributors +* \file op_util.h +* \brief operator helper functions +* \author Chris Olivier +*/ + +#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_OP_UTIL_H_ +#define CPP_PACKAGE_INCLUDE_MXNET_CPP_OP_UTIL_H_ + +#include + +#if defined(MXNET_USE_CAFFE) && MXNET_USE_CAFFE != 0 +#include +#include +#endif + +namespace mxnet { +namespace cpp { + +#if defined(MXNET_USE_CAFFE) && MXNET_USE_CAFFE != 0 + +inline ::caffe::LayerParameter textToCaffeLayerParameter(const std::string& text) { + caffe::NetParameter np; + const bool success = google::protobuf::TextFormat::ParseFromString(text, &np); + CHECK_EQ(success, true) << "Invalid protpbuf layer string: " << text; + return ::caffe::LayerParameter(np.layer(0)); +} + +template +inline StreamType& operator << (StreamType& os, const ::caffe::LayerParameter& op) { + std::string s; + caffe::NetParameter np; + // Avoid wasting time making a copy -- just push in out default object's pointer + np.mutable_layer()->AddAllocated(const_cast<::caffe::LayerParameter *>(&op)); + google::protobuf::TextFormat::PrintToString(np, &s); + np.mutable_layer()->ReleaseLast(); + os << s; + return os; +} +#endif + +} // namespace cpp +} // namespace mxnet + +#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_OP_UTIL_H_ diff --git a/cpp-package/include/mxnet-cpp/operator.h b/cpp-package/include/mxnet-cpp/operator.h new file mode 100644 index 000000000000..6677f86aeb75 --- /dev/null +++ b/cpp-package/include/mxnet-cpp/operator.h @@ -0,0 +1,188 @@ +/*! +* Copyright (c) 2016 by Contributors +* \file operator.h +* \brief definition of operator +* \author Chuntao Hong, Zhang Chen +*/ + +#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_OPERATOR_H_ +#define CPP_PACKAGE_INCLUDE_MXNET_CPP_OPERATOR_H_ + +#include +#include +#include +#include "mxnet-cpp/base.h" +#include "mxnet-cpp/op_map.h" +#include "mxnet-cpp/symbol.h" + +namespace mxnet { +namespace cpp { +class Mxnet; +/*! +* \brief Operator interface +*/ +class Operator { + public: + /*! + * \brief Operator constructor + * \param operator_name type of the operator + */ + explicit Operator(const std::string &operator_name); + Operator &operator=(const Operator &rhs); + /*! + * \brief set config parameters + * \param name name of the config parameter + * \param value value of the config parameter + * \return reference of self + */ + template + Operator &SetParam(const std::string &name, const T &value) { + std::string value_str; + std::stringstream ss; + ss << value; + ss >> value_str; + + params_[name] = value_str; + return *this; + } + /*! + * \brief set config parameters from positional inputs + * \param pos the position of parameter + * \param value value of the config parameter + * \return reference of self + */ + template + Operator &SetParam(int pos, const T &value) { + std::string value_str; + std::stringstream ss; + ss << value; + ss >> value_str; + + params_[arg_names_[pos]] = value_str; + return *this; + } + /*! + * \brief add an input symbol + * \param name name of the input symbol + * \param symbol the input symbol + * \return reference of self + */ + Operator &SetInput(const std::string &name, Symbol symbol); + /*! + * \brief add an input symbol + * \param symbol the input symbol + */ + template + void PushInput(const Symbol &symbol) { + input_symbols.push_back(symbol.GetHandle()); + } + /*! + * \brief add input symbols + * \return reference of self + */ + Operator &operator()() { return *this; } + /*! + * \brief add input symbols + * \param symbol the input symbol + * \return reference of self + */ + Operator &operator()(const Symbol &symbol) { + input_symbols.push_back(symbol.GetHandle()); + return *this; + } + /*! + * \brief add a list of input symbols + * \param symbols the vector of the input symbols + * \return reference of self + */ + Operator &operator()(const std::vector &symbols) { + for (auto &s : symbols) { + input_symbols.push_back(s.GetHandle()); + } + return *this; + } + /*! + * \brief create a Symbol from the current operator + * \param name the name of the operator + * \return the operator Symbol + */ + Symbol CreateSymbol(const std::string &name = ""); + + /*! + * \brief add an input ndarray + * \param name name of the input ndarray + * \param ndarray the input ndarray + * \return reference of self + */ + Operator &SetInput(const std::string &name, NDArray ndarray); + /*! + * \brief add an input ndarray + * \param ndarray the input ndarray + */ + template + void PushInput(const NDArray &ndarray) { + input_ndarrays.push_back(ndarray.GetHandle()); + } + /*! + * \brief add positional inputs + */ + template + void PushInput(const T &t, Args... args) { + SetParam(N, t); + PushInput(args...); + } + /*! + * \brief add the last positional input + */ + template + void PushInput(const T &t) { + SetParam(N, t); + } + /*! + * \brief add input ndarrays + * \param ndarray the input ndarray + * \return reference of self + */ + Operator &operator()(const NDArray &ndarray) { + input_ndarrays.push_back(ndarray.GetHandle()); + return *this; + } + /*! + * \brief add a list of input ndarrays + * \param ndarrays the vector of the input ndarrays + * \return reference of self + */ + Operator &operator()(const std::vector &ndarrays) { + for (auto &s : ndarrays) { + input_ndarrays.push_back(s.GetHandle()); + } + return *this; + } + /*! + * \brief add input ndarrays + * \return reference of self + */ + template + Operator &operator()(Args... args) { + PushInput(args...); + return *this; + } + std::vector Invoke(); + void Invoke(NDArray &output); + void Invoke(std::vector &outputs); + + private: + std::map params_desc_; + bool variable_params_ = false; + std::map params_; + std::vector input_symbols; + std::vector input_ndarrays; + std::vector input_keys; + std::vector arg_names_; + AtomicSymbolCreator handle_; + static OpMap*& op_map(); +}; +} // namespace cpp +} // namespace mxnet + +#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_OPERATOR_H_ diff --git a/cpp-package/include/mxnet-cpp/operator.hpp b/cpp-package/include/mxnet-cpp/operator.hpp new file mode 100644 index 000000000000..b979b7c56d73 --- /dev/null +++ b/cpp-package/include/mxnet-cpp/operator.hpp @@ -0,0 +1,158 @@ +/*! +* Copyright (c) 2016 by Contributors +* \file operator.hpp +* \brief implementation of operator +* \author Chuntao Hong, Zhang Chen +*/ + +#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_OPERATOR_HPP_ +#define CPP_PACKAGE_INCLUDE_MXNET_CPP_OPERATOR_HPP_ + +#include +#include +#include +#include +#include "mxnet-cpp/base.h" +#include "mxnet-cpp/op_map.h" +#include "mxnet-cpp/operator.h" + +namespace mxnet { +namespace cpp { + +/* + * Pushing NDArray or Symbol as inputs here to avoid partial specialization + * like PushInput, which is not allowed in C++ + */ +template <> +inline Operator& Operator::SetParam(int pos, const NDArray &value) { + input_ndarrays.push_back(value.GetHandle()); + return *this; +} +template <> +inline Operator& Operator::SetParam(int pos, const Symbol &value) { + input_symbols.push_back(value.GetHandle()); + return *this; +} + +inline OpMap*& Operator::op_map() { + static OpMap *op_map_ = new OpMap(); + return op_map_; +} + +inline Operator::Operator(const std::string &operator_name) { + handle_ = op_map()->GetSymbolCreator(operator_name); + const char *name; + const char *description; + mx_uint num_args; + const char **arg_names; + const char **arg_type_infos; + const char **arg_descriptions; + const char *key_var_num_args; + MXSymbolGetAtomicSymbolInfo(handle_, + &name, + &description, + &num_args, + &arg_names, + &arg_type_infos, + &arg_descriptions, + &key_var_num_args); + for (mx_uint i = 0; i < num_args; ++i) { + arg_names_.push_back(arg_names[i]); + } +} + +inline Symbol Operator::CreateSymbol(const std::string &name) { + if (input_keys.size() > 0) { + CHECK_EQ(input_keys.size(), input_symbols.size()); + } + const char *pname = name == "" ? nullptr : name.c_str(); + + SymbolHandle symbol_handle; + std::vector input_keys; + std::vector param_keys; + std::vector param_values; + + for (auto &data : params_) { + param_keys.push_back(data.first.c_str()); + param_values.push_back(data.second.c_str()); + } + for (auto &data : this->input_keys) { + input_keys.push_back(data.c_str()); + } + const char **input_keys_p = + (input_keys.size() > 0) ? input_keys.data() : nullptr; + + MXSymbolCreateAtomicSymbol(handle_, param_keys.size(), param_keys.data(), + param_values.data(), &symbol_handle); + MXSymbolCompose(symbol_handle, pname, input_symbols.size(), input_keys_p, + input_symbols.data()); + return Symbol(symbol_handle); +} + +inline void Operator::Invoke(std::vector &outputs) { + if (input_keys.size() > 0) { + CHECK_EQ(input_keys.size(), input_ndarrays.size()); + } + + std::vector input_keys; + std::vector param_keys; + std::vector param_values; + + for (auto &data : params_) { + param_keys.push_back(data.first.c_str()); + param_values.push_back(data.second.c_str()); + } + + int num_inputs = input_ndarrays.size(); + int num_outputs = outputs.size(); + std::vector output_handles; + std::transform(outputs.begin(), outputs.end(), + std::back_inserter(output_handles), [](NDArray& a) { + return a.GetHandle(); + }); + + NDArrayHandle *outputs_receiver = nullptr; + if (num_outputs > 0) { + outputs_receiver = output_handles.data(); + } + + MXImperativeInvoke(handle_, num_inputs, input_ndarrays.data(), + &num_outputs, &outputs_receiver, + param_keys.size(), param_keys.data(), param_values.data()); + + if (outputs.size() > 0) + return; + + std::transform(outputs_receiver, outputs_receiver+num_outputs, + std::back_inserter(outputs), [](const NDArrayHandle& handle) { + return NDArray(handle); + }); +} + +inline std::vector Operator::Invoke() { + std::vector outputs; + Invoke(outputs); + return outputs; +} + +inline void Operator::Invoke(NDArray &output) { + std::vector outputs{output}; + Invoke(outputs); +} + +inline Operator &Operator::SetInput(const std::string &name, Symbol symbol) { + input_keys.push_back(name.c_str()); + input_symbols.push_back(symbol.GetHandle()); + return *this; +} + +inline Operator &Operator::SetInput(const std::string &name, NDArray ndarray) { + input_keys.push_back(name.c_str()); + input_ndarrays.push_back(ndarray.GetHandle()); + return *this; +} + +} // namespace cpp +} // namespace mxnet + +#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_OPERATOR_HPP_ diff --git a/cpp-package/include/mxnet-cpp/optimizer.h b/cpp-package/include/mxnet-cpp/optimizer.h new file mode 100644 index 000000000000..80481fd282b4 --- /dev/null +++ b/cpp-package/include/mxnet-cpp/optimizer.h @@ -0,0 +1,122 @@ +/*! +* Copyright (c) 2016 by Contributors +* \file optimizer.h +* \brief definition of optimizer +* \author Chuntao Hong, Zhang Chen +*/ + +#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_OPTIMIZER_H_ +#define CPP_PACKAGE_INCLUDE_MXNET_CPP_OPTIMIZER_H_ + +#include +#include +#include +#include +#include +#include "mxnet-cpp/base.h" +#include "dmlc/logging.h" +#include "mxnet-cpp/ndarray.h" +#include "mxnet-cpp/op_map.h" + +namespace mxnet { +namespace cpp { + +/*! +* \brief Optimizer interface +*/ +class Optimizer { + public: + /*! + * \brief get optimizer type + * \return string of optimizer type + */ + virtual std::string GetType() const = 0; + /*! + * \brief destructor + */ + virtual ~Optimizer(); + /*! + * \brief set config parameters + * \param name name of the config parameter + * \param value value of the config parameter + * \return reference of self + */ + template + Optimizer *SetParam(const std::string &name, const T &value) { + std::string value_str; + std::stringstream ss; + ss << value; + ss >> value_str; + + params_[name] = value_str; + return this; + } + /*! + * \brief Update a weight with gradient. + * \param index the unique index for the weight. + * \param weight the weight to update. + * \param grad gradient for the weight. + * \param lr learning rate. + * \param wd weight decay. + */ + void Update(int index, NDArray weight, NDArray grad, mx_float lr, + mx_float wd); + /*! + * \brief Update a weight with gradient. + * \param index the unique index for the weight. + * \param weight the weight to update. + * \param grad gradient for the weight. + */ + virtual void Update(int index, NDArray weight, NDArray grad) = 0; + // TODO(zhangcheng-qinyinghua) + // implement Update a list of arrays, maybe in the form of map + // void Update(int index, std::vector weights, std::vector + // grad, mx_float lr); + + /*! + * \brief Serialize the optimizer parameters to a string. + * \return serialization + */ + std::string Serialize() const; + + protected: + std::map params_; + static OpMap*& op_map(); + const std::vector GetParamKeys_() const; + const std::vector GetParamValues_() const; +}; + +typedef std::function OptimizerCreator; + +class OptimizerRegistry { + public: + static Optimizer* Find(const std::string& name); + static int __REGISTER__(const std::string& name, OptimizerCreator creator); + private: + static std::map& cmap(); + OptimizerRegistry() = delete; + ~OptimizerRegistry() = delete; +}; + +#define MXNETCPP_REGISTER_OPTIMIZER(Name, OptimizerType) \ + static int __make_ ## OptimizerType ## _ ## Name ## __ = \ + OptimizerRegistry::__REGISTER__(#Name, [](){return new OptimizerType();}) + +class SGDOptimizer : public Optimizer { + public: + SGDOptimizer(); + virtual std::string GetType() const; + virtual void Update(int index, NDArray weight, NDArray grad); + private: + virtual ~SGDOptimizer(); + virtual void CreateState_(int index, NDArray weight); + std::map states_; + AtomicSymbolCreator update_handle_; + AtomicSymbolCreator mom_update_handle_; +}; + + +} // namespace cpp +} // namespace mxnet + +#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_OPTIMIZER_H_ diff --git a/cpp-package/include/mxnet-cpp/optimizer.hpp b/cpp-package/include/mxnet-cpp/optimizer.hpp new file mode 100644 index 000000000000..911989de1e95 --- /dev/null +++ b/cpp-package/include/mxnet-cpp/optimizer.hpp @@ -0,0 +1,139 @@ +/*! +* Copyright (c) 2016 by Contributors +* \file optimizer.hpp +* \brief implementation of optimizer +* \author Chuntao Hong, Zhang Chen +*/ + +#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_OPTIMIZER_HPP_ +#define CPP_PACKAGE_INCLUDE_MXNET_CPP_OPTIMIZER_HPP_ + +#include +#include +#include +#include +#include +#include +#include "mxnet-cpp/optimizer.h" +#include "mxnet-cpp/op.h" +#include "mxnet-cpp/op_map.h" + +namespace mxnet { +namespace cpp { + +inline std::map& OptimizerRegistry::cmap() { + static std::map cmap_; + return cmap_; +} + +inline OpMap*& Optimizer::op_map() { + static OpMap *op_map_ = new OpMap(); + return op_map_; +} + +inline Optimizer::~Optimizer() {} + +inline void Optimizer::Update(int index, NDArray weight, NDArray grad, mx_float lr, + mx_float wd) { + params_["lr"] = std::to_string(lr); + params_["wd"] = std::to_string(wd); + Update(index, weight, grad); +} + +inline std::string Optimizer::Serialize() const { + using ValueType = std::map::value_type; + auto params = params_; + params.emplace("opt_type", GetType()); + return std::accumulate(params.cbegin(), params.cend(), std::string(""), + [](const std::string& sum, const ValueType& i) { + return sum + '\n' + i.first + '=' + i.second; + }).substr(1); +} + +inline const std::vector Optimizer::GetParamKeys_() const { + std::vector keys; + for (auto& iter : params_) + keys.push_back(iter.first.c_str()); + return keys; +} + +inline const std::vector Optimizer::GetParamValues_() const { + std::vector values; + for (auto& iter : params_) + values.push_back(iter.second.c_str()); + return values; +} + +inline Optimizer* OptimizerRegistry::Find(const std::string& name) { + MXNETCPP_REGISTER_OPTIMIZER(sgd, SGDOptimizer); + MXNETCPP_REGISTER_OPTIMIZER(ccsgd, SGDOptimizer); // For backward compatibility + auto it = cmap().find(name); + if (it == cmap().end()) + return nullptr; + return it->second(); +} + +inline int OptimizerRegistry::__REGISTER__(const std::string& name, OptimizerCreator creator) { + CHECK_EQ(cmap().count(name), 0) << name << " already registered"; + cmap().emplace(name, std::move(creator)); + return 0; +} + +inline std::string SGDOptimizer::GetType() const { + return "sgd"; +} + +inline SGDOptimizer::SGDOptimizer() { + update_handle_ = op_map()->GetSymbolCreator("sgd_update"); + mom_update_handle_ = op_map()->GetSymbolCreator("sgd_mom_update"); +} + +inline SGDOptimizer::~SGDOptimizer() { + for (auto &it : states_) { + delete it.second; + } +} + +inline void SGDOptimizer::Update(int index, NDArray weight, NDArray grad) { + if (states_.count(index) == 0) { + CreateState_(index, weight); + } + + auto keys = GetParamKeys_(); + auto values = GetParamValues_(); + CHECK_EQ(keys.size(), values.size()); + + NDArrayHandle inputs[3]; + inputs[0] = weight.GetHandle(); + inputs[1] = grad.GetHandle(); + + int num_outputs = 1; + NDArrayHandle output = weight.GetHandle(); + NDArrayHandle *outputs = &output; + + if (states_[index] == nullptr) { + MXImperativeInvoke(update_handle_, 2, inputs, + &num_outputs, &outputs, + keys.size(), keys.data(), values.data()); + } else { + inputs[2] = states_[index]->GetHandle(); + MXImperativeInvoke(mom_update_handle_, 3, inputs, + &num_outputs, &outputs, + keys.size(), keys.data(), values.data()); + } +} + +inline void SGDOptimizer::CreateState_(int index, NDArray weight) { + if (params_.count("momentum") == 0) { + states_[index] = nullptr; + } else { + states_[index] = new NDArray(weight.GetShape(), weight.GetContext()); + *states_[index] = 0; + } +} + + +} // namespace cpp +} // namespace mxnet + +#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_OPTIMIZER_HPP_ diff --git a/cpp-package/include/mxnet-cpp/shape.h b/cpp-package/include/mxnet-cpp/shape.h new file mode 100644 index 000000000000..d8e3f2c95282 --- /dev/null +++ b/cpp-package/include/mxnet-cpp/shape.h @@ -0,0 +1,389 @@ +/*! +* Copyright (c) 2016 by Contributors +* \file shape.h +* \brief definition of shape +* \author Chuntao Hong, Zhang Chen +*/ + +#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_SHAPE_H_ +#define CPP_PACKAGE_INCLUDE_MXNET_CPP_SHAPE_H_ + +#include +#include +#include +#include +#include "mxnet-cpp/base.h" + +namespace mxnet { +namespace cpp { + +/*! +* \brief dynamic shape class that can hold shape +* of arbirary dimension +*/ +struct Shape { + public: + /*! \brief constructor */ + Shape() + : ndim_(0), + num_heap_allocated_(0), + data_heap_(NULL) {} + /*! + * \brief constructor from a vector of index_t + * \param v the vector + */ + explicit Shape(const std::vector &v) + : ndim_(v.size()) { + if (ndim_ <= kStackCache) { + data_heap_ = NULL; + num_heap_allocated_ = 0; + std::copy(v.begin(), v.end(), data_stack_); + } else { + data_heap_ = new index_t[ndim_]; + num_heap_allocated_ = ndim_; + std::copy(v.begin(), v.end(), data_heap_); + } + } + /*! + * \brief constructor one dimmension shape + * \param s1 size of the first dimmension + */ + explicit Shape(index_t s1) + : ndim_(1) { + if (ndim_ <= kStackCache) { + data_heap_ = NULL; + num_heap_allocated_ = 0; + data_stack_[0] = s1; + } else { + data_heap_ = new index_t[ndim_]; + num_heap_allocated_ = ndim_; + data_heap_[0] = s1; + } + } + /*! + * \brief constructor two dimmension shape + * \param s1 size of the first dimmension + * \param s2 size of the second dimmension + */ + Shape(index_t s1, index_t s2) + : ndim_(2) { + if (ndim_ <= kStackCache) { + data_heap_ = NULL; + num_heap_allocated_ = 0; + data_stack_[0] = s1; + data_stack_[1] = s2; + } else { + data_heap_ = new index_t[ndim_]; + num_heap_allocated_ = ndim_; + data_heap_[0] = s1; + data_heap_[1] = s2; + } + } + /*! + * \brief constructor three dimmension shape + * \param s1 size of the first dimmension + * \param s2 size of the second dimmension + * \param s3 size of the third dimmension + */ + Shape(index_t s1, index_t s2, index_t s3) + : ndim_(3) { + if (ndim_ <= kStackCache) { + data_heap_ = NULL; + num_heap_allocated_ = 0; + data_stack_[0] = s1; + data_stack_[1] = s2; + data_stack_[2] = s3; + } else { + data_heap_ = new index_t[ndim_]; + num_heap_allocated_ = ndim_; + data_heap_[0] = s1; + data_heap_[1] = s2; + data_heap_[2] = s3; + } + } + /*! + * \brief constructor four dimmension shape + * \param s1 size of the first dimmension + * \param s2 size of the second dimmension + * \param s3 size of the third dimmension + * \param s4 size of the fourth dimmension + */ + Shape(index_t s1, index_t s2, index_t s3, index_t s4) + : ndim_(4) { + if (ndim_ <= kStackCache) { + data_heap_ = NULL; + num_heap_allocated_ = 0; + data_stack_[0] = s1; + data_stack_[1] = s2; + data_stack_[2] = s3; + data_stack_[3] = s4; + } else { + data_heap_ = new index_t[ndim_]; + num_heap_allocated_ = ndim_; + data_heap_[0] = s1; + data_heap_[1] = s2; + data_heap_[2] = s3; + data_heap_[3] = s4; + } + } + /*! + * \brief constructor five dimmension shape + * \param s1 size of the first dimmension + * \param s2 size of the second dimmension + * \param s3 size of the third dimmension + * \param s4 size of the fourth dimmension + * \param s5 size of the fifth dimmension + */ + Shape(index_t s1, index_t s2, index_t s3, index_t s4, index_t s5) + : ndim_(5) { + if (ndim_ <= kStackCache) { + data_heap_ = NULL; + num_heap_allocated_ = 0; + data_stack_[0] = s1; + data_stack_[1] = s2; + data_stack_[2] = s3; + data_stack_[3] = s4; + data_stack_[4] = s5; + } else { + data_heap_ = new index_t[ndim_]; + num_heap_allocated_ = ndim_; + data_heap_[0] = s1; + data_heap_[1] = s2; + data_heap_[2] = s3; + data_heap_[3] = s4; + data_heap_[5] = s5; + } + } + /*! + * \brief constructor from Shape + * \param s the source shape + */ + Shape(const Shape &s) + : ndim_(s.ndim_) { + if (ndim_ <= kStackCache) { + data_heap_ = NULL; + num_heap_allocated_ = 0; + std::copy(s.data_stack_, s.data_stack_ + ndim_, data_stack_); + } else { + data_heap_ = new index_t[ndim_]; + num_heap_allocated_ = ndim_; + std::copy(s.data_heap_, s.data_heap_ + ndim_, data_heap_); + } + } +#if MSHADOW_IN_CXX11 + /*! + * \brief move constructor from Shape + * \param s the source shape + */ + Shape(Shape &&s) + : ndim_(s.ndim_), + num_heap_allocated_(s.num_heap_allocated_), + data_heap_(s.data_heap_) { + if (ndim_ <= kStackCache) { + std::copy(s.data_stack_, s.data_stack_ + ndim_, data_stack_); + } + // remove data heap space from s + s.data_heap_ = NULL; + } +#endif + /*! \brief destructor */ + ~Shape() { + // data_heap_ can be NULL + delete[] data_heap_; + } + /*! + * \brief copy shape from content betwen two iterators + * \param begin the beginning of iterator + * \param end the end of the iterator + * \tparam RandomAccessIterator iterator type + */ + template + inline void CopyFrom(RandomAccessIterator begin, + RandomAccessIterator end) { + this->SetDim(end - begin); + std::copy(begin, end, data()); + } + /*! + * \brief assignment from shape + * \param shape source shape + * \return reference of self + */ + inline Shape &operator=(const Shape &shape) { + this->SetDim(shape.ndim_); + const index_t *src = shape.data(); + std::copy(src, src + ndim_, data()); + return *this; + } + /*! + * \brief assignment from vector + * \param shape source shape + * \return reference of self + */ + inline Shape &operator=(const std::vector &shape) { + this->CopyFrom(shape.begin(), shape.end()); + return *this; + } + /*! \return the data content of the shape */ + inline const index_t *data() const { + return ndim_ <= kStackCache ? data_stack_ : data_heap_; + } + /*! \return the data content of the shape */ + inline index_t *data() { + return ndim_ <= kStackCache ? data_stack_ : data_heap_; + } + /*! \brief return number of dimension of the tensor inside */ + inline index_t ndim(void) const { + return ndim_; + } + /*! + * \brief get corresponding index + * \param i dimension index + * \return the corresponding dimension size + */ + inline index_t &operator[](index_t i) { + return data()[i]; + } + /*! + * \brief get corresponding index + * \param i dimension index + * \return the corresponding dimension size + */ + inline const index_t &operator[](index_t i) const { + return data()[i]; + } + /*! \brief total number of elements in the tensor */ + inline size_t Size(void) const { + size_t size = 1; + const index_t *d = this->data(); + for (index_t i = 0; i < ndim_; ++i) { + size *= d[i]; + } + return size; + } + /*! + * \return whether two shape equals + * \param s the shape to compare against + */ + inline bool operator==(const Shape &s) const { + if (ndim_ != s.ndim_) return false; + if (ndim_ <= kStackCache) { + for (index_t i = 0; i < ndim_; ++i) { + if (data_stack_[i] != s.data_stack_[i]) return false; + } + } else { + for (index_t i = 0; i < ndim_; ++i) { + if (data_heap_[i] != s.data_heap_[i]) return false; + } + } + return true; + } + /*! + * \return whether two shape not equals + * \param s the shape to compare against + */ + inline bool operator!=(const Shape &s) const { + return !(*this == s); + } + + friend std::ostream &operator<<(std::ostream &os, const Shape &shape); + friend std::istream &operator>>(std::istream &is, Shape &shape); + + private: + // the shape will be stored in data_stack_ + // when dimension is smaller than kStackCache + // when it is bigger, it will be stored in data_heap_; + /*! \brief size of in stack space */ + static const index_t kStackCache = 5; + /*! \brief number of dimnsion of the shape */ + index_t ndim_; + /*! \brief number of cells allocated in data_heap_ */ + index_t num_heap_allocated_; + /*! \brief in stack space used to store shape when it is small */ + index_t data_stack_[kStackCache]; + /*! \brief space to store shape when dimension is big*/ + index_t *data_heap_; + /*! + * \brief internal function to set the dimension + * \param dim the dimension of the shape + */ + inline void SetDim(index_t dim) { + if (dim > kStackCache && + dim > num_heap_allocated_) { + // data_heap_ can be NULL + delete[] data_heap_; + data_heap_ = new index_t[dim]; + num_heap_allocated_ = dim; + } + ndim_ = dim; + } +}; + +/*! +* \brief allow string printing of the shape +* \param os the output stream +* \param shape the shape +* \return the ostream +*/ +inline std::ostream &operator<<(std::ostream &os, const Shape &shape) { + os << '('; + for (index_t i = 0; i < shape.ndim(); ++i) { + if (i != 0) os << ','; + os << static_cast(shape[i]); // Supports negative Shape 'special codes' for inferring + } + // python style tuple + if (shape.ndim() == 1) os << ','; + os << ')'; + return os; +} + +/*! +* \brief read shape from the istream +* \param is the input stream +* \param shape the shape +* \return the istream +*/ +inline std::istream &operator>>(std::istream &is, Shape &shape) { + // get ( + while (true) { + char ch = is.get(); + if (ch == '(') break; + if (!isspace(ch)) { + is.setstate(std::ios::failbit); + return is; + } + } + index_t idx; + std::vector tmp; + while (is >> idx) { + tmp.push_back(idx); + char ch; + do { + ch = is.get(); + } while (isspace(ch)); + if (ch == ',') { + while (true) { + ch = is.peek(); + if (isspace(ch)) { + is.get(); continue; + } + if (ch == ')') { + is.get(); break; + } + break; + } + if (ch == ')') break; + } else if (ch == ')') { + break; + } else { + is.setstate(std::ios::failbit); + return is; + } + } + shape.CopyFrom(tmp.begin(), tmp.end()); + return is; +} + +} // namespace cpp +} // namespace mxnet + +#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_SHAPE_H_ diff --git a/cpp-package/include/mxnet-cpp/symbol.h b/cpp-package/include/mxnet-cpp/symbol.h new file mode 100644 index 000000000000..03a8409f8087 --- /dev/null +++ b/cpp-package/include/mxnet-cpp/symbol.h @@ -0,0 +1,257 @@ +/*! +* Copyright (c) 2016 by Contributors +* \file symbol.h +* \brief definition of symbol +* \author Chuntao Hong, Zhang Chen +*/ + +#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_SYMBOL_H_ +#define CPP_PACKAGE_INCLUDE_MXNET_CPP_SYMBOL_H_ + +#include +#include +#include +#include "mxnet-cpp/base.h" +#include "mxnet-cpp/ndarray.h" +#include "mxnet-cpp/op_map.h" + +namespace mxnet { +namespace cpp { + +class Executor; + +/*! +* \brief struct to store SymbolHandle +*/ +struct SymBlob { + public: + /*! + * \brief default constructor + */ + SymBlob() : handle_(nullptr) {} + /*! + * \brief construct with SymbolHandle to store + */ + explicit SymBlob(SymbolHandle handle) : handle_(handle) {} + /*! + * \brief destructor, free the SymbolHandle + */ + ~SymBlob() { MXSymbolFree(handle_); } + /*! + * \brief the SymbolHandle to store + */ + SymbolHandle handle_; + + private: + SymBlob(const SymBlob &); + SymBlob &operator=(const SymBlob &); +}; + +/*! +* \brief Symbol interface +*/ +class Symbol { + public: + Symbol() {} + /*! + * \brief construct a Symbol with SymbolHandle + * \param handle the given SymbolHandle + */ + explicit Symbol(SymbolHandle handle); + /*! + * \brief construct a variable Symbol + * \param name the name of the variable + */ + explicit Symbol(const char *name); + /*! + * \brief construct a variable Symbol + * \param name the name of the variable + */ + explicit Symbol(const std::string &name); + Symbol operator+(const Symbol &rhs) const; + Symbol operator-(const Symbol &rhs) const; + Symbol operator*(const Symbol &rhs) const; + Symbol operator/(const Symbol &rhs) const; + + Symbol operator+(mx_float scalar) const; + Symbol operator-(mx_float scalar) const; + Symbol operator*(mx_float scalar) const; + Symbol operator/(mx_float scalar) const; + Symbol Copy() const; + /*! + * \brief construct a variable Symbol + * \param name the name of the variable + */ + static Symbol Variable(const std::string &name = ""); + Symbol operator[](int index); + Symbol operator[](const std::string &index); + /*! + * \brief Create a symbol that groups symbols together + * \param symbols List of symbols to be groupe + */ + static Symbol Group(const std::vector &symbols); + /*! + * \brief load Symbol from a JSON file + * \param file_name the name of the file + */ + static Symbol Load(const std::string &file_name); + /*! + * \brief load Symbol from a JSON string + * \param json_str the JSON string + */ + static Symbol LoadJSON(const std::string &json_str); + /*! + * \brief save Symbol to a file + * \param file_name the name of the file + */ + void Save(const std::string &file_name) const; + /*! + * \brief save Symbol into a JSON string + */ + std::string ToJSON() const; + /*! + * \brief save Symbol into a JSON string + * \retutrn the symbol whose outputs are all the internals. + */ + Symbol GetInternals() const; + /*! + * \return the SymbolHandle + */ + SymbolHandle GetHandle() const { return blob_ptr_->handle_; } + /*! + * \brief construct an operator Symbol, with given input Symbol and config + * \param name the name of the Symbol + * \param input_keys the vector of keys of the input + * \param input_values the vector of the intput Symbols + * \param config_keys the vector of keys of the config + * \param config_values the vecotr of values of the config + */ + Symbol(const std::string &operator_name, const std::string &name, + std::vector input_keys, + std::vector input_values, + std::vector config_keys, + std::vector config_values); + /*! + * \brief infer the shapes by providing shapes of known argument shapes. + * \param arg_shapes map of argument name to shape of arguments with known + * shapes. + * \param in_shapes used to store infered shapes of input arguments. + * \param out_shapes used to store infered shapes of outputs. + * \param aux_shapes use to store the infered shapes of auxiliary states + */ + void InferShape( + const std::map > &arg_shapes, + std::vector > *in_shape, + std::vector > *aux_shape, + std::vector > *out_shape) const; + /*! + * \brief List the arguments names. + * + * The position of the returned list also corresponds to calling position in + *operator() + * \return the arguments list of this symbol, they can be either named or + *unnamed (empty string). + */ + std::vector ListArguments() const; + /*! \return get the descriptions of outputs for this symbol */ + std::vector ListOutputs() const; + /*! \return get the descriptions of auxiliary data for this symbol */ + std::vector ListAuxiliaryStates() const; + /*! + * \brief infer and construct all the arrays to bind to executor by providing + * some known arrays. + * \param context the context of all the infered arrays + * \param arg_arrays infered input arguments arrays. + * \param arad_arrays infered arrays to store the gradient output of the input + * arguments. + * \param aux_arrays infered arrays that is used as internal state in op. + * \param args_map map of some given arguments arrays. + * \param args_grad_store map of some gradient given store arrays. + * \param args_req_type map of some given type of gradient saving. Can only be + * in {kNullOp, kAddTo, kWriteTo}. + * \param aux_map NDArray that stores the internal state in op + */ + void InferExecutorArrays( + const Context &context, std::vector *arg_arrays, + std::vector *grad_arrays, std::vector *grad_reqs, + std::vector *aux_arrays, + const std::map &args_map, + const std::map &arg_grad_store = + std::map(), + const std::map &grad_req_type = + std::map(), + const std::map &aux_map = + std::map()) const; + /*! + * \brief infer and construct all the input arguments arrays to bind to + * executor by providing some known arguments arrays. + * \param context the context of all the infered arrays. + * \param args_map map of all the infered input arguments arrays. + * \param known_args map of some given arguments arrays. + */ + void InferArgsMap(const Context &context, + std::map *args_map, + const std::map &known_args) const; + /*! + * \brief Create an executor by bind symbol with context and arguments. + * If user do not want to compute the gradients of i-th argument, + *grad_req_type[i] can be kNullOp. + * The input arrays in the given maps should have the same name with the input + *symbol. + * Only need some of the necessary arrays, and the other arrays can be infered + *automatically. + * + * \param context the context of binding. + * \param args_map the NDArray that stores the input arguments to the symbol. + * \param arg_grad_store NDArray that is used to store the gradient output of + *the input arguments. + * \param grad_req_type requirment type of gradient saving. Can only be in + *{kNullOp, kAddTo, kWriteTo}. + * \param aux_map NDArray that stores the internal state in op + * \return a new executor, which need to be free manually. + */ + Executor *SimpleBind(const Context &context, + const std::map &args_map, + const std::map &arg_grad_store = + std::map(), + const std::map &grad_req_type = + std::map(), + const std::map &aux_map = + std::map()); + /*! + * \brief Create an executor by bind symbol with context and arguments. + * If user do not want to compute the gradients of i-th argument, + *grad_req_type[i] can be kNullOp. + * + * \param context the context of binding. + * \param arg_arrays the NDArray that stores the input arguments to the symbol. + * \param grad_arrays NDArray that is used to store the gradient output of the + *input arguments. + * \param grad_reqs requirment type of gradient saving. Can only be in + *{kNullOp, kAddTo, kWriteTo}. + * \param aux_arrays NDArray that is used as internal state in op + * \param group_to_ctx dict of string to mx.Context + * \param shared_exec Executor to share memory with. This is intended for + *runtime reshaping, variable length sequencesn etc. The returned executor + *shares state with shared_exec, and should not be used in parallel with it. + * \return a new executor, which need to be free manually. + */ + Executor *Bind(const Context &context, const std::vector &arg_arrays, + const std::vector &grad_arrays, + const std::vector &grad_reqs, + const std::vector &aux_arrays, + const std::map &group_to_ctx = + std::map(), + Executor *shared_exec = nullptr); + + private: + std::shared_ptr blob_ptr_; + static OpMap*& op_map(); +}; +Symbol operator+(mx_float lhs, const Symbol &rhs); +Symbol operator-(mx_float lhs, const Symbol &rhs); +Symbol operator*(mx_float lhs, const Symbol &rhs); +Symbol operator/(mx_float lhs, const Symbol &rhs); +} // namespace cpp +} // namespace mxnet +#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_SYMBOL_H_ diff --git a/cpp-package/include/mxnet-cpp/symbol.hpp b/cpp-package/include/mxnet-cpp/symbol.hpp new file mode 100644 index 000000000000..a2ab9cb87f30 --- /dev/null +++ b/cpp-package/include/mxnet-cpp/symbol.hpp @@ -0,0 +1,342 @@ +/*! + * Copyright (c) 2016 by Contributors + * \file symbol.hpp + * \brief implementation of the symbol + * \author Zhang Chen, Chuntao Hong + */ + +#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_SYMBOL_HPP_ +#define CPP_PACKAGE_INCLUDE_MXNET_CPP_SYMBOL_HPP_ + +#include +#include +#include +#include + +#include "dmlc/logging.h" +#include "mxnet-cpp/symbol.h" + +#include "mxnet-cpp/op_suppl.h" + +namespace mxnet { +namespace cpp { +inline OpMap*& Symbol::op_map() { + static OpMap* op_map_ = new OpMap(); + return op_map_; +} +inline Symbol::Symbol(SymbolHandle handle) { + blob_ptr_ = std::make_shared(handle); +} +inline Symbol::Symbol(const char *name) { + SymbolHandle handle; + CHECK_EQ(MXSymbolCreateVariable(name, &(handle)), 0); + blob_ptr_ = std::make_shared(handle); +} +inline Symbol::Symbol(const std::string &name) : Symbol(name.c_str()) {} +inline Symbol Symbol::Variable(const std::string &name) { return Symbol(name); } +inline Symbol Symbol::operator+(const Symbol &rhs) const { return _Plus(*this, rhs); } +inline Symbol Symbol::operator-(const Symbol &rhs) const { return _Minus(*this, rhs); } +inline Symbol Symbol::operator*(const Symbol &rhs) const { return _Mul(*this, rhs); } +inline Symbol Symbol::operator/(const Symbol &rhs) const { return _Div(*this, rhs); } +inline Symbol Symbol::operator+(mx_float scalar) const { + return _PlusScalar(*this, scalar); +} +inline Symbol Symbol::operator-(mx_float scalar) const { + return _MinusScalar(*this, scalar); +} +inline Symbol Symbol::operator*(mx_float scalar) const { + return _MulScalar(*this, scalar); +} +inline Symbol Symbol::operator/(mx_float scalar) const { + return _DivScalar(*this, scalar); +} +inline Symbol Symbol::operator[](int index) { + SymbolHandle out; + MXSymbolGetOutput(GetHandle(), index, &out); + return Symbol(out); +} +inline Symbol Symbol::operator[](const std::string &index) { + auto outputs = ListOutputs(); + for (mx_uint i = 0; i < outputs.size(); ++i) { + if (outputs[i] == index) { + return (*this)[i]; + } + } + LOG(FATAL) << "Cannot find output that matches name " << index; + return (*this)[0]; +} +inline Symbol Symbol::Group(const std::vector &symbols) { + SymbolHandle out; + std::vector handle_list; + for (const auto &t : symbols) { + handle_list.push_back(t.GetHandle()); + } + MXSymbolCreateGroup(handle_list.size(), handle_list.data(), &out); + return Symbol(out); +} +inline Symbol Symbol::Load(const std::string &file_name) { + SymbolHandle handle; + CHECK_EQ(MXSymbolCreateFromFile(file_name.c_str(), &(handle)), 0); + return Symbol(handle); +} +inline Symbol Symbol::LoadJSON(const std::string &json_str) { + SymbolHandle handle; + CHECK_EQ(MXSymbolCreateFromJSON(json_str.c_str(), &(handle)), 0); + return Symbol(handle); +} +inline void Symbol::Save(const std::string &file_name) const { + CHECK_EQ(MXSymbolSaveToFile(GetHandle(), file_name.c_str()), 0); +} +inline std::string Symbol::ToJSON() const { + const char *out_json; + CHECK_EQ(MXSymbolSaveToJSON(GetHandle(), &out_json), 0); + return std::string(out_json); +} +inline Symbol Symbol::GetInternals() const { + SymbolHandle handle; + CHECK_EQ(MXSymbolGetInternals(GetHandle(), &handle), 0); + return Symbol(handle); +} +inline Symbol::Symbol(const std::string &operator_name, const std::string &name, + std::vector input_keys, + std::vector input_values, + std::vector config_keys, + std::vector config_values) { + SymbolHandle handle; + AtomicSymbolCreator creator = op_map()->GetSymbolCreator(operator_name); + MXSymbolCreateAtomicSymbol(creator, config_keys.size(), config_keys.data(), + config_values.data(), &handle); + MXSymbolCompose(handle, operator_name.c_str(), input_keys.size(), + input_keys.data(), input_values.data()); + blob_ptr_ = std::make_shared(handle); +} + +inline Symbol Symbol::Copy() const { + SymbolHandle handle; + CHECK_EQ(MXSymbolCopy(GetHandle(), &handle), 0); + return Symbol(handle); +} + +inline std::vector Symbol::ListArguments() const { + std::vector ret; + mx_uint size; + const char **sarr; + MXSymbolListArguments(GetHandle(), &size, &sarr); + for (mx_uint i = 0; i < size; ++i) { + ret.push_back(std::string(sarr[i])); + } + return ret; +} +inline std::vector Symbol::ListOutputs() const { + std::vector ret; + mx_uint size; + const char **sarr; + MXSymbolListOutputs(GetHandle(), &size, &sarr); + for (mx_uint i = 0; i < size; ++i) { + ret.push_back(std::string(sarr[i])); + } + return ret; +} +inline std::vector Symbol::ListAuxiliaryStates() const { + std::vector ret; + mx_uint size; + const char **sarr; + MXSymbolListAuxiliaryStates(GetHandle(), &size, &sarr); + for (mx_uint i = 0; i < size; ++i) { + ret.push_back(std::string(sarr[i])); + } + return ret; +} + +inline void Symbol::InferShape( + const std::map > &arg_shapes, + std::vector > *in_shape, + std::vector > *aux_shape, + std::vector > *out_shape) const { + + std::vector keys; + std::vector arg_ind_ptr; + std::vector arg_shape_data; + + for (const auto &arg : arg_shapes) { + keys.push_back(arg.first.c_str()); + arg_ind_ptr.push_back(arg_shape_data.size()); + for (auto i : arg.second) { + arg_shape_data.push_back(i); + } + } + arg_ind_ptr.push_back(arg_shape_data.size()); + + mx_uint in_shape_size; + const mx_uint *in_shape_ndim; + const mx_uint **in_shape_data; + mx_uint out_shape_size; + const mx_uint *out_shape_ndim; + const mx_uint **out_shape_data; + mx_uint aux_shape_size; + const mx_uint *aux_shape_ndim; + const mx_uint **aux_shape_data; + int complete; + + CHECK_EQ(MXSymbolInferShape(GetHandle(), keys.size(), keys.data(), + arg_ind_ptr.data(), arg_shape_data.data(), + &in_shape_size, &in_shape_ndim, &in_shape_data, + &out_shape_size, &out_shape_ndim, &out_shape_data, + &aux_shape_size, &aux_shape_ndim, &aux_shape_data, + &complete), + 0); + + if (complete) { + for (mx_uint i = 0; i < in_shape_size; ++i) { + in_shape->push_back(std::vector()); + for (mx_uint j = 0; j < in_shape_ndim[i]; ++j) { + (*in_shape)[i].push_back(in_shape_data[i][j]); + } + } + for (mx_uint i = 0; i < aux_shape_size; ++i) { + aux_shape->push_back(std::vector()); + for (mx_uint j = 0; j < aux_shape_ndim[i]; ++j) { + (*aux_shape)[i].push_back(aux_shape_data[i][j]); + } + } + for (mx_uint i = 0; i < out_shape_size; ++i) { + out_shape->push_back(std::vector()); + for (mx_uint j = 0; j < out_shape_ndim[i]; ++j) { + (*out_shape)[i].push_back(out_shape_data[i][j]); + } + } + } +} + +inline void Symbol::InferExecutorArrays( + const Context &context, std::vector *arg_arrays, + std::vector *grad_arrays, std::vector *grad_reqs, + std::vector *aux_arrays, + const std::map &args_map, + const std::map &arg_grad_store, + const std::map &grad_req_type, + const std::map &aux_map) const { + + const auto arg_name_list = ListArguments(); + std::vector > in_shapes, aux_shapes, out_shapes; + std::map > arg_shapes; + + for (const auto &arg_name : arg_name_list) { + auto iter = args_map.find(arg_name); + if (iter != args_map.end()) { + arg_shapes[arg_name] = iter->second.GetShape(); + } + } + + InferShape(arg_shapes, &in_shapes, &aux_shapes, &out_shapes); + + for (size_t i = 0; i < in_shapes.size(); ++i) { + const auto &shape = in_shapes[i]; + const auto &arg_name = arg_name_list[i]; + auto iter_arg = args_map.find(arg_name); + if (iter_arg != args_map.end()) { + arg_arrays->push_back(iter_arg->second); + } else { + arg_arrays->push_back(NDArray(shape, context, false)); + NDArray::SampleGaussian(0, 1, &arg_arrays->back()); + } + auto iter_grad = arg_grad_store.find(arg_name); + if (iter_grad != arg_grad_store.end()) { + grad_arrays->push_back(iter_grad->second); + } else { + grad_arrays->push_back(NDArray(shape, context, false)); + } + auto iter_req = grad_req_type.find(arg_name); + if (iter_req != grad_req_type.end()) { + grad_reqs->push_back(iter_req->second); + } else if (arg_name.rfind("data") == arg_name.length() - 4 + || arg_name.rfind("label") == arg_name.length() - 5) { + grad_reqs->push_back(OpReqType::kNullOp); + } else { + grad_reqs->push_back(OpReqType::kWriteTo); + } + } + + const auto aux_name_list = ListAuxiliaryStates(); + for (size_t i = 0; i < aux_shapes.size(); ++i) { + const auto &shape = aux_shapes[i]; + const auto &aux_name = aux_name_list[i]; + auto iter_aux = aux_map.find(aux_name); + if (iter_aux != aux_map.end()) { + aux_arrays->push_back(iter_aux->second); + } else { + aux_arrays->push_back(NDArray(shape, context, false)); + NDArray::SampleGaussian(0, 1, &aux_arrays->back()); + } + } +} +inline void Symbol::InferArgsMap( + const Context &context, std::map *args_map, + const std::map &known_args) const { + + const auto arg_name_list = ListArguments(); + std::vector > in_shapes, aux_shapes, out_shapes; + std::map > arg_shapes; + + for (const auto &arg_name : arg_name_list) { + auto iter = known_args.find(arg_name); + if (iter != known_args.end()) { + arg_shapes[arg_name] = iter->second.GetShape(); + } + } + + InferShape(arg_shapes, &in_shapes, &aux_shapes, &out_shapes); + + for (size_t i = 0; i < in_shapes.size(); ++i) { + const auto &shape = in_shapes[i]; + const auto &arg_name = arg_name_list[i]; + auto iter_arg = known_args.find(arg_name); + if (iter_arg != known_args.end()) { + (*args_map)[arg_name] = iter_arg->second; + } else { + (*args_map)[arg_name] = NDArray(shape, context, false); + NDArray::SampleGaussian(0, 1, &(*args_map)[arg_name]); + } + } +} + +inline Executor *Symbol::SimpleBind( + const Context &context, const std::map &args_map, + const std::map &arg_grad_store, + const std::map &grad_req_type, + const std::map &aux_map) { + std::vector arg_arrays; + std::vector grad_arrays; + std::vector grad_reqs; + std::vector aux_arrays; + + InferExecutorArrays(context, &arg_arrays, &grad_arrays, &grad_reqs, + &aux_arrays, args_map, arg_grad_store, grad_req_type, + aux_map); + + return new Executor(*this, context, arg_arrays, grad_arrays, grad_reqs, + aux_arrays); +} + +inline Executor *Symbol::Bind(const Context &context, + const std::vector &arg_arrays, + const std::vector &grad_arrays, + const std::vector &grad_reqs, + const std::vector &aux_arrays, + const std::map &group_to_ctx, + Executor *shared_exec) { + return new Executor(*this, context, arg_arrays, grad_arrays, grad_reqs, + aux_arrays, group_to_ctx, shared_exec); +} +inline Symbol operator+(mx_float lhs, const Symbol &rhs) { return rhs + lhs; } +inline Symbol operator-(mx_float lhs, const Symbol &rhs) { + return mxnet::cpp::_RMinusScalar(lhs, rhs); +} +inline Symbol operator*(mx_float lhs, const Symbol &rhs) { return rhs * lhs; } +inline Symbol operator/(mx_float lhs, const Symbol &rhs) { + return mxnet::cpp::_RDivScalar(lhs, rhs); +} +} // namespace cpp +} // namespace mxnet + +#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_SYMBOL_HPP_ diff --git a/cpp-package/scripts/lint.py b/cpp-package/scripts/lint.py new file mode 100644 index 000000000000..89492eda4d82 --- /dev/null +++ b/cpp-package/scripts/lint.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python +# pylint: disable=protected-access, unused-variable, locally-disabled, redefined-variable-type +"""Lint helper to generate lint summary of source. +Copyright by Contributors +""" +import codecs +import sys +import re +import os +import cpplint +from cpplint import _cpplint_state +from pylint import epylint + +CXX_SUFFIX = set(['cc', 'c', 'cpp', 'h', 'cu', 'hpp']) +PYTHON_SUFFIX = set(['py']) + +class LintHelper(object): + """Class to help runing the lint and records summary""" + + @staticmethod + def _print_summary_map(strm, result_map, ftype): + """Print summary of certain result map.""" + if len(result_map) == 0: + return 0 + npass = len([x for k, x in result_map.iteritems() if len(x) == 0]) + strm.write('=====%d/%d %s files passed check=====\n' % (npass, len(result_map), ftype)) + for fname, emap in result_map.iteritems(): + if len(emap) == 0: + continue + strm.write('%s: %d Errors of %d Categories map=%s\n' % ( + fname, sum(emap.values()), len(emap), str(emap))) + return len(result_map) - npass + + def __init__(self): + self.project_name = None + self.cpp_header_map = {} + self.cpp_src_map = {} + self.python_map = {} + pylint_disable = ['superfluous-parens', + 'too-many-instance-attributes', + 'too-few-public-methods'] + # setup pylint + self.pylint_opts = ['--extension-pkg-whitelist=numpy', + '--disable=' + ','.join(pylint_disable)] + + self.pylint_cats = set(['error', 'warning', 'convention', 'refactor']) + # setup cpp lint + cpplint_args = ['.', '--extensions=' + (','.join(CXX_SUFFIX))] + _ = cpplint.ParseArguments(cpplint_args) + cpplint._SetFilters(','.join(['-build/c++11', + '-build/namespaces', + '-build/include', + '-build/header_guard', + '+build/include_what_you_use', + '+build/include_order'])) + cpplint._SetCountingStyle('toplevel') + cpplint._line_length = 100 + + def process_cpp(self, path, suffix): + """Process a cpp file.""" + _cpplint_state.ResetErrorCounts() + cpplint.ProcessFile(str(path), _cpplint_state.verbose_level) + _cpplint_state.PrintErrorCounts() + errors = _cpplint_state.errors_by_category.copy() + + if suffix == 'h': + self.cpp_header_map[str(path)] = errors + else: + self.cpp_src_map[str(path)] = errors + + def process_python(self, path): + """Process a python file.""" + (pylint_stdout, pylint_stderr) = epylint.py_run( + ' '.join([str(path)] + self.pylint_opts), return_std=True) + emap = {} + print pylint_stderr.read() + for line in pylint_stdout: + sys.stderr.write(line) + key = line.split(':')[-1].split('(')[0].strip() + if key not in self.pylint_cats: + continue + if key not in emap: + emap[key] = 1 + else: + emap[key] += 1 + sys.stderr.write('\n') + self.python_map[str(path)] = emap + + def print_summary(self, strm): + """Print summary of lint.""" + nerr = 0 + nerr += LintHelper._print_summary_map(strm, self.cpp_header_map, 'cpp-header') + nerr += LintHelper._print_summary_map(strm, self.cpp_src_map, 'cpp-soruce') + nerr += LintHelper._print_summary_map(strm, self.python_map, 'python') + if nerr == 0: + strm.write('All passed!\n') + else: + strm.write('%d files failed lint\n' % nerr) + return nerr + +# singleton helper for lint check +_HELPER = LintHelper() + +def get_header_guard_dmlc(filename): + """Get Header Guard Convention for DMLC Projects. + For headers in include, directly use the path + For headers in src, use project name plus path + Examples: with project-name = dmlc + include/dmlc/timer.h -> DMLC_TIMTER_H_ + src/io/libsvm_parser.h -> DMLC_IO_LIBSVM_PARSER_H_ + """ + fileinfo = cpplint.FileInfo(filename) + file_path_from_root = fileinfo.RepositoryName() + inc_list = ['include', 'api', 'wrapper'] + + if file_path_from_root.find('src/') != -1 and _HELPER.project_name is not None: + idx = file_path_from_root.find('src/') + file_path_from_root = _HELPER.project_name + file_path_from_root[idx + 3:] + else: + for spath in inc_list: + prefix = spath + os.sep + if file_path_from_root.startswith(prefix): + file_path_from_root = re.sub('^' + prefix, '', file_path_from_root) + break + return re.sub(r'[-./\s]', '_', file_path_from_root).upper() + '_' + +cpplint.GetHeaderGuardCPPVariable = get_header_guard_dmlc + +def process(fname, allow_type): + """Process a file.""" + fname = str(fname) + # HACK: ignore op.h which is automatically generated + if fname.endswith('op.h'): + return + arr = fname.rsplit('.', 1) + if fname.find('#') != -1 or arr[-1] not in allow_type: + return + if arr[-1] in CXX_SUFFIX: + _HELPER.process_cpp(fname, arr[-1]) + if arr[-1] in PYTHON_SUFFIX: + _HELPER.process_python(fname) + +def main(): + """Main entry function.""" + if len(sys.argv) < 3: + print('Usage: ') + print('\tfiletype can be python/cpp/all') + exit(-1) + _HELPER.project_name = sys.argv[1] + file_type = sys.argv[2] + allow_type = [] + if file_type == 'python' or file_type == 'all': + allow_type += [x for x in PYTHON_SUFFIX] + if file_type == 'cpp' or file_type == 'all': + allow_type += [x for x in CXX_SUFFIX] + allow_type = set(allow_type) + if os.name != 'nt': + sys.stderr = codecs.StreamReaderWriter(sys.stderr, + codecs.getreader('utf8'), + codecs.getwriter('utf8'), + 'replace') + for path in sys.argv[3:]: + if os.path.isfile(path): + process(path, allow_type) + else: + for root, dirs, files in os.walk(path): + for name in files: + process(os.path.join(root, name), allow_type) + + nerr = _HELPER.print_summary(sys.stderr) + sys.exit(nerr > 0) + +if __name__ == '__main__': + main() diff --git a/cpp-package/src/OpWrapperGenerator/OpWrapperGenerator.py b/cpp-package/src/OpWrapperGenerator/OpWrapperGenerator.py new file mode 100644 index 000000000000..b1fce38d5738 --- /dev/null +++ b/cpp-package/src/OpWrapperGenerator/OpWrapperGenerator.py @@ -0,0 +1,401 @@ +# -*- coding: utf-8 -*- +from ctypes import * +from ctypes.util import find_library +import os +import logging +import platform +import re +import sys +import tempfile + +class EnumType: + name = '' + enumValues = [] + def __init__(self, typeName = 'ElementWiseOpType', \ + typeString = "{'avg', 'max', 'sum'}"): + self.name = typeName + if (typeString[0] == '{'): # is a enum type + isEnum = True + # parse enum + self.enumValues = typeString[typeString.find('{') + 1:typeString.find('}')].split(',') + for i in range(0, len(self.enumValues)): + self.enumValues[i] = self.enumValues[i].strip().strip("'") + else: + logging.warn("trying to parse none-enum type as enum: %s" % typeString) + def GetDefinitionString(self, indent = 0): + indentStr = ' ' * indent + ret = indentStr + 'enum class %s {\n' % self.name + for i in range(0, len(self.enumValues)): + ret = ret + indentStr + ' %s = %d' % (self.enumValues[i], i) + if (i != len(self.enumValues) -1): + ret = ret + "," + ret = ret + "\n" + ret = ret + "};\n" + return ret + def GetDefaultValueString(self, value = ''): + return self.name + "::" + value + def GetEnumStringArray(self, indent = 0): + indentStr = ' ' * indent + ret = indentStr + 'static const char *%sValues[] = {\n' % self.name + for i in range(0, len(self.enumValues)): + ret = ret + indentStr + ' "%s"' % self.enumValues[i] + if (i != len(self.enumValues) -1): + ret = ret + "," + ret = ret + "\n" + ret = ret + indentStr + "};\n" + return ret + def GetConvertEnumVariableToString(self, variable=''): + return "%sValues[int(%s)]" % (self.name, variable) + + +class Arg: + typeDict = {'boolean':'bool',\ + 'Shape(tuple)':'Shape',\ + 'Symbol':'Symbol',\ + 'NDArray':'Symbol',\ + 'ndarray-or-symbol':'Symbol',\ + 'Symbol[]':'const std::vector&',\ + 'Symbol or Symbol[]':'const std::vector&',\ + 'NDArray[]':'const std::vector&',\ + 'ndarray-or-symbol[]':'const std::vector&',\ + 'caffe-layer-parameter':'::caffe::LayerParameter',\ + 'float':'mx_float',\ + 'real_t':'mx_float',\ + 'int':'int',\ + 'int (non-negative)': 'uint32_t',\ + 'long (non-negative)': 'uint64_t',\ + 'int or None':'dmlc::optional',\ + 'long':'int64_t',\ + 'double':'double',\ + 'string':'const std::string&'} + name = '' + type = '' + description = '' + isEnum = False + enum = None + hasDefault = False + defaultString = '' + def __init__(self, opName = '', argName = '', typeString = '', descString = ''): + self.name = argName + self.description = descString + if (typeString[0] == '{'): # is enum type + self.isEnum = True + self.enum = EnumType(self.ConstructEnumTypeName(opName, argName), typeString) + self.type = self.enum.name + else: + try: + self.type = self.typeDict[typeString.split(',')[0]] + except: + print 'argument "%s" of operator "%s" has unknown type "%s"' % (argName, opName, typeString) + pass + if typeString.find('default=') != -1: + self.hasDefault = True + self.defaultString = typeString.split('default=')[1].strip().strip("'") + if typeString.startswith('string'): + self.defaultString = self.MakeCString(self.defaultString) + elif self.isEnum: + self.defaultString = self.enum.GetDefaultValueString(self.defaultString) + elif self.defaultString == 'None': + self.defaultString = self.type + '()' + elif self.defaultString == 'False': + self.defaultString = 'false' + elif self.defaultString == 'True': + self.defaultString = 'true' + elif self.defaultString[0] == '(': + self.defaultString = 'Shape' + self.defaultString + elif self.type == 'dmlc::optional': + self.defaultString = self.type + '(' + self.defaultString + ')' + elif typeString.startswith('caffe-layer-parameter'): + self.defaultString = 'textToCaffeLayerParameter(' + self.MakeCString(self.defaultString) + ')' + hasCaffe = True + + def MakeCString(self, str): + str = str.replace('\n', "\\n") + str = str.replace('\t', "\\t") + return '\"' + str + '\"' + + def ConstructEnumTypeName(self, opName = '', argName = ''): + a = opName[0].upper() + # format ArgName so instead of act_type it returns ActType + argNameWords = argName.split('_') + argName = '' + for an in argNameWords: + argName = argName + an[0].upper() + an[1:] + typeName = a + opName[1:] + argName + return typeName + +class Op: + name = '' + description = '' + args = [] + + def __init__(self, name = '', description = '', args = []): + self.name = name + self.description = description + # add a 'name' argument + nameArg = Arg(self.name, \ + 'symbol_name', \ + 'string', \ + 'name of the resulting symbol') + args.insert(0, nameArg) + # reorder arguments, put those with default value to the end + orderedArgs = [] + for arg in args: + if not arg.hasDefault: + orderedArgs.append(arg) + for arg in args: + if arg.hasDefault: + orderedArgs.append(arg) + self.args = orderedArgs + + def WrapDescription(self, desc = ''): + ret = [] + sentences = desc.split('.') + lines = desc.split('\n') + for line in lines: + line = line.strip() + if len(line) <= 80: + ret.append(line.strip()) + else: + while len(line) > 80: + pos = line.rfind(' ', 0, 80)+1 + if pos <= 0: + pos = line.find(' ') + if pos < 0: + pos = len(line) + ret.append(line[:pos].strip()) + line = line[pos:] + return ret + + def GenDescription(self, desc = '', \ + firstLineHead = ' * \\brief ', \ + otherLineHead = ' * '): + ret = '' + descs = self.WrapDescription(desc) + ret = ret + firstLineHead + if len(descs) == 0: + return ret.rstrip() + ret = (ret + descs[0]).rstrip() + '\n' + for i in range(1, len(descs)): + ret = ret + (otherLineHead + descs[i]).rstrip() + '\n' + return ret + + def GetOpDefinitionString(self, use_name, indent=0): + ret = '' + indentStr = ' ' * indent + # define enums if any + for arg in self.args: + if arg.isEnum and use_name: + # comments + ret = ret + self.GenDescription(arg.description, \ + '/*! \\breif ', \ + ' * ') + ret = ret + " */\n" + # definition + ret = ret + arg.enum.GetDefinitionString(indent) + '\n' + # create function comments + ret = ret + self.GenDescription(self.description, \ + '/*!\n * \\breif ', \ + ' * ') + for arg in self.args: + if arg.name != 'symbol_name' or use_name: + ret = ret + self.GenDescription(arg.name + ' ' + arg.description, \ + ' * \\param ', \ + ' * ') + ret = ret + " * \\return new symbol\n" + ret = ret + " */\n" + # create function header + declFirstLine = indentStr + 'inline Symbol %s(' % self.name + ret = ret + declFirstLine + argIndentStr = ' ' * len(declFirstLine) + arg_start = 0 if use_name else 1 + if len(self.args) > arg_start: + ret = ret + self.GetArgString(self.args[arg_start]) + for i in range(arg_start+1, len(self.args)): + ret = ret + ',\n' + ret = ret + argIndentStr + self.GetArgString(self.args[i]) + ret = ret + ') {\n' + # create function body + # if there is enum, generate static enum<->string mapping + for arg in self.args: + if arg.isEnum: + ret = ret + arg.enum.GetEnumStringArray(indent + 2) + # now generate code + ret = ret + indentStr + ' return Operator(\"%s\")\n' % self.name + for arg in self.args: # set params + if arg.type == 'Symbol' or \ + arg.type == 'const std::string&' or \ + arg.type == 'const std::vector&': + continue + v = arg.name + if arg.isEnum: + v = arg.enum.GetConvertEnumVariableToString(v) + ret = ret + indentStr + ' ' * 11 + \ + '.SetParam(\"%s\", %s)\n' % (arg.name, v) + #ret = ret[:-1] # get rid of the last \n + symbols = '' + inputAlreadySet = False + for arg in self.args: # set inputs + if arg.type != 'Symbol': + continue + inputAlreadySet = True + #if symbols != '': + # symbols = symbols + ', ' + #symbols = symbols + arg.name + ret = ret + indentStr + ' ' * 11 + \ + '.SetInput(\"%s\", %s)\n' % (arg.name, arg.name) + for arg in self.args: # set input arrays vector + if arg.type != 'const std::vector&': + continue + if (inputAlreadySet): + logging.error("op %s has both Symbol[] and Symbol inputs!" % self.name) + inputAlreadySet = True + symbols = arg.name + ret = ret + '(%s)\n' % symbols + ret = ret + indentStr + ' ' * 11 + if use_name: + ret = ret + '.CreateSymbol(symbol_name);\n' + else: + ret = ret + '.CreateSymbol();\n' + ret = ret + indentStr + '}\n' + return ret + + def GetArgString(self, arg): + ret = '%s %s' % (arg.type, arg.name) + if arg.hasDefault: + ret = ret + ' = ' + arg.defaultString + return ret + + +def ParseAllOps(): + """ + MXNET_DLL int MXSymbolListAtomicSymbolCreators(mx_uint *out_size, + AtomicSymbolCreator **out_array); + + MXNET_DLL int MXSymbolGetAtomicSymbolInfo(AtomicSymbolCreator creator, + const char **name, + const char **description, + mx_uint *num_args, + const char ***arg_names, + const char ***arg_type_infos, + const char ***arg_descriptions, + const char **key_var_num_args); + """ + cdll.libmxnet = cdll.LoadLibrary(sys.argv[1]) + ListOP = cdll.libmxnet.MXSymbolListAtomicSymbolCreators + GetOpInfo = cdll.libmxnet.MXSymbolGetAtomicSymbolInfo + ListOP.argtypes=[POINTER(c_int), POINTER(POINTER(c_void_p))] + GetOpInfo.argtypes=[c_void_p, \ + POINTER(c_char_p), \ + POINTER(c_char_p), \ + POINTER(c_int), \ + POINTER(POINTER(c_char_p)), \ + POINTER(POINTER(c_char_p)), \ + POINTER(POINTER(c_char_p)), \ + POINTER(c_char_p), \ + POINTER(c_char_p) + ] + + nOps = c_int() + opHandlers = POINTER(c_void_p)() + r = ListOP(byref(nOps), byref(opHandlers)) + ret = '' + ret2 = '' + for i in range(0, nOps.value): + handler = opHandlers[i] + name = c_char_p() + description = c_char_p() + nArgs = c_int() + argNames = POINTER(c_char_p)() + argTypes = POINTER(c_char_p)() + argDescs = POINTER(c_char_p)() + varArgName = c_char_p() + return_type = c_char_p() + + GetOpInfo(handler, byref(name), byref(description), \ + byref(nArgs), byref(argNames), byref(argTypes), \ + byref(argDescs), byref(varArgName), byref(return_type)) + + if name.value[0]=='_': # get rid of functions like __init__ + continue + + args = [] + + for i in range(0, nArgs.value): + arg = Arg(name.value, + argNames[i], + argTypes[i], + argDescs[i]) + args.append(arg) + + op = Op(name.value, description.value, args) + + ret = ret + op.GetOpDefinitionString(True) + "\n" + ret2 = ret2 + op.GetOpDefinitionString(False) + "\n" + return ret + ret2 + +if __name__ == "__main__": + #et = EnumType(typeName = 'MyET') + reload(sys) + sys.setdefaultencoding('UTF8') + #print(et.GetDefinitionString()) + #print(et.GetEnumStringArray()) + #arg = Arg() + #print(arg.ConstructEnumTypeName('SoftmaxActivation', 'act_type')) + #arg = Arg(opName = 'FullConnected', argName='act_type', \ + # typeString="{'elu', 'leaky', 'prelu', 'rrelu'},optional, default='leaky'", \ + # descString='Activation function to be applied.') + #print(arg.isEnum) + #print(arg.defaultString) + #arg = Arg("fc", "alpha", "float, optional, default=0.0001", "alpha") + #decl = "%s %s" % (arg.type, arg.name) + #if arg.hasDefault: + # decl = decl + "=" + arg.defaultString + #print(decl) + + temp_file_name = "" + output_file = '../../include/mxnet-cpp/op.h' + try: + # generate file header + patternStr = ("/*!\n" + "* Copyright (c) 2016 by Contributors\n" + "* \\file op.h\n" + "* \\brief definition of all the operators\n" + "* \\author Chuntao Hong, Xin Li\n" + "*/\n" + "\n" + "#ifndef CPP_PACKAGE_INCLUDE_MXNET_CPP_OP_H_\n" + "#define CPP_PACKAGE_INCLUDE_MXNET_CPP_OP_H_\n" + "\n" + "#include \n" + "#include \n" + "#include \"mxnet-cpp/base.h\"\n" + "#include \"mxnet-cpp/shape.h\"\n" + "#include \"mxnet-cpp/op_util.h\"\n" + "#include \"mxnet-cpp/operator.h\"\n" + "#include \"dmlc/optional.h\"\n" + "\n" + "namespace mxnet {\n" + "namespace cpp {\n" + "\n" + "%s" + "} //namespace cpp\n" + "} //namespace mxnet\n" + "#endif // CPP_PACKAGE_INCLUDE_MXNET_CPP_OP_H_\n") + + # Generate a temporary file name + tf = tempfile.NamedTemporaryFile() + temp_file_name = tf.name + tf.close() + with open(temp_file_name, 'w') as f: + f.write(patternStr % ParseAllOps()) + + except Exception, e: + os.remove(output_file) + if len(temp_file_name) > 0: + os.remove(temp_file_name) + raise(e) + + os.system('./move-if-change.sh ' + temp_file_name + ' ' + output_file) + pass + diff --git a/cpp-package/src/OpWrapperGenerator/OpWrapperGenerator.pyproj b/cpp-package/src/OpWrapperGenerator/OpWrapperGenerator.pyproj new file mode 100644 index 000000000000..b2d8448b830d --- /dev/null +++ b/cpp-package/src/OpWrapperGenerator/OpWrapperGenerator.pyproj @@ -0,0 +1,28 @@ + + + + Debug + 2.0 + {027054bd-8dd3-4d2e-8032-22e339846ed1} + + OpWrapperGenerator.py + + . + . + {888888a0-9f3d-457c-b088-3a5042f75d52} + Standard Python launcher + + + + + + + 10.0 + $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)\Python Tools\Microsoft.PythonTools.targets + + + + + + + \ No newline at end of file diff --git a/cpp-package/src/OpWrapperGenerator/OpWrapperGenerator.sln b/cpp-package/src/OpWrapperGenerator/OpWrapperGenerator.sln new file mode 100644 index 000000000000..71dc32749769 --- /dev/null +++ b/cpp-package/src/OpWrapperGenerator/OpWrapperGenerator.sln @@ -0,0 +1,20 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 2013 +VisualStudioVersion = 12.0.40629.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{888888A0-9F3D-457C-B088-3A5042F75D52}") = "OpWrapperGenerator", "OpWrapperGenerator.pyproj", "{027054BD-8DD3-4D2E-8032-22E339846ED1}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {027054BD-8DD3-4D2E-8032-22E339846ED1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {027054BD-8DD3-4D2E-8032-22E339846ED1}.Release|Any CPU.ActiveCfg = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/cpp-package/src/OpWrapperGenerator/README.md b/cpp-package/src/OpWrapperGenerator/README.md new file mode 100644 index 000000000000..8fb45ec661f2 --- /dev/null +++ b/cpp-package/src/OpWrapperGenerator/README.md @@ -0,0 +1 @@ +## This is a python script that generates operator wrappers such as FullyConnected, based on current libmxnet.dll. This script is written so that we don't need to write new operator wrappers when new ones are added to the library. diff --git a/cpp-package/src/OpWrapperGenerator/move-if-change.sh b/cpp-package/src/OpWrapperGenerator/move-if-change.sh new file mode 100755 index 000000000000..c475fae5a847 --- /dev/null +++ b/cpp-package/src/OpWrapperGenerator/move-if-change.sh @@ -0,0 +1,18 @@ +#!/bin/sh +if [ -z "$1" ] || [ -z "$2" ]; then + echo "Usage: $0 " +fi + +if [ ! -f "$2" ]; then + mv -v "$1" "$2" + exit 0 +fi + +diff "$1" "$2" >/dev/null + +if [ $? -ne 0 ]; then + mv -v "$1" "$2" +else + rm -f "$1" +fi + diff --git a/cpp-package/tests/travis/run_test.sh b/cpp-package/tests/travis/run_test.sh new file mode 100755 index 000000000000..27506584f40c --- /dev/null +++ b/cpp-package/tests/travis/run_test.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +if [ ${TASK} == "lint" ]; then + make lint || exit -1 + echo "Check documentations of c++ code..." + make doc 2>log.txt + (cat log.txt| grep -v ENABLE_PREPROCESSING |grep -v "unsupported tag") > logclean.txt + echo "---------Error Log----------" + cat logclean.txt + echo "----------------------------" + (cat logclean.txt|grep warning) && exit -1 + (cat logclean.txt|grep error) && exit -1 + exit 0 +fi + +if [ ${TRAVIS_OS_NAME} == "linux" ]; then + # use g++-4.8 in linux + export CXX=g++-4.8 +fi + +if [ ${TASK} == "build" ]; then + make + exit $? +fi diff --git a/cpp-package/tests/travis/setup.sh b/cpp-package/tests/travis/setup.sh new file mode 100755 index 000000000000..4238c7654fe4 --- /dev/null +++ b/cpp-package/tests/travis/setup.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +if [ ${TASK} == "lint" ]; then + pip install cpplint 'pylint==1.4.4' 'astroid==1.3.6' --user +fi diff --git a/dmlc-core b/dmlc-core index a79b0df25c42..b5bec5481df8 160000 --- a/dmlc-core +++ b/dmlc-core @@ -1 +1 @@ -Subproject commit a79b0df25c42b9612dcdfd950d91ce0928a394cd +Subproject commit b5bec5481df86e8e6728d8bd80a61d87ef3b2cd5 diff --git a/docker/.gitignore b/docker/.gitignore new file mode 100644 index 000000000000..2a377effe731 --- /dev/null +++ b/docker/.gitignore @@ -0,0 +1,2 @@ +Dockerfile.* +!Dockerfile.in.* diff --git a/docker/Dockerfiles/Dockerfile.in.julia b/docker/Dockerfiles/Dockerfile.in.julia new file mode 100644 index 000000000000..42422ddbed54 --- /dev/null +++ b/docker/Dockerfiles/Dockerfile.in.julia @@ -0,0 +1,7 @@ +# -*- mode: dockerfile -*- +# part of the dockerfile to install the julia binding + +COPY install/julia.sh install/ +RUN install/julia.sh +ENV MXNET_HOME /mxnet +RUN julia -e 'Pkg.add("MXNet")' diff --git a/docker/Dockerfiles/Dockerfile.in.lib.cpu b/docker/Dockerfiles/Dockerfile.in.lib.cpu new file mode 100644 index 000000000000..002e2d1e4209 --- /dev/null +++ b/docker/Dockerfiles/Dockerfile.in.lib.cpu @@ -0,0 +1,10 @@ +# -*- mode: dockerfile -*- +# dockerfile to build libmxnet.so on CPU +FROM ubuntu:14.04 + +COPY install/cpp.sh install/ +RUN install/cpp.sh + +RUN git clone --recursive https://github.com/dmlc/mxnet && cd mxnet && \ + make -j$(nproc) && \ + rm -r build diff --git a/docker/Dockerfiles/Dockerfile.in.lib.gpu b/docker/Dockerfiles/Dockerfile.in.lib.gpu new file mode 100644 index 000000000000..2185babf085c --- /dev/null +++ b/docker/Dockerfiles/Dockerfile.in.lib.gpu @@ -0,0 +1,9 @@ +# -*- mode: dockerfile -*- +# dockerfile to build libmxnet.so on GPU +FROM nvidia/cuda:8.0-cudnn5-devel + +COPY install/cpp.sh install/ +RUN install/cpp.sh + +RUN git clone --recursive https://github.com/dmlc/mxnet && cd mxnet && \ + make -j$(nproc) USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 diff --git a/docker/Dockerfiles/Dockerfile.in.python b/docker/Dockerfiles/Dockerfile.in.python new file mode 100644 index 000000000000..b7979b231d7d --- /dev/null +++ b/docker/Dockerfiles/Dockerfile.in.python @@ -0,0 +1,6 @@ +# -*- mode: dockerfile -*- +# part of the dockerfile to install the python binding + +COPY install/python.sh install/ +RUN install/python.sh +ENV PYTHONPATH=/mxnet/python diff --git a/docker/Dockerfiles/Dockerfile.in.r-lang b/docker/Dockerfiles/Dockerfile.in.r-lang new file mode 100644 index 000000000000..321094ec6c63 --- /dev/null +++ b/docker/Dockerfiles/Dockerfile.in.r-lang @@ -0,0 +1,7 @@ +# -*- mode: dockerfile -*- +# part of the dockerfile to install the r binding + +COPY install/r.sh install/ +ADD https://raw.githubusercontent.com/dmlc/mxnet/master/R-package/DESCRIPTION install/ +RUN install/r.sh +RUN cd mxnet && make rpkg && R CMD INSTALL mxnet_current_r.tar.gz diff --git a/docker/Dockerfiles/Dockerfile.in.scala b/docker/Dockerfiles/Dockerfile.in.scala new file mode 100644 index 000000000000..a45fbfdaaab6 --- /dev/null +++ b/docker/Dockerfiles/Dockerfile.in.scala @@ -0,0 +1,7 @@ +# -*- mode: dockerfile -*- +# part of the dockerfile to install the scala binding + +COPY install/scala.sh install/ +RUN install/scala.sh + +RUN cd mxnet && make scalapkg diff --git a/docker/README.md b/docker/README.md index 5b2897ade43f..95fa668e97d7 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,58 +1,113 @@ # Docker images for MXNET -Pre-built docker images are available at https://hub.docker.com/r/dmlc/mxnet/ - ## How to use -1. First pull the pre-built image +First make sure [docker](https://docs.docker.com/engine/installation/) is +installed. The docker plugin +[nvidia-docker](https://github.com/NVIDIA/nvidia-docker) is required to run on +Nvidia GPUs. + +Pre-built docker containers are available at https://hub.docker.com/r/mxnet/ + +For example, the following command launches a container with the Python package +installed. It will pull the docker images from docker hub if it does not exist +locally. + +```bash +docker run -ti --rm mxnet/python +``` + +Then you can run MXNet in python, e.g.: + +```bash +# python -c 'import mxnet as mx; a = mx.nd.ones((2,3)); print((a*2).asnumpy())' +[[ 2. 2. 2.] + [ 2. 2. 2.]] +``` + +If the host machine has at least one GPU installed and `nvidia-docker` is installed, namely +`nvidia-docker run --rm nvidia/cuda nvidia-smi` runs successfully, then you can +run a container with GPU supports + +```bash +nvidia-docker run -ti --rm mxnet/python:gpu +``` + +Now you can run the above example in `GPU 0`: + +```bash +# python -c 'import mxnet as mx; a = mx.nd.ones((2,3), mx.gpu(0)); print((a*2).asnumpy())' +[[ 2. 2. 2.] + [ 2. 2. 2.]] +``` + +## Hosted containers + +All images are based on Ubuntu 14.04. The `gpu` tag is built with CUDA 8.0 and +cuDNN 5. + +### Python - ```bash - docker pull dmlc/mxnet - ``` -2. Then we can run the python shell in the docker +Hosted at https://hub.docker.com/r/mxnet/python/ - ```bash - docker run -ti dmlc/mxnet python - ``` - For example - ```bash - $ docker run -ti dmlc/mxnet python - Python 2.7.6 (default, Jun 22 2015, 17:58:13) - [GCC 4.8.2] on linux2 - Type "help", "copyright", "credits" or "license" for more information. - >>> import mxnet as mx - import mxnet as mx - >>> quit() - quit() - ``` +Python versions: 2.7.12 and 3.5.2. - Note: One may get the error message `libdc1394 error: Failed to initialize - libdc1394`, which is due to opencv and can be ignored. +Available tags: -3. Train a model on MNIST to check everything works +- mxnet/python +- mxnet/python:gpu - ``` - docker run dmlc/mxnet python /mxnet/example/image-classification/train_mnist.py - ``` +### R -If the host machine has Nvidia GPUs, we can use `dmlc/mxnet:cuda`, which has both CUDA and CUDNN installed. -To launch the docker, we need to install [nvidia-docker](https://github.com/NVIDIA/nvidia-docker) first. +Hosted at https://hub.docker.com/r/mxnet/r-lang/ -1. Pull the image +R version: 3.3.3 - ```bash - docker pull dmlc/mxnet:cuda - ``` +Available tags: -2. Train MNIST on GPU 0 +- mxnet/r-lang +- mxnet/r-lang:gpu - ```bash - nvidia-docker run dmlc/mxnet:cuda python /mxnet/example/image-classification/train_mnist.py --gpus 0 - ``` + +### Julia + +Hosted at https://hub.docker.com/r/mxnet/julia/ + +Julia version: 0.5.1 + +Available tags: + +- mxnet/julia +- mxnet/julia:gpu + +#### Scala + +Hosted at https://hub.docker.com/r/mxnet/scala/ + +Scala version: 2.11.8 + +Available tags: + +- mxnet/scala ## How to build +The following command build the default Python package + +```bash +./tool.sh build python cpu +``` + +Run `./tool.sh` for more details. Use + + +Tips: The following commands stop all docker containers and delete all docker images. + +```bash +docker stop $(docker ps -a -q) +docker rm $(docker ps -a -q) +``` + ```bash -docker build -t dmlc/mxnet:cpu cpu -docker build -t dmlc/mxnet:cuda cuda +docker rmi $(docker images -a -q) ``` diff --git a/docker/cpu/Dockerfile b/docker/cpu/Dockerfile deleted file mode 100644 index 1e5a956450dc..000000000000 --- a/docker/cpu/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -FROM ubuntu:14.04 -MAINTAINER Mu Li - -# install the core library -RUN apt-get update && apt-get install -y build-essential git libopenblas-dev libopencv-dev -RUN git clone --recursive https://github.com/dmlc/mxnet/ && cd mxnet && \ - cp make/config.mk . && \ - echo "USE_BLAS=openblas" >>config.mk && \ - make -j$(nproc) - -# python pakcage -RUN apt-get install -y python-numpy wget unzip -ENV PYTHONPATH /mxnet/python diff --git a/docker/cuda/7.5/Dockerfile b/docker/cuda/7.5/Dockerfile deleted file mode 100644 index ff0b0bbb2cd6..000000000000 --- a/docker/cuda/7.5/Dockerfile +++ /dev/null @@ -1,25 +0,0 @@ -FROM nvidia/cuda:7.5-cudnn5-devel -MAINTAINER Qingsong Liu - -RUN apt-get update && apt-get install -y \ - git \ - libopenblas-dev \ - libopencv-dev \ - python-dev \ - python-numpy \ - python-setuptools \ - wget \ - python-pip \ - unzip - -RUN cd /root && git clone --recursive https://github.com/dmlc/mxnet && cd mxnet && \ - cp make/config.mk . && \ - sed -i 's/USE_BLAS = atlas/USE_BLAS = openblas/g' config.mk && \ - sed -i 's/USE_CUDA = 0/USE_CUDA = 1/g' config.mk && \ - sed -i 's/USE_CUDA_PATH = NONE/USE_CUDA_PATH = \/usr\/local\/cuda/g' config.mk && \ - sed -i 's/USE_CUDNN = 0/USE_CUDNN = 1/g' config.mk && \ - make -j"$(nproc)" - -ENV PYTHONPATH /root/mxnet/python - -WORKDIR /root/mxnet diff --git a/docker/cuda/8.0/Dockerfile b/docker/cuda/8.0/Dockerfile deleted file mode 100644 index c67375576f03..000000000000 --- a/docker/cuda/8.0/Dockerfile +++ /dev/null @@ -1,25 +0,0 @@ -FROM nvidia/cuda:8.0-cudnn5-devel -MAINTAINER Qingsong Liu - -RUN apt-get update && apt-get install -y \ - git \ - libopenblas-dev \ - libopencv-dev \ - python-dev \ - python-numpy \ - python-setuptools \ - wget \ - python-pip \ - unzip - -RUN cd /root && git clone --recursive https://github.com/dmlc/mxnet && cd mxnet && \ - cp make/config.mk . && \ - sed -i 's/USE_BLAS = atlas/USE_BLAS = openblas/g' config.mk && \ - sed -i 's/USE_CUDA = 0/USE_CUDA = 1/g' config.mk && \ - sed -i 's/USE_CUDA_PATH = NONE/USE_CUDA_PATH = \/usr\/local\/cuda/g' config.mk && \ - sed -i 's/USE_CUDNN = 0/USE_CUDNN = 1/g' config.mk && \ - make -j"$(nproc)" - -ENV PYTHONPATH /root/mxnet/python - -WORKDIR /root/mxnet diff --git a/docker/install/cpp.sh b/docker/install/cpp.sh new file mode 100755 index 000000000000..91b8b8db0607 --- /dev/null +++ b/docker/install/cpp.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# libraries for building mxnet c++ core on ubuntu + +apt-get update && apt-get install -y \ + build-essential git libatlas-base-dev libopencv-dev \ + libcurl4-openssl-dev libgtest-dev cmake wget unzip + +cd /usr/src/gtest && cmake CMakeLists.txt && make && cp *.a /usr/lib diff --git a/docker/install/julia.sh b/docker/install/julia.sh new file mode 100755 index 000000000000..604a1bc2c234 --- /dev/null +++ b/docker/install/julia.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# install libraries for mxnet's julia package on ubuntu + +# the julia version shipped with ubuntu (version 0.4) is too low. so download a +# new version +# apt-get install -y julia + +wget -q https://julialang.s3.amazonaws.com/bin/linux/x64/0.5/julia-0.5.1-linux-x86_64.tar.gz +tar -zxf julia-0.5.1-linux-x86_64.tar.gz +rm julia-0.5.1-linux-x86_64.tar.gz +ln -s $(pwd)/julia-6445c82d00/bin/julia /usr/bin/julia diff --git a/docker/install/python.sh b/docker/install/python.sh new file mode 100755 index 000000000000..0459bb9198c4 --- /dev/null +++ b/docker/install/python.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# install libraries for mxnet's python package on ubuntu + +apt-get update && apt-get install -y python-dev python3-dev + +# the version of the pip shipped with ubuntu may be too lower, install a recent version here +cd /tmp && wget https://bootstrap.pypa.io/get-pip.py && python3 get-pip.py && python2 get-pip.py + +pip2 install nose pylint numpy nose-timer requests +pip3 install nose pylint numpy nose-timer requests diff --git a/docker/install/r.sh b/docker/install/r.sh new file mode 100755 index 000000000000..9351763ddcee --- /dev/null +++ b/docker/install/r.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# install libraries for mxnet's r package on ubuntu + +echo "deb http://cran.rstudio.com/bin/linux/ubuntu trusty/" >> /etc/apt/sources.list +gpg --keyserver keyserver.ubuntu.com --recv-key E084DAB9 +gpg -a --export E084DAB9 | apt-key add - + +apt-get update +apt-get install -y r-base r-base-dev libxml2-dev libxt-dev libssl-dev + +cd "$(dirname "${BASH_SOURCE[0]}")" + +if [ ! -f "./DESCRIPTION" ]; then + cp ../../R-package/DESCRIPTION . +fi + +Rscript -e "install.packages('devtools', repo = 'https://cran.rstudio.com')" +Rscript -e "library(devtools); library(methods); options(repos=c(CRAN='https://cran.rstudio.com')); install_deps(dependencies = TRUE)" diff --git a/docker/install/scala.sh b/docker/install/scala.sh new file mode 100755 index 000000000000..8cbe91199463 --- /dev/null +++ b/docker/install/scala.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# install libraries for mxnet's scala package on ubuntu + +apt-get install -y maven default-jdk + +wget http://downloads.lightbend.com/scala/2.11.8/scala-2.11.8.deb +dpkg -i scala-2.11.8.deb +rm scala-2.11.8.deb diff --git a/docker/run.sh b/docker/run.sh new file mode 100644 index 000000000000..0037ab1926d7 --- /dev/null +++ b/docker/run.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Build and push all docker containers + +DEVICES=('cpu' 'gpu') +LANGUAGES=('python' 'julia' 'r-lang' 'scala') +for DEV in "${DEVICES[@]}"; do + for LANG in "${LANGUAGES[@]}"; do + ./tool.sh build ${LANG} ${DEV} + ./tool.sh push ${LANG} ${DEV} + done +done diff --git a/docker/tool.sh b/docker/tool.sh new file mode 100755 index 000000000000..31a98822350d --- /dev/null +++ b/docker/tool.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# +# Script to build, test and push a docker container +# +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +function show_usage() { + echo "" + echo "Usage: $(basename $0) COMMAND DEVICE LANGUAGE " + echo "" + echo " COMMAND: build or commit." + echo " commit needs logined in docker hub" + echo " DEVICE: targed device, e.g. cpu, or gpu" + echo " LANGUAGE: the language binding to buld, e.g. python, r-lang, julia, or scala" + echo "" +} + +if (( $# < 3 )); then + show_usage + exit -1 +fi + +COMMAND=$( echo "$1" | tr '[:upper:]' '[:lower:]' ) +shift 1 +LANGUAGE=$( echo "$1" | tr '[:upper:]' '[:lower:]' ) +shift 1 +DEVICE=$( echo "$1" | tr '[:upper:]' '[:lower:]' ) +shift 1 + +DOCKERFILE_LIB="${SCRIPT_DIR}/Dockerfiles/Dockerfile.in.lib.${DEVICE}" +if [ ! -e ${DOCKERFILE_LIB} ]; then + echo "Error DEVICE=${DEVICE}, failed to find ${DOCKERFILE_LIB}" + show_usage + exit 1 +fi + +DOCKERFILE_LANG="${SCRIPT_DIR}/Dockerfiles/Dockerfile.in.${LANGUAGE}" +if [ ! -e ${DOCKERFILE_LANG} ]; then + echo "Error LANGUAGE=${LANGUAGE}, failed to find ${DOCKERFILE_LANG}" + show_usage + exit 1 +fi + +if [[ "${DEVICE}" == *"gpu"* ]] && [[ "{COMMAND}" == "test" ]]; then + DOCKER_BINARY="nvidia-docker" +else + DOCKER_BINARY="docker" +fi + +DOCKER_TAG="mxnet/${LANGUAGE}" +if [ "${DEVICE}" != 'cpu' ]; then + DOCKER_TAG="${DOCKER_TAG}:${DEVICE}" +fi +DOCKERFILE="Dockerfile.${LANGUAGE}.${DEVICE}" + +# print arguments +echo "DOCKER_BINARY: ${DOCKER_BINARY}" +echo "DOCKERFILE: ${DOCKERFILE}" +echo "DOCKER_TAG: ${DOCKER_TAG}" + +if [[ "${COMMAND}" == "build" ]]; then + rm -rf ${DOCKERFILE} + cp ${DOCKERFILE_LIB} ${DOCKERFILE} + cat ${DOCKERFILE_LANG} >>${DOCKERFILE} + # To remove the following error caused by opencv + # libdc1394 error: Failed to initialize libdc1394" + CMD="sh -c 'ln -s /dev/null /dev/raw1394';" + # setup scala classpath + if [[ "${LANGUAGE}" == "scala" ]]; then + CMD+="CLASSPATH=\${CLASSPATH}:\`ls /mxnet/scala-package/assembly/linux-x86_64-*/target/*.jar | paste -sd \":\"\` " + fi + echo "CMD ${CMD} bash" >>${DOCKERFILE} + ${DOCKER_BINARY} build -t ${DOCKER_TAG} -f ${DOCKERFILE} . +elif [[ "${COMMAND}" == "push" ]]; then + ${DOCKER_BINARY} push ${DOCKER_TAG} +else + echo "Unknow COMMAND=${COMMAND}" + show_usage + exit 1 +fi diff --git a/docs/README.md b/docs/README.md index 173b9a4b1de1..8e88358c8b05 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,50 +1,26 @@ # MXNet documentation -A built version of document is available at http://mxnet.io +MXNet's documents can be built by running `make html` in this folder. -## To build the docs with Docker +A built version of document is available at http://mxnet.io -The `Dockerfile` in this directory encapsulates all the dependencies needed -to build the docs. The default entry-point builds the docs and serves them -through a simple HTTP server for previewing. +To build the documents locally, the easiest way is by using `docker`. First make +sure [docker](docker.com) is installed. Then use the following commands to clone and +build MXNet's documents (not including jupyter notebooks and API documents +execept for Python): +```bash +git clone --recursive https://github.com/dmlc/mxnet +cd mxnet +tests/ci_build/ci_build.sh doc DEV=1 make -C docs/ html ``` -docker build -t mxnet/docs . -docker run -it -p 8008:8008 mxnet/docs -open http://localhost:8008/ -``` - -### Faster iterative development -If you are working on the docs and want to rebuild them without creating a new -docker image each time, you can do this with - -``` -docker run -it -p 8008:8008 -v `pwd`:/opt/mxnet/docs mxnet/docs -``` +The built documents will be available at `docs/_build/html/`. -which maps your current directory into the docker image to get any local -changes. - -**NOTE:** Any changes to the API reference will not get rebuilt this way. -The API reference docs are introspected from the built binaries, which -in this Dockerfile are pulled from github/dmlc/master. To work-around -this, map a volume with your code changes into the container, and rebuild -MXNet in the container before doing the doc build. Or use the local -build described below. - -## Local build - -To build the documentation without docker on your local machine, first -install the required packages for Ubuntu 14.04. These are approximately: - -``` -sudo apt-get install doxygen python-pip -sudo pip install sphinx==1.3.5 CommonMark==0.5.4 breathe mock==1.0.1 recommonmark -``` +Note: -(Refer to the Dockerfile for a more reliable description of the dependencies.) -Once the MXNet binaries are built, and you have the dependencies installed, -you can build the docs with: +- If C++ codes have been changed, we suggest to remove the previous results before + building, namely run `rm -rf docs/_build/html/`. -```make html``` +- If CSS or javascript are changed, we often need to do a *force refresh* in the + browser to clear the cache. diff --git a/docs/_static/js/options.js b/docs/_static/js/options.js new file mode 100644 index 000000000000..77ef94074c57 --- /dev/null +++ b/docs/_static/js/options.js @@ -0,0 +1,23 @@ +$(document).ready(function () { + function label(lbl) { + return lbl.replace(/[ .]/g, '-').toLowerCase(); + } + function showContent() { + $('.opt-group .opt').each(function(){ + $('.'+label($(this).text())).hide(); + $('.highlight-'+label($(this).text())).hide(); + }); + $('.opt-group .active').each(function(){ + $('.'+label($(this).text())).show(); + $('.highlight-'+label($(this).text())).show(); + }); + } + showContent(); + function setContent() { + var el = $(this); + el.siblings().removeClass('active'); + el.addClass('active'); + showContent(); + } + $('.opt-group').on('click', '.opt', setContent); +}); diff --git a/docs/_static/mxnet-theme/index.html b/docs/_static/mxnet-theme/index.html index a9d5c5660d3b..04e00b4daef9 100644 --- a/docs/_static/mxnet-theme/index.html +++ b/docs/_static/mxnet-theme/index.html @@ -34,7 +34,7 @@

Portable

Multiple Languages

-

Supports multiple languages, including C++, Python, R, Scala, Julia, Matlab and Javascript - All with the same amazing performance.

+

Supports multiple languages, including C++, Python, R, Scala, Julia, Perl, Matlab and Javascript - All with the same amazing performance.

Auto-Differentiation

@@ -62,7 +62,7 @@

Performance

MXNet and sponsoring its major developers (alphabetical order).

- +
@@ -72,7 +72,7 @@

Performance

- +
@@ -86,17 +86,17 @@

Performance

- +
- +
- +
- +
@@ -106,15 +106,15 @@

Performance

- +
- +
- +
- +
diff --git a/docs/_static/mxnet-theme/layout.html b/docs/_static/mxnet-theme/layout.html index 85e7bdbbbe47..d4926b993cac 100644 --- a/docs/_static/mxnet-theme/layout.html +++ b/docs/_static/mxnet-theme/layout.html @@ -65,6 +65,19 @@ + + + @@ -92,7 +105,7 @@ {{ metatags }} {%- block htmltitle %} {%- if pagename != 'index' %} - + {{ title|striptags|e }}{{ titlesuffix }} {%- else %} MXNet Documents diff --git a/docs/_static/mxnet-theme/navbar.html b/docs/_static/mxnet-theme/navbar.html index f5b2cb7f0de8..eff7d0810a3e 100644 --- a/docs/_static/mxnet-theme/navbar.html +++ b/docs/_static/mxnet-theme/navbar.html @@ -21,7 +21,7 @@
  • {{name}}
      - {% for lang in ['Python', 'R', 'Julia', 'C++', 'Scala'] %} + {% for lang in ['Python', 'R', 'Julia', 'C++', 'Scala', 'Perl'] %}
    • {{lang}}
      • @@ -42,8 +42,6 @@ Previous Navbar Layout End -->
      -Fork me on GitHub